Context Navigation

source: trunk/kernel/mm/vmm.c @ 587

Last change on this file since 587 was 585, checked in by alain, 6 years ago
Introduce two separate vmm_handle_page_fault() & vmm_handle_cow() functions in vmm.c
File size: 72.8 KB

Rev	Line
[1]	1	/*
	2	* vmm.c - virtual memory manager related operations interface.
	3	*
	4	* Authors Ghassan Almaless (2008,2009,2010,2011, 2012)
	5	* Mohamed Lamine Karaoui (2015)
	6	* Alain Greiner (2016)
[21]	7	*
[1]	8	* Copyright (c) UPMC Sorbonne Universites
	9	*
	10	* This file is part of ALMOS-MKH.
	11	*
	12	* ALMOS-MKH is free software; you can redistribute it and/or modify it
	13	* under the terms of the GNU General Public License as published by
	14	* the Free Software Foundation; version 2.0 of the License.
	15	*
	16	* ALMOS-MKH is distributed in the hope that it will be useful, but
	17	* WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU General Public License
	22	* along with ALMOS-MKH; if not, write to the Free Software Foundation,
	23	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	24	*/
	25
[14]	26	#include <kernel_config.h>
[457]	27	#include <hal_kernel_types.h>
[1]	28	#include <hal_special.h>
	29	#include <hal_gpt.h>
[409]	30	#include <hal_vmm.h>
[577]	31	#include <hal_macros.h>
[1]	32	#include <printk.h>
[23]	33	#include <memcpy.h>
[567]	34	#include <remote_rwlock.h>
	35	#include <remote_queuelock.h>
[1]	36	#include <list.h>
[408]	37	#include <xlist.h>
[1]	38	#include <bits.h>
	39	#include <process.h>
	40	#include <thread.h>
	41	#include <vseg.h>
	42	#include <cluster.h>
	43	#include <scheduler.h>
	44	#include <vfs.h>
	45	#include <mapper.h>
	46	#include <page.h>
	47	#include <kmem.h>
	48	#include <vmm.h>
[585]	49	#include <hal_exception.h>
[1]	50
	51	//////////////////////////////////////////////////////////////////////////////////
	52	// Extern global variables
	53	//////////////////////////////////////////////////////////////////////////////////
	54
[567]	55	extern process_t process_zero; // allocated in cluster.c
[1]	56
[415]	57	///////////////////////////////////////
	58	error_t vmm_init( process_t * process )
[21]	59	{
[1]	60	error_t error;
	61	vseg_t * vseg_kentry;
	62	vseg_t * vseg_args;
	63	vseg_t * vseg_envs;
	64	intptr_t base;
	65	intptr_t size;
	66
[438]	67	#if DEBUG_VMM_INIT
[567]	68	thread_t * this = CURRENT_THREAD;
[433]	69	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	70	if( DEBUG_VMM_INIT )
[567]	71	printk("\n[DBG] %s : thread %x in process %x enter for process %x / cycle %d\n",
	72	__FUNCTION__ , this->trdid, this->process->pid, process->pid , cycle );
[433]	73	#endif
[204]	74
[1]	75	// get pointer on VMM
	76	vmm_t * vmm = &process->vmm;
	77
[407]	78	// initialize local list of vsegs
	79	vmm->vsegs_nr = 0;
[408]	80	xlist_root_init( XPTR( local_cxy , &vmm->vsegs_root ) );
[580]	81	remote_rwlock_init( XPTR( local_cxy , &vmm->vsegs_lock ) , LOCK_VMM_VSL );
[407]	82
[567]	83	assert( ((CONFIG_VMM_KENTRY_SIZE + CONFIG_VMM_ARGS_SIZE + CONFIG_VMM_ENVS_SIZE)
	84	<= CONFIG_VMM_ELF_BASE) , "UTILS zone too small\n" );
[21]	85
[567]	86	assert( (CONFIG_THREADS_MAX_PER_CLUSTER <= 32) ,
	87	"no more than 32 threads per cluster for a single process\n");
[1]	88
[567]	89	assert( ((CONFIG_VMM_STACK_SIZE * CONFIG_THREADS_MAX_PER_CLUSTER) <=
	90	(CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE)) ,
	91	"STACK zone too small\n");
[1]	92
[409]	93	// register kentry vseg in VSL
[406]	94	base = CONFIG_VMM_KENTRY_BASE << CONFIG_PPM_PAGE_SHIFT;
[1]	95	size = CONFIG_VMM_KENTRY_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	96
[407]	97	vseg_kentry = vmm_create_vseg( process,
	98	VSEG_TYPE_CODE,
	99	base,
	100	size,
	101	0, // file_offset unused
	102	0, // file_size unused
	103	XPTR_NULL, // mapper_xp unused
	104	local_cxy );
[204]	105
[415]	106	if( vseg_kentry == NULL )
	107	{
	108	printk("\n[ERROR] in %s : cannot register kentry vseg\n", __FUNCTION__ );
	109	return -1;
	110	}
[204]	111
[406]	112	vmm->kent_vpn_base = base;
[1]	113
[409]	114	// register args vseg in VSL
[406]	115	base = (CONFIG_VMM_KENTRY_BASE +
	116	CONFIG_VMM_KENTRY_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
[1]	117	size = CONFIG_VMM_ARGS_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	118
[407]	119	vseg_args = vmm_create_vseg( process,
	120	VSEG_TYPE_DATA,
	121	base,
	122	size,
	123	0, // file_offset unused
	124	0, // file_size unused
	125	XPTR_NULL, // mapper_xp unused
	126	local_cxy );
[204]	127
[415]	128	if( vseg_args == NULL )
	129	{
	130	printk("\n[ERROR] in %s : cannot register args vseg\n", __FUNCTION__ );
	131	return -1;
	132	}
[204]	133
[406]	134	vmm->args_vpn_base = base;
[1]	135
[409]	136	// register the envs vseg in VSL
[406]	137	base = (CONFIG_VMM_KENTRY_BASE +
	138	CONFIG_VMM_KENTRY_SIZE +
	139	CONFIG_VMM_ARGS_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
[1]	140	size = CONFIG_VMM_ENVS_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	141
[407]	142	vseg_envs = vmm_create_vseg( process,
	143	VSEG_TYPE_DATA,
	144	base,
	145	size,
	146	0, // file_offset unused
	147	0, // file_size unused
	148	XPTR_NULL, // mapper_xp unused
	149	local_cxy );
[204]	150
[415]	151	if( vseg_envs == NULL )
	152	{
	153	printk("\n[ERROR] in %s : cannot register envs vseg\n", __FUNCTION__ );
	154	return -1;
	155	}
[204]	156
[406]	157	vmm->envs_vpn_base = base;
[1]	158
[409]	159	// create GPT (empty)
[1]	160	error = hal_gpt_create( &vmm->gpt );
	161
[415]	162	if( error )
	163	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
[204]	164
[585]	165	// initialize GPT lock
	166	remote_rwlock_init( XPTR( local_cxy , &vmm->gpt_lock ) , LOCK_VMM_GPT );
	167
	168	// architecture specic GPT initialisation
[409]	169	// (For TSAR, identity map the kentry_vseg)
	170	error = hal_vmm_init( vmm );
	171
[415]	172	if( error )
	173	printk("\n[ERROR] in %s : cannot initialize GPT\n", __FUNCTION__ );
[409]	174
[1]	175	// initialize STACK allocator
	176	vmm->stack_mgr.bitmap = 0;
	177	vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
[567]	178	busylock_init( &vmm->stack_mgr.lock , LOCK_VMM_STACK );
[1]	179
	180	// initialize MMAP allocator
[407]	181	vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
	182	vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
	183	vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
[567]	184	busylock_init( &vmm->mmap_mgr.lock , LOCK_VMM_MMAP );
[457]	185
[1]	186	uint32_t i;
	187	for( i = 0 ; i < 32 ; i++ ) list_root_init( &vmm->mmap_mgr.zombi_list[i] );
	188
[21]	189	// initialize instrumentation counters
[409]	190	vmm->pgfault_nr = 0;
[1]	191
[124]	192	hal_fence();
[1]	193
[438]	194	#if DEBUG_VMM_INIT
[433]	195	cycle = (uint32_t)hal_get_cycles();
[438]	196	if( DEBUG_VMM_INIT )
[567]	197	printk("\n[DBG] %s : thread %x in process %x exit / process %x / entry_point = %x / cycle %d\n",
	198	__FUNCTION__, this->trdid, this->process->pid, process->pid , process->vmm.entry_point , cycle );
[433]	199	#endif
[204]	200
[415]	201	return 0;
	202
[204]	203	} // end vmm_init()
	204
[407]	205	//////////////////////////////////////
	206	void vmm_display( process_t * process,
	207	bool_t mapping )
	208	{
	209	vmm_t * vmm = &process->vmm;
	210	gpt_t * gpt = &vmm->gpt;
	211
[457]	212	printk("\n***** VSL and GPT(%x) for process %x in cluster %x\n\n",
	213	process->vmm.gpt.ptr , process->pid , local_cxy );
[407]	214
[585]	215	// get lock protecting the VSL and the GPT
[567]	216	remote_rwlock_rd_acquire( XPTR( local_cxy , &vmm->vsegs_lock ) );
[585]	217	remote_rwlock_rd_acquire( XPTR( local_cxy , &vmm->gpt_lock ) );
[407]	218
	219	// scan the list of vsegs
[408]	220	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	221	xptr_t iter_xp;
	222	xptr_t vseg_xp;
[407]	223	vseg_t * vseg;
[408]	224	XLIST_FOREACH( root_xp , iter_xp )
[407]	225	{
[408]	226	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	227	vseg = GET_PTR( vseg_xp );
[408]	228
[407]	229	printk(" - %s : base = %X / size = %X / npages = %d\n",
	230	vseg_type_str( vseg->type ) , vseg->min , vseg->max - vseg->min , vseg->vpn_size );
	231
	232	if( mapping )
	233	{
	234	vpn_t vpn;
	235	ppn_t ppn;
	236	uint32_t attr;
	237	vpn_t base = vseg->vpn_base;
	238	vpn_t size = vseg->vpn_size;
	239	for( vpn = base ; vpn < (base+size) ; vpn++ )
	240	{
[585]	241	hal_gpt_get_pte( XPTR( local_cxy , gpt ) , vpn , &attr , &ppn );
[407]	242	if( attr & GPT_MAPPED )
	243	{
	244	printk(" . vpn = %X / attr = %X / ppn = %X\n", vpn , attr , ppn );
	245	}
	246	}
	247	}
	248	}
	249
[585]	250	// release the locks
[567]	251	remote_rwlock_rd_release( XPTR( local_cxy , &vmm->vsegs_lock ) );
[585]	252	remote_rwlock_rd_release( XPTR( local_cxy , &vmm->gpt_lock ) );
[407]	253
[408]	254	} // vmm_display()
	255
[567]	256	///////////////////////////////////
	257	void vmm_vseg_attach( vmm_t * vmm,
	258	vseg_t * vseg )
	259	{
	260	// build extended pointer on rwlock protecting VSL
	261	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	262
	263	// get rwlock in write mode
	264	remote_rwlock_wr_acquire( lock_xp );
	265
	266	// update vseg descriptor
	267	vseg->vmm = vmm;
	268
	269	// add vseg in vmm list
	270	xlist_add_last( XPTR( local_cxy , &vmm->vsegs_root ),
	271	XPTR( local_cxy , &vseg->xlist ) );
	272
	273	// release rwlock in write mode
	274	remote_rwlock_wr_release( lock_xp );
	275	}
	276
	277	///////////////////////////////////
	278	void vmm_vseg_detach( vmm_t * vmm,
	279	vseg_t * vseg )
	280	{
	281	// build extended pointer on rwlock protecting VSL
	282	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	283
	284	// get rwlock in write mode
	285	remote_rwlock_wr_acquire( lock_xp );
	286
	287	// update vseg descriptor
	288	vseg->vmm = NULL;
	289
	290	// remove vseg from vmm list
	291	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
	292
	293	// release rwlock in write mode
	294	remote_rwlock_wr_release( lock_xp );
	295	}
	296
[433]	297	/////////////////////i//////////////////////////
	298	void vmm_global_update_pte( process_t * process,
	299	vpn_t vpn,
	300	uint32_t attr,
	301	ppn_t ppn )
[23]	302	{
[408]	303	xlist_entry_t * process_root_ptr;
	304	xptr_t process_root_xp;
	305	xptr_t process_iter_xp;
[23]	306
[408]	307	xptr_t remote_process_xp;
	308	cxy_t remote_process_cxy;
	309	process_t * remote_process_ptr;
	310	xptr_t remote_gpt_xp;
[23]	311
[408]	312	pid_t pid;
	313	cxy_t owner_cxy;
	314	lpid_t owner_lpid;
[23]	315
[438]	316	#if DEBUG_VMM_UPDATE_PTE
[433]	317	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	318	if( DEBUG_VMM_UPDATE_PTE < cycle )
[433]	319	printk("\n[DBG] %s : thread %x enter for process %x / vpn %x / cycle %d\n",
	320	__FUNCTION__ , CURRENT_THREAD , process->pid , vpn , cycle );
	321	#endif
	322
[567]	323	// check cluster is reference
[585]	324	assert( (GET_CXY( process->ref_xp ) == local_cxy) , "not called in reference cluster\n");
[433]	325
[408]	326	// get extended pointer on root of process copies xlist in owner cluster
	327	pid = process->pid;
	328	owner_cxy = CXY_FROM_PID( pid );
	329	owner_lpid = LPID_FROM_PID( pid );
	330	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
	331	process_root_xp = XPTR( owner_cxy , process_root_ptr );
[23]	332
[408]	333	// loop on destination process copies
	334	XLIST_FOREACH( process_root_xp , process_iter_xp )
	335	{
	336	// get cluster and local pointer on remote process
	337	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
[433]	338	remote_process_ptr = GET_PTR( remote_process_xp );
[408]	339	remote_process_cxy = GET_CXY( remote_process_xp );
[407]	340
[438]	341	#if (DEBUG_VMM_UPDATE_PTE & 0x1)
	342	if( DEBUG_VMM_UPDATE_PTE < cycle )
[433]	343	printk("\n[DBG] %s : thread %x handling process %x in cluster %x\n",
	344	__FUNCTION__ , CURRENT_THREAD , process->pid , remote_process_cxy );
	345	#endif
	346
[408]	347	// get extended pointer on remote gpt
	348	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
	349
[433]	350	// update remote GPT
	351	hal_gpt_update_pte( remote_gpt_xp, vpn, attr, ppn );
[408]	352	}
	353
[438]	354	#if DEBUG_VMM_UPDATE_PTE
[433]	355	cycle = (uint32_t)hal_get_cycles();
[438]	356	if( DEBUG_VMM_UPDATE_PTE < cycle )
[433]	357	printk("\n[DBG] %s : thread %x exit for process %x / vpn %x / cycle %d\n",
	358	__FUNCTION__ , CURRENT_THREAD , process->pid , vpn , cycle );
	359	#endif
	360
	361	} // end vmm_global_update_pte()
	362
[408]	363	///////////////////////////////////////
	364	void vmm_set_cow( process_t * process )
	365	{
	366	vmm_t * vmm;
	367
	368	xlist_entry_t * process_root_ptr;
	369	xptr_t process_root_xp;
	370	xptr_t process_iter_xp;
	371
	372	xptr_t remote_process_xp;
	373	cxy_t remote_process_cxy;
	374	process_t * remote_process_ptr;
	375	xptr_t remote_gpt_xp;
	376
	377	xptr_t vseg_root_xp;
	378	xptr_t vseg_iter_xp;
	379
	380	xptr_t vseg_xp;
	381	vseg_t * vseg;
	382
	383	pid_t pid;
	384	cxy_t owner_cxy;
	385	lpid_t owner_lpid;
	386
[438]	387	#if DEBUG_VMM_SET_COW
[433]	388	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	389	if( DEBUG_VMM_SET_COW < cycle )
[433]	390	printk("\n[DBG] %s : thread %x enter for process %x / cycle %d\n",
	391	__FUNCTION__ , CURRENT_THREAD , process->pid , cycle );
	392	#endif
[408]	393
[567]	394	// check cluster is reference
	395	assert( (GET_CXY( process->ref_xp ) == local_cxy) ,
	396	"local cluster is not process reference cluster\n");
[408]	397
	398	// get pointer on reference VMM
	399	vmm = &process->vmm;
	400
	401	// get extended pointer on root of process copies xlist in owner cluster
	402	pid = process->pid;
	403	owner_cxy = CXY_FROM_PID( pid );
	404	owner_lpid = LPID_FROM_PID( pid );
	405	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
	406	process_root_xp = XPTR( owner_cxy , process_root_ptr );
	407
	408	// get extended pointer on root of vsegs xlist from reference VMM
	409	vseg_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	410
	411	// loop on destination process copies
	412	XLIST_FOREACH( process_root_xp , process_iter_xp )
	413	{
	414	// get cluster and local pointer on remote process
	415	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
[433]	416	remote_process_ptr = GET_PTR( remote_process_xp );
[408]	417	remote_process_cxy = GET_CXY( remote_process_xp );
	418
[438]	419	#if (DEBUG_VMM_SET_COW &0x1)
	420	if( DEBUG_VMM_SET_COW < cycle )
[433]	421	printk("\n[DBG] %s : thread %x handling process %x in cluster %x\n",
	422	__FUNCTION__ , CURRENT_THREAD , process->pid , remote_process_cxy );
	423	#endif
[408]	424
	425	// get extended pointer on remote gpt
	426	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
	427
	428	// loop on vsegs in (local) reference process VSL
	429	XLIST_FOREACH( vseg_root_xp , vseg_iter_xp )
	430	{
	431	// get pointer on vseg
	432	vseg_xp = XLIST_ELEMENT( vseg_iter_xp , vseg_t , xlist );
[433]	433	vseg = GET_PTR( vseg_xp );
[408]	434
[567]	435	assert( (GET_CXY( vseg_xp ) == local_cxy) ,
	436	"all vsegs in reference VSL must be local\n" );
[408]	437
	438	// get vseg type, base and size
	439	uint32_t type = vseg->type;
	440	vpn_t vpn_base = vseg->vpn_base;
	441	vpn_t vpn_size = vseg->vpn_size;
	442
[438]	443	#if (DEBUG_VMM_SET_COW & 0x1)
	444	if( DEBUG_VMM_SET_COW < cycle )
[433]	445	printk("\n[DBG] %s : thread %x handling vseg %s / vpn_base = %x / vpn_size = %x\n",
	446	__FUNCTION__, CURRENT_THREAD , vseg_type_str(type), vpn_base, vpn_size );
	447	#endif
	448	// only DATA, ANON and REMOTE vsegs
[408]	449	if( (type == VSEG_TYPE_DATA) \|\|
	450	(type == VSEG_TYPE_ANON) \|\|
	451	(type == VSEG_TYPE_REMOTE) )
	452	{
[433]	453	vpn_t vpn;
	454	uint32_t attr;
	455	ppn_t ppn;
	456	xptr_t page_xp;
	457	cxy_t page_cxy;
	458	page_t * page_ptr;
	459	xptr_t forks_xp;
[469]	460	xptr_t lock_xp;
[433]	461
	462	// update flags in remote GPT
	463	hal_gpt_set_cow( remote_gpt_xp,
	464	vpn_base,
	465	vpn_size );
	466
	467	// atomically increment pending forks counter in physical pages,
	468	// for all vseg pages that are mapped in reference cluster
	469	if( remote_process_cxy == local_cxy )
	470	{
	471	// scan all pages in vseg
	472	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
	473	{
	474	// get page attributes and PPN from reference GPT
[585]	475	hal_gpt_get_pte( remote_gpt_xp , vpn , &attr , &ppn );
[433]	476
	477	// atomically update pending forks counter if page is mapped
	478	if( attr & GPT_MAPPED )
	479	{
[469]	480	// get pointers and cluster on page descriptor
[433]	481	page_xp = ppm_ppn2page( ppn );
	482	page_cxy = GET_CXY( page_xp );
	483	page_ptr = GET_PTR( page_xp );
[469]	484
	485	// get extended pointers on "forks" and "lock"
[433]	486	forks_xp = XPTR( page_cxy , &page_ptr->forks );
[469]	487	lock_xp = XPTR( page_cxy , &page_ptr->lock );
	488
[567]	489	// take lock protecting "forks" counter
	490	remote_busylock_acquire( lock_xp );
	491
[469]	492	// increment "forks"
[433]	493	hal_remote_atomic_add( forks_xp , 1 );
[567]	494
	495	// release lock protecting "forks" counter
	496	remote_busylock_release( lock_xp );
[433]	497	}
	498	} // end loop on vpn
	499	} // end if local
	500	} // end if vseg type
	501	} // end loop on vsegs
[408]	502	} // end loop on process copies
	503
[438]	504	#if DEBUG_VMM_SET_COW
[433]	505	cycle = (uint32_t)hal_get_cycles();
[438]	506	if( DEBUG_VMM_SET_COW < cycle )
[433]	507	printk("\n[DBG] %s : thread %x exit for process %x / cycle %d\n",
	508	__FUNCTION__ , CURRENT_THREAD , process->pid , cycle );
	509	#endif
[408]	510
	511	} // end vmm_set-cow()
	512
	513	/////////////////////////////////////////////////
	514	error_t vmm_fork_copy( process_t * child_process,
	515	xptr_t parent_process_xp )
	516	{
	517	error_t error;
	518	cxy_t parent_cxy;
	519	process_t * parent_process;
	520	vmm_t * parent_vmm;
	521	xptr_t parent_lock_xp;
	522	vmm_t * child_vmm;
	523	xptr_t iter_xp;
	524	xptr_t parent_vseg_xp;
	525	vseg_t * parent_vseg;
	526	vseg_t * child_vseg;
	527	uint32_t type;
	528	bool_t cow;
	529	vpn_t vpn;
	530	vpn_t vpn_base;
	531	vpn_t vpn_size;
[469]	532	xptr_t page_xp; // extended pointer on page descriptor
[408]	533	page_t * page_ptr;
	534	cxy_t page_cxy;
[469]	535	xptr_t forks_xp; // extended pointer on forks counter in page descriptor
	536	xptr_t lock_xp; // extended pointer on lock protecting the forks counter
[408]	537	xptr_t parent_root_xp;
	538	bool_t mapped;
	539	ppn_t ppn;
	540
[438]	541	#if DEBUG_VMM_FORK_COPY
[433]	542	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	543	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	544	printk("\n[DBG] %s : thread %x enter / cycle %d\n",
	545	__FUNCTION__ , CURRENT_THREAD, cycle );
	546	#endif
[408]	547
	548	// get parent process cluster and local pointer
	549	parent_cxy = GET_CXY( parent_process_xp );
[433]	550	parent_process = GET_PTR( parent_process_xp );
[408]	551
	552	// get local pointers on parent and child VMM
	553	parent_vmm = &parent_process->vmm;
	554	child_vmm = &child_process->vmm;
	555
	556	// get extended pointer on lock protecting the parent VSL
	557	parent_lock_xp = XPTR( parent_cxy , &parent_vmm->vsegs_lock );
	558
	559	// initialize the lock protecting the child VSL
[567]	560	remote_rwlock_init( XPTR( local_cxy , &child_vmm->vsegs_lock ), LOCK_VMM_STACK );
[408]	561
	562	// initialize the child VSL as empty
	563	xlist_root_init( XPTR( local_cxy, &child_vmm->vsegs_root ) );
	564	child_vmm->vsegs_nr = 0;
	565
[415]	566	// create child GPT
[408]	567	error = hal_gpt_create( &child_vmm->gpt );
[415]	568
[407]	569	if( error )
	570	{
[408]	571	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
	572	return -1;
[407]	573	}
	574
[408]	575	// build extended pointer on parent VSL
	576	parent_root_xp = XPTR( parent_cxy , &parent_vmm->vsegs_root );
	577
[567]	578	// take the lock protecting the parent VSL in read mode
	579	remote_rwlock_rd_acquire( parent_lock_xp );
[415]	580
[408]	581	// loop on parent VSL xlist
	582	XLIST_FOREACH( parent_root_xp , iter_xp )
[23]	583	{
[408]	584	// get local and extended pointers on current parent vseg
	585	parent_vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	586	parent_vseg = GET_PTR( parent_vseg_xp );
[23]	587
[408]	588	// get vseg type
[567]	589	type = hal_remote_l32( XPTR( parent_cxy , &parent_vseg->type ) );
[408]	590
[438]	591	#if DEBUG_VMM_FORK_COPY
[433]	592	cycle = (uint32_t)hal_get_cycles();
[438]	593	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	594	printk("\n[DBG] %s : thread %x found parent vseg %s / vpn_base = %x / cycle %d\n",
	595	__FUNCTION__ , CURRENT_THREAD, vseg_type_str(type),
[567]	596	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
[433]	597	#endif
[23]	598
[408]	599	// all parent vsegs - but STACK - must be copied in child VSL
	600	if( type != VSEG_TYPE_STACK )
[23]	601	{
[408]	602	// allocate memory for a new child vseg
	603	child_vseg = vseg_alloc();
	604	if( child_vseg == NULL ) // release all allocated vsegs
[23]	605	{
[408]	606	vmm_destroy( child_process );
	607	printk("\n[ERROR] in %s : cannot create vseg for child\n", __FUNCTION__ );
	608	return -1;
[23]	609	}
	610
[408]	611	// copy parent vseg to child vseg
	612	vseg_init_from_ref( child_vseg , parent_vseg_xp );
[23]	613
[408]	614	// register child vseg in child VSL
[567]	615	vmm_vseg_attach( child_vmm , child_vseg );
[407]	616
[438]	617	#if DEBUG_VMM_FORK_COPY
[433]	618	cycle = (uint32_t)hal_get_cycles();
[438]	619	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	620	printk("\n[DBG] %s : thread %x copied vseg %s / vpn_base = %x to child VSL / cycle %d\n",
	621	__FUNCTION__ , CURRENT_THREAD , vseg_type_str(type),
[567]	622	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
[433]	623	#endif
[23]	624
[408]	625	// copy DATA, MMAP, REMOTE, FILE parent GPT entries to child GPT
	626	if( type != VSEG_TYPE_CODE )
	627	{
	628	// activate the COW for DATA, MMAP, REMOTE vsegs only
	629	cow = ( type != VSEG_TYPE_FILE );
[23]	630
[408]	631	vpn_base = child_vseg->vpn_base;
	632	vpn_size = child_vseg->vpn_size;
[23]	633
[408]	634	// scan pages in parent vseg
	635	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
	636	{
	637	error = hal_gpt_pte_copy( &child_vmm->gpt,
	638	XPTR( parent_cxy , &parent_vmm->gpt ),
	639	vpn,
	640	cow,
	641	&ppn,
	642	&mapped );
	643	if( error )
	644	{
	645	vmm_destroy( child_process );
	646	printk("\n[ERROR] in %s : cannot copy GPT\n", __FUNCTION__ );
	647	return -1;
	648	}
	649
[433]	650	// increment pending forks counter in page if mapped
[408]	651	if( mapped )
	652	{
[469]	653	// get pointers and cluster on page descriptor
	654	page_xp = ppm_ppn2page( ppn );
[408]	655	page_cxy = GET_CXY( page_xp );
[433]	656	page_ptr = GET_PTR( page_xp );
[408]	657
[469]	658	// get extended pointers on "forks" and "lock"
	659	forks_xp = XPTR( page_cxy , &page_ptr->forks );
	660	lock_xp = XPTR( page_cxy , &page_ptr->lock );
	661
[567]	662	// get lock protecting "forks" counter
	663	remote_busylock_acquire( lock_xp );
	664
[469]	665	// increment "forks"
	666	hal_remote_atomic_add( forks_xp , 1 );
	667
[567]	668	// release lock protecting "forks" counter
	669	remote_busylock_release( lock_xp );
	670
[438]	671	#if DEBUG_VMM_FORK_COPY
[433]	672	cycle = (uint32_t)hal_get_cycles();
[438]	673	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	674	printk("\n[DBG] %s : thread %x copied vpn %x to child GPT / cycle %d\n",
	675	__FUNCTION__ , CURRENT_THREAD , vpn , cycle );
	676	#endif
[408]	677	}
	678	}
	679	} // end if no code & no stack
	680	} // end if no stack
	681	} // end loop on vsegs
	682
[567]	683	// release the parent VSL lock in read mode
	684	remote_rwlock_rd_release( parent_lock_xp );
[408]	685
[415]	686	// initialize child GPT (architecture specic)
	687	// => For TSAR, identity map the kentry_vseg
	688	error = hal_vmm_init( child_vmm );
	689
	690	if( error )
	691	{
	692	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
	693	return -1;
	694	}
	695
[408]	696	// initialize the child VMM STACK allocator
	697	child_vmm->stack_mgr.bitmap = 0;
	698	child_vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
	699
	700	// initialize the child VMM MMAP allocator
[23]	701	uint32_t i;
[408]	702	child_vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
	703	child_vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
	704	child_vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
	705	for( i = 0 ; i < 32 ; i++ ) list_root_init( &child_vmm->mmap_mgr.zombi_list[i] );
[23]	706
[178]	707	// initialize instrumentation counters
[408]	708	child_vmm->pgfault_nr = 0;
[23]	709
[408]	710	// copy base addresses from parent VMM to child VMM
	711	child_vmm->kent_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->kent_vpn_base));
	712	child_vmm->args_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->args_vpn_base));
	713	child_vmm->envs_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->envs_vpn_base));
	714	child_vmm->heap_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->heap_vpn_base));
	715	child_vmm->code_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->code_vpn_base));
	716	child_vmm->data_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->data_vpn_base));
[23]	717
[408]	718	child_vmm->entry_point = (intptr_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->entry_point));
[23]	719
[124]	720	hal_fence();
[23]	721
[438]	722	#if DEBUG_VMM_FORK_COPY
[433]	723	cycle = (uint32_t)hal_get_cycles();
[438]	724	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	725	printk("\n[DBG] %s : thread %x exit successfully / cycle %d\n",
	726	__FUNCTION__ , CURRENT_THREAD , cycle );
	727	#endif
	728
[23]	729	return 0;
	730
[408]	731	} // vmm_fork_copy()
[204]	732
[1]	733	///////////////////////////////////////
	734	void vmm_destroy( process_t * process )
	735	{
[408]	736	xptr_t vseg_xp;
[1]	737	vseg_t * vseg;
	738
[438]	739	#if DEBUG_VMM_DESTROY
[433]	740	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	741	if( DEBUG_VMM_DESTROY < cycle )
[443]	742	printk("\n[DBG] %s : thread %x enter for process %x in cluster %x / cycle %d\n",
[469]	743	__FUNCTION__, CURRENT_THREAD->trdid, process->pid, local_cxy, cycle );
[433]	744	#endif
[416]	745
[438]	746	#if (DEBUG_VMM_DESTROY & 1 )
[443]	747	if( DEBUG_VMM_DESTROY < cycle )
[437]	748	vmm_display( process , true );
	749	#endif
	750
[433]	751	// get pointer on local VMM
[1]	752	vmm_t * vmm = &process->vmm;
	753
[408]	754	// get extended pointer on VSL root and VSL lock
	755	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	756
[409]	757	// remove all user vsegs registered in VSL
[408]	758	while( !xlist_is_empty( root_xp ) )
[1]	759	{
[409]	760	// get pointer on first vseg in VSL
[567]	761	vseg_xp = XLIST_FIRST( root_xp , vseg_t , xlist );
[433]	762	vseg = GET_PTR( vseg_xp );
[409]	763
[437]	764	// unmap and release physical pages
[409]	765	vmm_unmap_vseg( process , vseg );
	766
	767	// remove vseg from VSL
[567]	768	vmm_vseg_detach( vmm , vseg );
[409]	769
	770	// release memory allocated to vseg descriptor
[1]	771	vseg_free( vseg );
[443]	772
	773	#if( DEBUG_VMM_DESTROY & 1 )
	774	if( DEBUG_VMM_DESTROY < cycle )
	775	printk("\n[DBG] %s : %s vseg released / vpn_base %x / vpn_size %d\n",
	776	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	777	#endif
	778
[1]	779	}
	780
	781	// remove all vsegs from zombi_lists in MMAP allocator
	782	uint32_t i;
	783	for( i = 0 ; i<32 ; i++ )
	784	{
	785	while( !list_is_empty( &vmm->mmap_mgr.zombi_list[i] ) )
	786	{
[408]	787	vseg = LIST_FIRST( &vmm->mmap_mgr.zombi_list[i] , vseg_t , zlist );
[443]	788
	789	#if( DEBUG_VMM_DESTROY & 1 )
	790	if( DEBUG_VMM_DESTROY < cycle )
	791	printk("\n[DBG] %s : found zombi vseg / vpn_base %x / vpn_size %d\n",
	792	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	793	#endif
[567]	794	vmm_vseg_detach( vmm , vseg );
[1]	795	vseg_free( vseg );
[443]	796
	797	#if( DEBUG_VMM_DESTROY & 1 )
	798	if( DEBUG_VMM_DESTROY < cycle )
	799	printk("\n[DBG] %s : zombi vseg released / vpn_base %x / vpn_size %d\n",
	800	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	801	#endif
[1]	802	}
	803	}
	804
[409]	805	// release memory allocated to the GPT itself
[1]	806	hal_gpt_destroy( &vmm->gpt );
	807
[438]	808	#if DEBUG_VMM_DESTROY
[433]	809	cycle = (uint32_t)hal_get_cycles();
[438]	810	if( DEBUG_VMM_DESTROY < cycle )
[443]	811	printk("\n[DBG] %s : thread %x exit for process %x in cluster %x / cycle %d\n",
[469]	812	__FUNCTION__, CURRENT_THREAD->trdid, process->pid, local_cxy , cycle );
[433]	813	#endif
[416]	814
[204]	815	} // end vmm_destroy()
	816
[1]	817	/////////////////////////////////////////////////
	818	vseg_t * vmm_check_conflict( process_t * process,
[21]	819	vpn_t vpn_base,
[1]	820	vpn_t vpn_size )
	821	{
	822	vmm_t * vmm = &process->vmm;
[408]	823
	824	// scan the VSL
[1]	825	vseg_t * vseg;
[408]	826	xptr_t iter_xp;
	827	xptr_t vseg_xp;
	828	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
[1]	829
[408]	830	XLIST_FOREACH( root_xp , iter_xp )
[1]	831	{
[408]	832	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	833	vseg = GET_PTR( vseg_xp );
[204]	834
[21]	835	if( ((vpn_base + vpn_size) > vseg->vpn_base) &&
	836	(vpn_base < (vseg->vpn_base + vseg->vpn_size)) ) return vseg;
[1]	837	}
	838	return NULL;
	839
[204]	840	} // end vmm_check_conflict()
	841
[1]	842	////////////////////////////////////////////////////////////////////////////////////////////
	843	// This static function is called by the vmm_create_vseg() function, and implements
	844	// the VMM stack_vseg specific allocator.
	845	////////////////////////////////////////////////////////////////////////////////////////////
	846	// @ vmm : pointer on VMM.
[21]	847	// @ vpn_base : (return value) first allocated page
[1]	848	// @ vpn_size : (return value) number of allocated pages
	849	////////////////////////////////////////////////////////////////////////////////////////////
	850	static error_t vmm_stack_alloc( vmm_t * vmm,
	851	vpn_t * vpn_base,
	852	vpn_t * vpn_size )
	853	{
	854	// get stack allocator pointer
	855	stack_mgr_t * mgr = &vmm->stack_mgr;
	856
	857	// get lock on stack allocator
[567]	858	busylock_acquire( &mgr->lock );
[1]	859
	860	// get first free slot index in bitmap
	861	int32_t index = bitmap_ffc( &mgr->bitmap , 4 );
[179]	862	if( (index < 0) \|\| (index > 31) )
	863	{
[567]	864	busylock_release( &mgr->lock );
	865	return 0xFFFFFFFF;
[179]	866	}
[1]	867
	868	// update bitmap
	869	bitmap_set( &mgr->bitmap , index );
[21]	870
[1]	871	// release lock on stack allocator
[567]	872	busylock_release( &mgr->lock );
[1]	873
[21]	874	// returns vpn_base, vpn_size (one page non allocated)
[1]	875	vpn_base = mgr->vpn_base + index CONFIG_VMM_STACK_SIZE + 1;
	876	*vpn_size = CONFIG_VMM_STACK_SIZE - 1;
	877	return 0;
	878
[204]	879	} // end vmm_stack_alloc()
	880
[1]	881	////////////////////////////////////////////////////////////////////////////////////////////
	882	// This static function is called by the vmm_create_vseg() function, and implements
	883	// the VMM MMAP specific allocator.
	884	////////////////////////////////////////////////////////////////////////////////////////////
	885	// @ vmm : [in] pointer on VMM.
	886	// @ npages : [in] requested number of pages.
[21]	887	// @ vpn_base : [out] first allocated page.
[1]	888	// @ vpn_size : [out] actual number of allocated pages.
	889	////////////////////////////////////////////////////////////////////////////////////////////
	890	static error_t vmm_mmap_alloc( vmm_t * vmm,
	891	vpn_t npages,
	892	vpn_t * vpn_base,
	893	vpn_t * vpn_size )
	894	{
	895	uint32_t index;
	896	vseg_t * vseg;
	897	vpn_t base;
	898	vpn_t size;
[21]	899	vpn_t free;
[1]	900
[21]	901	// mmap vseg size must be power of 2
[1]	902	// compute actual size and index in zombi_list array
	903	size = POW2_ROUNDUP( npages );
	904	index = bits_log2( size );
	905
	906	// get mmap allocator pointer
	907	mmap_mgr_t * mgr = &vmm->mmap_mgr;
	908
	909	// get lock on mmap allocator
[567]	910	busylock_acquire( &mgr->lock );
[1]	911
	912	// get vseg from zombi_list or from mmap zone
	913	if( list_is_empty( &mgr->zombi_list[index] ) ) // from mmap zone
	914	{
	915	// check overflow
	916	free = mgr->first_free_vpn;
	917	if( (free + size) > mgr->vpn_size ) return ENOMEM;
	918
	919	// update STACK allocator
	920	mgr->first_free_vpn += size;
	921
	922	// compute base
	923	base = free;
	924	}
	925	else // from zombi_list
	926	{
	927	// get pointer on zombi vseg from zombi_list
[408]	928	vseg = LIST_FIRST( &mgr->zombi_list[index] , vseg_t , zlist );
[1]	929
	930	// remove vseg from free-list
[408]	931	list_unlink( &vseg->zlist );
[1]	932
	933	// compute base
	934	base = vseg->vpn_base;
[21]	935	}
	936
[1]	937	// release lock on mmap allocator
[567]	938	busylock_release( &mgr->lock );
[1]	939
	940	// returns vpn_base, vpn_size
	941	*vpn_base = base;
	942	*vpn_size = size;
	943	return 0;
	944
[204]	945	} // end vmm_mmap_alloc()
	946
[407]	947	////////////////////////////////////////////////
	948	vseg_t * vmm_create_vseg( process_t * process,
	949	vseg_type_t type,
	950	intptr_t base,
	951	uint32_t size,
	952	uint32_t file_offset,
	953	uint32_t file_size,
	954	xptr_t mapper_xp,
	955	cxy_t cxy )
[1]	956	{
	957	vseg_t * vseg; // created vseg pointer
[204]	958	vpn_t vpn_base; // first page index
[1]	959	vpn_t vpn_size; // number of pages
	960	error_t error;
	961
[438]	962	#if DEBUG_VMM_CREATE_VSEG
[433]	963	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	964	if( DEBUG_VMM_CREATE_VSEG < cycle )
[433]	965	printk("\n[DBG] %s : thread %x enter / process %x / base %x / size %x / %s / cxy %x / cycle %d\n",
	966	__FUNCTION__, CURRENT_THREAD, process->pid, base, size, vseg_type_str(type), cxy, cycle );
	967	#endif
[21]	968
[407]	969	// get pointer on VMM
	970	vmm_t * vmm = &process->vmm;
[21]	971
[204]	972	// compute base, size, vpn_base, vpn_size, depending on vseg type
[407]	973	// we use the VMM specific allocators for "stack", "file", "anon", & "remote" vsegs
[1]	974	if( type == VSEG_TYPE_STACK )
	975	{
	976	// get vpn_base and vpn_size from STACK allocator
	977	error = vmm_stack_alloc( vmm , &vpn_base , &vpn_size );
	978	if( error )
	979	{
[407]	980	printk("\n[ERROR] in %s : no space for stack vseg / process %x in cluster %x\n",
	981	__FUNCTION__ , process->pid , local_cxy );
[1]	982	return NULL;
	983	}
	984
	985	// compute vseg base and size from vpn_base and vpn_size
	986	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
	987	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
	988	}
[21]	989	else if( (type == VSEG_TYPE_ANON) \|\|
	990	(type == VSEG_TYPE_FILE) \|\|
[1]	991	(type == VSEG_TYPE_REMOTE) )
	992	{
	993	// get vpn_base and vpn_size from MMAP allocator
	994	vpn_t npages = size >> CONFIG_PPM_PAGE_SHIFT;
	995	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
	996	if( error )
	997	{
	998	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
	999	__FUNCTION__ , process->pid , local_cxy );
	1000	return NULL;
	1001	}
	1002
	1003	// compute vseg base and size from vpn_base and vpn_size
	1004	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
	1005	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
	1006	}
	1007	else
	1008	{
[204]	1009	uint32_t vpn_min = base >> CONFIG_PPM_PAGE_SHIFT;
	1010	uint32_t vpn_max = (base + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1011
	1012	vpn_base = vpn_min;
	1013	vpn_size = vpn_max - vpn_min + 1;
[1]	1014	}
	1015
	1016	// check collisions
	1017	vseg = vmm_check_conflict( process , vpn_base , vpn_size );
	1018	if( vseg != NULL )
	1019	{
[21]	1020	printk("\n[ERROR] in %s for process %x : new vseg [vpn_base = %x / vpn_size = %x]\n"
[1]	1021	" overlap existing vseg [vpn_base = %x / vpn_size = %x]\n",
[407]	1022	__FUNCTION__ , process->pid, vpn_base, vpn_size, vseg->vpn_base, vseg->vpn_size );
[1]	1023	return NULL;
	1024	}
	1025
	1026	// allocate physical memory for vseg descriptor
	1027	vseg = vseg_alloc();
	1028	if( vseg == NULL )
	1029	{
	1030	printk("\n[ERROR] in %s for process %x : cannot allocate memory for vseg\n",
[407]	1031	__FUNCTION__ , process->pid );
[1]	1032	return NULL;
	1033	}
	1034
	1035	// initialize vseg descriptor
[407]	1036	vseg_init( vseg,
	1037	type,
	1038	base,
	1039	size,
	1040	vpn_base,
	1041	vpn_size,
	1042	file_offset,
	1043	file_size,
	1044	mapper_xp,
	1045	cxy );
[1]	1046
[408]	1047	// attach vseg to VSL
[567]	1048	vmm_vseg_attach( vmm , vseg );
[1]	1049
[438]	1050	#if DEBUG_VMM_CREATE_VSEG
[433]	1051	cycle = (uint32_t)hal_get_cycles();
[438]	1052	if( DEBUG_VMM_CREATE_VSEG < cycle )
[433]	1053	printk("\n[DBG] %s : thread %x exit / process %x / %s / cxy %x / cycle %d\n",
	1054	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str(type), cxy, cycle );
	1055	#endif
[21]	1056
[1]	1057	return vseg;
	1058
[406]	1059	} // vmm_create_vseg()
	1060
[1]	1061	/////////////////////////////////////
	1062	void vmm_remove_vseg( vseg_t * vseg )
	1063	{
	1064	// get pointers on calling process and VMM
	1065	thread_t * this = CURRENT_THREAD;
	1066	vmm_t * vmm = &this->process->vmm;
	1067	uint32_t type = vseg->type;
	1068
[408]	1069	// detach vseg from VSL
[567]	1070	vmm_vseg_detach( vmm , vseg );
[1]	1071
	1072	// release the stack slot to VMM stack allocator if STACK type
	1073	if( type == VSEG_TYPE_STACK )
	1074	{
	1075	// get pointer on stack allocator
	1076	stack_mgr_t * mgr = &vmm->stack_mgr;
	1077
	1078	// compute slot index
	1079	uint32_t index = ((vseg->vpn_base - mgr->vpn_base - 1) / CONFIG_VMM_STACK_SIZE);
	1080
	1081	// update stacks_bitmap
[567]	1082	busylock_acquire( &mgr->lock );
[1]	1083	bitmap_clear( &mgr->bitmap , index );
[567]	1084	busylock_release( &mgr->lock );
[1]	1085	}
	1086
	1087	// release the vseg to VMM mmap allocator if MMAP type
	1088	if( (type == VSEG_TYPE_ANON) \|\| (type == VSEG_TYPE_FILE) \|\| (type == VSEG_TYPE_REMOTE) )
	1089	{
	1090	// get pointer on mmap allocator
	1091	mmap_mgr_t * mgr = &vmm->mmap_mgr;
	1092
	1093	// compute zombi_list index
	1094	uint32_t index = bits_log2( vseg->vpn_size );
	1095
	1096	// update zombi_list
[567]	1097	busylock_acquire( &mgr->lock );
[408]	1098	list_add_first( &mgr->zombi_list[index] , &vseg->zlist );
[567]	1099	busylock_release( &mgr->lock );
[1]	1100	}
	1101
	1102	// release physical memory allocated for vseg descriptor if no MMAP type
	1103	if( (type != VSEG_TYPE_ANON) && (type != VSEG_TYPE_FILE) && (type != VSEG_TYPE_REMOTE) )
	1104	{
	1105	vseg_free( vseg );
	1106	}
[407]	1107	} // end vmm_remove_vseg()
[1]	1108
	1109	/////////////////////////////////////////
	1110	void vmm_unmap_vseg( process_t * process,
	1111	vseg_t * vseg )
	1112	{
[21]	1113	vpn_t vpn; // VPN of current PTE
	1114	vpn_t vpn_min; // VPN of first PTE
[1]	1115	vpn_t vpn_max; // VPN of last PTE (excluded)
[409]	1116	ppn_t ppn; // current PTE ppn value
	1117	uint32_t attr; // current PTE attributes
	1118	kmem_req_t req; // request to release memory
	1119	xptr_t page_xp; // extended pointer on page descriptor
	1120	cxy_t page_cxy; // page descriptor cluster
	1121	page_t * page_ptr; // page descriptor pointer
[433]	1122	xptr_t forks_xp; // extended pointer on pending forks counter
[469]	1123	xptr_t lock_xp; // extended pointer on lock protecting forks counter
	1124	uint32_t forks; // actual number of pendinf forks
[1]	1125
[438]	1126	#if DEBUG_VMM_UNMAP_VSEG
[433]	1127	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	1128	if( DEBUG_VMM_UNMAP_VSEG < cycle )
[433]	1129	printk("\n[DBG] %s : thread %x enter / process %x / vseg %s / base %x / cycle %d\n",
	1130	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str( vseg->type ), vseg->vpn_base, cycle );
	1131	#endif
[409]	1132
[433]	1133	// get pointer on local GPT
[1]	1134	gpt_t * gpt = &process->vmm.gpt;
	1135
	1136	// loop on pages in vseg
	1137	vpn_min = vseg->vpn_base;
	1138	vpn_max = vpn_min + vseg->vpn_size;
	1139	for( vpn = vpn_min ; vpn < vpn_max ; vpn++ )
	1140	{
[409]	1141	// get GPT entry
[585]	1142	hal_gpt_get_pte( XPTR( local_cxy , gpt ) , vpn , &attr , &ppn );
[409]	1143
	1144	if( attr & GPT_MAPPED ) // entry is mapped
	1145	{
[437]	1146
[438]	1147	#if( DEBUG_VMM_UNMAP_VSEG & 1 )
	1148	if( DEBUG_VMM_UNMAP_VSEG < cycle )
[437]	1149	printk("- vpn %x / ppn %x\n" , vpn , ppn );
	1150	#endif
	1151
[567]	1152	// check small page
[585]	1153	assert( (attr & GPT_SMALL) , "an user vseg must use small pages" );
[409]	1154
[585]	1155	// unmap GPT entry in local GPT
[409]	1156	hal_gpt_reset_pte( gpt , vpn );
	1157
[433]	1158	// handle pending forks counter if
	1159	// 1) not identity mapped
[567]	1160	// 2) reference cluster
[433]	1161	if( ((vseg->flags & VSEG_IDENT) == 0) &&
	1162	(GET_CXY( process->ref_xp ) == local_cxy) )
[409]	1163	{
[433]	1164	// get extended pointer on physical page descriptor
[409]	1165	page_xp = ppm_ppn2page( ppn );
	1166	page_cxy = GET_CXY( page_xp );
[433]	1167	page_ptr = GET_PTR( page_xp );
[409]	1168
[469]	1169	// get extended pointers on forks and lock fields
	1170	forks_xp = XPTR( page_cxy , &page_ptr->forks );
	1171	lock_xp = XPTR( page_cxy , &page_ptr->lock );
[433]	1172
	1173	// get pending forks counter
[567]	1174	forks = hal_remote_l32( forks_xp );
[433]	1175
[469]	1176	if( forks ) // decrement pending forks counter
[409]	1177	{
[433]	1178	hal_remote_atomic_add( forks_xp , -1 );
	1179	}
	1180	else // release physical page to relevant cluster
[409]	1181	{
[433]	1182	if( page_cxy == local_cxy ) // local cluster
	1183	{
	1184	req.type = KMEM_PAGE;
	1185	req.ptr = page_ptr;
	1186	kmem_free( &req );
	1187	}
	1188	else // remote cluster
	1189	{
	1190	rpc_pmem_release_pages_client( page_cxy , page_ptr );
	1191	}
[409]	1192	}
	1193	}
	1194	}
[1]	1195	}
[433]	1196
[438]	1197	#if DEBUG_VMM_UNMAP_VSEG
[433]	1198	cycle = (uint32_t)hal_get_cycles();
[438]	1199	if( DEBUG_VMM_UNMAP_VSEG < cycle )
[433]	1200	printk("\n[DBG] %s : thread %x exit / process %x / vseg %s / base %x / cycle %d\n",
	1201	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str( vseg->type ), vseg->vpn_base, cycle );
	1202	#endif
	1203
[409]	1204	} // end vmm_unmap_vseg()
[1]	1205
[407]	1206	//////////////////////////////////////////////////////////////////////////////////////////
[440]	1207	// This low-level static function is called by the vmm_get_vseg(), vmm_get_pte(),
	1208	// and vmm_resize_vseg() functions. It scan the local VSL to find the unique vseg
	1209	// containing a given virtual address.
[407]	1210	//////////////////////////////////////////////////////////////////////////////////////////
[406]	1211	// @ vmm : pointer on the process VMM.
	1212	// @ vaddr : virtual address.
	1213	// @ return vseg pointer if success / return NULL if not found.
[407]	1214	//////////////////////////////////////////////////////////////////////////////////////////
[406]	1215	static vseg_t * vseg_from_vaddr( vmm_t * vmm,
	1216	intptr_t vaddr )
	1217	{
[408]	1218	xptr_t iter_xp;
	1219	xptr_t vseg_xp;
	1220	vseg_t * vseg;
[406]	1221
[408]	1222	// get extended pointers on VSL lock and root
	1223	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	1224	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
[406]	1225
[408]	1226	// get lock protecting the VSL
[567]	1227	remote_rwlock_rd_acquire( lock_xp );
[408]	1228
	1229	// scan the list of vsegs in VSL
	1230	XLIST_FOREACH( root_xp , iter_xp )
[406]	1231	{
[408]	1232	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	1233	vseg = GET_PTR( vseg_xp );
[408]	1234	if( (vaddr >= vseg->min) && (vaddr < vseg->max) )
	1235	{
	1236	// return success
[567]	1237	remote_rwlock_rd_release( lock_xp );
[408]	1238	return vseg;
	1239	}
[406]	1240	}
	1241
[408]	1242	// return failure
[567]	1243	remote_rwlock_rd_release( lock_xp );
[408]	1244	return NULL;
[406]	1245
[408]	1246	} // end vseg_from_vaddr()
[406]	1247
[1]	1248	/////////////////////////////////////////////
	1249	error_t vmm_resize_vseg( process_t * process,
	1250	intptr_t base,
	1251	intptr_t size )
	1252	{
[406]	1253	error_t error;
	1254	vseg_t * new;
	1255	vpn_t vpn_min;
	1256	vpn_t vpn_max;
[1]	1257
	1258	// get pointer on process VMM
	1259	vmm_t * vmm = &process->vmm;
	1260
	1261	intptr_t addr_min = base;
	1262	intptr_t addr_max = base + size;
	1263
	1264	// get pointer on vseg
[406]	1265	vseg_t * vseg = vseg_from_vaddr( vmm , base );
[1]	1266
	1267	if( vseg == NULL) return EINVAL;
[21]	1268
[408]	1269	// get extended pointer on VSL lock
	1270	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
[21]	1271
[408]	1272	// get lock protecting VSL
[567]	1273	remote_rwlock_wr_acquire( lock_xp );
[408]	1274
[1]	1275	if( (vseg->min > addr_min) \|\| (vseg->max < addr_max) ) // region not included in vseg
	1276	{
	1277	error = EINVAL;
	1278	}
	1279	else if( (vseg->min == addr_min) && (vseg->max == addr_max) ) // vseg must be removed
	1280	{
	1281	vmm_remove_vseg( vseg );
	1282	error = 0;
	1283	}
[406]	1284	else if( vseg->min == addr_min ) // vseg must be resized
[1]	1285	{
[406]	1286	// update vseg base address
	1287	vseg->min = addr_max;
	1288
	1289	// update vpn_base and vpn_size
	1290	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1291	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1292	vseg->vpn_base = vpn_min;
	1293	vseg->vpn_size = vpn_max - vpn_min + 1;
	1294	error = 0;
[1]	1295	}
[406]	1296	else if( vseg->max == addr_max ) // vseg must be resized
[1]	1297	{
[406]	1298	// update vseg max address
	1299	vseg->max = addr_min;
	1300
	1301	// update vpn_base and vpn_size
	1302	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1303	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1304	vseg->vpn_base = vpn_min;
	1305	vseg->vpn_size = vpn_max - vpn_min + 1;
	1306	error = 0;
[1]	1307	}
[406]	1308	else // vseg cut in three regions
[1]	1309	{
[406]	1310	// resize existing vseg
	1311	vseg->max = addr_min;
	1312
	1313	// update vpn_base and vpn_size
	1314	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1315	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1316	vseg->vpn_base = vpn_min;
	1317	vseg->vpn_size = vpn_max - vpn_min + 1;
	1318
	1319	// create new vseg
[407]	1320	new = vmm_create_vseg( process,
	1321	vseg->type,
	1322	addr_min,
	1323	(vseg->max - addr_max),
	1324	vseg->file_offset,
	1325	vseg->file_size,
	1326	vseg->mapper_xp,
	1327	vseg->cxy );
	1328
[406]	1329	if( new == NULL ) error = EINVAL;
	1330	else error = 0;
[1]	1331	}
	1332
	1333	// release VMM lock
[567]	1334	remote_rwlock_wr_release( lock_xp );
[1]	1335
	1336	return error;
	1337
[406]	1338	} // vmm_resize_vseg()
	1339
[1]	1340	///////////////////////////////////////////
[388]	1341	error_t vmm_get_vseg( process_t * process,
[394]	1342	intptr_t vaddr,
[388]	1343	vseg_t ** found_vseg )
[1]	1344	{
[440]	1345	xptr_t vseg_xp;
	1346	error_t error;
	1347	vseg_t * vseg;
	1348	vmm_t * vmm;
[1]	1349
[440]	1350	// get pointer on local VMM
	1351	vmm = &process->vmm;
[1]	1352
[440]	1353	// try to get vseg from local VMM
	1354	vseg = vseg_from_vaddr( vmm , vaddr );
	1355
[388]	1356	if( vseg == NULL ) // vseg not found in local cluster => try to get it from ref
	1357	{
	1358	// get extended pointer on reference process
	1359	xptr_t ref_xp = process->ref_xp;
[1]	1360
[388]	1361	// get cluster and local pointer on reference process
	1362	cxy_t ref_cxy = GET_CXY( ref_xp );
[433]	1363	process_t * ref_ptr = GET_PTR( ref_xp );
[388]	1364
	1365	if( local_cxy == ref_cxy ) return -1; // local cluster is the reference
	1366
	1367	// get extended pointer on reference vseg
[394]	1368	rpc_vmm_get_vseg_client( ref_cxy , ref_ptr , vaddr , &vseg_xp , &error );
[388]	1369
[440]	1370	if( error ) return -1; // vseg not found => illegal user vaddr
[388]	1371
	1372	// allocate a vseg in local cluster
	1373	vseg = vseg_alloc();
	1374
[440]	1375	if( vseg == NULL ) return -1; // cannot allocate a local vseg
[388]	1376
	1377	// initialise local vseg from reference
	1378	vseg_init_from_ref( vseg , vseg_xp );
	1379
	1380	// register local vseg in local VMM
[567]	1381	vmm_vseg_attach( &process->vmm , vseg );
[388]	1382	}
	1383
	1384	// success
	1385	*found_vseg = vseg;
[394]	1386	return 0;
[388]	1387
	1388	} // end vmm_get_vseg()
	1389
[407]	1390	//////////////////////////////////////////////////////////////////////////////////////
	1391	// This static function compute the target cluster to allocate a physical page
	1392	// for a given <vpn> in a given <vseg>, allocates the page (with an RPC if required)
	1393	// and returns an extended pointer on the allocated page descriptor.
[585]	1394	// It can be called by a thread running in any cluster.
[407]	1395	// The vseg cannot have the FILE type.
	1396	//////////////////////////////////////////////////////////////////////////////////////
	1397	static xptr_t vmm_page_allocate( vseg_t * vseg,
	1398	vpn_t vpn )
	1399	{
[433]	1400
[438]	1401	#if DEBUG_VMM_ALLOCATE_PAGE
	1402	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
[433]	1403	printk("\n[DBG] in %s : thread %x enter for vpn %x\n",
	1404	__FUNCTION__ , CURRENT_THREAD, vpn );
	1405	#endif
	1406
[407]	1407	// compute target cluster
	1408	page_t * page_ptr;
	1409	cxy_t page_cxy;
	1410	kmem_req_t req;
[577]	1411	uint32_t index;
[407]	1412
[577]	1413	uint32_t type = vseg->type;
	1414	uint32_t flags = vseg->flags;
	1415	uint32_t x_size = LOCAL_CLUSTER->x_size;
	1416	uint32_t y_size = LOCAL_CLUSTER->y_size;
[407]	1417
[567]	1418	// check vseg type
	1419	assert( ( type != VSEG_TYPE_FILE ) , "illegal vseg type\n" );
[407]	1420
	1421	if( flags & VSEG_DISTRIB ) // distributed => cxy depends on vpn LSB
	1422	{
[577]	1423	index = vpn & ((x_size * y_size) - 1);
	1424	page_cxy = HAL_CXY_FROM_XY( (index / y_size) , (index % y_size) );
[561]	1425
[577]	1426	// If the cluster selected from VPN's LSBs is empty, we select one randomly
	1427	if ( cluster_is_active( page_cxy ) == false )
	1428	{
	1429	page_cxy = cluster_random_select();
[561]	1430	}
[407]	1431	}
	1432	else // other cases => cxy specified in vseg
	1433	{
[561]	1434	page_cxy = vseg->cxy;
[407]	1435	}
	1436
	1437	// allocate a physical page from target cluster
	1438	if( page_cxy == local_cxy ) // target cluster is the local cluster
	1439	{
	1440	req.type = KMEM_PAGE;
	1441	req.size = 0;
	1442	req.flags = AF_NONE;
	1443	page_ptr = (page_t *)kmem_alloc( &req );
	1444	}
	1445	else // target cluster is not the local cluster
	1446	{
	1447	rpc_pmem_get_pages_client( page_cxy , 0 , &page_ptr );
	1448	}
	1449
[438]	1450	#if DEBUG_VMM_ALLOCATE_PAGE
	1451	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
[433]	1452	printk("\n[DBG] in %s : thread %x exit for vpn = %d / ppn = %x\n",
	1453	__FUNCTION__ , CURRENT_THREAD, vpn, ppm_page2ppn( XPTR( page_cxy , page_ptr ) ) );
	1454	#endif
	1455
[407]	1456	if( page_ptr == NULL ) return XPTR_NULL;
	1457	else return XPTR( page_cxy , page_ptr );
	1458
	1459	} // end vmm_page_allocate()
	1460
[313]	1461	////////////////////////////////////////
	1462	error_t vmm_get_one_ppn( vseg_t * vseg,
	1463	vpn_t vpn,
	1464	ppn_t * ppn )
	1465	{
	1466	error_t error;
[407]	1467	xptr_t page_xp; // extended pointer on physical page descriptor
[313]	1468	page_t * page_ptr; // local pointer on physical page descriptor
[406]	1469	uint32_t index; // missing page index in vseg mapper
	1470	uint32_t type; // vseg type;
[313]	1471
[406]	1472	type = vseg->type;
	1473	index = vpn - vseg->vpn_base;
[313]	1474
[438]	1475	#if DEBUG_VMM_GET_ONE_PPN
[441]	1476	thread_t * this = CURRENT_THREAD;
[469]	1477	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1478	printk("\n[DBG] %s : thread %x enter for vpn = %x / type = %s / index = %d\n",
[441]	1479	__FUNCTION__, this, vpn, vseg_type_str(type), index );
[433]	1480	#endif
[313]	1481
[406]	1482	// FILE type : get the physical page from the file mapper
[313]	1483	if( type == VSEG_TYPE_FILE )
	1484	{
[406]	1485	// get extended pointer on mapper
[407]	1486	xptr_t mapper_xp = vseg->mapper_xp;
[313]	1487
[567]	1488	assert( (mapper_xp != XPTR_NULL),
	1489	"mapper not defined for a FILE vseg\n" );
[406]	1490
	1491	// get mapper cluster and local pointer
	1492	cxy_t mapper_cxy = GET_CXY( mapper_xp );
[433]	1493	mapper_t * mapper_ptr = GET_PTR( mapper_xp );
[406]	1494
[313]	1495	// get page descriptor from mapper
	1496	if( mapper_cxy == local_cxy ) // mapper is local
	1497	{
	1498	page_ptr = mapper_get_page( mapper_ptr , index );
	1499	}
	1500	else // mapper is remote
	1501	{
	1502	rpc_mapper_get_page_client( mapper_cxy , mapper_ptr , index , &page_ptr );
	1503	}
	1504
	1505	if ( page_ptr == NULL ) return EINVAL;
	1506
[407]	1507	page_xp = XPTR( mapper_cxy , page_ptr );
[313]	1508	}
	1509
[406]	1510	// Other types : allocate a physical page from target cluster,
[407]	1511	// as defined by vseg type and vpn value
[313]	1512	else
	1513	{
[433]	1514	// allocate one physical page
[407]	1515	page_xp = vmm_page_allocate( vseg , vpn );
[406]	1516
[407]	1517	if( page_xp == XPTR_NULL ) return ENOMEM;
[313]	1518
[406]	1519	// initialise missing page from .elf file mapper for DATA and CODE types
[440]	1520	// the vseg->mapper_xp field is an extended pointer on the .elf file mapper
[313]	1521	if( (type == VSEG_TYPE_CODE) \|\| (type == VSEG_TYPE_DATA) )
	1522	{
[406]	1523	// get extended pointer on mapper
	1524	xptr_t mapper_xp = vseg->mapper_xp;
[313]	1525
[567]	1526	assert( (mapper_xp != XPTR_NULL),
	1527	"mapper not defined for a CODE or DATA vseg\n" );
[406]	1528
	1529	// get mapper cluster and local pointer
	1530	cxy_t mapper_cxy = GET_CXY( mapper_xp );
[433]	1531	mapper_t * mapper_ptr = GET_PTR( mapper_xp );
[406]	1532
	1533	// compute missing page offset in vseg
	1534	uint32_t offset = index << CONFIG_PPM_PAGE_SHIFT;
	1535
[313]	1536	// compute missing page offset in .elf file
[406]	1537	uint32_t elf_offset = vseg->file_offset + offset;
[313]	1538
[438]	1539	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1540	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1541	printk("\n[DBG] %s : thread %x for vpn = %x / elf_offset = %x\n",
[441]	1542	__FUNCTION__, this, vpn, elf_offset );
[433]	1543	#endif
[406]	1544	// compute extended pointer on page base
[407]	1545	xptr_t base_xp = ppm_page2base( page_xp );
[313]	1546
[406]	1547	// file_size (in .elf mapper) can be smaller than vseg_size (BSS)
	1548	uint32_t file_size = vseg->file_size;
	1549
	1550	if( file_size < offset ) // missing page fully in BSS
[313]	1551	{
[406]	1552
[438]	1553	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1554	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1555	printk("\n[DBG] %s : thread%x for vpn = %x / fully in BSS\n",
[441]	1556	__FUNCTION__, this, vpn );
[433]	1557	#endif
[407]	1558	if( GET_CXY( page_xp ) == local_cxy )
[313]	1559	{
[315]	1560	memset( GET_PTR( base_xp ) , 0 , CONFIG_PPM_PAGE_SIZE );
[313]	1561	}
	1562	else
	1563	{
[315]	1564	hal_remote_memset( base_xp , 0 , CONFIG_PPM_PAGE_SIZE );
[313]	1565	}
	1566	}
[406]	1567	else if( file_size >= (offset + CONFIG_PPM_PAGE_SIZE) ) // fully in mapper
[315]	1568	{
[406]	1569
[438]	1570	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1571	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1572	printk("\n[DBG] %s : thread %x, for vpn = %x / fully in mapper\n",
[441]	1573	__FUNCTION__, this, vpn );
[433]	1574	#endif
[313]	1575	if( mapper_cxy == local_cxy )
	1576	{
	1577	error = mapper_move_kernel( mapper_ptr,
	1578	true, // to_buffer
[406]	1579	elf_offset,
[313]	1580	base_xp,
[315]	1581	CONFIG_PPM_PAGE_SIZE );
[313]	1582	}
	1583	else
	1584	{
	1585	rpc_mapper_move_buffer_client( mapper_cxy,
	1586	mapper_ptr,
	1587	true, // to buffer
	1588	false, // kernel buffer
[406]	1589	elf_offset,
	1590	base_xp,
[315]	1591	CONFIG_PPM_PAGE_SIZE,
[313]	1592	&error );
	1593	}
	1594	if( error ) return EINVAL;
	1595	}
[406]	1596	else // both in mapper and in BSS :
	1597	// - (file_size - offset) bytes from mapper
	1598	// - (page_size + offset - file_size) bytes from BSS
[313]	1599	{
[406]	1600
[438]	1601	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1602	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1603	printk("\n[DBG] %s : thread %x for vpn = %x / both mapper & BSS\n"
	1604	" %d bytes from mapper / %d bytes from BSS\n",
[441]	1605	__FUNCTION__, this, vpn,
[407]	1606	file_size - offset , offset + CONFIG_PPM_PAGE_SIZE - file_size );
[433]	1607	#endif
[313]	1608	// initialize mapper part
[315]	1609	if( mapper_cxy == local_cxy )
[313]	1610	{
	1611	error = mapper_move_kernel( mapper_ptr,
[315]	1612	true, // to buffer
[406]	1613	elf_offset,
[313]	1614	base_xp,
[406]	1615	file_size - offset );
[313]	1616	}
[315]	1617	else
[313]	1618	{
	1619	rpc_mapper_move_buffer_client( mapper_cxy,
	1620	mapper_ptr,
[315]	1621	true, // to buffer
[313]	1622	false, // kernel buffer
[406]	1623	elf_offset,
	1624	base_xp,
	1625	file_size - offset,
[313]	1626	&error );
	1627	}
	1628	if( error ) return EINVAL;
	1629
	1630	// initialize BSS part
[407]	1631	if( GET_CXY( page_xp ) == local_cxy )
[313]	1632	{
[406]	1633	memset( GET_PTR( base_xp ) + file_size - offset , 0 ,
	1634	offset + CONFIG_PPM_PAGE_SIZE - file_size );
[313]	1635	}
	1636	else
	1637	{
[406]	1638	hal_remote_memset( base_xp + file_size - offset , 0 ,
	1639	offset + CONFIG_PPM_PAGE_SIZE - file_size );
[313]	1640	}
	1641	}
	1642	} // end initialisation for CODE or DATA types
	1643	}
	1644
	1645	// return ppn
[407]	1646	*ppn = ppm_page2ppn( page_xp );
[406]	1647
[438]	1648	#if DEBUG_VMM_GET_ONE_PPN
[469]	1649	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1650	printk("\n[DBG] %s : thread %x exit for vpn = %x / ppn = %x\n",
[441]	1651	__FUNCTION__ , this , vpn , *ppn );
[433]	1652	#endif
[406]	1653
[313]	1654	return 0;
	1655
	1656	} // end vmm_get_one_ppn()
	1657
[585]	1658	///////////////////////////////////////////////////
	1659	error_t vmm_handle_page_fault( process_t * process,
	1660	vpn_t vpn )
[1]	1661	{
[585]	1662	vseg_t * vseg; // vseg containing vpn
	1663	uint32_t new_attr; // new PTE_ATTR value
	1664	ppn_t new_ppn; // new PTE_PPN value
	1665	uint32_t ref_attr; // PTE_ATTR value in reference GPT
	1666	ppn_t ref_ppn; // PTE_PPN value in reference GPT
	1667	cxy_t ref_cxy; // reference cluster for missing vpn
	1668	process_t * ref_ptr; // reference process for missing vpn
	1669	xptr_t local_gpt_xp; // extended pointer on local GPT
	1670	xptr_t local_lock_xp; // extended pointer on local GPT lock
	1671	xptr_t ref_gpt_xp; // extended pointer on reference GPT
	1672	xptr_t ref_lock_xp; // extended pointer on reference GPT lock
	1673	error_t error; // value returned by called functions
[1]	1674
[585]	1675	// get local vseg (access to reference VSL can be required)
	1676	error = vmm_get_vseg( process,
	1677	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
	1678	&vseg );
[441]	1679
[585]	1680	if( error )
	1681	{
	1682	printk("\n[ERROR] in %s : illegal VPN %x in process %x\n",
	1683	__FUNCTION__ , vpn , process->pid );
	1684
	1685	return EXCP_USER_ERROR;
	1686	}
	1687
	1688	#if DEBUG_VMM_HANDLE_PAGE_FAULT
	1689	uint32_t cycle = (uint32_t)hal_get_cycles();
[567]	1690	thread_t * this = CURRENT_THREAD;
[585]	1691	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
	1692	printk("\n[DBG] %s : threadr[%x,%x] enter for vpn %x / %s / cycle %d\n",
	1693	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(vseg->type), cycle );
[433]	1694	#endif
[406]	1695
[585]	1696	//////////////// private vseg => access only the local GPT
	1697	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
[438]	1698	{
[585]	1699	// build extended pointer on local GPT and local GPT lock
	1700	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
	1701	local_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
[407]	1702
[585]	1703	// take local GPT lock in write mode
	1704	remote_rwlock_wr_acquire( local_lock_xp );
[407]	1705
[585]	1706	// check VPN still unmapped in local GPT
	1707	// do nothing if VPN has been mapped by a a concurrent page_fault
	1708	hal_gpt_get_pte( local_gpt_xp,
	1709	vpn,
	1710	&new_attr,
	1711	&new_ppn );
[407]	1712
[585]	1713	if( (new_attr & GPT_MAPPED) == 0 ) // VPN still unmapped
	1714	{
	1715	// allocate and initialise a physical page depending on the vseg type
	1716	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
[407]	1717
[585]	1718	if( error )
[408]	1719	{
	1720	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
	1721	__FUNCTION__ , process->pid , vpn );
[1]	1722
[585]	1723	// release local GPT lock in write mode
	1724	remote_rwlock_wr_release( local_lock_xp );
[406]	1725
[585]	1726	return EXCP_KERNEL_PANIC;
[407]	1727	}
	1728
[408]	1729	// define new_attr from vseg flags
[407]	1730	new_attr = GPT_MAPPED \| GPT_SMALL;
	1731	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
	1732	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
	1733	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
	1734	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
	1735
[585]	1736	// set PTE (PPN & attribute) to local GPT
	1737	error = hal_gpt_set_pte( local_gpt_xp,
[408]	1738	vpn,
	1739	new_attr,
	1740	new_ppn );
[585]	1741	if ( error )
[407]	1742	{
[585]	1743	printk("\n[ERROR] in %s : cannot update local GPT / process %x / vpn = %x\n",
[407]	1744	__FUNCTION__ , process->pid , vpn );
[585]	1745
	1746	// release local GPT lock in write mode
	1747	remote_rwlock_wr_release( local_lock_xp );
	1748
	1749	return EXCP_KERNEL_PANIC;
[407]	1750	}
	1751	}
[585]	1752
	1753	// release local GPT lock in write mode
	1754	remote_rwlock_wr_release( local_lock_xp );
	1755
	1756	#if DEBUG_VMM_HANDLE_PAGE_FAULT
	1757	cycle = (uint32_t)hal_get_cycles();
	1758	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
	1759	printk("\n[DBG] %s : private page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
	1760	__FUNCTION__, vpn, new_ppn, new_attr, cycle );
	1761	#endif
	1762	return EXCP_NON_FATAL;
	1763
	1764	} // end local GPT access
	1765
	1766	//////////// public vseg => access reference GPT
	1767	else
	1768	{
	1769	// get reference process cluster and local pointer
	1770	ref_cxy = GET_CXY( process->ref_xp );
	1771	ref_ptr = GET_PTR( process->ref_xp );
	1772
	1773	// build extended pointer on reference GPT and reference GPT lock
	1774	ref_gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
	1775	ref_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt_lock );
	1776
	1777	// build extended pointer on local GPT and local GPT lock
	1778	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
	1779	local_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
	1780
	1781	// take reference GPT lock in read mode
	1782	remote_rwlock_rd_acquire( ref_lock_xp );
	1783
	1784	// get directly PPN & attributes from reference GPT
	1785	// this can avoids a costly RPC for a false page fault
	1786	hal_gpt_get_pte( ref_gpt_xp,
	1787	vpn,
	1788	&ref_attr,
	1789	&ref_ppn );
	1790
	1791	// release reference GPT lock in read mode
	1792	remote_rwlock_rd_release( ref_lock_xp );
	1793
	1794	if( ref_attr & GPT_MAPPED ) // false page fault => update local GPT
[1]	1795	{
[585]	1796	// take local GPT lock in write mode
	1797	remote_rwlock_wr_acquire( local_lock_xp );
	1798
	1799	// check VPN still unmapped in local GPT
	1800	hal_gpt_get_pte( local_gpt_xp,
	1801	vpn,
	1802	&new_attr,
	1803	&new_ppn );
[1]	1804
[585]	1805	if( (new_attr & GPT_MAPPED) == 0 ) // VPN still unmapped
	1806	{
	1807	// update local GPT from reference GPT
	1808	error = hal_gpt_set_pte( local_gpt_xp,
	1809	vpn,
	1810	ref_attr,
	1811	ref_ppn );
	1812	if( error )
	1813	{
	1814	printk("\n[ERROR] in %s : cannot update local GPT / process %x / vpn = %x\n",
	1815	__FUNCTION__ , process->pid , vpn );
	1816
	1817	// release local GPT lock in write mode
	1818	remote_rwlock_wr_release( local_lock_xp );
	1819
	1820	return EXCP_KERNEL_PANIC;
	1821	}
	1822	}
	1823	else // VPN has been mapped by a a concurrent page_fault
	1824	{
	1825	// keep PTE from local GPT
	1826	ref_attr = new_attr;
	1827	ref_ppn = new_ppn;
	1828	}
	1829
	1830	// release local GPT lock in write mode
	1831	remote_rwlock_wr_release( local_lock_xp );
	1832
	1833	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[433]	1834	cycle = (uint32_t)hal_get_cycles();
[585]	1835	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
	1836	printk("\n[DBG] %s : false page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
	1837	__FUNCTION__, vpn, ref_ppn, ref_attr, cycle );
[433]	1838	#endif
[585]	1839	return EXCP_NON_FATAL;
	1840	}
	1841	else // true page fault => update reference GPT
	1842	{
	1843	// take reference GPT lock in write mode
	1844	remote_rwlock_wr_acquire( ref_lock_xp );
	1845
	1846	// check VPN still unmapped in reference GPT
	1847	// do nothing if VPN has been mapped by a a concurrent page_fault
	1848	hal_gpt_get_pte( ref_gpt_xp,
	1849	vpn,
	1850	&ref_attr,
	1851	&ref_ppn );
[406]	1852
[585]	1853	if( (ref_attr & GPT_MAPPED) == 0 ) // VPN actually unmapped
	1854	{
	1855	// allocate and initialise a physical page depending on the vseg type
	1856	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
[1]	1857
[585]	1858	if( error )
	1859	{
	1860	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
	1861	__FUNCTION__ , process->pid , vpn );
[313]	1862
[585]	1863	// release reference GPT lock in write mode
	1864	remote_rwlock_wr_release( ref_lock_xp );
	1865
	1866	return EXCP_KERNEL_PANIC;
	1867	}
[1]	1868
[585]	1869	// define new_attr from vseg flags
	1870	new_attr = GPT_MAPPED \| GPT_SMALL;
	1871	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
	1872	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
	1873	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
	1874	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
[440]	1875
[585]	1876	// update reference GPT
	1877	error = hal_gpt_set_pte( ref_gpt_xp,
	1878	vpn,
	1879	new_attr,
	1880	new_ppn );
	1881
	1882	// update local GPT (protected by reference GPT lock)
	1883	error \|= hal_gpt_set_pte( local_gpt_xp,
	1884	vpn,
	1885	new_attr,
	1886	new_ppn );
	1887
	1888	if( error )
	1889	{
	1890	printk("\n[ERROR] in %s : cannot update GPT / process %x / vpn = %x\n",
	1891	__FUNCTION__ , process->pid , vpn );
	1892
	1893	// release reference GPT lock in write mode
	1894	remote_rwlock_wr_release( ref_lock_xp );
	1895
	1896	return EXCP_KERNEL_PANIC;
	1897	}
	1898	}
	1899
	1900	// release reference GPT lock in write mode
	1901	remote_rwlock_wr_release( ref_lock_xp );
	1902
[440]	1903	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[585]	1904	cycle = (uint32_t)hal_get_cycles();
[469]	1905	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
[585]	1906	printk("\n[DBG] %s : true page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
	1907	__FUNCTION__, vpn, new_ppn, new_attr, cycle );
[435]	1908	#endif
[585]	1909	return EXCP_NON_FATAL;
	1910	}
	1911	}
	1912	} // end vmm_handle_page_fault()
[435]	1913
[585]	1914	////////////////////////////////////////////
	1915	error_t vmm_handle_cow( process_t * process,
	1916	vpn_t vpn )
	1917	{
	1918	vseg_t * vseg; // vseg containing vpn
	1919	cxy_t ref_cxy; // reference cluster for missing vpn
	1920	process_t * ref_ptr; // reference process for missing vpn
	1921	xptr_t gpt_xp; // extended pointer on GPT
	1922	xptr_t gpt_lock_xp; // extended pointer on GPT lock
	1923	uint32_t old_attr; // current PTE_ATTR value
	1924	ppn_t old_ppn; // current PTE_PPN value
	1925	uint32_t new_attr; // new PTE_ATTR value
	1926	ppn_t new_ppn; // new PTE_PPN value
	1927	error_t error;
[1]	1928
[585]	1929	#if DEBUG_VMM_HANDLE_COW
	1930	uint32_t cycle = (uint32_t)hal_get_cycles();
	1931	thread_t * this = CURRENT_THREAD;
	1932	if( DEBUG_VMM_HANDLE_COW < cycle )
	1933	printk("\n[DBG] %s : thread[%x,%x] enter for vpn %x / core[%x,%d] / cycle %d\n",
	1934	__FUNCTION__, process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
	1935	#endif
	1936
	1937	// get local vseg
	1938	error = vmm_get_vseg( process,
	1939	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
	1940	&vseg );
	1941
[440]	1942	if( error )
[1]	1943	{
[585]	1944	printk("\n[PANIC] in %s : VPN %x in process %x not in registered vseg\n",
	1945	__FUNCTION__, vpn, process->pid );
	1946
	1947	return EXCP_KERNEL_PANIC;
[440]	1948	}
[407]	1949
[585]	1950	// get reference GPT cluster and local pointer
	1951	ref_cxy = GET_CXY( process->ref_xp );
	1952	ref_ptr = GET_PTR( process->ref_xp );
[407]	1953
[585]	1954	// build relevant extended pointers on GPT and GPT lock
	1955	// - access local GPT for a private vseg
	1956	// - access reference GPT for a public vseg
	1957	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
[440]	1958	{
[585]	1959	gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
	1960	gpt_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
[1]	1961	}
[440]	1962	else
[1]	1963	{
[585]	1964	gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
	1965	gpt_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt_lock );
[1]	1966	}
	1967
[585]	1968	// take GPT lock in write mode
	1969	remote_rwlock_wr_acquire( gpt_lock_xp );
[441]	1970
[585]	1971	// get current PTE from reference GPT
	1972	hal_gpt_get_pte( gpt_xp,
	1973	vpn,
	1974	&old_attr,
	1975	&old_ppn );
[441]	1976
[585]	1977	// the PTE must be mapped for a COW
	1978	if( (old_attr & GPT_MAPPED) == 0 )
	1979	{
	1980	printk("\n[PANIC] in %s : VPN %x in process %x unmapped\n",
	1981	__FUNCTION__, vpn, process->pid );
[407]	1982
[585]	1983	// release GPT lock in write mode
	1984	remote_rwlock_wr_acquire( gpt_lock_xp );
[407]	1985
[585]	1986	return EXCP_KERNEL_PANIC;
[407]	1987	}
	1988
[585]	1989	// get extended pointer, cluster and local pointer on physical page descriptor
	1990	xptr_t page_xp = ppm_ppn2page( old_ppn );
	1991	cxy_t page_cxy = GET_CXY( page_xp );
	1992	page_t * page_ptr = GET_PTR( page_xp );
[435]	1993
[585]	1994	// get extended pointers on forks and lock field in page descriptor
	1995	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
	1996	xptr_t forks_lock_xp = XPTR( page_cxy , &page_ptr->lock );
[407]	1997
[585]	1998	// take lock protecting "forks" counter
	1999	remote_busylock_acquire( forks_lock_xp );
[407]	2000
[585]	2001	// get number of pending forks from page descriptor
	2002	uint32_t forks = hal_remote_l32( forks_xp );
[441]	2003
[585]	2004	if( forks ) // pending fork => allocate a new page, and copy old to new
	2005	{
	2006	// allocate a new physical page
	2007	page_xp = vmm_page_allocate( vseg , vpn );
	2008	if( page_xp == XPTR_NULL )
	2009	{
	2010	printk("\n[PANIC] in %s : no memory for vpn %x in process %x\n",
	2011	__FUNCTION__ , vpn, process->pid );
[441]	2012
[585]	2013	// release GPT lock in write mode
	2014	remote_rwlock_wr_acquire( gpt_lock_xp );
[441]	2015
[585]	2016	// release lock protecting "forks" counter
	2017	remote_busylock_release( forks_lock_xp );
[441]	2018
[585]	2019	return EXCP_KERNEL_PANIC;
	2020	}
[441]	2021
[585]	2022	// compute allocated page PPN
	2023	new_ppn = ppm_page2ppn( page_xp );
[441]	2024
[585]	2025	// copy old page content to new page
	2026	xptr_t old_base_xp = ppm_ppn2base( old_ppn );
	2027	xptr_t new_base_xp = ppm_ppn2base( new_ppn );
	2028	memcpy( GET_PTR( new_base_xp ),
	2029	GET_PTR( old_base_xp ),
	2030	CONFIG_PPM_PAGE_SIZE );
[441]	2031
[585]	2032	// decrement pending forks counter in page descriptor
	2033	hal_remote_atomic_add( forks_xp , -1 );
[441]	2034
[585]	2035	#if(DEBUG_VMM_HANDLE_COW & 1)
	2036	if( DEBUG_VMM_HANDLE_COW < cycle )
	2037	printk("\n[DBG] %s : thread[%x,%x] : pending forks => allocate a new PPN %x\n",
	2038	__FUNCTION__, process->pid, this->trdid, new_ppn );
	2039	#endif
[440]	2040
[585]	2041	}
	2042	else // no pending fork => keep the existing page
	2043	{
[1]	2044
[585]	2045	#if(DEBUG_VMM_HANDLE_COW & 1)
	2046	if( DEBUG_VMM_HANDLE_COW < cycle )
	2047	printk("\n[DBG] %s : thread[%x,%x] no pending forks => keep existing PPN %x\n",
	2048	__FUNCTION__, process->pid, this->trdid, new_ppn );
	2049	#endif
	2050	new_ppn = old_ppn;
	2051	}
[1]	2052
[585]	2053	// release lock protecting "forks" counter
	2054	remote_busylock_release( forks_lock_xp );
	2055
	2056	// build new_attr : reset COW and set WRITABLE,
	2057	new_attr = (old_attr \| GPT_WRITABLE) & (~GPT_COW);
	2058
	2059	// update the relevan GPT
	2060	// - private vseg => update local GPT
	2061	// - public vseg => update all GPT copies
	2062	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
[1]	2063	{
[585]	2064	hal_gpt_set_pte( gpt_xp,
	2065	vpn,
	2066	new_attr,
	2067	new_ppn );
[1]	2068	}
[585]	2069	else
[1]	2070	{
[585]	2071	if( ref_cxy == local_cxy ) // reference cluster is local
	2072	{
	2073	vmm_global_update_pte( process,
	2074	vpn,
	2075	new_attr,
	2076	new_ppn );
	2077	}
	2078	else // reference cluster is remote
	2079	{
	2080	rpc_vmm_global_update_pte_client( ref_cxy,
	2081	ref_ptr,
	2082	vpn,
	2083	new_attr,
	2084	new_ppn );
	2085	}
[1]	2086	}
	2087
[585]	2088	// release GPT lock in write mode
	2089	remote_rwlock_wr_release( gpt_lock_xp );
[21]	2090
[585]	2091	#if DEBUG_VMM_HANDLE_COW
	2092	cycle = (uint32_t)hal_get_cycles();
	2093	if( DEBUG_VMM_HANDLE_COW < cycle )
	2094	printk("\n[DBG] %s : thread[%x,%x] exit for vpn %x / core[%x,%d] / cycle %d\n",
	2095	__FUNCTION__, process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
	2096	#endif
[313]	2097
[585]	2098	return EXCP_NON_FATAL;
[1]	2099
[585]	2100	} // end vmm_handle_cow()
	2101

Note: See TracBrowser for help on using the repository browser.

Download in other formats: