Context Navigation

source: trunk/kernel/mm/vmm.c @ 433

Last change on this file since 433 was 433, checked in by alain, 6 years ago
blip
File size: 65.6 KB

Rev	Line
[1]	1	/*
	2	* vmm.c - virtual memory manager related operations interface.
	3	*
	4	* Authors Ghassan Almaless (2008,2009,2010,2011, 2012)
	5	* Mohamed Lamine Karaoui (2015)
	6	* Alain Greiner (2016)
[21]	7	*
[1]	8	* Copyright (c) UPMC Sorbonne Universites
	9	*
	10	* This file is part of ALMOS-MKH.
	11	*
	12	* ALMOS-MKH is free software; you can redistribute it and/or modify it
	13	* under the terms of the GNU General Public License as published by
	14	* the Free Software Foundation; version 2.0 of the License.
	15	*
	16	* ALMOS-MKH is distributed in the hope that it will be useful, but
	17	* WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU General Public License
	22	* along with ALMOS-MKH; if not, write to the Free Software Foundation,
	23	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	24	*/
	25
[14]	26	#include <kernel_config.h>
[1]	27	#include <hal_types.h>
	28	#include <hal_special.h>
	29	#include <hal_gpt.h>
[409]	30	#include <hal_vmm.h>
[1]	31	#include <printk.h>
[23]	32	#include <memcpy.h>
[1]	33	#include <rwlock.h>
	34	#include <list.h>
[408]	35	#include <xlist.h>
[1]	36	#include <bits.h>
	37	#include <process.h>
	38	#include <thread.h>
	39	#include <vseg.h>
	40	#include <cluster.h>
	41	#include <scheduler.h>
	42	#include <vfs.h>
	43	#include <mapper.h>
	44	#include <page.h>
	45	#include <kmem.h>
	46	#include <vmm.h>
	47
	48	//////////////////////////////////////////////////////////////////////////////////
	49	// Extern global variables
	50	//////////////////////////////////////////////////////////////////////////////////
	51
	52	extern process_t process_zero; // defined in cluster.c file
	53
	54
[415]	55	///////////////////////////////////////
	56	error_t vmm_init( process_t * process )
[21]	57	{
[1]	58	error_t error;
	59	vseg_t * vseg_kentry;
	60	vseg_t * vseg_args;
	61	vseg_t * vseg_envs;
	62	intptr_t base;
	63	intptr_t size;
	64
[433]	65	#if CONFIG_DEBUG_VMM_INIT
	66	uint32_t cycle = (uint32_t)hal_get_cycles();
	67	if( CONFIG_DEBUG_VMM_INIT )
	68	printk("\n[DBG] %s : thread %x enter for process %x / cycle %d\n",
	69	__FUNCTION__ , CURRENT_THREAD , process->pid , cycle );
	70	#endif
[204]	71
[1]	72	// get pointer on VMM
	73	vmm_t * vmm = &process->vmm;
	74
[407]	75	// initialize local list of vsegs
	76	vmm->vsegs_nr = 0;
[408]	77	xlist_root_init( XPTR( local_cxy , &vmm->vsegs_root ) );
	78	remote_rwlock_init( XPTR( local_cxy , &vmm->vsegs_lock ) );
[407]	79
[204]	80	assert( ((CONFIG_VMM_KENTRY_SIZE + CONFIG_VMM_ARGS_SIZE + CONFIG_VMM_ENVS_SIZE)
	81	<= CONFIG_VMM_ELF_BASE) , __FUNCTION__ , "UTILS zone too small\n" );
[21]	82
[204]	83	assert( (CONFIG_THREAD_MAX_PER_CLUSTER <= 32) , __FUNCTION__ ,
	84	"no more than 32 threads per cluster for a single process\n");
[1]	85
[204]	86	assert( ((CONFIG_VMM_STACK_SIZE * CONFIG_THREAD_MAX_PER_CLUSTER) <=
	87	(CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE)) , __FUNCTION__ ,
	88	"STACK zone too small\n");
[1]	89
[409]	90	// register kentry vseg in VSL
[406]	91	base = CONFIG_VMM_KENTRY_BASE << CONFIG_PPM_PAGE_SHIFT;
[1]	92	size = CONFIG_VMM_KENTRY_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	93
[407]	94	vseg_kentry = vmm_create_vseg( process,
	95	VSEG_TYPE_CODE,
	96	base,
	97	size,
	98	0, // file_offset unused
	99	0, // file_size unused
	100	XPTR_NULL, // mapper_xp unused
	101	local_cxy );
[204]	102
[415]	103	if( vseg_kentry == NULL )
	104	{
	105	printk("\n[ERROR] in %s : cannot register kentry vseg\n", __FUNCTION__ );
	106	return -1;
	107	}
[204]	108
[406]	109	vmm->kent_vpn_base = base;
[1]	110
[409]	111	// register args vseg in VSL
[406]	112	base = (CONFIG_VMM_KENTRY_BASE +
	113	CONFIG_VMM_KENTRY_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
[1]	114	size = CONFIG_VMM_ARGS_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	115
[407]	116	vseg_args = vmm_create_vseg( process,
	117	VSEG_TYPE_DATA,
	118	base,
	119	size,
	120	0, // file_offset unused
	121	0, // file_size unused
	122	XPTR_NULL, // mapper_xp unused
	123	local_cxy );
[204]	124
[415]	125	if( vseg_args == NULL )
	126	{
	127	printk("\n[ERROR] in %s : cannot register args vseg\n", __FUNCTION__ );
	128	return -1;
	129	}
[204]	130
[406]	131	vmm->args_vpn_base = base;
[1]	132
[409]	133	// register the envs vseg in VSL
[406]	134	base = (CONFIG_VMM_KENTRY_BASE +
	135	CONFIG_VMM_KENTRY_SIZE +
	136	CONFIG_VMM_ARGS_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
[1]	137	size = CONFIG_VMM_ENVS_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	138
[407]	139	vseg_envs = vmm_create_vseg( process,
	140	VSEG_TYPE_DATA,
	141	base,
	142	size,
	143	0, // file_offset unused
	144	0, // file_size unused
	145	XPTR_NULL, // mapper_xp unused
	146	local_cxy );
[204]	147
[415]	148	if( vseg_envs == NULL )
	149	{
	150	printk("\n[ERROR] in %s : cannot register envs vseg\n", __FUNCTION__ );
	151	return -1;
	152	}
[204]	153
[406]	154	vmm->envs_vpn_base = base;
[1]	155
[409]	156	// create GPT (empty)
[1]	157	error = hal_gpt_create( &vmm->gpt );
	158
[415]	159	if( error )
	160	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
[204]	161
[415]	162	// initialize GPT (architecture specic)
[409]	163	// (For TSAR, identity map the kentry_vseg)
	164	error = hal_vmm_init( vmm );
	165
[415]	166	if( error )
	167	printk("\n[ERROR] in %s : cannot initialize GPT\n", __FUNCTION__ );
[409]	168
[1]	169	// initialize STACK allocator
	170	vmm->stack_mgr.bitmap = 0;
	171	vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
	172
	173	// initialize MMAP allocator
[407]	174	vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
	175	vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
	176	vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
[1]	177	uint32_t i;
	178	for( i = 0 ; i < 32 ; i++ ) list_root_init( &vmm->mmap_mgr.zombi_list[i] );
	179
[21]	180	// initialize instrumentation counters
[409]	181	vmm->pgfault_nr = 0;
[1]	182
[124]	183	hal_fence();
[1]	184
[433]	185	#if CONFIG_DEBUG_VMM_INIT
	186	cycle = (uint32_t)hal_get_cycles();
	187	if( CONFIG_DEBUG_VMM_INIT )
	188	printk("\n[DBG] %s : thread %x exit for process %x / entry_point = %x / cycle %d\n",
	189	__FUNCTION__ , CURRENT_THREAD , process->pid , process->vmm.entry_point , cycle );
	190	#endif
[204]	191
[415]	192	return 0;
	193
[204]	194	} // end vmm_init()
	195
[407]	196	//////////////////////////////////////
	197	void vmm_display( process_t * process,
	198	bool_t mapping )
	199	{
[429]	200	assert( (process->ref_xp == XPTR( local_cxy , process )) , __FUNCTION__,
	201	"this function must be executed in reference cluster" );
	202
[407]	203	vmm_t * vmm = &process->vmm;
	204	gpt_t * gpt = &vmm->gpt;
	205
	206	printk("\n***** VSL and GPT for process %x\n\n",
	207	process->pid );
	208
	209	// get lock protecting the vseg list
[408]	210	remote_rwlock_rd_lock( XPTR( local_cxy , &vmm->vsegs_lock ) );
[407]	211
	212	// scan the list of vsegs
[408]	213	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	214	xptr_t iter_xp;
	215	xptr_t vseg_xp;
[407]	216	vseg_t * vseg;
[408]	217	XLIST_FOREACH( root_xp , iter_xp )
[407]	218	{
[408]	219	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	220	vseg = GET_PTR( vseg_xp );
[408]	221
[407]	222	printk(" - %s : base = %X / size = %X / npages = %d\n",
	223	vseg_type_str( vseg->type ) , vseg->min , vseg->max - vseg->min , vseg->vpn_size );
	224
	225	if( mapping )
	226	{
	227	vpn_t vpn;
	228	ppn_t ppn;
	229	uint32_t attr;
	230	vpn_t base = vseg->vpn_base;
	231	vpn_t size = vseg->vpn_size;
	232	for( vpn = base ; vpn < (base+size) ; vpn++ )
	233	{
	234	hal_gpt_get_pte( gpt , vpn , &attr , &ppn );
	235	if( attr & GPT_MAPPED )
	236	{
	237	printk(" . vpn = %X / attr = %X / ppn = %X\n", vpn , attr , ppn );
	238	}
	239	}
	240	}
	241	}
	242
	243	// release the lock
[408]	244	remote_rwlock_rd_unlock( XPTR( local_cxy , &vmm->vsegs_lock ) );
[407]	245
[408]	246	} // vmm_display()
	247
[433]	248	/////////////////////i//////////////////////////
	249	void vmm_global_update_pte( process_t * process,
	250	vpn_t vpn,
	251	uint32_t attr,
	252	ppn_t ppn )
[23]	253	{
	254
[408]	255	xlist_entry_t * process_root_ptr;
	256	xptr_t process_root_xp;
	257	xptr_t process_iter_xp;
[23]	258
[408]	259	xptr_t remote_process_xp;
	260	cxy_t remote_process_cxy;
	261	process_t * remote_process_ptr;
	262	xptr_t remote_gpt_xp;
[23]	263
[408]	264	pid_t pid;
	265	cxy_t owner_cxy;
	266	lpid_t owner_lpid;
[23]	267
[433]	268	#if CONFIG_DEBUG_VMM_UPDATE_PTE
	269	uint32_t cycle = (uint32_t)hal_get_cycles();
	270	if( CONFIG_DEBUG_VMM_UPDATE_PTE < cycle )
	271	printk("\n[DBG] %s : thread %x enter for process %x / vpn %x / cycle %d\n",
	272	__FUNCTION__ , CURRENT_THREAD , process->pid , vpn , cycle );
	273	#endif
	274
	275	// check cluster is reference
	276	assert( (GET_CXY( process->ref_xp ) == local_cxy) , __FUNCTION__,
	277	"not called in reference cluster\n");
	278
[408]	279	// get extended pointer on root of process copies xlist in owner cluster
	280	pid = process->pid;
	281	owner_cxy = CXY_FROM_PID( pid );
	282	owner_lpid = LPID_FROM_PID( pid );
	283	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
	284	process_root_xp = XPTR( owner_cxy , process_root_ptr );
[23]	285
[408]	286	// loop on destination process copies
	287	XLIST_FOREACH( process_root_xp , process_iter_xp )
	288	{
	289	// get cluster and local pointer on remote process
	290	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
[433]	291	remote_process_ptr = GET_PTR( remote_process_xp );
[408]	292	remote_process_cxy = GET_CXY( remote_process_xp );
[407]	293
[433]	294	#if (CONFIG_DEBUG_VMM_UPDATE_PTE & 0x1)
	295	if( CONFIG_DEBUG_VMM_UPDATE_PTE < cycle )
	296	printk("\n[DBG] %s : thread %x handling process %x in cluster %x\n",
	297	__FUNCTION__ , CURRENT_THREAD , process->pid , remote_process_cxy );
	298	#endif
	299
[408]	300	// get extended pointer on remote gpt
	301	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
	302
[433]	303	// update remote GPT
	304	hal_gpt_update_pte( remote_gpt_xp, vpn, attr, ppn );
[408]	305	}
	306
[433]	307	#if CONFIG_DEBUG_VMM_UPDATE_PTE
	308	cycle = (uint32_t)hal_get_cycles();
	309	if( CONFIG_DEBUG_VMM_UPDATE_PTE < cycle )
	310	printk("\n[DBG] %s : thread %x exit for process %x / vpn %x / cycle %d\n",
	311	__FUNCTION__ , CURRENT_THREAD , process->pid , vpn , cycle );
	312	#endif
	313
	314	} // end vmm_global_update_pte()
	315
[408]	316	///////////////////////////////////////
	317	void vmm_set_cow( process_t * process )
	318	{
	319	vmm_t * vmm;
	320
	321	xlist_entry_t * process_root_ptr;
	322	xptr_t process_root_xp;
	323	xptr_t process_iter_xp;
	324
	325	xptr_t remote_process_xp;
	326	cxy_t remote_process_cxy;
	327	process_t * remote_process_ptr;
	328	xptr_t remote_gpt_xp;
	329
	330	xptr_t vseg_root_xp;
	331	xptr_t vseg_iter_xp;
	332
	333	xptr_t vseg_xp;
	334	vseg_t * vseg;
	335
	336	pid_t pid;
	337	cxy_t owner_cxy;
	338	lpid_t owner_lpid;
	339
[433]	340	#if CONFIG_DEBUG_VMM_SET_COW
	341	uint32_t cycle = (uint32_t)hal_get_cycles();
	342	if( CONFIG_DEBUG_VMM_SET_COW < cycle )
	343	printk("\n[DBG] %s : thread %x enter for process %x / cycle %d\n",
	344	__FUNCTION__ , CURRENT_THREAD , process->pid , cycle );
	345	#endif
[408]	346
	347	// check cluster is reference
	348	assert( (GET_CXY( process->ref_xp ) == local_cxy) , __FUNCTION__,
	349	"local cluster is not process reference cluster\n");
	350
	351	// get pointer on reference VMM
	352	vmm = &process->vmm;
	353
	354	// get extended pointer on root of process copies xlist in owner cluster
	355	pid = process->pid;
	356	owner_cxy = CXY_FROM_PID( pid );
	357	owner_lpid = LPID_FROM_PID( pid );
	358	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
	359	process_root_xp = XPTR( owner_cxy , process_root_ptr );
	360
	361	// get extended pointer on root of vsegs xlist from reference VMM
	362	vseg_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	363
	364	// loop on destination process copies
	365	XLIST_FOREACH( process_root_xp , process_iter_xp )
	366	{
	367	// get cluster and local pointer on remote process
	368	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
[433]	369	remote_process_ptr = GET_PTR( remote_process_xp );
[408]	370	remote_process_cxy = GET_CXY( remote_process_xp );
	371
[433]	372	#if (CONFIG_DEBUG_VMM_SET_COW &0x1)
	373	if( CONFIG_DEBUG_VMM_SET_COW < cycle )
	374	printk("\n[DBG] %s : thread %x handling process %x in cluster %x\n",
	375	__FUNCTION__ , CURRENT_THREAD , process->pid , remote_process_cxy );
	376	#endif
[408]	377
	378	// get extended pointer on remote gpt
	379	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
	380
	381	// loop on vsegs in (local) reference process VSL
	382	XLIST_FOREACH( vseg_root_xp , vseg_iter_xp )
	383	{
	384	// get pointer on vseg
	385	vseg_xp = XLIST_ELEMENT( vseg_iter_xp , vseg_t , xlist );
[433]	386	vseg = GET_PTR( vseg_xp );
[408]	387
	388	assert( (GET_CXY( vseg_xp ) == local_cxy) , __FUNCTION__,
	389	"all vsegs in reference VSL must be local\n" );
	390
	391	// get vseg type, base and size
	392	uint32_t type = vseg->type;
	393	vpn_t vpn_base = vseg->vpn_base;
	394	vpn_t vpn_size = vseg->vpn_size;
	395
[433]	396	#if (CONFIG_DEBUG_VMM_SET_COW & 0x1)
	397	if( CONFIG_DEBUG_VMM_SET_COW < cycle )
	398	printk("\n[DBG] %s : thread %x handling vseg %s / vpn_base = %x / vpn_size = %x\n",
	399	__FUNCTION__, CURRENT_THREAD , vseg_type_str(type), vpn_base, vpn_size );
	400	#endif
	401	// only DATA, ANON and REMOTE vsegs
[408]	402	if( (type == VSEG_TYPE_DATA) \|\|
	403	(type == VSEG_TYPE_ANON) \|\|
	404	(type == VSEG_TYPE_REMOTE) )
	405	{
[433]	406	vpn_t vpn;
	407	uint32_t attr;
	408	ppn_t ppn;
	409	xptr_t page_xp;
	410	cxy_t page_cxy;
	411	page_t * page_ptr;
	412	xptr_t forks_xp;
	413
	414	// update flags in remote GPT
	415	hal_gpt_set_cow( remote_gpt_xp,
	416	vpn_base,
	417	vpn_size );
	418
	419	// atomically increment pending forks counter in physical pages,
	420	// for all vseg pages that are mapped in reference cluster
	421	if( remote_process_cxy == local_cxy )
	422	{
	423	// the reference GPT is the local GPT
	424	gpt_t * gpt = GET_PTR( remote_gpt_xp );
	425
	426	// scan all pages in vseg
	427	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
	428	{
	429	// get page attributes and PPN from reference GPT
	430	hal_gpt_get_pte( gpt , vpn , &attr , &ppn );
	431
	432	// atomically update pending forks counter if page is mapped
	433	if( attr & GPT_MAPPED )
	434	{
	435	page_xp = ppm_ppn2page( ppn );
	436	page_cxy = GET_CXY( page_xp );
	437	page_ptr = GET_PTR( page_xp );
	438	forks_xp = XPTR( page_cxy , &page_ptr->forks );
	439	hal_remote_atomic_add( forks_xp , 1 );
	440	}
	441	} // end loop on vpn
	442	} // end if local
	443	} // end if vseg type
	444	} // end loop on vsegs
[408]	445	} // end loop on process copies
	446
[433]	447	#if CONFIG_DEBUG_VMM_SET_COW
	448	cycle = (uint32_t)hal_get_cycles();
	449	if( CONFIG_DEBUG_VMM_SET_COW < cycle )
	450	printk("\n[DBG] %s : thread %x exit for process %x / cycle %d\n",
	451	__FUNCTION__ , CURRENT_THREAD , process->pid , cycle );
	452	#endif
[408]	453
	454	} // end vmm_set-cow()
	455
	456	/////////////////////////////////////////////////
	457	error_t vmm_fork_copy( process_t * child_process,
	458	xptr_t parent_process_xp )
	459	{
	460	error_t error;
	461	cxy_t parent_cxy;
	462	process_t * parent_process;
	463	vmm_t * parent_vmm;
	464	xptr_t parent_lock_xp;
	465	vmm_t * child_vmm;
	466	xptr_t iter_xp;
	467	xptr_t parent_vseg_xp;
	468	vseg_t * parent_vseg;
	469	vseg_t * child_vseg;
	470	uint32_t type;
	471	bool_t cow;
	472	vpn_t vpn;
	473	vpn_t vpn_base;
	474	vpn_t vpn_size;
	475	xptr_t page_xp;
	476	page_t * page_ptr;
	477	cxy_t page_cxy;
	478	xptr_t parent_root_xp;
	479	bool_t mapped;
	480	ppn_t ppn;
	481
[433]	482	#if CONFIG_DEBUG_VMM_FORK_COPY
	483	uint32_t cycle = (uint32_t)hal_get_cycles();
	484	if( CONFIG_DEBUG_VMM_FORK_COPY < cycle )
	485	printk("\n[DBG] %s : thread %x enter / cycle %d\n",
	486	__FUNCTION__ , CURRENT_THREAD, cycle );
	487	#endif
[408]	488
	489	// get parent process cluster and local pointer
	490	parent_cxy = GET_CXY( parent_process_xp );
[433]	491	parent_process = GET_PTR( parent_process_xp );
[408]	492
	493	// get local pointers on parent and child VMM
	494	parent_vmm = &parent_process->vmm;
	495	child_vmm = &child_process->vmm;
	496
	497	// get extended pointer on lock protecting the parent VSL
	498	parent_lock_xp = XPTR( parent_cxy , &parent_vmm->vsegs_lock );
	499
	500	// initialize the lock protecting the child VSL
	501	remote_rwlock_init( XPTR( local_cxy , &child_vmm->vsegs_lock ) );
	502
	503	// initialize the child VSL as empty
	504	xlist_root_init( XPTR( local_cxy, &child_vmm->vsegs_root ) );
	505	child_vmm->vsegs_nr = 0;
	506
[415]	507	// create child GPT
[408]	508	error = hal_gpt_create( &child_vmm->gpt );
[415]	509
[407]	510	if( error )
	511	{
[408]	512	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
	513	return -1;
[407]	514	}
	515
[408]	516	// build extended pointer on parent VSL
	517	parent_root_xp = XPTR( parent_cxy , &parent_vmm->vsegs_root );
	518
[415]	519	// take the lock protecting the parent VSL
	520	remote_rwlock_rd_lock( parent_lock_xp );
	521
[408]	522	// loop on parent VSL xlist
	523	XLIST_FOREACH( parent_root_xp , iter_xp )
[23]	524	{
[408]	525	// get local and extended pointers on current parent vseg
	526	parent_vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	527	parent_vseg = GET_PTR( parent_vseg_xp );
[23]	528
[408]	529	// get vseg type
	530	type = hal_remote_lw( XPTR( parent_cxy , &parent_vseg->type ) );
	531
[433]	532	#if CONFIG_DEBUG_VMM_FORK_COPY
	533	cycle = (uint32_t)hal_get_cycles();
	534	if( CONFIG_DEBUG_VMM_FORK_COPY < cycle )
	535	printk("\n[DBG] %s : thread %x found parent vseg %s / vpn_base = %x / cycle %d\n",
	536	__FUNCTION__ , CURRENT_THREAD, vseg_type_str(type),
	537	hal_remote_lw( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
	538	#endif
[23]	539
[408]	540	// all parent vsegs - but STACK - must be copied in child VSL
	541	if( type != VSEG_TYPE_STACK )
[23]	542	{
[408]	543	// allocate memory for a new child vseg
	544	child_vseg = vseg_alloc();
	545	if( child_vseg == NULL ) // release all allocated vsegs
[23]	546	{
[408]	547	vmm_destroy( child_process );
	548	printk("\n[ERROR] in %s : cannot create vseg for child\n", __FUNCTION__ );
	549	return -1;
[23]	550	}
	551
[408]	552	// copy parent vseg to child vseg
	553	vseg_init_from_ref( child_vseg , parent_vseg_xp );
[23]	554
[408]	555	// register child vseg in child VSL
	556	vseg_attach( child_vmm , child_vseg );
[407]	557
[433]	558	#if CONFIG_DEBUG_VMM_FORK_COPY
	559	cycle = (uint32_t)hal_get_cycles();
	560	if( CONFIG_DEBUG_VMM_FORK_COPY < cycle )
	561	printk("\n[DBG] %s : thread %x copied vseg %s / vpn_base = %x to child VSL / cycle %d\n",
	562	__FUNCTION__ , CURRENT_THREAD , vseg_type_str(type),
	563	hal_remote_lw( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
	564	#endif
[23]	565
[408]	566	// copy DATA, MMAP, REMOTE, FILE parent GPT entries to child GPT
	567	if( type != VSEG_TYPE_CODE )
	568	{
	569	// activate the COW for DATA, MMAP, REMOTE vsegs only
	570	cow = ( type != VSEG_TYPE_FILE );
[23]	571
[408]	572	vpn_base = child_vseg->vpn_base;
	573	vpn_size = child_vseg->vpn_size;
[23]	574
[408]	575	// scan pages in parent vseg
	576	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
	577	{
	578	error = hal_gpt_pte_copy( &child_vmm->gpt,
	579	XPTR( parent_cxy , &parent_vmm->gpt ),
	580	vpn,
	581	cow,
	582	&ppn,
	583	&mapped );
	584	if( error )
	585	{
	586	vmm_destroy( child_process );
	587	printk("\n[ERROR] in %s : cannot copy GPT\n", __FUNCTION__ );
	588	return -1;
	589	}
	590
[433]	591	// increment pending forks counter in page if mapped
[408]	592	if( mapped )
	593	{
	594	page_xp = ppm_ppn2page( ppn );
	595	page_cxy = GET_CXY( page_xp );
[433]	596	page_ptr = GET_PTR( page_xp );
	597	hal_remote_atomic_add( XPTR( page_cxy , &page_ptr->forks ) , 1 );
[408]	598
[433]	599	#if CONFIG_DEBUG_VMM_FORK_COPY
	600	cycle = (uint32_t)hal_get_cycles();
	601	if( CONFIG_DEBUG_VMM_FORK_COPY < cycle )
	602	printk("\n[DBG] %s : thread %x copied vpn %x to child GPT / cycle %d\n",
	603	__FUNCTION__ , CURRENT_THREAD , vpn , cycle );
	604	#endif
[408]	605
	606	}
	607	}
	608	} // end if no code & no stack
	609	} // end if no stack
	610	} // end loop on vsegs
	611
	612	// release the parent vsegs lock
	613	remote_rwlock_rd_unlock( parent_lock_xp );
	614
[415]	615	// initialize child GPT (architecture specic)
	616	// => For TSAR, identity map the kentry_vseg
	617	error = hal_vmm_init( child_vmm );
	618
	619	if( error )
	620	{
	621	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
	622	return -1;
	623	}
	624
[408]	625	// initialize the child VMM STACK allocator
	626	child_vmm->stack_mgr.bitmap = 0;
	627	child_vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
	628
	629	// initialize the child VMM MMAP allocator
[23]	630	uint32_t i;
[408]	631	child_vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
	632	child_vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
	633	child_vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
	634	for( i = 0 ; i < 32 ; i++ ) list_root_init( &child_vmm->mmap_mgr.zombi_list[i] );
[23]	635
[178]	636	// initialize instrumentation counters
[408]	637	child_vmm->pgfault_nr = 0;
[23]	638
[408]	639	// copy base addresses from parent VMM to child VMM
	640	child_vmm->kent_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->kent_vpn_base));
	641	child_vmm->args_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->args_vpn_base));
	642	child_vmm->envs_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->envs_vpn_base));
	643	child_vmm->heap_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->heap_vpn_base));
	644	child_vmm->code_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->code_vpn_base));
	645	child_vmm->data_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->data_vpn_base));
[23]	646
[408]	647	child_vmm->entry_point = (intptr_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->entry_point));
[23]	648
[124]	649	hal_fence();
[23]	650
[433]	651	#if CONFIG_DEBUG_VMM_FORK_COPY
	652	cycle = (uint32_t)hal_get_cycles();
	653	if( CONFIG_DEBUG_VMM_FORK_COPY < cycle )
	654	printk("\n[DBG] %s : thread %x exit successfully / cycle %d\n",
	655	__FUNCTION__ , CURRENT_THREAD , cycle );
	656	#endif
	657
[23]	658	return 0;
	659
[408]	660	} // vmm_fork_copy()
[204]	661
[1]	662	///////////////////////////////////////
	663	void vmm_destroy( process_t * process )
	664	{
[408]	665	xptr_t vseg_xp;
[1]	666	vseg_t * vseg;
	667
[433]	668	#if CONFIG_DEBUG_VMM_DESTROY
	669	uint32_t cycle = (uint32_t)hal_get_cycles();
	670	if( CONFIG_DEBUG_VMM_DESTROY < cycle )
	671	printk("\n[DBG] %s : thread %x enter for process %x / cycle %d\n",
	672	__FUNCTION__ , CURRENT_THREAD , process->pid , cycle );
	673	#endif
[416]	674
[433]	675	// get pointer on local VMM
[1]	676	vmm_t * vmm = &process->vmm;
	677
[408]	678	// get extended pointer on VSL root and VSL lock
	679	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	680	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	681
[1]	682	// get lock protecting vseg list
[408]	683	remote_rwlock_wr_lock( lock_xp );
[1]	684
[409]	685	// remove all user vsegs registered in VSL
[408]	686	while( !xlist_is_empty( root_xp ) )
[1]	687	{
[409]	688	// get pointer on first vseg in VSL
[408]	689	vseg_xp = XLIST_FIRST_ELEMENT( root_xp , vseg_t , xlist );
[433]	690	vseg = GET_PTR( vseg_xp );
[409]	691
[433]	692	// unmap rand release physical pages if required)
[409]	693	vmm_unmap_vseg( process , vseg );
	694
	695	// remove vseg from VSL
[1]	696	vseg_detach( vmm , vseg );
[409]	697
	698	// release memory allocated to vseg descriptor
[1]	699	vseg_free( vseg );
	700	}
	701
[433]	702	// release lock protecting VSL
[408]	703	remote_rwlock_wr_unlock( lock_xp );
[1]	704
	705	// remove all vsegs from zombi_lists in MMAP allocator
	706	uint32_t i;
	707	for( i = 0 ; i<32 ; i++ )
	708	{
	709	while( !list_is_empty( &vmm->mmap_mgr.zombi_list[i] ) )
	710	{
[408]	711	vseg = LIST_FIRST( &vmm->mmap_mgr.zombi_list[i] , vseg_t , zlist );
[1]	712	vseg_detach( vmm , vseg );
	713	vseg_free( vseg );
	714	}
	715	}
	716
[409]	717	// release memory allocated to the GPT itself
[1]	718	hal_gpt_destroy( &vmm->gpt );
	719
[433]	720	#if CONFIG_DEBUG_VMM_DESTROY
	721	cycle = (uint32_t)hal_get_cycles();
	722	if( CONFIG_DEBUG_VMM_DESTROY < cycle )
	723	printk("\n[DBG] %s : thread %x exit / cycle %d\n",
	724	__FUNCTION__ , CURRENT_THREAD , cycle );
	725	#endif
[416]	726
[204]	727	} // end vmm_destroy()
	728
[1]	729	/////////////////////////////////////////////////
	730	vseg_t * vmm_check_conflict( process_t * process,
[21]	731	vpn_t vpn_base,
[1]	732	vpn_t vpn_size )
	733	{
	734	vmm_t * vmm = &process->vmm;
[408]	735
	736	// scan the VSL
[1]	737	vseg_t * vseg;
[408]	738	xptr_t iter_xp;
	739	xptr_t vseg_xp;
	740	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
[1]	741
[408]	742	XLIST_FOREACH( root_xp , iter_xp )
[1]	743	{
[408]	744	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	745	vseg = GET_PTR( vseg_xp );
[204]	746
[21]	747	if( ((vpn_base + vpn_size) > vseg->vpn_base) &&
	748	(vpn_base < (vseg->vpn_base + vseg->vpn_size)) ) return vseg;
[1]	749	}
	750	return NULL;
	751
[204]	752	} // end vmm_check_conflict()
	753
[1]	754	////////////////////////////////////////////////////////////////////////////////////////////
	755	// This static function is called by the vmm_create_vseg() function, and implements
	756	// the VMM stack_vseg specific allocator.
	757	////////////////////////////////////////////////////////////////////////////////////////////
	758	// @ vmm : pointer on VMM.
[21]	759	// @ vpn_base : (return value) first allocated page
[1]	760	// @ vpn_size : (return value) number of allocated pages
	761	////////////////////////////////////////////////////////////////////////////////////////////
	762	static error_t vmm_stack_alloc( vmm_t * vmm,
	763	vpn_t * vpn_base,
	764	vpn_t * vpn_size )
	765	{
	766	// get stack allocator pointer
	767	stack_mgr_t * mgr = &vmm->stack_mgr;
	768
	769	// get lock on stack allocator
	770	spinlock_lock( &mgr->lock );
	771
	772	// get first free slot index in bitmap
	773	int32_t index = bitmap_ffc( &mgr->bitmap , 4 );
[179]	774	if( (index < 0) \|\| (index > 31) )
	775	{
	776	spinlock_unlock( &mgr->lock );
	777	return ENOMEM;
	778	}
[1]	779
	780	// update bitmap
	781	bitmap_set( &mgr->bitmap , index );
[21]	782
[1]	783	// release lock on stack allocator
	784	spinlock_unlock( &mgr->lock );
	785
[21]	786	// returns vpn_base, vpn_size (one page non allocated)
[1]	787	vpn_base = mgr->vpn_base + index CONFIG_VMM_STACK_SIZE + 1;
	788	*vpn_size = CONFIG_VMM_STACK_SIZE - 1;
	789	return 0;
	790
[204]	791	} // end vmm_stack_alloc()
	792
[1]	793	////////////////////////////////////////////////////////////////////////////////////////////
	794	// This static function is called by the vmm_create_vseg() function, and implements
	795	// the VMM MMAP specific allocator.
	796	////////////////////////////////////////////////////////////////////////////////////////////
	797	// @ vmm : [in] pointer on VMM.
	798	// @ npages : [in] requested number of pages.
[21]	799	// @ vpn_base : [out] first allocated page.
[1]	800	// @ vpn_size : [out] actual number of allocated pages.
	801	////////////////////////////////////////////////////////////////////////////////////////////
	802	static error_t vmm_mmap_alloc( vmm_t * vmm,
	803	vpn_t npages,
	804	vpn_t * vpn_base,
	805	vpn_t * vpn_size )
	806	{
	807	uint32_t index;
	808	vseg_t * vseg;
	809	vpn_t base;
	810	vpn_t size;
[21]	811	vpn_t free;
[1]	812
[21]	813	// mmap vseg size must be power of 2
[1]	814	// compute actual size and index in zombi_list array
	815	size = POW2_ROUNDUP( npages );
	816	index = bits_log2( size );
	817
	818	// get mmap allocator pointer
	819	mmap_mgr_t * mgr = &vmm->mmap_mgr;
	820
	821	// get lock on mmap allocator
	822	spinlock_lock( &mgr->lock );
	823
	824	// get vseg from zombi_list or from mmap zone
	825	if( list_is_empty( &mgr->zombi_list[index] ) ) // from mmap zone
	826	{
	827	// check overflow
	828	free = mgr->first_free_vpn;
	829	if( (free + size) > mgr->vpn_size ) return ENOMEM;
	830
	831	// update STACK allocator
	832	mgr->first_free_vpn += size;
	833
	834	// compute base
	835	base = free;
	836	}
	837	else // from zombi_list
	838	{
	839	// get pointer on zombi vseg from zombi_list
[408]	840	vseg = LIST_FIRST( &mgr->zombi_list[index] , vseg_t , zlist );
[1]	841
	842	// remove vseg from free-list
[408]	843	list_unlink( &vseg->zlist );
[1]	844
	845	// compute base
	846	base = vseg->vpn_base;
[21]	847	}
	848
[1]	849	// release lock on mmap allocator
	850	spinlock_unlock( &mgr->lock );
	851
	852	// returns vpn_base, vpn_size
	853	*vpn_base = base;
	854	*vpn_size = size;
	855	return 0;
	856
[204]	857	} // end vmm_mmap_alloc()
	858
[407]	859	////////////////////////////////////////////////
	860	vseg_t * vmm_create_vseg( process_t * process,
	861	vseg_type_t type,
	862	intptr_t base,
	863	uint32_t size,
	864	uint32_t file_offset,
	865	uint32_t file_size,
	866	xptr_t mapper_xp,
	867	cxy_t cxy )
[1]	868	{
	869	vseg_t * vseg; // created vseg pointer
[204]	870	vpn_t vpn_base; // first page index
[1]	871	vpn_t vpn_size; // number of pages
	872	error_t error;
	873
[433]	874	#if CONFIG_DEBUG_VMM_CREATE_VSEG
	875	uint32_t cycle = (uint32_t)hal_get_cycles();
	876	if( CONFIG_DEBUG_VMM_CREATE_VSEG < cycle )
	877	printk("\n[DBG] %s : thread %x enter / process %x / base %x / size %x / %s / cxy %x / cycle %d\n",
	878	__FUNCTION__, CURRENT_THREAD, process->pid, base, size, vseg_type_str(type), cxy, cycle );
	879	#endif
[21]	880
[407]	881	// get pointer on VMM
	882	vmm_t * vmm = &process->vmm;
[21]	883
[204]	884	// compute base, size, vpn_base, vpn_size, depending on vseg type
[407]	885	// we use the VMM specific allocators for "stack", "file", "anon", & "remote" vsegs
[1]	886	if( type == VSEG_TYPE_STACK )
	887	{
	888	// get vpn_base and vpn_size from STACK allocator
	889	error = vmm_stack_alloc( vmm , &vpn_base , &vpn_size );
	890	if( error )
	891	{
[407]	892	printk("\n[ERROR] in %s : no space for stack vseg / process %x in cluster %x\n",
	893	__FUNCTION__ , process->pid , local_cxy );
[1]	894	return NULL;
	895	}
	896
	897	// compute vseg base and size from vpn_base and vpn_size
	898	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
	899	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
	900	}
[21]	901	else if( (type == VSEG_TYPE_ANON) \|\|
	902	(type == VSEG_TYPE_FILE) \|\|
[1]	903	(type == VSEG_TYPE_REMOTE) )
	904	{
	905	// get vpn_base and vpn_size from MMAP allocator
	906	vpn_t npages = size >> CONFIG_PPM_PAGE_SHIFT;
	907	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
	908	if( error )
	909	{
	910	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
	911	__FUNCTION__ , process->pid , local_cxy );
	912	return NULL;
	913	}
	914
	915	// compute vseg base and size from vpn_base and vpn_size
	916	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
	917	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
	918	}
	919	else
	920	{
[204]	921	uint32_t vpn_min = base >> CONFIG_PPM_PAGE_SHIFT;
	922	uint32_t vpn_max = (base + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
	923
	924	vpn_base = vpn_min;
	925	vpn_size = vpn_max - vpn_min + 1;
[1]	926	}
	927
	928	// check collisions
	929	vseg = vmm_check_conflict( process , vpn_base , vpn_size );
	930	if( vseg != NULL )
	931	{
[21]	932	printk("\n[ERROR] in %s for process %x : new vseg [vpn_base = %x / vpn_size = %x]\n"
[1]	933	" overlap existing vseg [vpn_base = %x / vpn_size = %x]\n",
[407]	934	__FUNCTION__ , process->pid, vpn_base, vpn_size, vseg->vpn_base, vseg->vpn_size );
[1]	935	return NULL;
	936	}
	937
	938	// allocate physical memory for vseg descriptor
	939	vseg = vseg_alloc();
	940	if( vseg == NULL )
	941	{
	942	printk("\n[ERROR] in %s for process %x : cannot allocate memory for vseg\n",
[407]	943	__FUNCTION__ , process->pid );
[1]	944	return NULL;
	945	}
	946
	947	// initialize vseg descriptor
[407]	948	vseg_init( vseg,
	949	type,
	950	base,
	951	size,
	952	vpn_base,
	953	vpn_size,
	954	file_offset,
	955	file_size,
	956	mapper_xp,
	957	cxy );
[1]	958
[408]	959	// attach vseg to VSL
	960	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	961	remote_rwlock_wr_lock( lock_xp );
[1]	962	vseg_attach( vmm , vseg );
[408]	963	remote_rwlock_wr_unlock( lock_xp );
[1]	964
[433]	965	#if CONFIG_DEBUG_VMM_CREATE_VSEG
	966	cycle = (uint32_t)hal_get_cycles();
	967	if( CONFIG_DEBUG_VMM_CREATE_VSEG < cycle )
	968	printk("\n[DBG] %s : thread %x exit / process %x / %s / cxy %x / cycle %d\n",
	969	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str(type), cxy, cycle );
	970	#endif
[21]	971
[1]	972	return vseg;
	973
[406]	974	} // vmm_create_vseg()
	975
[1]	976	/////////////////////////////////////
	977	void vmm_remove_vseg( vseg_t * vseg )
	978	{
	979	// get pointers on calling process and VMM
	980	thread_t * this = CURRENT_THREAD;
	981	process_t * process = this->process;
	982	vmm_t * vmm = &this->process->vmm;
	983	uint32_t type = vseg->type;
	984
[408]	985	// detach vseg from VSL
	986	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	987	remote_rwlock_wr_lock( lock_xp );
	988	vseg_detach( &process->vmm , vseg );
	989	remote_rwlock_wr_unlock( lock_xp );
[1]	990
	991	// release the stack slot to VMM stack allocator if STACK type
	992	if( type == VSEG_TYPE_STACK )
	993	{
	994	// get pointer on stack allocator
	995	stack_mgr_t * mgr = &vmm->stack_mgr;
	996
	997	// compute slot index
	998	uint32_t index = ((vseg->vpn_base - mgr->vpn_base - 1) / CONFIG_VMM_STACK_SIZE);
	999
	1000	// update stacks_bitmap
	1001	spinlock_lock( &mgr->lock );
	1002	bitmap_clear( &mgr->bitmap , index );
	1003	spinlock_unlock( &mgr->lock );
	1004	}
	1005
	1006	// release the vseg to VMM mmap allocator if MMAP type
	1007	if( (type == VSEG_TYPE_ANON) \|\| (type == VSEG_TYPE_FILE) \|\| (type == VSEG_TYPE_REMOTE) )
	1008	{
	1009	// get pointer on mmap allocator
	1010	mmap_mgr_t * mgr = &vmm->mmap_mgr;
	1011
	1012	// compute zombi_list index
	1013	uint32_t index = bits_log2( vseg->vpn_size );
	1014
	1015	// update zombi_list
	1016	spinlock_lock( &mgr->lock );
[408]	1017	list_add_first( &mgr->zombi_list[index] , &vseg->zlist );
[1]	1018	spinlock_unlock( &mgr->lock );
	1019	}
	1020
	1021	// release physical memory allocated for vseg descriptor if no MMAP type
	1022	if( (type != VSEG_TYPE_ANON) && (type != VSEG_TYPE_FILE) && (type != VSEG_TYPE_REMOTE) )
	1023	{
	1024	vseg_free( vseg );
	1025	}
[407]	1026	} // end vmm_remove_vseg()
[1]	1027
[68]	1028	//////////////////////////////////////////////
	1029	error_t vmm_map_kernel_vseg( vseg_t * vseg,
	1030	uint32_t attr )
[1]	1031	{
	1032	vpn_t vpn; // VPN of PTE to be set
	1033	vpn_t vpn_min; // VPN of first PTE to be set
	1034	vpn_t vpn_max; // VPN of last PTE to be set (excluded)
	1035	ppn_t ppn; // PPN of allocated physical page
	1036	uint32_t order; // ln( number of small pages for one single PTE )
	1037	page_t * page;
	1038	error_t error;
	1039
[68]	1040	// check vseg type : must be a kernel vseg
[1]	1041	uint32_t type = vseg->type;
[68]	1042	assert( ((type==VSEG_TYPE_KCODE) \|\| (type==VSEG_TYPE_KDATA) \|\| (type==VSEG_TYPE_KDEV)),
	1043	__FUNCTION__ , "not a kernel vseg\n" );
[1]	1044
	1045	// get pointer on page table
	1046	gpt_t * gpt = &process_zero.vmm.gpt;
	1047
[21]	1048	// define number of small pages per PTE
[1]	1049	if( attr & GPT_SMALL ) order = 0; // 1 small page
	1050	else order = 9; // 512 small pages
	1051
	1052	// loop on pages in vseg
	1053	vpn_min = vseg->vpn_base;
	1054	vpn_max = vpn_min + vseg->vpn_size;
	1055	for( vpn = vpn_min ; vpn < vpn_max ; vpn++ )
	1056	{
[68]	1057	// allocate a physical page from local PPM
[1]	1058	kmem_req_t req;
	1059	req.type = KMEM_PAGE;
	1060	req.size = order;
	1061	req.flags = AF_KERNEL \| AF_ZERO;
	1062	page = (page_t *)kmem_alloc( &req );
[21]	1063	if( page == NULL )
[1]	1064	{
	1065	printk("\n[ERROR] in %s : cannot allocate physical memory\n", __FUNCTION__ );
	1066	return ENOMEM;
	1067	}
	1068
	1069	// set page table entry
[315]	1070	ppn = ppm_page2ppn( XPTR( local_cxy , page ) );
[408]	1071	error = hal_gpt_set_pte( gpt,
	1072	vpn,
	1073	attr,
	1074	ppn );
[21]	1075	if( error )
[1]	1076	{
	1077	printk("\n[ERROR] in %s : cannot register PPE\n", __FUNCTION__ );
	1078	return ENOMEM;
	1079	}
	1080	}
[21]	1081
[1]	1082	return 0;
	1083
[408]	1084	} // end vmm_map_kernel_vseg()
	1085
[1]	1086	/////////////////////////////////////////
	1087	void vmm_unmap_vseg( process_t * process,
	1088	vseg_t * vseg )
	1089	{
[21]	1090	vpn_t vpn; // VPN of current PTE
	1091	vpn_t vpn_min; // VPN of first PTE
[1]	1092	vpn_t vpn_max; // VPN of last PTE (excluded)
[409]	1093	ppn_t ppn; // current PTE ppn value
	1094	uint32_t attr; // current PTE attributes
	1095	kmem_req_t req; // request to release memory
	1096	xptr_t page_xp; // extended pointer on page descriptor
	1097	cxy_t page_cxy; // page descriptor cluster
	1098	page_t * page_ptr; // page descriptor pointer
[433]	1099	xptr_t forks_xp; // extended pointer on pending forks counter
	1100	uint32_t count; // actual number of pendinf forks
[1]	1101
[433]	1102	#if CONFIG_DEBUG_VMM_UNMAP_VSEG
	1103	uint32_t cycle = (uint32_t)hal_get_cycles();
	1104	if( CONFIG_DEBUG_VMM_UNMAP_VSEG < cycle )
	1105	printk("\n[DBG] %s : thread %x enter / process %x / vseg %s / base %x / cycle %d\n",
	1106	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str( vseg->type ), vseg->vpn_base, cycle );
	1107	#endif
[409]	1108
[433]	1109	// get pointer on local GPT
[1]	1110	gpt_t * gpt = &process->vmm.gpt;
	1111
	1112	// loop on pages in vseg
	1113	vpn_min = vseg->vpn_base;
	1114	vpn_max = vpn_min + vseg->vpn_size;
	1115	for( vpn = vpn_min ; vpn < vpn_max ; vpn++ )
	1116	{
[409]	1117	// get GPT entry
	1118	hal_gpt_get_pte( gpt , vpn , &attr , &ppn );
	1119
	1120	if( attr & GPT_MAPPED ) // entry is mapped
	1121	{
	1122	// check small page
	1123	assert( (attr & GPT_SMALL) , __FUNCTION__ ,
	1124	"an user vseg must use small pages" );
	1125
[433]	1126	// unmap GPT entry in all GPT copies
[409]	1127	hal_gpt_reset_pte( gpt , vpn );
	1128
[433]	1129	// handle pending forks counter if
	1130	// 1) not identity mapped
	1131	// 2) running in reference cluster
	1132	if( ((vseg->flags & VSEG_IDENT) == 0) &&
	1133	(GET_CXY( process->ref_xp ) == local_cxy) )
[409]	1134	{
[433]	1135	// get extended pointer on physical page descriptor
[409]	1136	page_xp = ppm_ppn2page( ppn );
	1137	page_cxy = GET_CXY( page_xp );
[433]	1138	page_ptr = GET_PTR( page_xp );
[409]	1139
[433]	1140	// FIXME lock the physical page
	1141
	1142	// get extended pointer on pending forks counter
	1143	forks_xp = XPTR( page_cxy , &page_ptr->forks );
	1144
	1145	// get pending forks counter
	1146	count = hal_remote_lw( forks_xp );
	1147
	1148	if( count ) // decrement pending forks counter
[409]	1149	{
[433]	1150	hal_remote_atomic_add( forks_xp , -1 );
	1151	}
	1152	else // release physical page to relevant cluster
[409]	1153	{
[433]	1154	if( page_cxy == local_cxy ) // local cluster
	1155	{
	1156	req.type = KMEM_PAGE;
	1157	req.ptr = page_ptr;
	1158	kmem_free( &req );
	1159	}
	1160	else // remote cluster
	1161	{
	1162	rpc_pmem_release_pages_client( page_cxy , page_ptr );
	1163	}
[409]	1164	}
[433]	1165
	1166	// FIXME unlock the physical page
[409]	1167	}
	1168	}
[1]	1169	}
[433]	1170
	1171	#if CONFIG_DEBUG_VMM_UNMAP_VSEG
	1172	cycle = (uint32_t)hal_get_cycles();
	1173	if( CONFIG_DEBUG_VMM_UNMAP_VSEG < cycle )
	1174	printk("\n[DBG] %s : thread %x exit / process %x / vseg %s / base %x / cycle %d\n",
	1175	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str( vseg->type ), vseg->vpn_base, cycle );
	1176	#endif
	1177
[409]	1178	} // end vmm_unmap_vseg()
[1]	1179
[407]	1180	//////////////////////////////////////////////////////////////////////////////////////////
[406]	1181	// This low-level static function is called by the vmm_get_vseg() and vmm_resize_vseg()
	1182	// functions. It scan the list of registered vsegs to find the unique vseg containing
	1183	// a given virtual address.
[407]	1184	//////////////////////////////////////////////////////////////////////////////////////////
[406]	1185	// @ vmm : pointer on the process VMM.
	1186	// @ vaddr : virtual address.
	1187	// @ return vseg pointer if success / return NULL if not found.
[407]	1188	//////////////////////////////////////////////////////////////////////////////////////////
[406]	1189	static vseg_t * vseg_from_vaddr( vmm_t * vmm,
	1190	intptr_t vaddr )
	1191	{
[408]	1192	xptr_t iter_xp;
	1193	xptr_t vseg_xp;
	1194	vseg_t * vseg;
[406]	1195
[408]	1196	// get extended pointers on VSL lock and root
	1197	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	1198	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
[406]	1199
[408]	1200	// get lock protecting the VSL
	1201	remote_rwlock_rd_lock( lock_xp );
	1202
	1203	// scan the list of vsegs in VSL
	1204	XLIST_FOREACH( root_xp , iter_xp )
[406]	1205	{
[408]	1206	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	1207	vseg = GET_PTR( vseg_xp );
[408]	1208	if( (vaddr >= vseg->min) && (vaddr < vseg->max) )
	1209	{
	1210	// return success
	1211	remote_rwlock_rd_unlock( lock_xp );
	1212	return vseg;
	1213	}
[406]	1214	}
	1215
[408]	1216	// return failure
	1217	remote_rwlock_rd_unlock( lock_xp );
	1218	return NULL;
[406]	1219
[408]	1220	} // end vseg_from_vaddr()
[406]	1221
[1]	1222	/////////////////////////////////////////////
	1223	error_t vmm_resize_vseg( process_t * process,
	1224	intptr_t base,
	1225	intptr_t size )
	1226	{
[406]	1227	error_t error;
	1228	vseg_t * new;
	1229	vpn_t vpn_min;
	1230	vpn_t vpn_max;
[1]	1231
	1232	// get pointer on process VMM
	1233	vmm_t * vmm = &process->vmm;
	1234
	1235	intptr_t addr_min = base;
	1236	intptr_t addr_max = base + size;
	1237
	1238	// get pointer on vseg
[406]	1239	vseg_t * vseg = vseg_from_vaddr( vmm , base );
[1]	1240
	1241	if( vseg == NULL) return EINVAL;
[21]	1242
[408]	1243	// get extended pointer on VSL lock
	1244	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
[21]	1245
[408]	1246	// get lock protecting VSL
	1247	remote_rwlock_wr_lock( lock_xp );
	1248
[1]	1249	if( (vseg->min > addr_min) \|\| (vseg->max < addr_max) ) // region not included in vseg
	1250	{
	1251	error = EINVAL;
	1252	}
	1253	else if( (vseg->min == addr_min) && (vseg->max == addr_max) ) // vseg must be removed
	1254	{
	1255	vmm_remove_vseg( vseg );
	1256	error = 0;
	1257	}
[406]	1258	else if( vseg->min == addr_min ) // vseg must be resized
[1]	1259	{
[406]	1260	// update vseg base address
	1261	vseg->min = addr_max;
	1262
	1263	// update vpn_base and vpn_size
	1264	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1265	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1266	vseg->vpn_base = vpn_min;
	1267	vseg->vpn_size = vpn_max - vpn_min + 1;
	1268	error = 0;
[1]	1269	}
[406]	1270	else if( vseg->max == addr_max ) // vseg must be resized
[1]	1271	{
[406]	1272	// update vseg max address
	1273	vseg->max = addr_min;
	1274
	1275	// update vpn_base and vpn_size
	1276	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1277	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1278	vseg->vpn_base = vpn_min;
	1279	vseg->vpn_size = vpn_max - vpn_min + 1;
	1280	error = 0;
[1]	1281	}
[406]	1282	else // vseg cut in three regions
[1]	1283	{
[406]	1284	// resize existing vseg
	1285	vseg->max = addr_min;
	1286
	1287	// update vpn_base and vpn_size
	1288	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1289	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1290	vseg->vpn_base = vpn_min;
	1291	vseg->vpn_size = vpn_max - vpn_min + 1;
	1292
	1293	// create new vseg
[407]	1294	new = vmm_create_vseg( process,
	1295	vseg->type,
	1296	addr_min,
	1297	(vseg->max - addr_max),
	1298	vseg->file_offset,
	1299	vseg->file_size,
	1300	vseg->mapper_xp,
	1301	vseg->cxy );
	1302
[406]	1303	if( new == NULL ) error = EINVAL;
	1304	else error = 0;
[1]	1305	}
	1306
	1307	// release VMM lock
[408]	1308	remote_rwlock_wr_unlock( lock_xp );
[1]	1309
	1310	return error;
	1311
[406]	1312	} // vmm_resize_vseg()
	1313
[1]	1314	///////////////////////////////////////////
[388]	1315	error_t vmm_get_vseg( process_t * process,
[394]	1316	intptr_t vaddr,
[388]	1317	vseg_t ** found_vseg )
[1]	1318	{
[406]	1319	vmm_t * vmm = &process->vmm;
[1]	1320
[406]	1321	// get vseg from vaddr
	1322	vseg_t * vseg = vseg_from_vaddr( vmm , vaddr );
[1]	1323
[388]	1324	if( vseg == NULL ) // vseg not found in local cluster => try to get it from ref
	1325	{
	1326	// get extended pointer on reference process
	1327	xptr_t ref_xp = process->ref_xp;
[1]	1328
[388]	1329	// get cluster and local pointer on reference process
	1330	cxy_t ref_cxy = GET_CXY( ref_xp );
[433]	1331	process_t * ref_ptr = GET_PTR( ref_xp );
[388]	1332
	1333	if( local_cxy == ref_cxy ) return -1; // local cluster is the reference
	1334
	1335	// get extended pointer on reference vseg
	1336	xptr_t vseg_xp;
	1337	error_t error;
[406]	1338
[394]	1339	rpc_vmm_get_vseg_client( ref_cxy , ref_ptr , vaddr , &vseg_xp , &error );
[388]	1340
	1341	if( error ) return -1; // vseg not found => illegal user vaddr
	1342
	1343	// allocate a vseg in local cluster
	1344	vseg = vseg_alloc();
	1345
[406]	1346	if( vseg == NULL ) return -1;
[388]	1347
	1348	// initialise local vseg from reference
	1349	vseg_init_from_ref( vseg , vseg_xp );
	1350
	1351	// register local vseg in local VMM
[406]	1352	vseg_attach( &process->vmm , vseg );
[388]	1353	}
	1354
	1355	// success
	1356	*found_vseg = vseg;
[394]	1357	return 0;
[388]	1358
	1359	} // end vmm_get_vseg()
	1360
[407]	1361	//////////////////////////////////////////////////////////////////////////////////////
	1362	// This static function compute the target cluster to allocate a physical page
	1363	// for a given <vpn> in a given <vseg>, allocates the page (with an RPC if required)
	1364	// and returns an extended pointer on the allocated page descriptor.
	1365	// The vseg cannot have the FILE type.
	1366	//////////////////////////////////////////////////////////////////////////////////////
	1367	static xptr_t vmm_page_allocate( vseg_t * vseg,
	1368	vpn_t vpn )
	1369	{
[433]	1370
	1371	#if CONFIG_DEBUG_VMM_ALLOCATE_PAGE
	1372	if( CONFIG_DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
	1373	printk("\n[DBG] in %s : thread %x enter for vpn %x\n",
	1374	__FUNCTION__ , CURRENT_THREAD, vpn );
	1375	#endif
	1376
[407]	1377	// compute target cluster
	1378	page_t * page_ptr;
	1379	cxy_t page_cxy;
	1380	kmem_req_t req;
	1381
	1382	uint32_t type = vseg->type;
	1383	uint32_t flags = vseg->flags;
	1384
	1385	assert( ( type != VSEG_TYPE_FILE ) , __FUNCTION__ , "illegal vseg type\n" );
	1386
	1387	if( flags & VSEG_DISTRIB ) // distributed => cxy depends on vpn LSB
	1388	{
	1389	uint32_t x_size = LOCAL_CLUSTER->x_size;
	1390	uint32_t y_size = LOCAL_CLUSTER->y_size;
	1391	uint32_t y_width = LOCAL_CLUSTER->y_width;
	1392	uint32_t index = vpn & ((x_size * y_size) - 1);
	1393	uint32_t x = index / y_size;
	1394	uint32_t y = index % y_size;
	1395	page_cxy = (x<<y_width) + y;
	1396	}
	1397	else // other cases => cxy specified in vseg
	1398	{
	1399	page_cxy = vseg->cxy;
	1400	}
	1401
	1402	// allocate a physical page from target cluster
	1403	if( page_cxy == local_cxy ) // target cluster is the local cluster
	1404	{
	1405	req.type = KMEM_PAGE;
	1406	req.size = 0;
	1407	req.flags = AF_NONE;
	1408	page_ptr = (page_t *)kmem_alloc( &req );
	1409	}
	1410	else // target cluster is not the local cluster
	1411	{
	1412	rpc_pmem_get_pages_client( page_cxy , 0 , &page_ptr );
	1413	}
	1414
[433]	1415	#if CONFIG_DEBUG_VMM_ALLOCATE_PAGE
	1416	if( CONFIG_DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
	1417	printk("\n[DBG] in %s : thread %x exit for vpn = %d / ppn = %x\n",
	1418	__FUNCTION__ , CURRENT_THREAD, vpn, ppm_page2ppn( XPTR( page_cxy , page_ptr ) ) );
	1419	#endif
	1420
[407]	1421	if( page_ptr == NULL ) return XPTR_NULL;
	1422	else return XPTR( page_cxy , page_ptr );
	1423
	1424	} // end vmm_page_allocate()
	1425
[313]	1426	////////////////////////////////////////
	1427	error_t vmm_get_one_ppn( vseg_t * vseg,
	1428	vpn_t vpn,
	1429	ppn_t * ppn )
	1430	{
	1431	error_t error;
[407]	1432	xptr_t page_xp; // extended pointer on physical page descriptor
[313]	1433	page_t * page_ptr; // local pointer on physical page descriptor
[406]	1434	uint32_t index; // missing page index in vseg mapper
	1435	uint32_t type; // vseg type;
[313]	1436
[406]	1437	type = vseg->type;
	1438	index = vpn - vseg->vpn_base;
[313]	1439
[433]	1440	#if CONFIG_DEBUG_VMM_GET_ONE_PPN
	1441	if( CONFIG_DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
	1442	printk("\n[DBG] %s : thread %x enter for vpn = %x / type = %s / index = %d\n",
	1443	__FUNCTION__, CURRENT_THREAD, vpn, vseg_type_str(type), index );
	1444	#endif
[313]	1445
[406]	1446	// FILE type : get the physical page from the file mapper
[313]	1447	if( type == VSEG_TYPE_FILE )
	1448	{
[406]	1449	// get extended pointer on mapper
[407]	1450	xptr_t mapper_xp = vseg->mapper_xp;
[313]	1451
[406]	1452	assert( (mapper_xp != XPTR_NULL), __FUNCTION__,
	1453	"mapper not defined for a FILE vseg\n" );
	1454
	1455	// get mapper cluster and local pointer
	1456	cxy_t mapper_cxy = GET_CXY( mapper_xp );
[433]	1457	mapper_t * mapper_ptr = GET_PTR( mapper_xp );
[406]	1458
[313]	1459	// get page descriptor from mapper
	1460	if( mapper_cxy == local_cxy ) // mapper is local
	1461	{
	1462	page_ptr = mapper_get_page( mapper_ptr , index );
	1463	}
	1464	else // mapper is remote
	1465	{
	1466	rpc_mapper_get_page_client( mapper_cxy , mapper_ptr , index , &page_ptr );
	1467	}
	1468
	1469	if ( page_ptr == NULL ) return EINVAL;
	1470
[407]	1471	page_xp = XPTR( mapper_cxy , page_ptr );
[313]	1472	}
	1473
[406]	1474	// Other types : allocate a physical page from target cluster,
[407]	1475	// as defined by vseg type and vpn value
[313]	1476	else
	1477	{
[433]	1478	// allocate one physical page
[407]	1479	page_xp = vmm_page_allocate( vseg , vpn );
[406]	1480
[407]	1481	if( page_xp == XPTR_NULL ) return ENOMEM;
[313]	1482
[406]	1483	// initialise missing page from .elf file mapper for DATA and CODE types
[433]	1484	// (the vseg->mapper_xp field is an extended pointer on the .elf file mapper)
[313]	1485	if( (type == VSEG_TYPE_CODE) \|\| (type == VSEG_TYPE_DATA) )
	1486	{
[406]	1487	// get extended pointer on mapper
	1488	xptr_t mapper_xp = vseg->mapper_xp;
[313]	1489
[406]	1490	assert( (mapper_xp != XPTR_NULL), __FUNCTION__,
	1491	"mapper not defined for a CODE or DATA vseg\n" );
	1492
	1493	// get mapper cluster and local pointer
	1494	cxy_t mapper_cxy = GET_CXY( mapper_xp );
[433]	1495	mapper_t * mapper_ptr = GET_PTR( mapper_xp );
[406]	1496
	1497	// compute missing page offset in vseg
	1498	uint32_t offset = index << CONFIG_PPM_PAGE_SHIFT;
	1499
[313]	1500	// compute missing page offset in .elf file
[406]	1501	uint32_t elf_offset = vseg->file_offset + offset;
[313]	1502
[433]	1503	#if (CONFIG_DEBUG_VMM_GET_ONE_PPN & 0x1)
	1504	if( CONFIG_DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
	1505	printk("\n[DBG] %s : thread %x for vpn = %x / elf_offset = %x\n",
	1506	__FUNCTION__, CURRENT_THREAD, vpn, elf_offset );
	1507	#endif
[313]	1508
[406]	1509	// compute extended pointer on page base
[407]	1510	xptr_t base_xp = ppm_page2base( page_xp );
[313]	1511
[406]	1512	// file_size (in .elf mapper) can be smaller than vseg_size (BSS)
	1513	uint32_t file_size = vseg->file_size;
	1514
	1515	if( file_size < offset ) // missing page fully in BSS
[313]	1516	{
[406]	1517
[433]	1518	#if (CONFIG_DEBUG_VMM_GET_ONE_PPN & 0x1)
	1519	if( CONFIG_DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
	1520	printk("\n[DBG] %s : thread%x for vpn = %x / fully in BSS\n",
	1521	__FUNCTION__, CURRENT_THREAD, vpn );
	1522	#endif
	1523
[407]	1524	if( GET_CXY( page_xp ) == local_cxy )
[313]	1525	{
[315]	1526	memset( GET_PTR( base_xp ) , 0 , CONFIG_PPM_PAGE_SIZE );
[313]	1527	}
	1528	else
	1529	{
[315]	1530	hal_remote_memset( base_xp , 0 , CONFIG_PPM_PAGE_SIZE );
[313]	1531	}
	1532	}
[406]	1533	else if( file_size >= (offset + CONFIG_PPM_PAGE_SIZE) ) // fully in mapper
[315]	1534	{
[406]	1535
[433]	1536	#if (CONFIG_DEBUG_VMM_GET_ONE_PPN & 0x1)
	1537	if( CONFIG_DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
	1538	printk("\n[DBG] %s : thread %x, for vpn = %x / fully in mapper\n",
	1539	__FUNCTION__, CURRENT_THREAD, vpn );
	1540	#endif
[407]	1541
[313]	1542	if( mapper_cxy == local_cxy )
	1543	{
	1544	error = mapper_move_kernel( mapper_ptr,
	1545	true, // to_buffer
[406]	1546	elf_offset,
[313]	1547	base_xp,
[315]	1548	CONFIG_PPM_PAGE_SIZE );
[313]	1549	}
	1550	else
	1551	{
	1552	rpc_mapper_move_buffer_client( mapper_cxy,
	1553	mapper_ptr,
	1554	true, // to buffer
	1555	false, // kernel buffer
[406]	1556	elf_offset,
	1557	base_xp,
[315]	1558	CONFIG_PPM_PAGE_SIZE,
[313]	1559	&error );
	1560	}
	1561	if( error ) return EINVAL;
	1562	}
[406]	1563	else // both in mapper and in BSS :
	1564	// - (file_size - offset) bytes from mapper
	1565	// - (page_size + offset - file_size) bytes from BSS
[313]	1566	{
[406]	1567
[433]	1568	#if (CONFIG_DEBUG_VMM_GET_ONE_PPN & 0x1)
	1569	if( CONFIG_DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
	1570	printk("\n[DBG] %s : thread %x for vpn = %x / both mapper & BSS\n"
	1571	" %d bytes from mapper / %d bytes from BSS\n",
	1572	__FUNCTION__, CURRENT_THREAD, vpn,
[407]	1573	file_size - offset , offset + CONFIG_PPM_PAGE_SIZE - file_size );
[433]	1574	#endif
[313]	1575	// initialize mapper part
[315]	1576	if( mapper_cxy == local_cxy )
[313]	1577	{
	1578	error = mapper_move_kernel( mapper_ptr,
[315]	1579	true, // to buffer
[406]	1580	elf_offset,
[313]	1581	base_xp,
[406]	1582	file_size - offset );
[313]	1583	}
[315]	1584	else
[313]	1585	{
	1586	rpc_mapper_move_buffer_client( mapper_cxy,
	1587	mapper_ptr,
[315]	1588	true, // to buffer
[313]	1589	false, // kernel buffer
[406]	1590	elf_offset,
	1591	base_xp,
	1592	file_size - offset,
[313]	1593	&error );
	1594	}
	1595	if( error ) return EINVAL;
	1596
	1597	// initialize BSS part
[407]	1598	if( GET_CXY( page_xp ) == local_cxy )
[313]	1599	{
[406]	1600	memset( GET_PTR( base_xp ) + file_size - offset , 0 ,
	1601	offset + CONFIG_PPM_PAGE_SIZE - file_size );
[313]	1602	}
	1603	else
	1604	{
[406]	1605	hal_remote_memset( base_xp + file_size - offset , 0 ,
	1606	offset + CONFIG_PPM_PAGE_SIZE - file_size );
[313]	1607	}
	1608	}
	1609	} // end initialisation for CODE or DATA types
	1610	}
	1611
	1612	// return ppn
[407]	1613	*ppn = ppm_page2ppn( page_xp );
[406]	1614
[433]	1615	#if CONFIG_DEBUG_VMM_GET_ONE_PPN
	1616	if( CONFIG_DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
	1617	printk("\n[DBG] %s : thread %x exit for vpn = %x / ppn = %x\n",
	1618	__FUNCTION__ , CURRENT_THREAD , vpn , *ppn );
	1619	#endif
[406]	1620
[313]	1621	return 0;
	1622
	1623	} // end vmm_get_one_ppn()
	1624
[1]	1625	/////////////////////////////////////////
	1626	error_t vmm_get_pte( process_t * process,
	1627	vpn_t vpn,
[407]	1628	bool_t cow,
	1629	uint32_t * attr,
	1630	ppn_t * ppn )
[1]	1631	{
[433]	1632	vseg_t * vseg; // vseg containing VPN
[407]	1633	ppn_t old_ppn; // current PTE_PPN
	1634	uint32_t old_attr; // current PTE_ATTR
	1635	ppn_t new_ppn; // new PTE_PPN
	1636	uint32_t new_attr; // new PTE_ATTR
[1]	1637	error_t error;
	1638
[178]	1639	// this function must be called by a thread running in the reference cluster
[68]	1640	assert( (GET_CXY( process->ref_xp ) == local_cxy ) , __FUNCTION__ ,
[406]	1641	"not called in the reference cluster\n" );
[1]	1642
[433]	1643	#if CONFIG_DEBUG_VMM_GET_PTE
	1644	uint32_t cycle = (uint32_t)hal_get_cycles();
	1645	if( CONFIG_DEBUG_VMM_GET_PTE > cycle )
	1646	printk("\n[DBG] %s : thread %x enter for vpn = %x / process %x / cow = %d / cycle %d\n",
	1647	__FUNCTION__ , CURRENT_THREAD , vpn , process->pid , cow , cycle );
	1648	#endif
[406]	1649
[1]	1650	// get VMM pointer
	1651	vmm_t * vmm = &process->vmm;
	1652
[433]	1653	// get vseg pointer from reference VSL
[407]	1654	error = vmm_get_vseg( process , vpn<<CONFIG_PPM_PAGE_SHIFT , &vseg );
[1]	1655
[407]	1656	if( error )
[1]	1657	{
[407]	1658	printk("\n[ERROR] in %s : out of segment / process = %x / vpn = %x\n",
	1659	__FUNCTION__ , process->pid , vpn );
	1660	return error;
	1661	}
[406]	1662
[433]	1663	#if CONFIG_DEBUG_VMM_GET_PTE
	1664	cycle = (uint32_t)hal_get_cycles();
	1665	if( CONFIG_DEBUG_VMM_GET_PTE < cycle )
	1666	printk("\n[DBG] %s : thread %x found vseg %s / vpn_base = %x / vpn_size = %x\n",
	1667	__FUNCTION__, CURRENT_THREAD, vseg_type_str(vseg->type), vseg->vpn_base, vseg->vpn_size );
	1668	#endif
[1]	1669
[407]	1670	// access GPT to get current PTE attributes and PPN
	1671	hal_gpt_get_pte( &vmm->gpt , vpn , &old_attr , &old_ppn );
	1672
[408]	1673	// for both "copy_on_write" and "page_fault" events, allocate a physical page,
	1674	// initialize it, register it in the reference GPT, update GPT copies in all
	1675	// clusters containing a copy, and return the new_ppn and new_attr
[407]	1676
[433]	1677	if( cow ) /////////////////////////// copy_on_write request //////////////////////
[407]	1678	{
[408]	1679	assert( (old_attr & GPT_MAPPED) , __FUNCTION__ ,
	1680	"PTE must be mapped for a copy-on-write exception\n" );
[407]	1681
[433]	1682	#if CONFIG_DEBUG_VMM_GET_PTE
	1683	cycle = (uint32_t)hal_get_cycles();
	1684	if( CONFIG_DEBUG_VMM_GET_PTE < cycle )
	1685	printk("\n[DBG] %s : thread %x handling COW for vpn %x in process %x\n",
	1686	__FUNCTION__, CURRENT_THREAD, vpn, process->pid );
	1687	#endif
[407]	1688
[433]	1689	// get extended pointer, cluster and local pointer on physical page descriptor
[408]	1690	xptr_t page_xp = ppm_ppn2page( old_ppn );
	1691	cxy_t page_cxy = GET_CXY( page_xp );
[433]	1692	page_t * page_ptr = GET_PTR( page_xp );
[407]	1693
[408]	1694	// get number of pending forks in page descriptor
[433]	1695	uint32_t forks = hal_remote_lw( XPTR( page_cxy , &page_ptr->forks ) );
[408]	1696
[433]	1697	if( forks ) // pending fork => allocate a new page, copy old to new
[1]	1698	{
[408]	1699	// allocate a new physical page
	1700	page_xp = vmm_page_allocate( vseg , vpn );
	1701	if( page_xp == XPTR_NULL )
	1702	{
	1703	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
	1704	__FUNCTION__ , process->pid , vpn );
	1705	return -1;
	1706	}
[1]	1707
[408]	1708	// compute allocated page PPN
	1709	new_ppn = ppm_page2ppn( page_xp );
[406]	1710
[408]	1711	// copy old page content to new page
	1712	xptr_t old_base_xp = ppm_ppn2base( old_ppn );
	1713	xptr_t new_base_xp = ppm_ppn2base( new_ppn );
	1714	memcpy( GET_PTR( new_base_xp ),
	1715	GET_PTR( old_base_xp ),
	1716	CONFIG_PPM_PAGE_SIZE );
	1717	}
	1718	else // no pending fork => keep the existing page, reset COW
	1719	{
	1720	new_ppn = old_ppn;
	1721	}
[1]	1722
[408]	1723	// build new_attr : reset COW and set WRITABLE,
	1724	new_attr = (old_attr \| GPT_WRITABLE) & (~GPT_COW);
[407]	1725
[408]	1726	// update GPT[vpn] for all GPT copies
[433]	1727	vmm_global_update_pte( process, vpn, new_attr, new_ppn );
[407]	1728
[433]	1729	// decrement pending forks counter in page descriptor
	1730	hal_remote_atomic_add( XPTR( page_cxy , &page_ptr->forks ) , -1 );
[407]	1731	}
[433]	1732	else ////////////////////////////////// page_fault request ////////////////////////
[407]	1733	{
[408]	1734	if( (old_attr & GPT_MAPPED) == 0 ) // true page_fault => map it
[407]	1735	{
[1]	1736
[433]	1737	#if CONFIG_DEBUG_VMM_GET_PTE
	1738	cycle = (uint32_t)hal_get_cycles();
	1739	if( CONFIG_DEBUG_VMM_GET_PTE < cycle )
	1740	printk("\n[DBG] %s : thread %x handling page fault for vpn %x in process %x\n",
	1741	__FUNCTION__, CURRENT_THREAD, vpn, process->pid );
	1742	#endif
[1]	1743
[408]	1744	// allocate new_ppn, depending on vseg type
[407]	1745	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
	1746	if( error )
	1747	{
	1748	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
	1749	__FUNCTION__ , process->pid , vpn );
[408]	1750	return -1;
[407]	1751	}
	1752
[408]	1753	// define new_attr from vseg flags
[407]	1754	new_attr = GPT_MAPPED \| GPT_SMALL;
	1755	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
	1756	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
	1757	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
	1758	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
	1759
[408]	1760	// register new PTE in reference GPT
	1761	// on demand policy => no update of GPT copies
	1762	error = hal_gpt_set_pte( &vmm->gpt,
	1763	vpn,
	1764	new_attr,
	1765	new_ppn );
[407]	1766	if( error )
	1767	{
	1768	printk("\n[ERROR] in %s : cannot update GPT / process = %x / vpn = %x\n",
	1769	__FUNCTION__ , process->pid , vpn );
[408]	1770	return -1;
[407]	1771	}
	1772	}
[408]	1773	else // mapped in reference GPT => get it
[1]	1774	{
[408]	1775	new_ppn = old_ppn;
[407]	1776	new_attr = old_attr;
[1]	1777	}
[407]	1778	}
[1]	1779
[433]	1780	#if CONFIG_DEBUG_VMM_GET_PTE
	1781	cycle = (uint32_t)hal_get_cycles();
	1782	if( CONFIG_DEBUG_VMM_GET_PTE < cycle )
	1783	printk("\n[DBG] %s : thread,%x exit for vpn %x in process %x / ppn = %x / attr = %x / cycle %d\n",
	1784	__FUNCTION__, CURRENT_THREAD, vpn, process->pid, new_ppn, new_attr, cycle );
	1785	#endif
[406]	1786
[433]	1787	// return success
[407]	1788	*ppn = new_ppn;
	1789	*attr = new_attr;
[1]	1790	return 0;
	1791
[313]	1792	} // end vmm_get_pte()
	1793
[1]	1794	///////////////////////////////////////////////////
	1795	error_t vmm_handle_page_fault( process_t * process,
	1796	vpn_t vpn )
	1797	{
	1798	uint32_t attr; // missing page attributes
[21]	1799	ppn_t ppn; // missing page PPN
[407]	1800	error_t error;
[1]	1801
	1802	// get reference process cluster and local pointer
	1803	cxy_t ref_cxy = GET_CXY( process->ref_xp );
[433]	1804	process_t * ref_ptr = GET_PTR( process->ref_xp );
[1]	1805
[313]	1806	// get missing PTE attributes and PPN from reference cluster
[407]	1807	if( local_cxy != ref_cxy )
[1]	1808	{
[407]	1809	rpc_vmm_get_pte_client( ref_cxy,
	1810	ref_ptr,
	1811	vpn,
	1812	false, // page_fault
	1813	&attr,
	1814	&ppn,
	1815	&error );
	1816
	1817	// get local VMM pointer
	1818	vmm_t * vmm = &process->vmm;
	1819
	1820	// update local GPT
[408]	1821	error \|= hal_gpt_set_pte( &vmm->gpt,
	1822	vpn,
	1823	attr,
	1824	ppn );
[1]	1825	}
[407]	1826	else // local cluster is the reference cluster
[1]	1827	{
[407]	1828	error = vmm_get_pte( process,
	1829	vpn,
	1830	false, // page-fault
	1831	&attr,
	1832	&ppn );
[1]	1833	}
	1834
[388]	1835	return error;
[1]	1836
[313]	1837	} // end vmm_handle_page_fault()
	1838
[408]	1839	////////////////////////////////////////////
	1840	error_t vmm_handle_cow( process_t * process,
	1841	vpn_t vpn )
[407]	1842	{
[433]	1843	uint32_t attr; // page attributes
	1844	ppn_t ppn; // page PPN
[407]	1845	error_t error;
[313]	1846
[433]	1847
[407]	1848	// get reference process cluster and local pointer
	1849	cxy_t ref_cxy = GET_CXY( process->ref_xp );
[433]	1850	process_t * ref_ptr = GET_PTR( process->ref_xp );
[407]	1851
	1852	// get new PTE attributes and PPN from reference cluster
	1853	if( local_cxy != ref_cxy )
	1854	{
	1855	rpc_vmm_get_pte_client( ref_cxy,
	1856	ref_ptr,
	1857	vpn,
	1858	true, // copy-on-write
	1859	&attr,
	1860	&ppn,
	1861	&error );
	1862
	1863	// get local VMM pointer
	1864	vmm_t * vmm = &process->vmm;
	1865
	1866	// update local GPT
[408]	1867	error \|= hal_gpt_set_pte( &vmm->gpt,
	1868	vpn,
	1869	attr,
	1870	ppn );
[407]	1871	}
	1872	else // local cluster is the reference cluster
	1873	{
	1874	error = vmm_get_pte( process,
	1875	vpn,
	1876	true, // copy-on-write
	1877	&attr,
	1878	&ppn );
	1879	}
	1880
	1881	return error;
	1882
[408]	1883	} // end vmm_handle_cow()
[407]	1884
[1]	1885	///////////////////////////////////////////
	1886	error_t vmm_v2p_translate( bool_t ident,
	1887	void * ptr,
	1888	paddr_t * paddr )
	1889	{
[23]	1890	process_t * process = CURRENT_THREAD->process;
[1]	1891
	1892	if( ident ) // identity mapping
	1893	{
[23]	1894	*paddr = (paddr_t)PADDR( local_cxy , (lpa_t)ptr );
[1]	1895	return 0;
	1896	}
	1897
[21]	1898	// access page table
[1]	1899	error_t error;
	1900	vpn_t vpn;
	1901	uint32_t attr;
	1902	ppn_t ppn;
	1903	uint32_t offset;
	1904
[23]	1905	vpn = (vpn_t)( (intptr_t)ptr >> CONFIG_PPM_PAGE_SHIFT );
	1906	offset = (uint32_t)( ((intptr_t)ptr) & CONFIG_PPM_PAGE_MASK );
[1]	1907
[50]	1908	if( local_cxy == GET_CXY( process->ref_xp) ) // calling process is reference process
[1]	1909	{
[407]	1910	error = vmm_get_pte( process, vpn , false , &attr , &ppn );
[1]	1911	}
[50]	1912	else // calling process is not reference process
[1]	1913	{
	1914	cxy_t ref_cxy = GET_CXY( process->ref_xp );
[433]	1915	process_t * ref_ptr = GET_PTR( process->ref_xp );
[407]	1916	rpc_vmm_get_pte_client( ref_cxy , ref_ptr , vpn , false , &attr , &ppn , &error );
[1]	1917	}
	1918
[23]	1919	// set paddr
[1]	1920	*paddr = (((paddr_t)ppn) << CONFIG_PPM_PAGE_SHIFT) \| offset;
[21]	1921
[23]	1922	return error;
[313]	1923
[315]	1924	} // end vmm_v2p_translate()
[1]	1925
[315]	1926

Note: See TracBrowser for help on using the repository browser.

Download in other formats: