Context Navigation

source: trunk/kernel/mm/vmm.c @ 561

Last change on this file since 561 was 561, checked in by nicolas.van.phan@…, 6 years ago
Remove y_max in all functions except dqdt_init()
File size: 66.1 KB

Rev	Line
[1]	1	/*
	2	* vmm.c - virtual memory manager related operations interface.
	3	*
	4	* Authors Ghassan Almaless (2008,2009,2010,2011, 2012)
	5	* Mohamed Lamine Karaoui (2015)
	6	* Alain Greiner (2016)
[21]	7	*
[1]	8	* Copyright (c) UPMC Sorbonne Universites
	9	*
	10	* This file is part of ALMOS-MKH.
	11	*
	12	* ALMOS-MKH is free software; you can redistribute it and/or modify it
	13	* under the terms of the GNU General Public License as published by
	14	* the Free Software Foundation; version 2.0 of the License.
	15	*
	16	* ALMOS-MKH is distributed in the hope that it will be useful, but
	17	* WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU General Public License
	22	* along with ALMOS-MKH; if not, write to the Free Software Foundation,
	23	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	24	*/
	25
[14]	26	#include <kernel_config.h>
[457]	27	#include <hal_kernel_types.h>
[1]	28	#include <hal_special.h>
	29	#include <hal_gpt.h>
[409]	30	#include <hal_vmm.h>
[1]	31	#include <printk.h>
[23]	32	#include <memcpy.h>
[1]	33	#include <rwlock.h>
	34	#include <list.h>
[408]	35	#include <xlist.h>
[1]	36	#include <bits.h>
	37	#include <process.h>
	38	#include <thread.h>
	39	#include <vseg.h>
	40	#include <cluster.h>
	41	#include <scheduler.h>
	42	#include <vfs.h>
	43	#include <mapper.h>
	44	#include <page.h>
	45	#include <kmem.h>
	46	#include <vmm.h>
[561]	47	#include <cluster_info.h>
[1]	48
	49	//////////////////////////////////////////////////////////////////////////////////
	50	// Extern global variables
	51	//////////////////////////////////////////////////////////////////////////////////
	52
	53	extern process_t process_zero; // defined in cluster.c file
	54
	55
[415]	56	///////////////////////////////////////
	57	error_t vmm_init( process_t * process )
[21]	58	{
[1]	59	error_t error;
	60	vseg_t * vseg_kentry;
	61	vseg_t * vseg_args;
	62	vseg_t * vseg_envs;
	63	intptr_t base;
	64	intptr_t size;
	65
[438]	66	#if DEBUG_VMM_INIT
[433]	67	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	68	if( DEBUG_VMM_INIT )
[433]	69	printk("\n[DBG] %s : thread %x enter for process %x / cycle %d\n",
	70	__FUNCTION__ , CURRENT_THREAD , process->pid , cycle );
	71	#endif
[204]	72
[1]	73	// get pointer on VMM
	74	vmm_t * vmm = &process->vmm;
	75
[407]	76	// initialize local list of vsegs
	77	vmm->vsegs_nr = 0;
[408]	78	xlist_root_init( XPTR( local_cxy , &vmm->vsegs_root ) );
	79	remote_rwlock_init( XPTR( local_cxy , &vmm->vsegs_lock ) );
[407]	80
[204]	81	assert( ((CONFIG_VMM_KENTRY_SIZE + CONFIG_VMM_ARGS_SIZE + CONFIG_VMM_ENVS_SIZE)
[492]	82	<= CONFIG_VMM_ELF_BASE) , "UTILS zone too small\n" );
[21]	83
[492]	84	assert( (CONFIG_THREAD_MAX_PER_CLUSTER <= 32) ,
[204]	85	"no more than 32 threads per cluster for a single process\n");
[1]	86
[204]	87	assert( ((CONFIG_VMM_STACK_SIZE * CONFIG_THREAD_MAX_PER_CLUSTER) <=
[492]	88	(CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE)) ,
[204]	89	"STACK zone too small\n");
[1]	90
[409]	91	// register kentry vseg in VSL
[406]	92	base = CONFIG_VMM_KENTRY_BASE << CONFIG_PPM_PAGE_SHIFT;
[1]	93	size = CONFIG_VMM_KENTRY_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	94
[407]	95	vseg_kentry = vmm_create_vseg( process,
	96	VSEG_TYPE_CODE,
	97	base,
	98	size,
	99	0, // file_offset unused
	100	0, // file_size unused
	101	XPTR_NULL, // mapper_xp unused
	102	local_cxy );
[204]	103
[415]	104	if( vseg_kentry == NULL )
	105	{
	106	printk("\n[ERROR] in %s : cannot register kentry vseg\n", __FUNCTION__ );
	107	return -1;
	108	}
[204]	109
[406]	110	vmm->kent_vpn_base = base;
[1]	111
[409]	112	// register args vseg in VSL
[406]	113	base = (CONFIG_VMM_KENTRY_BASE +
	114	CONFIG_VMM_KENTRY_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
[1]	115	size = CONFIG_VMM_ARGS_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	116
[407]	117	vseg_args = vmm_create_vseg( process,
	118	VSEG_TYPE_DATA,
	119	base,
	120	size,
	121	0, // file_offset unused
	122	0, // file_size unused
	123	XPTR_NULL, // mapper_xp unused
	124	local_cxy );
[204]	125
[415]	126	if( vseg_args == NULL )
	127	{
	128	printk("\n[ERROR] in %s : cannot register args vseg\n", __FUNCTION__ );
	129	return -1;
	130	}
[204]	131
[406]	132	vmm->args_vpn_base = base;
[1]	133
[409]	134	// register the envs vseg in VSL
[406]	135	base = (CONFIG_VMM_KENTRY_BASE +
	136	CONFIG_VMM_KENTRY_SIZE +
	137	CONFIG_VMM_ARGS_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
[1]	138	size = CONFIG_VMM_ENVS_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	139
[407]	140	vseg_envs = vmm_create_vseg( process,
	141	VSEG_TYPE_DATA,
	142	base,
	143	size,
	144	0, // file_offset unused
	145	0, // file_size unused
	146	XPTR_NULL, // mapper_xp unused
	147	local_cxy );
[204]	148
[415]	149	if( vseg_envs == NULL )
	150	{
	151	printk("\n[ERROR] in %s : cannot register envs vseg\n", __FUNCTION__ );
	152	return -1;
	153	}
[204]	154
[406]	155	vmm->envs_vpn_base = base;
[1]	156
[409]	157	// create GPT (empty)
[1]	158	error = hal_gpt_create( &vmm->gpt );
	159
[415]	160	if( error )
	161	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
[204]	162
[415]	163	// initialize GPT (architecture specic)
[409]	164	// (For TSAR, identity map the kentry_vseg)
	165	error = hal_vmm_init( vmm );
	166
[415]	167	if( error )
	168	printk("\n[ERROR] in %s : cannot initialize GPT\n", __FUNCTION__ );
[409]	169
[1]	170	// initialize STACK allocator
	171	vmm->stack_mgr.bitmap = 0;
	172	vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
[457]	173	spinlock_init( &vmm->stack_mgr.lock );
[1]	174
	175	// initialize MMAP allocator
[407]	176	vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
	177	vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
	178	vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
[457]	179	spinlock_init( &vmm->mmap_mgr.lock );
	180
[1]	181	uint32_t i;
	182	for( i = 0 ; i < 32 ; i++ ) list_root_init( &vmm->mmap_mgr.zombi_list[i] );
	183
[21]	184	// initialize instrumentation counters
[409]	185	vmm->pgfault_nr = 0;
[1]	186
[124]	187	hal_fence();
[1]	188
[438]	189	#if DEBUG_VMM_INIT
[433]	190	cycle = (uint32_t)hal_get_cycles();
[438]	191	if( DEBUG_VMM_INIT )
[433]	192	printk("\n[DBG] %s : thread %x exit for process %x / entry_point = %x / cycle %d\n",
	193	__FUNCTION__ , CURRENT_THREAD , process->pid , process->vmm.entry_point , cycle );
	194	#endif
[204]	195
[415]	196	return 0;
	197
[204]	198	} // end vmm_init()
	199
[407]	200	//////////////////////////////////////
	201	void vmm_display( process_t * process,
	202	bool_t mapping )
	203	{
	204	vmm_t * vmm = &process->vmm;
	205	gpt_t * gpt = &vmm->gpt;
	206
[457]	207	printk("\n***** VSL and GPT(%x) for process %x in cluster %x\n\n",
	208	process->vmm.gpt.ptr , process->pid , local_cxy );
[407]	209
	210	// get lock protecting the vseg list
[408]	211	remote_rwlock_rd_lock( XPTR( local_cxy , &vmm->vsegs_lock ) );
[407]	212
	213	// scan the list of vsegs
[408]	214	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	215	xptr_t iter_xp;
	216	xptr_t vseg_xp;
[407]	217	vseg_t * vseg;
[408]	218	XLIST_FOREACH( root_xp , iter_xp )
[407]	219	{
[408]	220	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	221	vseg = GET_PTR( vseg_xp );
[408]	222
[407]	223	printk(" - %s : base = %X / size = %X / npages = %d\n",
	224	vseg_type_str( vseg->type ) , vseg->min , vseg->max - vseg->min , vseg->vpn_size );
	225
	226	if( mapping )
	227	{
	228	vpn_t vpn;
	229	ppn_t ppn;
	230	uint32_t attr;
	231	vpn_t base = vseg->vpn_base;
	232	vpn_t size = vseg->vpn_size;
	233	for( vpn = base ; vpn < (base+size) ; vpn++ )
	234	{
	235	hal_gpt_get_pte( gpt , vpn , &attr , &ppn );
	236	if( attr & GPT_MAPPED )
	237	{
	238	printk(" . vpn = %X / attr = %X / ppn = %X\n", vpn , attr , ppn );
	239	}
	240	}
	241	}
	242	}
	243
	244	// release the lock
[408]	245	remote_rwlock_rd_unlock( XPTR( local_cxy , &vmm->vsegs_lock ) );
[407]	246
[408]	247	} // vmm_display()
	248
[433]	249	/////////////////////i//////////////////////////
	250	void vmm_global_update_pte( process_t * process,
	251	vpn_t vpn,
	252	uint32_t attr,
	253	ppn_t ppn )
[23]	254	{
	255
[408]	256	xlist_entry_t * process_root_ptr;
	257	xptr_t process_root_xp;
	258	xptr_t process_iter_xp;
[23]	259
[408]	260	xptr_t remote_process_xp;
	261	cxy_t remote_process_cxy;
	262	process_t * remote_process_ptr;
	263	xptr_t remote_gpt_xp;
[23]	264
[408]	265	pid_t pid;
	266	cxy_t owner_cxy;
	267	lpid_t owner_lpid;
[23]	268
[438]	269	#if DEBUG_VMM_UPDATE_PTE
[433]	270	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	271	if( DEBUG_VMM_UPDATE_PTE < cycle )
[433]	272	printk("\n[DBG] %s : thread %x enter for process %x / vpn %x / cycle %d\n",
	273	__FUNCTION__ , CURRENT_THREAD , process->pid , vpn , cycle );
	274	#endif
	275
	276	// check cluster is reference
[492]	277	assert( (GET_CXY( process->ref_xp ) == local_cxy) ,
[433]	278	"not called in reference cluster\n");
	279
[408]	280	// get extended pointer on root of process copies xlist in owner cluster
	281	pid = process->pid;
	282	owner_cxy = CXY_FROM_PID( pid );
	283	owner_lpid = LPID_FROM_PID( pid );
	284	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
	285	process_root_xp = XPTR( owner_cxy , process_root_ptr );
[23]	286
[408]	287	// loop on destination process copies
	288	XLIST_FOREACH( process_root_xp , process_iter_xp )
	289	{
	290	// get cluster and local pointer on remote process
	291	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
[433]	292	remote_process_ptr = GET_PTR( remote_process_xp );
[408]	293	remote_process_cxy = GET_CXY( remote_process_xp );
[407]	294
[438]	295	#if (DEBUG_VMM_UPDATE_PTE & 0x1)
	296	if( DEBUG_VMM_UPDATE_PTE < cycle )
[433]	297	printk("\n[DBG] %s : thread %x handling process %x in cluster %x\n",
	298	__FUNCTION__ , CURRENT_THREAD , process->pid , remote_process_cxy );
	299	#endif
	300
[408]	301	// get extended pointer on remote gpt
	302	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
	303
[433]	304	// update remote GPT
	305	hal_gpt_update_pte( remote_gpt_xp, vpn, attr, ppn );
[408]	306	}
	307
[438]	308	#if DEBUG_VMM_UPDATE_PTE
[433]	309	cycle = (uint32_t)hal_get_cycles();
[438]	310	if( DEBUG_VMM_UPDATE_PTE < cycle )
[433]	311	printk("\n[DBG] %s : thread %x exit for process %x / vpn %x / cycle %d\n",
	312	__FUNCTION__ , CURRENT_THREAD , process->pid , vpn , cycle );
	313	#endif
	314
	315	} // end vmm_global_update_pte()
	316
[408]	317	///////////////////////////////////////
	318	void vmm_set_cow( process_t * process )
	319	{
	320	vmm_t * vmm;
	321
	322	xlist_entry_t * process_root_ptr;
	323	xptr_t process_root_xp;
	324	xptr_t process_iter_xp;
	325
	326	xptr_t remote_process_xp;
	327	cxy_t remote_process_cxy;
	328	process_t * remote_process_ptr;
	329	xptr_t remote_gpt_xp;
	330
	331	xptr_t vseg_root_xp;
	332	xptr_t vseg_iter_xp;
	333
	334	xptr_t vseg_xp;
	335	vseg_t * vseg;
	336
	337	pid_t pid;
	338	cxy_t owner_cxy;
	339	lpid_t owner_lpid;
	340
[438]	341	#if DEBUG_VMM_SET_COW
[433]	342	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	343	if( DEBUG_VMM_SET_COW < cycle )
[433]	344	printk("\n[DBG] %s : thread %x enter for process %x / cycle %d\n",
	345	__FUNCTION__ , CURRENT_THREAD , process->pid , cycle );
	346	#endif
[408]	347
	348	// check cluster is reference
[492]	349	assert( (GET_CXY( process->ref_xp ) == local_cxy) ,
[408]	350	"local cluster is not process reference cluster\n");
	351
	352	// get pointer on reference VMM
	353	vmm = &process->vmm;
	354
	355	// get extended pointer on root of process copies xlist in owner cluster
	356	pid = process->pid;
	357	owner_cxy = CXY_FROM_PID( pid );
	358	owner_lpid = LPID_FROM_PID( pid );
	359	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
	360	process_root_xp = XPTR( owner_cxy , process_root_ptr );
	361
	362	// get extended pointer on root of vsegs xlist from reference VMM
	363	vseg_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	364
	365	// loop on destination process copies
	366	XLIST_FOREACH( process_root_xp , process_iter_xp )
	367	{
	368	// get cluster and local pointer on remote process
	369	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
[433]	370	remote_process_ptr = GET_PTR( remote_process_xp );
[408]	371	remote_process_cxy = GET_CXY( remote_process_xp );
	372
[438]	373	#if (DEBUG_VMM_SET_COW &0x1)
	374	if( DEBUG_VMM_SET_COW < cycle )
[433]	375	printk("\n[DBG] %s : thread %x handling process %x in cluster %x\n",
	376	__FUNCTION__ , CURRENT_THREAD , process->pid , remote_process_cxy );
	377	#endif
[408]	378
	379	// get extended pointer on remote gpt
	380	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
	381
	382	// loop on vsegs in (local) reference process VSL
	383	XLIST_FOREACH( vseg_root_xp , vseg_iter_xp )
	384	{
	385	// get pointer on vseg
	386	vseg_xp = XLIST_ELEMENT( vseg_iter_xp , vseg_t , xlist );
[433]	387	vseg = GET_PTR( vseg_xp );
[408]	388
[492]	389	assert( (GET_CXY( vseg_xp ) == local_cxy) ,
[408]	390	"all vsegs in reference VSL must be local\n" );
	391
	392	// get vseg type, base and size
	393	uint32_t type = vseg->type;
	394	vpn_t vpn_base = vseg->vpn_base;
	395	vpn_t vpn_size = vseg->vpn_size;
	396
[438]	397	#if (DEBUG_VMM_SET_COW & 0x1)
	398	if( DEBUG_VMM_SET_COW < cycle )
[433]	399	printk("\n[DBG] %s : thread %x handling vseg %s / vpn_base = %x / vpn_size = %x\n",
	400	__FUNCTION__, CURRENT_THREAD , vseg_type_str(type), vpn_base, vpn_size );
	401	#endif
	402	// only DATA, ANON and REMOTE vsegs
[408]	403	if( (type == VSEG_TYPE_DATA) \|\|
	404	(type == VSEG_TYPE_ANON) \|\|
	405	(type == VSEG_TYPE_REMOTE) )
	406	{
[433]	407	vpn_t vpn;
	408	uint32_t attr;
	409	ppn_t ppn;
	410	xptr_t page_xp;
	411	cxy_t page_cxy;
	412	page_t * page_ptr;
	413	xptr_t forks_xp;
[469]	414	xptr_t lock_xp;
[433]	415
	416	// update flags in remote GPT
	417	hal_gpt_set_cow( remote_gpt_xp,
	418	vpn_base,
	419	vpn_size );
	420
	421	// atomically increment pending forks counter in physical pages,
	422	// for all vseg pages that are mapped in reference cluster
	423	if( remote_process_cxy == local_cxy )
	424	{
	425	// the reference GPT is the local GPT
	426	gpt_t * gpt = GET_PTR( remote_gpt_xp );
	427
	428	// scan all pages in vseg
	429	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
	430	{
	431	// get page attributes and PPN from reference GPT
	432	hal_gpt_get_pte( gpt , vpn , &attr , &ppn );
	433
	434	// atomically update pending forks counter if page is mapped
	435	if( attr & GPT_MAPPED )
	436	{
[469]	437	// get pointers and cluster on page descriptor
[433]	438	page_xp = ppm_ppn2page( ppn );
	439	page_cxy = GET_CXY( page_xp );
	440	page_ptr = GET_PTR( page_xp );
[469]	441
	442	// get extended pointers on "forks" and "lock"
[433]	443	forks_xp = XPTR( page_cxy , &page_ptr->forks );
[469]	444	lock_xp = XPTR( page_cxy , &page_ptr->lock );
	445
	446	// increment "forks"
	447	remote_spinlock_lock( lock_xp );
[433]	448	hal_remote_atomic_add( forks_xp , 1 );
[469]	449	remote_spinlock_unlock( lock_xp );
[433]	450	}
	451	} // end loop on vpn
	452	} // end if local
	453	} // end if vseg type
	454	} // end loop on vsegs
[408]	455	} // end loop on process copies
	456
[438]	457	#if DEBUG_VMM_SET_COW
[433]	458	cycle = (uint32_t)hal_get_cycles();
[438]	459	if( DEBUG_VMM_SET_COW < cycle )
[433]	460	printk("\n[DBG] %s : thread %x exit for process %x / cycle %d\n",
	461	__FUNCTION__ , CURRENT_THREAD , process->pid , cycle );
	462	#endif
[408]	463
	464	} // end vmm_set-cow()
	465
	466	/////////////////////////////////////////////////
	467	error_t vmm_fork_copy( process_t * child_process,
	468	xptr_t parent_process_xp )
	469	{
	470	error_t error;
	471	cxy_t parent_cxy;
	472	process_t * parent_process;
	473	vmm_t * parent_vmm;
	474	xptr_t parent_lock_xp;
	475	vmm_t * child_vmm;
	476	xptr_t iter_xp;
	477	xptr_t parent_vseg_xp;
	478	vseg_t * parent_vseg;
	479	vseg_t * child_vseg;
	480	uint32_t type;
	481	bool_t cow;
	482	vpn_t vpn;
	483	vpn_t vpn_base;
	484	vpn_t vpn_size;
[469]	485	xptr_t page_xp; // extended pointer on page descriptor
[408]	486	page_t * page_ptr;
	487	cxy_t page_cxy;
[469]	488	xptr_t forks_xp; // extended pointer on forks counter in page descriptor
	489	xptr_t lock_xp; // extended pointer on lock protecting the forks counter
[408]	490	xptr_t parent_root_xp;
	491	bool_t mapped;
	492	ppn_t ppn;
	493
[438]	494	#if DEBUG_VMM_FORK_COPY
[433]	495	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	496	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	497	printk("\n[DBG] %s : thread %x enter / cycle %d\n",
	498	__FUNCTION__ , CURRENT_THREAD, cycle );
	499	#endif
[408]	500
	501	// get parent process cluster and local pointer
	502	parent_cxy = GET_CXY( parent_process_xp );
[433]	503	parent_process = GET_PTR( parent_process_xp );
[408]	504
	505	// get local pointers on parent and child VMM
	506	parent_vmm = &parent_process->vmm;
	507	child_vmm = &child_process->vmm;
	508
	509	// get extended pointer on lock protecting the parent VSL
	510	parent_lock_xp = XPTR( parent_cxy , &parent_vmm->vsegs_lock );
	511
	512	// initialize the lock protecting the child VSL
	513	remote_rwlock_init( XPTR( local_cxy , &child_vmm->vsegs_lock ) );
	514
	515	// initialize the child VSL as empty
	516	xlist_root_init( XPTR( local_cxy, &child_vmm->vsegs_root ) );
	517	child_vmm->vsegs_nr = 0;
	518
[415]	519	// create child GPT
[408]	520	error = hal_gpt_create( &child_vmm->gpt );
[415]	521
[407]	522	if( error )
	523	{
[408]	524	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
	525	return -1;
[407]	526	}
	527
[408]	528	// build extended pointer on parent VSL
	529	parent_root_xp = XPTR( parent_cxy , &parent_vmm->vsegs_root );
	530
[415]	531	// take the lock protecting the parent VSL
	532	remote_rwlock_rd_lock( parent_lock_xp );
	533
[408]	534	// loop on parent VSL xlist
	535	XLIST_FOREACH( parent_root_xp , iter_xp )
[23]	536	{
[408]	537	// get local and extended pointers on current parent vseg
	538	parent_vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	539	parent_vseg = GET_PTR( parent_vseg_xp );
[23]	540
[408]	541	// get vseg type
	542	type = hal_remote_lw( XPTR( parent_cxy , &parent_vseg->type ) );
	543
[438]	544	#if DEBUG_VMM_FORK_COPY
[433]	545	cycle = (uint32_t)hal_get_cycles();
[438]	546	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	547	printk("\n[DBG] %s : thread %x found parent vseg %s / vpn_base = %x / cycle %d\n",
	548	__FUNCTION__ , CURRENT_THREAD, vseg_type_str(type),
	549	hal_remote_lw( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
	550	#endif
[23]	551
[408]	552	// all parent vsegs - but STACK - must be copied in child VSL
	553	if( type != VSEG_TYPE_STACK )
[23]	554	{
[408]	555	// allocate memory for a new child vseg
	556	child_vseg = vseg_alloc();
	557	if( child_vseg == NULL ) // release all allocated vsegs
[23]	558	{
[408]	559	vmm_destroy( child_process );
	560	printk("\n[ERROR] in %s : cannot create vseg for child\n", __FUNCTION__ );
	561	return -1;
[23]	562	}
	563
[408]	564	// copy parent vseg to child vseg
	565	vseg_init_from_ref( child_vseg , parent_vseg_xp );
[23]	566
[408]	567	// register child vseg in child VSL
	568	vseg_attach( child_vmm , child_vseg );
[407]	569
[438]	570	#if DEBUG_VMM_FORK_COPY
[433]	571	cycle = (uint32_t)hal_get_cycles();
[438]	572	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	573	printk("\n[DBG] %s : thread %x copied vseg %s / vpn_base = %x to child VSL / cycle %d\n",
	574	__FUNCTION__ , CURRENT_THREAD , vseg_type_str(type),
	575	hal_remote_lw( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
	576	#endif
[23]	577
[408]	578	// copy DATA, MMAP, REMOTE, FILE parent GPT entries to child GPT
	579	if( type != VSEG_TYPE_CODE )
	580	{
	581	// activate the COW for DATA, MMAP, REMOTE vsegs only
	582	cow = ( type != VSEG_TYPE_FILE );
[23]	583
[408]	584	vpn_base = child_vseg->vpn_base;
	585	vpn_size = child_vseg->vpn_size;
[23]	586
[408]	587	// scan pages in parent vseg
	588	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
	589	{
	590	error = hal_gpt_pte_copy( &child_vmm->gpt,
	591	XPTR( parent_cxy , &parent_vmm->gpt ),
	592	vpn,
	593	cow,
	594	&ppn,
	595	&mapped );
	596	if( error )
	597	{
	598	vmm_destroy( child_process );
	599	printk("\n[ERROR] in %s : cannot copy GPT\n", __FUNCTION__ );
	600	return -1;
	601	}
	602
[433]	603	// increment pending forks counter in page if mapped
[408]	604	if( mapped )
	605	{
[469]	606	// get pointers and cluster on page descriptor
	607	page_xp = ppm_ppn2page( ppn );
[408]	608	page_cxy = GET_CXY( page_xp );
[433]	609	page_ptr = GET_PTR( page_xp );
[408]	610
[469]	611	// get extended pointers on "forks" and "lock"
	612	forks_xp = XPTR( page_cxy , &page_ptr->forks );
	613	lock_xp = XPTR( page_cxy , &page_ptr->lock );
	614
	615	// increment "forks"
	616	remote_spinlock_lock( lock_xp );
	617	hal_remote_atomic_add( forks_xp , 1 );
	618	remote_spinlock_unlock( lock_xp );
	619
[438]	620	#if DEBUG_VMM_FORK_COPY
[433]	621	cycle = (uint32_t)hal_get_cycles();
[438]	622	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	623	printk("\n[DBG] %s : thread %x copied vpn %x to child GPT / cycle %d\n",
	624	__FUNCTION__ , CURRENT_THREAD , vpn , cycle );
	625	#endif
[408]	626	}
	627	}
	628	} // end if no code & no stack
	629	} // end if no stack
	630	} // end loop on vsegs
	631
	632	// release the parent vsegs lock
	633	remote_rwlock_rd_unlock( parent_lock_xp );
	634
[415]	635	// initialize child GPT (architecture specic)
	636	// => For TSAR, identity map the kentry_vseg
	637	error = hal_vmm_init( child_vmm );
	638
	639	if( error )
	640	{
	641	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
	642	return -1;
	643	}
	644
[408]	645	// initialize the child VMM STACK allocator
	646	child_vmm->stack_mgr.bitmap = 0;
	647	child_vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
	648
	649	// initialize the child VMM MMAP allocator
[23]	650	uint32_t i;
[408]	651	child_vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
	652	child_vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
	653	child_vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
	654	for( i = 0 ; i < 32 ; i++ ) list_root_init( &child_vmm->mmap_mgr.zombi_list[i] );
[23]	655
[178]	656	// initialize instrumentation counters
[408]	657	child_vmm->pgfault_nr = 0;
[23]	658
[408]	659	// copy base addresses from parent VMM to child VMM
	660	child_vmm->kent_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->kent_vpn_base));
	661	child_vmm->args_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->args_vpn_base));
	662	child_vmm->envs_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->envs_vpn_base));
	663	child_vmm->heap_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->heap_vpn_base));
	664	child_vmm->code_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->code_vpn_base));
	665	child_vmm->data_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->data_vpn_base));
[23]	666
[408]	667	child_vmm->entry_point = (intptr_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->entry_point));
[23]	668
[124]	669	hal_fence();
[23]	670
[438]	671	#if DEBUG_VMM_FORK_COPY
[433]	672	cycle = (uint32_t)hal_get_cycles();
[438]	673	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	674	printk("\n[DBG] %s : thread %x exit successfully / cycle %d\n",
	675	__FUNCTION__ , CURRENT_THREAD , cycle );
	676	#endif
	677
[23]	678	return 0;
	679
[408]	680	} // vmm_fork_copy()
[204]	681
[1]	682	///////////////////////////////////////
	683	void vmm_destroy( process_t * process )
	684	{
[408]	685	xptr_t vseg_xp;
[1]	686	vseg_t * vseg;
	687
[438]	688	#if DEBUG_VMM_DESTROY
[433]	689	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	690	if( DEBUG_VMM_DESTROY < cycle )
[443]	691	printk("\n[DBG] %s : thread %x enter for process %x in cluster %x / cycle %d\n",
[469]	692	__FUNCTION__, CURRENT_THREAD->trdid, process->pid, local_cxy, cycle );
[433]	693	#endif
[416]	694
[438]	695	#if (DEBUG_VMM_DESTROY & 1 )
[443]	696	if( DEBUG_VMM_DESTROY < cycle )
[437]	697	vmm_display( process , true );
	698	#endif
	699
[433]	700	// get pointer on local VMM
[1]	701	vmm_t * vmm = &process->vmm;
	702
[408]	703	// get extended pointer on VSL root and VSL lock
	704	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	705	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	706
[1]	707	// get lock protecting vseg list
[408]	708	remote_rwlock_wr_lock( lock_xp );
[1]	709
[409]	710	// remove all user vsegs registered in VSL
[408]	711	while( !xlist_is_empty( root_xp ) )
[1]	712	{
[409]	713	// get pointer on first vseg in VSL
[408]	714	vseg_xp = XLIST_FIRST_ELEMENT( root_xp , vseg_t , xlist );
[433]	715	vseg = GET_PTR( vseg_xp );
[409]	716
[437]	717	// unmap and release physical pages
[409]	718	vmm_unmap_vseg( process , vseg );
	719
	720	// remove vseg from VSL
[473]	721	vseg_detach( vseg );
[409]	722
	723	// release memory allocated to vseg descriptor
[1]	724	vseg_free( vseg );
[443]	725
	726	#if( DEBUG_VMM_DESTROY & 1 )
	727	if( DEBUG_VMM_DESTROY < cycle )
	728	printk("\n[DBG] %s : %s vseg released / vpn_base %x / vpn_size %d\n",
	729	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	730	#endif
	731
[1]	732	}
	733
[433]	734	// release lock protecting VSL
[408]	735	remote_rwlock_wr_unlock( lock_xp );
[1]	736
	737	// remove all vsegs from zombi_lists in MMAP allocator
	738	uint32_t i;
	739	for( i = 0 ; i<32 ; i++ )
	740	{
	741	while( !list_is_empty( &vmm->mmap_mgr.zombi_list[i] ) )
	742	{
[408]	743	vseg = LIST_FIRST( &vmm->mmap_mgr.zombi_list[i] , vseg_t , zlist );
[443]	744
	745	#if( DEBUG_VMM_DESTROY & 1 )
	746	if( DEBUG_VMM_DESTROY < cycle )
	747	printk("\n[DBG] %s : found zombi vseg / vpn_base %x / vpn_size %d\n",
	748	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	749	#endif
[473]	750	vseg_detach( vseg );
[1]	751	vseg_free( vseg );
[443]	752
	753	#if( DEBUG_VMM_DESTROY & 1 )
	754	if( DEBUG_VMM_DESTROY < cycle )
	755	printk("\n[DBG] %s : zombi vseg released / vpn_base %x / vpn_size %d\n",
	756	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	757	#endif
[1]	758	}
	759	}
	760
[409]	761	// release memory allocated to the GPT itself
[1]	762	hal_gpt_destroy( &vmm->gpt );
	763
[438]	764	#if DEBUG_VMM_DESTROY
[433]	765	cycle = (uint32_t)hal_get_cycles();
[438]	766	if( DEBUG_VMM_DESTROY < cycle )
[443]	767	printk("\n[DBG] %s : thread %x exit for process %x in cluster %x / cycle %d\n",
[469]	768	__FUNCTION__, CURRENT_THREAD->trdid, process->pid, local_cxy , cycle );
[433]	769	#endif
[416]	770
[204]	771	} // end vmm_destroy()
	772
[1]	773	/////////////////////////////////////////////////
	774	vseg_t * vmm_check_conflict( process_t * process,
[21]	775	vpn_t vpn_base,
[1]	776	vpn_t vpn_size )
	777	{
	778	vmm_t * vmm = &process->vmm;
[408]	779
	780	// scan the VSL
[1]	781	vseg_t * vseg;
[408]	782	xptr_t iter_xp;
	783	xptr_t vseg_xp;
	784	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
[1]	785
[408]	786	XLIST_FOREACH( root_xp , iter_xp )
[1]	787	{
[408]	788	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	789	vseg = GET_PTR( vseg_xp );
[204]	790
[21]	791	if( ((vpn_base + vpn_size) > vseg->vpn_base) &&
	792	(vpn_base < (vseg->vpn_base + vseg->vpn_size)) ) return vseg;
[1]	793	}
	794	return NULL;
	795
[204]	796	} // end vmm_check_conflict()
	797
[1]	798	////////////////////////////////////////////////////////////////////////////////////////////
	799	// This static function is called by the vmm_create_vseg() function, and implements
	800	// the VMM stack_vseg specific allocator.
	801	////////////////////////////////////////////////////////////////////////////////////////////
	802	// @ vmm : pointer on VMM.
[21]	803	// @ vpn_base : (return value) first allocated page
[1]	804	// @ vpn_size : (return value) number of allocated pages
	805	////////////////////////////////////////////////////////////////////////////////////////////
	806	static error_t vmm_stack_alloc( vmm_t * vmm,
	807	vpn_t * vpn_base,
	808	vpn_t * vpn_size )
	809	{
	810	// get stack allocator pointer
	811	stack_mgr_t * mgr = &vmm->stack_mgr;
	812
	813	// get lock on stack allocator
	814	spinlock_lock( &mgr->lock );
	815
	816	// get first free slot index in bitmap
	817	int32_t index = bitmap_ffc( &mgr->bitmap , 4 );
[179]	818	if( (index < 0) \|\| (index > 31) )
	819	{
	820	spinlock_unlock( &mgr->lock );
	821	return ENOMEM;
	822	}
[1]	823
	824	// update bitmap
	825	bitmap_set( &mgr->bitmap , index );
[21]	826
[1]	827	// release lock on stack allocator
	828	spinlock_unlock( &mgr->lock );
	829
[21]	830	// returns vpn_base, vpn_size (one page non allocated)
[1]	831	vpn_base = mgr->vpn_base + index CONFIG_VMM_STACK_SIZE + 1;
	832	*vpn_size = CONFIG_VMM_STACK_SIZE - 1;
	833	return 0;
	834
[204]	835	} // end vmm_stack_alloc()
	836
[1]	837	////////////////////////////////////////////////////////////////////////////////////////////
	838	// This static function is called by the vmm_create_vseg() function, and implements
	839	// the VMM MMAP specific allocator.
	840	////////////////////////////////////////////////////////////////////////////////////////////
	841	// @ vmm : [in] pointer on VMM.
	842	// @ npages : [in] requested number of pages.
[21]	843	// @ vpn_base : [out] first allocated page.
[1]	844	// @ vpn_size : [out] actual number of allocated pages.
	845	////////////////////////////////////////////////////////////////////////////////////////////
	846	static error_t vmm_mmap_alloc( vmm_t * vmm,
	847	vpn_t npages,
	848	vpn_t * vpn_base,
	849	vpn_t * vpn_size )
	850	{
	851	uint32_t index;
	852	vseg_t * vseg;
	853	vpn_t base;
	854	vpn_t size;
[21]	855	vpn_t free;
[1]	856
[21]	857	// mmap vseg size must be power of 2
[1]	858	// compute actual size and index in zombi_list array
	859	size = POW2_ROUNDUP( npages );
	860	index = bits_log2( size );
	861
	862	// get mmap allocator pointer
	863	mmap_mgr_t * mgr = &vmm->mmap_mgr;
	864
	865	// get lock on mmap allocator
	866	spinlock_lock( &mgr->lock );
	867
	868	// get vseg from zombi_list or from mmap zone
	869	if( list_is_empty( &mgr->zombi_list[index] ) ) // from mmap zone
	870	{
	871	// check overflow
	872	free = mgr->first_free_vpn;
	873	if( (free + size) > mgr->vpn_size ) return ENOMEM;
	874
	875	// update STACK allocator
	876	mgr->first_free_vpn += size;
	877
	878	// compute base
	879	base = free;
	880	}
	881	else // from zombi_list
	882	{
	883	// get pointer on zombi vseg from zombi_list
[408]	884	vseg = LIST_FIRST( &mgr->zombi_list[index] , vseg_t , zlist );
[1]	885
	886	// remove vseg from free-list
[408]	887	list_unlink( &vseg->zlist );
[1]	888
	889	// compute base
	890	base = vseg->vpn_base;
[21]	891	}
	892
[1]	893	// release lock on mmap allocator
	894	spinlock_unlock( &mgr->lock );
	895
	896	// returns vpn_base, vpn_size
	897	*vpn_base = base;
	898	*vpn_size = size;
	899	return 0;
	900
[204]	901	} // end vmm_mmap_alloc()
	902
[407]	903	////////////////////////////////////////////////
	904	vseg_t * vmm_create_vseg( process_t * process,
	905	vseg_type_t type,
	906	intptr_t base,
	907	uint32_t size,
	908	uint32_t file_offset,
	909	uint32_t file_size,
	910	xptr_t mapper_xp,
	911	cxy_t cxy )
[1]	912	{
	913	vseg_t * vseg; // created vseg pointer
[204]	914	vpn_t vpn_base; // first page index
[1]	915	vpn_t vpn_size; // number of pages
	916	error_t error;
	917
[438]	918	#if DEBUG_VMM_CREATE_VSEG
[433]	919	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	920	if( DEBUG_VMM_CREATE_VSEG < cycle )
[433]	921	printk("\n[DBG] %s : thread %x enter / process %x / base %x / size %x / %s / cxy %x / cycle %d\n",
	922	__FUNCTION__, CURRENT_THREAD, process->pid, base, size, vseg_type_str(type), cxy, cycle );
	923	#endif
[21]	924
[407]	925	// get pointer on VMM
	926	vmm_t * vmm = &process->vmm;
[21]	927
[204]	928	// compute base, size, vpn_base, vpn_size, depending on vseg type
[407]	929	// we use the VMM specific allocators for "stack", "file", "anon", & "remote" vsegs
[1]	930	if( type == VSEG_TYPE_STACK )
	931	{
	932	// get vpn_base and vpn_size from STACK allocator
	933	error = vmm_stack_alloc( vmm , &vpn_base , &vpn_size );
	934	if( error )
	935	{
[407]	936	printk("\n[ERROR] in %s : no space for stack vseg / process %x in cluster %x\n",
	937	__FUNCTION__ , process->pid , local_cxy );
[1]	938	return NULL;
	939	}
	940
	941	// compute vseg base and size from vpn_base and vpn_size
	942	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
	943	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
	944	}
[21]	945	else if( (type == VSEG_TYPE_ANON) \|\|
	946	(type == VSEG_TYPE_FILE) \|\|
[1]	947	(type == VSEG_TYPE_REMOTE) )
	948	{
	949	// get vpn_base and vpn_size from MMAP allocator
	950	vpn_t npages = size >> CONFIG_PPM_PAGE_SHIFT;
	951	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
	952	if( error )
	953	{
	954	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
	955	__FUNCTION__ , process->pid , local_cxy );
	956	return NULL;
	957	}
	958
	959	// compute vseg base and size from vpn_base and vpn_size
	960	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
	961	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
	962	}
	963	else
	964	{
[204]	965	uint32_t vpn_min = base >> CONFIG_PPM_PAGE_SHIFT;
	966	uint32_t vpn_max = (base + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
	967
	968	vpn_base = vpn_min;
	969	vpn_size = vpn_max - vpn_min + 1;
[1]	970	}
	971
	972	// check collisions
	973	vseg = vmm_check_conflict( process , vpn_base , vpn_size );
	974	if( vseg != NULL )
	975	{
[21]	976	printk("\n[ERROR] in %s for process %x : new vseg [vpn_base = %x / vpn_size = %x]\n"
[1]	977	" overlap existing vseg [vpn_base = %x / vpn_size = %x]\n",
[407]	978	__FUNCTION__ , process->pid, vpn_base, vpn_size, vseg->vpn_base, vseg->vpn_size );
[1]	979	return NULL;
	980	}
	981
	982	// allocate physical memory for vseg descriptor
	983	vseg = vseg_alloc();
	984	if( vseg == NULL )
	985	{
	986	printk("\n[ERROR] in %s for process %x : cannot allocate memory for vseg\n",
[407]	987	__FUNCTION__ , process->pid );
[1]	988	return NULL;
	989	}
	990
	991	// initialize vseg descriptor
[407]	992	vseg_init( vseg,
	993	type,
	994	base,
	995	size,
	996	vpn_base,
	997	vpn_size,
	998	file_offset,
	999	file_size,
	1000	mapper_xp,
	1001	cxy );
[1]	1002
[408]	1003	// attach vseg to VSL
	1004	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	1005	remote_rwlock_wr_lock( lock_xp );
[1]	1006	vseg_attach( vmm , vseg );
[408]	1007	remote_rwlock_wr_unlock( lock_xp );
[1]	1008
[438]	1009	#if DEBUG_VMM_CREATE_VSEG
[433]	1010	cycle = (uint32_t)hal_get_cycles();
[438]	1011	if( DEBUG_VMM_CREATE_VSEG < cycle )
[433]	1012	printk("\n[DBG] %s : thread %x exit / process %x / %s / cxy %x / cycle %d\n",
	1013	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str(type), cxy, cycle );
	1014	#endif
[21]	1015
[1]	1016	return vseg;
	1017
[406]	1018	} // vmm_create_vseg()
	1019
[1]	1020	/////////////////////////////////////
	1021	void vmm_remove_vseg( vseg_t * vseg )
	1022	{
	1023	// get pointers on calling process and VMM
	1024	thread_t * this = CURRENT_THREAD;
	1025	vmm_t * vmm = &this->process->vmm;
	1026	uint32_t type = vseg->type;
	1027
[408]	1028	// detach vseg from VSL
	1029	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	1030	remote_rwlock_wr_lock( lock_xp );
[473]	1031	vseg_detach( vseg );
[408]	1032	remote_rwlock_wr_unlock( lock_xp );
[1]	1033
	1034	// release the stack slot to VMM stack allocator if STACK type
	1035	if( type == VSEG_TYPE_STACK )
	1036	{
	1037	// get pointer on stack allocator
	1038	stack_mgr_t * mgr = &vmm->stack_mgr;
	1039
	1040	// compute slot index
	1041	uint32_t index = ((vseg->vpn_base - mgr->vpn_base - 1) / CONFIG_VMM_STACK_SIZE);
	1042
	1043	// update stacks_bitmap
	1044	spinlock_lock( &mgr->lock );
	1045	bitmap_clear( &mgr->bitmap , index );
	1046	spinlock_unlock( &mgr->lock );
	1047	}
	1048
	1049	// release the vseg to VMM mmap allocator if MMAP type
	1050	if( (type == VSEG_TYPE_ANON) \|\| (type == VSEG_TYPE_FILE) \|\| (type == VSEG_TYPE_REMOTE) )
	1051	{
	1052	// get pointer on mmap allocator
	1053	mmap_mgr_t * mgr = &vmm->mmap_mgr;
	1054
	1055	// compute zombi_list index
	1056	uint32_t index = bits_log2( vseg->vpn_size );
	1057
	1058	// update zombi_list
	1059	spinlock_lock( &mgr->lock );
[408]	1060	list_add_first( &mgr->zombi_list[index] , &vseg->zlist );
[1]	1061	spinlock_unlock( &mgr->lock );
	1062	}
	1063
	1064	// release physical memory allocated for vseg descriptor if no MMAP type
	1065	if( (type != VSEG_TYPE_ANON) && (type != VSEG_TYPE_FILE) && (type != VSEG_TYPE_REMOTE) )
	1066	{
	1067	vseg_free( vseg );
	1068	}
[407]	1069	} // end vmm_remove_vseg()
[1]	1070
	1071	/////////////////////////////////////////
	1072	void vmm_unmap_vseg( process_t * process,
	1073	vseg_t * vseg )
	1074	{
[21]	1075	vpn_t vpn; // VPN of current PTE
	1076	vpn_t vpn_min; // VPN of first PTE
[1]	1077	vpn_t vpn_max; // VPN of last PTE (excluded)
[409]	1078	ppn_t ppn; // current PTE ppn value
	1079	uint32_t attr; // current PTE attributes
	1080	kmem_req_t req; // request to release memory
	1081	xptr_t page_xp; // extended pointer on page descriptor
	1082	cxy_t page_cxy; // page descriptor cluster
	1083	page_t * page_ptr; // page descriptor pointer
[433]	1084	xptr_t forks_xp; // extended pointer on pending forks counter
[469]	1085	xptr_t lock_xp; // extended pointer on lock protecting forks counter
	1086	uint32_t forks; // actual number of pendinf forks
[1]	1087
[438]	1088	#if DEBUG_VMM_UNMAP_VSEG
[433]	1089	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	1090	if( DEBUG_VMM_UNMAP_VSEG < cycle )
[433]	1091	printk("\n[DBG] %s : thread %x enter / process %x / vseg %s / base %x / cycle %d\n",
	1092	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str( vseg->type ), vseg->vpn_base, cycle );
	1093	#endif
[409]	1094
[433]	1095	// get pointer on local GPT
[1]	1096	gpt_t * gpt = &process->vmm.gpt;
	1097
	1098	// loop on pages in vseg
	1099	vpn_min = vseg->vpn_base;
	1100	vpn_max = vpn_min + vseg->vpn_size;
	1101	for( vpn = vpn_min ; vpn < vpn_max ; vpn++ )
	1102	{
[409]	1103	// get GPT entry
	1104	hal_gpt_get_pte( gpt , vpn , &attr , &ppn );
	1105
	1106	if( attr & GPT_MAPPED ) // entry is mapped
	1107	{
[437]	1108
[438]	1109	#if( DEBUG_VMM_UNMAP_VSEG & 1 )
	1110	if( DEBUG_VMM_UNMAP_VSEG < cycle )
[437]	1111	printk("- vpn %x / ppn %x\n" , vpn , ppn );
	1112	#endif
	1113
[409]	1114	// check small page
[492]	1115	assert( (attr & GPT_SMALL) ,
[409]	1116	"an user vseg must use small pages" );
	1117
[433]	1118	// unmap GPT entry in all GPT copies
[409]	1119	hal_gpt_reset_pte( gpt , vpn );
	1120
[433]	1121	// handle pending forks counter if
	1122	// 1) not identity mapped
	1123	// 2) running in reference cluster
	1124	if( ((vseg->flags & VSEG_IDENT) == 0) &&
	1125	(GET_CXY( process->ref_xp ) == local_cxy) )
[409]	1126	{
[433]	1127	// get extended pointer on physical page descriptor
[409]	1128	page_xp = ppm_ppn2page( ppn );
	1129	page_cxy = GET_CXY( page_xp );
[433]	1130	page_ptr = GET_PTR( page_xp );
[409]	1131
[469]	1132	// get extended pointers on forks and lock fields
	1133	forks_xp = XPTR( page_cxy , &page_ptr->forks );
	1134	lock_xp = XPTR( page_cxy , &page_ptr->lock );
[433]	1135
[469]	1136	// get lock protecting page descriptor
	1137	remote_spinlock_lock( lock_xp );
	1138
[433]	1139	// get pending forks counter
[469]	1140	forks = hal_remote_lw( forks_xp );
[433]	1141
[469]	1142	if( forks ) // decrement pending forks counter
[409]	1143	{
[433]	1144	hal_remote_atomic_add( forks_xp , -1 );
	1145	}
	1146	else // release physical page to relevant cluster
[409]	1147	{
[433]	1148	if( page_cxy == local_cxy ) // local cluster
	1149	{
	1150	req.type = KMEM_PAGE;
	1151	req.ptr = page_ptr;
	1152	kmem_free( &req );
	1153	}
	1154	else // remote cluster
	1155	{
	1156	rpc_pmem_release_pages_client( page_cxy , page_ptr );
	1157	}
[409]	1158	}
[433]	1159
[469]	1160	// release lock protecting page descriptor
	1161	remote_spinlock_unlock( lock_xp );
[409]	1162	}
	1163	}
[1]	1164	}
[433]	1165
[438]	1166	#if DEBUG_VMM_UNMAP_VSEG
[433]	1167	cycle = (uint32_t)hal_get_cycles();
[438]	1168	if( DEBUG_VMM_UNMAP_VSEG < cycle )
[433]	1169	printk("\n[DBG] %s : thread %x exit / process %x / vseg %s / base %x / cycle %d\n",
	1170	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str( vseg->type ), vseg->vpn_base, cycle );
	1171	#endif
	1172
[409]	1173	} // end vmm_unmap_vseg()
[1]	1174
[407]	1175	//////////////////////////////////////////////////////////////////////////////////////////
[440]	1176	// This low-level static function is called by the vmm_get_vseg(), vmm_get_pte(),
	1177	// and vmm_resize_vseg() functions. It scan the local VSL to find the unique vseg
	1178	// containing a given virtual address.
[407]	1179	//////////////////////////////////////////////////////////////////////////////////////////
[406]	1180	// @ vmm : pointer on the process VMM.
	1181	// @ vaddr : virtual address.
	1182	// @ return vseg pointer if success / return NULL if not found.
[407]	1183	//////////////////////////////////////////////////////////////////////////////////////////
[406]	1184	static vseg_t * vseg_from_vaddr( vmm_t * vmm,
	1185	intptr_t vaddr )
	1186	{
[408]	1187	xptr_t iter_xp;
	1188	xptr_t vseg_xp;
	1189	vseg_t * vseg;
[406]	1190
[408]	1191	// get extended pointers on VSL lock and root
	1192	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	1193	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
[406]	1194
[408]	1195	// get lock protecting the VSL
	1196	remote_rwlock_rd_lock( lock_xp );
	1197
	1198	// scan the list of vsegs in VSL
	1199	XLIST_FOREACH( root_xp , iter_xp )
[406]	1200	{
[408]	1201	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	1202	vseg = GET_PTR( vseg_xp );
[408]	1203	if( (vaddr >= vseg->min) && (vaddr < vseg->max) )
	1204	{
	1205	// return success
	1206	remote_rwlock_rd_unlock( lock_xp );
	1207	return vseg;
	1208	}
[406]	1209	}
	1210
[408]	1211	// return failure
	1212	remote_rwlock_rd_unlock( lock_xp );
	1213	return NULL;
[406]	1214
[408]	1215	} // end vseg_from_vaddr()
[406]	1216
[1]	1217	/////////////////////////////////////////////
	1218	error_t vmm_resize_vseg( process_t * process,
	1219	intptr_t base,
	1220	intptr_t size )
	1221	{
[406]	1222	error_t error;
	1223	vseg_t * new;
	1224	vpn_t vpn_min;
	1225	vpn_t vpn_max;
[1]	1226
	1227	// get pointer on process VMM
	1228	vmm_t * vmm = &process->vmm;
	1229
	1230	intptr_t addr_min = base;
	1231	intptr_t addr_max = base + size;
	1232
	1233	// get pointer on vseg
[406]	1234	vseg_t * vseg = vseg_from_vaddr( vmm , base );
[1]	1235
	1236	if( vseg == NULL) return EINVAL;
[21]	1237
[408]	1238	// get extended pointer on VSL lock
	1239	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
[21]	1240
[408]	1241	// get lock protecting VSL
	1242	remote_rwlock_wr_lock( lock_xp );
	1243
[1]	1244	if( (vseg->min > addr_min) \|\| (vseg->max < addr_max) ) // region not included in vseg
	1245	{
	1246	error = EINVAL;
	1247	}
	1248	else if( (vseg->min == addr_min) && (vseg->max == addr_max) ) // vseg must be removed
	1249	{
	1250	vmm_remove_vseg( vseg );
	1251	error = 0;
	1252	}
[406]	1253	else if( vseg->min == addr_min ) // vseg must be resized
[1]	1254	{
[406]	1255	// update vseg base address
	1256	vseg->min = addr_max;
	1257
	1258	// update vpn_base and vpn_size
	1259	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1260	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1261	vseg->vpn_base = vpn_min;
	1262	vseg->vpn_size = vpn_max - vpn_min + 1;
	1263	error = 0;
[1]	1264	}
[406]	1265	else if( vseg->max == addr_max ) // vseg must be resized
[1]	1266	{
[406]	1267	// update vseg max address
	1268	vseg->max = addr_min;
	1269
	1270	// update vpn_base and vpn_size
	1271	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1272	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1273	vseg->vpn_base = vpn_min;
	1274	vseg->vpn_size = vpn_max - vpn_min + 1;
	1275	error = 0;
[1]	1276	}
[406]	1277	else // vseg cut in three regions
[1]	1278	{
[406]	1279	// resize existing vseg
	1280	vseg->max = addr_min;
	1281
	1282	// update vpn_base and vpn_size
	1283	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1284	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1285	vseg->vpn_base = vpn_min;
	1286	vseg->vpn_size = vpn_max - vpn_min + 1;
	1287
	1288	// create new vseg
[407]	1289	new = vmm_create_vseg( process,
	1290	vseg->type,
	1291	addr_min,
	1292	(vseg->max - addr_max),
	1293	vseg->file_offset,
	1294	vseg->file_size,
	1295	vseg->mapper_xp,
	1296	vseg->cxy );
	1297
[406]	1298	if( new == NULL ) error = EINVAL;
	1299	else error = 0;
[1]	1300	}
	1301
	1302	// release VMM lock
[408]	1303	remote_rwlock_wr_unlock( lock_xp );
[1]	1304
	1305	return error;
	1306
[406]	1307	} // vmm_resize_vseg()
	1308
[1]	1309	///////////////////////////////////////////
[388]	1310	error_t vmm_get_vseg( process_t * process,
[394]	1311	intptr_t vaddr,
[388]	1312	vseg_t ** found_vseg )
[1]	1313	{
[440]	1314	xptr_t vseg_xp;
	1315	error_t error;
	1316	vseg_t * vseg;
	1317	vmm_t * vmm;
[1]	1318
[440]	1319	// get pointer on local VMM
	1320	vmm = &process->vmm;
[1]	1321
[440]	1322	// try to get vseg from local VMM
	1323	vseg = vseg_from_vaddr( vmm , vaddr );
	1324
[388]	1325	if( vseg == NULL ) // vseg not found in local cluster => try to get it from ref
	1326	{
	1327	// get extended pointer on reference process
	1328	xptr_t ref_xp = process->ref_xp;
[1]	1329
[388]	1330	// get cluster and local pointer on reference process
	1331	cxy_t ref_cxy = GET_CXY( ref_xp );
[433]	1332	process_t * ref_ptr = GET_PTR( ref_xp );
[388]	1333
	1334	if( local_cxy == ref_cxy ) return -1; // local cluster is the reference
	1335
	1336	// get extended pointer on reference vseg
[394]	1337	rpc_vmm_get_vseg_client( ref_cxy , ref_ptr , vaddr , &vseg_xp , &error );
[388]	1338
[440]	1339	if( error ) return -1; // vseg not found => illegal user vaddr
[388]	1340
	1341	// allocate a vseg in local cluster
	1342	vseg = vseg_alloc();
	1343
[440]	1344	if( vseg == NULL ) return -1; // cannot allocate a local vseg
[388]	1345
	1346	// initialise local vseg from reference
	1347	vseg_init_from_ref( vseg , vseg_xp );
	1348
	1349	// register local vseg in local VMM
[406]	1350	vseg_attach( &process->vmm , vseg );
[388]	1351	}
	1352
	1353	// success
	1354	*found_vseg = vseg;
[394]	1355	return 0;
[388]	1356
	1357	} // end vmm_get_vseg()
	1358
[407]	1359	//////////////////////////////////////////////////////////////////////////////////////
	1360	// This static function compute the target cluster to allocate a physical page
	1361	// for a given <vpn> in a given <vseg>, allocates the page (with an RPC if required)
	1362	// and returns an extended pointer on the allocated page descriptor.
	1363	// The vseg cannot have the FILE type.
	1364	//////////////////////////////////////////////////////////////////////////////////////
	1365	static xptr_t vmm_page_allocate( vseg_t * vseg,
	1366	vpn_t vpn )
	1367	{
[433]	1368
[438]	1369	#if DEBUG_VMM_ALLOCATE_PAGE
	1370	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
[433]	1371	printk("\n[DBG] in %s : thread %x enter for vpn %x\n",
	1372	__FUNCTION__ , CURRENT_THREAD, vpn );
	1373	#endif
	1374
[407]	1375	// compute target cluster
	1376	page_t * page_ptr;
	1377	cxy_t page_cxy;
	1378	kmem_req_t req;
	1379
	1380	uint32_t type = vseg->type;
	1381	uint32_t flags = vseg->flags;
	1382
[492]	1383	assert( ( type != VSEG_TYPE_FILE ) , "illegal vseg type\n" );
[407]	1384
	1385	if( flags & VSEG_DISTRIB ) // distributed => cxy depends on vpn LSB
	1386	{
[561]	1387	uint32_t x_size = LOCAL_CLUSTER->x_size;
	1388	uint32_t y_size = LOCAL_CLUSTER->y_size;
[407]	1389	uint32_t y_width = LOCAL_CLUSTER->y_width;
[561]	1390	uint32_t index = vpn & ((x_size * y_size) - 1);
	1391	uint32_t x = index / y_size;
	1392	uint32_t y = index % y_size;
	1393
	1394	// If the cluster selected from VPN's LSBs is empty, then we select one randomly
	1395	// cluster_random_select() ensures that its randomly selected cluster is not empty
	1396	if ( cluster_info_is_active( LOCAL_CLUSTER->cluster_info[x][y] ) == 0 ) {
	1397	cxy_t cxy = cluster_random_select();
	1398	x = ( ( cxy >> y_width ) & 0xF);
	1399	y = ( cxy & 0xF );
	1400	}
	1401	page_cxy = ( x << y_width ) + y;
[407]	1402	}
	1403	else // other cases => cxy specified in vseg
	1404	{
[561]	1405	page_cxy = vseg->cxy;
[407]	1406	}
	1407
	1408	// allocate a physical page from target cluster
	1409	if( page_cxy == local_cxy ) // target cluster is the local cluster
	1410	{
	1411	req.type = KMEM_PAGE;
	1412	req.size = 0;
	1413	req.flags = AF_NONE;
	1414	page_ptr = (page_t *)kmem_alloc( &req );
	1415	}
	1416	else // target cluster is not the local cluster
	1417	{
	1418	rpc_pmem_get_pages_client( page_cxy , 0 , &page_ptr );
	1419	}
	1420
[438]	1421	#if DEBUG_VMM_ALLOCATE_PAGE
	1422	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
[433]	1423	printk("\n[DBG] in %s : thread %x exit for vpn = %d / ppn = %x\n",
	1424	__FUNCTION__ , CURRENT_THREAD, vpn, ppm_page2ppn( XPTR( page_cxy , page_ptr ) ) );
	1425	#endif
	1426
[407]	1427	if( page_ptr == NULL ) return XPTR_NULL;
	1428	else return XPTR( page_cxy , page_ptr );
	1429
	1430	} // end vmm_page_allocate()
	1431
[313]	1432	////////////////////////////////////////
	1433	error_t vmm_get_one_ppn( vseg_t * vseg,
	1434	vpn_t vpn,
	1435	ppn_t * ppn )
	1436	{
	1437	error_t error;
[407]	1438	xptr_t page_xp; // extended pointer on physical page descriptor
[313]	1439	page_t * page_ptr; // local pointer on physical page descriptor
[406]	1440	uint32_t index; // missing page index in vseg mapper
	1441	uint32_t type; // vseg type;
[313]	1442
[406]	1443	type = vseg->type;
	1444	index = vpn - vseg->vpn_base;
[313]	1445
[438]	1446	#if DEBUG_VMM_GET_ONE_PPN
[441]	1447	thread_t * this = CURRENT_THREAD;
[469]	1448	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1449	printk("\n[DBG] %s : thread %x enter for vpn = %x / type = %s / index = %d\n",
[441]	1450	__FUNCTION__, this, vpn, vseg_type_str(type), index );
[433]	1451	#endif
[313]	1452
[406]	1453	// FILE type : get the physical page from the file mapper
[313]	1454	if( type == VSEG_TYPE_FILE )
	1455	{
[406]	1456	// get extended pointer on mapper
[407]	1457	xptr_t mapper_xp = vseg->mapper_xp;
[313]	1458
[492]	1459	assert( (mapper_xp != XPTR_NULL),
[406]	1460	"mapper not defined for a FILE vseg\n" );
	1461
	1462	// get mapper cluster and local pointer
	1463	cxy_t mapper_cxy = GET_CXY( mapper_xp );
[433]	1464	mapper_t * mapper_ptr = GET_PTR( mapper_xp );
[406]	1465
[313]	1466	// get page descriptor from mapper
	1467	if( mapper_cxy == local_cxy ) // mapper is local
	1468	{
	1469	page_ptr = mapper_get_page( mapper_ptr , index );
	1470	}
	1471	else // mapper is remote
	1472	{
	1473	rpc_mapper_get_page_client( mapper_cxy , mapper_ptr , index , &page_ptr );
	1474	}
	1475
	1476	if ( page_ptr == NULL ) return EINVAL;
	1477
[407]	1478	page_xp = XPTR( mapper_cxy , page_ptr );
[313]	1479	}
	1480
[406]	1481	// Other types : allocate a physical page from target cluster,
[407]	1482	// as defined by vseg type and vpn value
[313]	1483	else
	1484	{
[433]	1485	// allocate one physical page
[407]	1486	page_xp = vmm_page_allocate( vseg , vpn );
[406]	1487
[407]	1488	if( page_xp == XPTR_NULL ) return ENOMEM;
[313]	1489
[406]	1490	// initialise missing page from .elf file mapper for DATA and CODE types
[440]	1491	// the vseg->mapper_xp field is an extended pointer on the .elf file mapper
[313]	1492	if( (type == VSEG_TYPE_CODE) \|\| (type == VSEG_TYPE_DATA) )
	1493	{
[406]	1494	// get extended pointer on mapper
	1495	xptr_t mapper_xp = vseg->mapper_xp;
[313]	1496
[492]	1497	assert( (mapper_xp != XPTR_NULL),
[406]	1498	"mapper not defined for a CODE or DATA vseg\n" );
	1499
	1500	// get mapper cluster and local pointer
	1501	cxy_t mapper_cxy = GET_CXY( mapper_xp );
[433]	1502	mapper_t * mapper_ptr = GET_PTR( mapper_xp );
[406]	1503
	1504	// compute missing page offset in vseg
	1505	uint32_t offset = index << CONFIG_PPM_PAGE_SHIFT;
	1506
[313]	1507	// compute missing page offset in .elf file
[406]	1508	uint32_t elf_offset = vseg->file_offset + offset;
[313]	1509
[438]	1510	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1511	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1512	printk("\n[DBG] %s : thread %x for vpn = %x / elf_offset = %x\n",
[441]	1513	__FUNCTION__, this, vpn, elf_offset );
[433]	1514	#endif
[313]	1515
[440]	1516
[406]	1517	// compute extended pointer on page base
[407]	1518	xptr_t base_xp = ppm_page2base( page_xp );
[313]	1519
[406]	1520	// file_size (in .elf mapper) can be smaller than vseg_size (BSS)
	1521	uint32_t file_size = vseg->file_size;
	1522
	1523	if( file_size < offset ) // missing page fully in BSS
[313]	1524	{
[406]	1525
[438]	1526	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1527	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1528	printk("\n[DBG] %s : thread%x for vpn = %x / fully in BSS\n",
[441]	1529	__FUNCTION__, this, vpn );
[433]	1530	#endif
	1531
[440]	1532
[407]	1533	if( GET_CXY( page_xp ) == local_cxy )
[313]	1534	{
[315]	1535	memset( GET_PTR( base_xp ) , 0 , CONFIG_PPM_PAGE_SIZE );
[313]	1536	}
	1537	else
	1538	{
[315]	1539	hal_remote_memset( base_xp , 0 , CONFIG_PPM_PAGE_SIZE );
[313]	1540	}
	1541	}
[406]	1542	else if( file_size >= (offset + CONFIG_PPM_PAGE_SIZE) ) // fully in mapper
[315]	1543	{
[406]	1544
[438]	1545	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1546	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1547	printk("\n[DBG] %s : thread %x, for vpn = %x / fully in mapper\n",
[441]	1548	__FUNCTION__, this, vpn );
[433]	1549	#endif
[313]	1550	if( mapper_cxy == local_cxy )
	1551	{
	1552	error = mapper_move_kernel( mapper_ptr,
	1553	true, // to_buffer
[406]	1554	elf_offset,
[313]	1555	base_xp,
[315]	1556	CONFIG_PPM_PAGE_SIZE );
[313]	1557	}
	1558	else
	1559	{
	1560	rpc_mapper_move_buffer_client( mapper_cxy,
	1561	mapper_ptr,
	1562	true, // to buffer
	1563	false, // kernel buffer
[406]	1564	elf_offset,
	1565	base_xp,
[315]	1566	CONFIG_PPM_PAGE_SIZE,
[313]	1567	&error );
	1568	}
	1569	if( error ) return EINVAL;
	1570	}
[406]	1571	else // both in mapper and in BSS :
	1572	// - (file_size - offset) bytes from mapper
	1573	// - (page_size + offset - file_size) bytes from BSS
[313]	1574	{
[406]	1575
[438]	1576	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1577	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1578	printk("\n[DBG] %s : thread %x for vpn = %x / both mapper & BSS\n"
	1579	" %d bytes from mapper / %d bytes from BSS\n",
[441]	1580	__FUNCTION__, this, vpn,
[407]	1581	file_size - offset , offset + CONFIG_PPM_PAGE_SIZE - file_size );
[433]	1582	#endif
[313]	1583	// initialize mapper part
[315]	1584	if( mapper_cxy == local_cxy )
[313]	1585	{
	1586	error = mapper_move_kernel( mapper_ptr,
[315]	1587	true, // to buffer
[406]	1588	elf_offset,
[313]	1589	base_xp,
[406]	1590	file_size - offset );
[313]	1591	}
[315]	1592	else
[313]	1593	{
	1594	rpc_mapper_move_buffer_client( mapper_cxy,
	1595	mapper_ptr,
[315]	1596	true, // to buffer
[313]	1597	false, // kernel buffer
[406]	1598	elf_offset,
	1599	base_xp,
	1600	file_size - offset,
[313]	1601	&error );
	1602	}
	1603	if( error ) return EINVAL;
	1604
	1605	// initialize BSS part
[407]	1606	if( GET_CXY( page_xp ) == local_cxy )
[313]	1607	{
[406]	1608	memset( GET_PTR( base_xp ) + file_size - offset , 0 ,
	1609	offset + CONFIG_PPM_PAGE_SIZE - file_size );
[313]	1610	}
	1611	else
	1612	{
[406]	1613	hal_remote_memset( base_xp + file_size - offset , 0 ,
	1614	offset + CONFIG_PPM_PAGE_SIZE - file_size );
[313]	1615	}
	1616	}
	1617	} // end initialisation for CODE or DATA types
	1618	}
	1619
	1620	// return ppn
[407]	1621	*ppn = ppm_page2ppn( page_xp );
[406]	1622
[438]	1623	#if DEBUG_VMM_GET_ONE_PPN
[469]	1624	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1625	printk("\n[DBG] %s : thread %x exit for vpn = %x / ppn = %x\n",
[441]	1626	__FUNCTION__ , this , vpn , *ppn );
[433]	1627	#endif
[406]	1628
[313]	1629	return 0;
	1630
	1631	} // end vmm_get_one_ppn()
	1632
[1]	1633	/////////////////////////////////////////
	1634	error_t vmm_get_pte( process_t * process,
	1635	vpn_t vpn,
[407]	1636	bool_t cow,
	1637	uint32_t * attr,
	1638	ppn_t * ppn )
[1]	1639	{
[440]	1640	ppn_t old_ppn; // current PTE_PPN
	1641	uint32_t old_attr; // current PTE_ATTR
	1642	ppn_t new_ppn; // new PTE_PPN
	1643	uint32_t new_attr; // new PTE_ATTR
	1644	vmm_t * vmm;
	1645	vseg_t * vseg;
	1646	error_t error;
[1]	1647
[441]	1648	thread_t * this = CURRENT_THREAD;
	1649
[438]	1650	#if DEBUG_VMM_GET_PTE
[441]	1651	uint32_t cycle = (uint32_t)hal_get_cycles();
[469]	1652	if( DEBUG_VMM_GET_PTE < cycle )
	1653	printk("\n[DBG] %s : thread %x in process %x enter / vpn %x / cow %d / cycle %d\n",
	1654	__FUNCTION__, this->trdid, process->pid, vpn, cow, cycle );
[433]	1655	#endif
[406]	1656
[1]	1657	// get VMM pointer
[440]	1658	vmm = &process->vmm;
[1]	1659
[440]	1660	// get local vseg descriptor
	1661	error = vmm_get_vseg( process,
	1662	((intptr_t)vpn << CONFIG_PPM_PAGE_SHIFT),
	1663	&vseg );
[1]	1664
[440]	1665	// vseg has been checked by the vmm_handle_page_fault() function
[492]	1666	assert( (vseg != NULL) , "vseg undefined / vpn %x\n");
[406]	1667
[438]	1668	if( cow ) //////////////// copy_on_write request //////////////////////
[440]	1669	// get PTE from local GPT
[438]	1670	// allocate a new physical page if there is pending forks,
	1671	// initialize it from old physical page content,
	1672	// update PTE in all GPT copies,
	1673	{
[440]	1674	// access local GPT to get current PTE attributes and PPN
[438]	1675	hal_gpt_get_pte( &vmm->gpt , vpn , &old_attr , &old_ppn );
[407]	1676
[492]	1677	assert( (old_attr & GPT_MAPPED),
	1678	"PTE unmapped for a COW exception / vpn %x\n" );
[407]	1679
[438]	1680	#if( DEBUG_VMM_GET_PTE & 1 )
[469]	1681	if( DEBUG_VMM_GET_PTE < cycle )
	1682	printk("\n[DBG] %s : thread %x in process %x handling COW for vpn %x\n",
	1683	__FUNCTION__, this->trdid, process->pid, vpn );
[433]	1684	#endif
[407]	1685
[433]	1686	// get extended pointer, cluster and local pointer on physical page descriptor
[408]	1687	xptr_t page_xp = ppm_ppn2page( old_ppn );
	1688	cxy_t page_cxy = GET_CXY( page_xp );
[433]	1689	page_t * page_ptr = GET_PTR( page_xp );
[407]	1690
[469]	1691	// get extended pointers on forks and lock field in page descriptor
	1692	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
	1693	xptr_t lock_xp = XPTR( page_cxy , &page_ptr->lock );
	1694
	1695	// take lock protecting page descriptor
	1696	remote_spinlock_lock( lock_xp );
	1697
[408]	1698	// get number of pending forks in page descriptor
[469]	1699	uint32_t forks = hal_remote_lw( forks_xp );
[408]	1700
[433]	1701	if( forks ) // pending fork => allocate a new page, copy old to new
[1]	1702	{
[408]	1703	// allocate a new physical page
	1704	page_xp = vmm_page_allocate( vseg , vpn );
	1705	if( page_xp == XPTR_NULL )
	1706	{
	1707	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
	1708	__FUNCTION__ , process->pid , vpn );
	1709	return -1;
	1710	}
[1]	1711
[408]	1712	// compute allocated page PPN
	1713	new_ppn = ppm_page2ppn( page_xp );
[406]	1714
[408]	1715	// copy old page content to new page
	1716	xptr_t old_base_xp = ppm_ppn2base( old_ppn );
	1717	xptr_t new_base_xp = ppm_ppn2base( new_ppn );
	1718	memcpy( GET_PTR( new_base_xp ),
	1719	GET_PTR( old_base_xp ),
	1720	CONFIG_PPM_PAGE_SIZE );
[469]	1721
	1722	// decrement pending forks counter in page descriptor
	1723	hal_remote_atomic_add( forks_xp , -1 );
[408]	1724	}
[469]	1725	else // no pending fork => keep the existing page
[408]	1726	{
	1727	new_ppn = old_ppn;
	1728	}
[1]	1729
[469]	1730	// release lock protecting page descriptor
	1731	remote_spinlock_unlock( lock_xp );
	1732
[408]	1733	// build new_attr : reset COW and set WRITABLE,
	1734	new_attr = (old_attr \| GPT_WRITABLE) & (~GPT_COW);
[407]	1735
[408]	1736	// update GPT[vpn] for all GPT copies
[433]	1737	vmm_global_update_pte( process, vpn, new_attr, new_ppn );
[407]	1738	}
[438]	1739	else //////////// page_fault request ///////////////////////////
[440]	1740	// get PTE from local GPT
[438]	1741	// allocate a physical page if it is a true page fault,
[440]	1742	// initialize it if type is FILE, CODE, or DATA,
[438]	1743	// register in reference GPT, but don't update GPT copies
[407]	1744	{
[440]	1745	// access local GPT to get current PTE
[438]	1746	hal_gpt_get_pte( &vmm->gpt , vpn , &old_attr , &old_ppn );
	1747
[408]	1748	if( (old_attr & GPT_MAPPED) == 0 ) // true page_fault => map it
[407]	1749	{
[1]	1750
[438]	1751	#if( DEBUG_VMM_GET_PTE & 1 )
[469]	1752	if( DEBUG_VMM_GET_PTE < cycle )
	1753	printk("\n[DBG] %s : thread %x in process %x handling page fault for vpn %x\n",
	1754	__FUNCTION__, this->trdid, process->pid, vpn );
[433]	1755	#endif
[440]	1756	// allocate new_ppn, and initialize the new page
[407]	1757	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
	1758	if( error )
	1759	{
	1760	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
	1761	__FUNCTION__ , process->pid , vpn );
[408]	1762	return -1;
[407]	1763	}
	1764
[408]	1765	// define new_attr from vseg flags
[407]	1766	new_attr = GPT_MAPPED \| GPT_SMALL;
	1767	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
	1768	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
	1769	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
	1770	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
	1771
[408]	1772	// register new PTE in reference GPT
	1773	// on demand policy => no update of GPT copies
	1774	error = hal_gpt_set_pte( &vmm->gpt,
	1775	vpn,
	1776	new_attr,
	1777	new_ppn );
[407]	1778	if( error )
	1779	{
	1780	printk("\n[ERROR] in %s : cannot update GPT / process = %x / vpn = %x\n",
	1781	__FUNCTION__ , process->pid , vpn );
[408]	1782	return -1;
[407]	1783	}
	1784	}
[408]	1785	else // mapped in reference GPT => get it
[1]	1786	{
[408]	1787	new_ppn = old_ppn;
[407]	1788	new_attr = old_attr;
[1]	1789	}
[407]	1790	}
[1]	1791
[438]	1792	#if DEBUG_VMM_GET_PTE
[433]	1793	cycle = (uint32_t)hal_get_cycles();
[469]	1794	if( DEBUG_VMM_GET_PTE < cycle )
	1795	printk("\n[DBG] %s : thread %x in process %x exit / vpn %x / ppn %x / attr %x / cycle %d\n",
	1796	__FUNCTION__, this->trdid, process->pid, vpn, new_ppn, new_attr, cycle );
[433]	1797	#endif
[406]	1798
[438]	1799	// return PPN and flags
[407]	1800	*ppn = new_ppn;
	1801	*attr = new_attr;
[1]	1802	return 0;
	1803
[313]	1804	} // end vmm_get_pte()
	1805
[1]	1806	///////////////////////////////////////////////////
	1807	error_t vmm_handle_page_fault( process_t * process,
[440]	1808	vpn_t vpn,
	1809	bool_t is_cow )
[1]	1810	{
	1811	uint32_t attr; // missing page attributes
[21]	1812	ppn_t ppn; // missing page PPN
[440]	1813	vseg_t * vseg; // vseg containing vpn
	1814	uint32_t type; // vseg type
	1815	cxy_t ref_cxy; // reference cluster for missing vpn
	1816	process_t * ref_ptr; // reference process for missing vpn
[407]	1817	error_t error;
[1]	1818
[440]	1819	thread_t * this = CURRENT_THREAD;
	1820
	1821	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[435]	1822	uint32_t cycle = (uint32_t)hal_get_cycles();
[469]	1823	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
[441]	1824	printk("\n[DBG] %s : thread %x in process %x enter for vpn %x / core[%x,%d] / cycle %d\n",
	1825	__FUNCTION__, this, process->pid, vpn, local_cxy, this->core->lid, cycle );
[435]	1826	#endif
	1827
[440]	1828	// get local vseg (access reference VSL if required)
	1829	error = vmm_get_vseg( process , vpn<<CONFIG_PPM_PAGE_SHIFT , &vseg );
[1]	1830
[440]	1831	if( error )
[1]	1832	{
[440]	1833	printk("\n[ERROR] in %s : vpn %x / process %x / thread %x / core[%x,%d] / cycle %d\n",
	1834	__FUNCTION__, vpn, process->pid, this->trdid, local_cxy, this->core->lid,
	1835	(uint32_t)hal_get_cycles() );
	1836	return error;
	1837	}
[407]	1838
[440]	1839	// get segment type
	1840	type = vseg->type;
[407]	1841
[440]	1842	// get reference process cluster and local pointer
	1843	// for private vsegs (CODE and DATA type),
	1844	// the reference is the local process descriptor.
	1845	if( (type == VSEG_TYPE_STACK) \|\| (type == VSEG_TYPE_CODE) )
	1846	{
	1847	ref_cxy = local_cxy;
	1848	ref_ptr = process;
[1]	1849	}
[440]	1850	else
[1]	1851	{
[440]	1852	ref_cxy = GET_CXY( process->ref_xp );
	1853	ref_ptr = GET_PTR( process->ref_xp );
[1]	1854	}
	1855
[440]	1856	// get missing PTE attributes and PPN
	1857	if( local_cxy != ref_cxy )
[407]	1858	{
[441]	1859
	1860	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[469]	1861	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
[441]	1862	printk("\n[DBG] %s : thread %x in process %x call RPC_VMM_GET_PTE\n",
	1863	__FUNCTION__, this, process->pid );
	1864	#endif
	1865
[407]	1866	rpc_vmm_get_pte_client( ref_cxy,
	1867	ref_ptr,
	1868	vpn,
[440]	1869	is_cow,
[407]	1870	&attr,
	1871	&ppn,
	1872	&error );
	1873
	1874	// get local VMM pointer
	1875	vmm_t * vmm = &process->vmm;
	1876
	1877	// update local GPT
[408]	1878	error \|= hal_gpt_set_pte( &vmm->gpt,
	1879	vpn,
	1880	attr,
	1881	ppn );
[407]	1882	}
	1883	else // local cluster is the reference cluster
	1884	{
	1885	error = vmm_get_pte( process,
	1886	vpn,
[440]	1887	is_cow,
[407]	1888	&attr,
	1889	&ppn );
	1890	}
	1891
[440]	1892	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[435]	1893	cycle = (uint32_t)hal_get_cycles();
[469]	1894	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
[441]	1895	printk("\n[DBG] %s : thread %x in process %x exit for vpn %x / core[%x,%d] / cycle %d\n",
	1896	__FUNCTION__, this, process->pid, vpn, local_cxy, this->core->lid, cycle );
[435]	1897	#endif
	1898
[407]	1899	return error;
	1900
[441]	1901	} // end vmm_handle_page_fault()
[407]	1902
[441]	1903
	1904
	1905
	1906
	1907
	1908
	1909
	1910
[440]	1911	/* deprecated April 2018 [AG]
	1912
	1913	error_t vmm_v2p_translate( process_t * process,
	1914	void * ptr,
	1915	paddr_t * paddr )
[1]	1916	{
[21]	1917	// access page table
[1]	1918	error_t error;
	1919	vpn_t vpn;
	1920	uint32_t attr;
	1921	ppn_t ppn;
	1922	uint32_t offset;
	1923
[23]	1924	vpn = (vpn_t)( (intptr_t)ptr >> CONFIG_PPM_PAGE_SHIFT );
	1925	offset = (uint32_t)( ((intptr_t)ptr) & CONFIG_PPM_PAGE_MASK );
[1]	1926
[440]	1927	if( local_cxy == GET_CXY( process->ref_xp) ) // local process is reference process
[1]	1928	{
[407]	1929	error = vmm_get_pte( process, vpn , false , &attr , &ppn );
[1]	1930	}
[50]	1931	else // calling process is not reference process
[1]	1932	{
	1933	cxy_t ref_cxy = GET_CXY( process->ref_xp );
[433]	1934	process_t * ref_ptr = GET_PTR( process->ref_xp );
[407]	1935	rpc_vmm_get_pte_client( ref_cxy , ref_ptr , vpn , false , &attr , &ppn , &error );
[1]	1936	}
	1937
[23]	1938	// set paddr
[1]	1939	*paddr = (((paddr_t)ppn) << CONFIG_PPM_PAGE_SHIFT) \| offset;
[21]	1940
[23]	1941	return error;
[313]	1942
[315]	1943	} // end vmm_v2p_translate()
[1]	1944
[440]	1945	*/

Note: See TracBrowser for help on using the repository browser.

Download in other formats: