Context Navigation

source: trunk/kernel/mm/vmm.c @ 577

Last change on this file since 577 was 577, checked in by alain, 6 years ago
Fix small bugs.
File size: 66.6 KB

Rev	Line
[1]	1	/*
	2	* vmm.c - virtual memory manager related operations interface.
	3	*
	4	* Authors Ghassan Almaless (2008,2009,2010,2011, 2012)
	5	* Mohamed Lamine Karaoui (2015)
	6	* Alain Greiner (2016)
[21]	7	*
[1]	8	* Copyright (c) UPMC Sorbonne Universites
	9	*
	10	* This file is part of ALMOS-MKH.
	11	*
	12	* ALMOS-MKH is free software; you can redistribute it and/or modify it
	13	* under the terms of the GNU General Public License as published by
	14	* the Free Software Foundation; version 2.0 of the License.
	15	*
	16	* ALMOS-MKH is distributed in the hope that it will be useful, but
	17	* WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU General Public License
	22	* along with ALMOS-MKH; if not, write to the Free Software Foundation,
	23	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	24	*/
	25
[14]	26	#include <kernel_config.h>
[457]	27	#include <hal_kernel_types.h>
[1]	28	#include <hal_special.h>
	29	#include <hal_gpt.h>
[409]	30	#include <hal_vmm.h>
[577]	31	#include <hal_macros.h>
[1]	32	#include <printk.h>
[23]	33	#include <memcpy.h>
[567]	34	#include <remote_rwlock.h>
	35	#include <remote_queuelock.h>
[1]	36	#include <list.h>
[408]	37	#include <xlist.h>
[1]	38	#include <bits.h>
	39	#include <process.h>
	40	#include <thread.h>
	41	#include <vseg.h>
	42	#include <cluster.h>
	43	#include <scheduler.h>
	44	#include <vfs.h>
	45	#include <mapper.h>
	46	#include <page.h>
	47	#include <kmem.h>
	48	#include <vmm.h>
	49
	50	//////////////////////////////////////////////////////////////////////////////////
	51	// Extern global variables
	52	//////////////////////////////////////////////////////////////////////////////////
	53
[567]	54	extern process_t process_zero; // allocated in cluster.c
[1]	55
[415]	56	///////////////////////////////////////
	57	error_t vmm_init( process_t * process )
[21]	58	{
[1]	59	error_t error;
	60	vseg_t * vseg_kentry;
	61	vseg_t * vseg_args;
	62	vseg_t * vseg_envs;
	63	intptr_t base;
	64	intptr_t size;
	65
[438]	66	#if DEBUG_VMM_INIT
[567]	67	thread_t * this = CURRENT_THREAD;
[433]	68	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	69	if( DEBUG_VMM_INIT )
[567]	70	printk("\n[DBG] %s : thread %x in process %x enter for process %x / cycle %d\n",
	71	__FUNCTION__ , this->trdid, this->process->pid, process->pid , cycle );
[433]	72	#endif
[204]	73
[1]	74	// get pointer on VMM
	75	vmm_t * vmm = &process->vmm;
	76
[407]	77	// initialize local list of vsegs
	78	vmm->vsegs_nr = 0;
[408]	79	xlist_root_init( XPTR( local_cxy , &vmm->vsegs_root ) );
[567]	80	remote_rwlock_init( XPTR( local_cxy , &vmm->vsegs_lock ),LOCK_VMM_VSL );
[407]	81
[567]	82	assert( ((CONFIG_VMM_KENTRY_SIZE + CONFIG_VMM_ARGS_SIZE + CONFIG_VMM_ENVS_SIZE)
	83	<= CONFIG_VMM_ELF_BASE) , "UTILS zone too small\n" );
[21]	84
[567]	85	assert( (CONFIG_THREADS_MAX_PER_CLUSTER <= 32) ,
	86	"no more than 32 threads per cluster for a single process\n");
[1]	87
[567]	88	assert( ((CONFIG_VMM_STACK_SIZE * CONFIG_THREADS_MAX_PER_CLUSTER) <=
	89	(CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE)) ,
	90	"STACK zone too small\n");
[1]	91
[409]	92	// register kentry vseg in VSL
[406]	93	base = CONFIG_VMM_KENTRY_BASE << CONFIG_PPM_PAGE_SHIFT;
[1]	94	size = CONFIG_VMM_KENTRY_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	95
[407]	96	vseg_kentry = vmm_create_vseg( process,
	97	VSEG_TYPE_CODE,
	98	base,
	99	size,
	100	0, // file_offset unused
	101	0, // file_size unused
	102	XPTR_NULL, // mapper_xp unused
	103	local_cxy );
[204]	104
[415]	105	if( vseg_kentry == NULL )
	106	{
	107	printk("\n[ERROR] in %s : cannot register kentry vseg\n", __FUNCTION__ );
	108	return -1;
	109	}
[204]	110
[406]	111	vmm->kent_vpn_base = base;
[1]	112
[409]	113	// register args vseg in VSL
[406]	114	base = (CONFIG_VMM_KENTRY_BASE +
	115	CONFIG_VMM_KENTRY_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
[1]	116	size = CONFIG_VMM_ARGS_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	117
[407]	118	vseg_args = vmm_create_vseg( process,
	119	VSEG_TYPE_DATA,
	120	base,
	121	size,
	122	0, // file_offset unused
	123	0, // file_size unused
	124	XPTR_NULL, // mapper_xp unused
	125	local_cxy );
[204]	126
[415]	127	if( vseg_args == NULL )
	128	{
	129	printk("\n[ERROR] in %s : cannot register args vseg\n", __FUNCTION__ );
	130	return -1;
	131	}
[204]	132
[406]	133	vmm->args_vpn_base = base;
[1]	134
[409]	135	// register the envs vseg in VSL
[406]	136	base = (CONFIG_VMM_KENTRY_BASE +
	137	CONFIG_VMM_KENTRY_SIZE +
	138	CONFIG_VMM_ARGS_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
[1]	139	size = CONFIG_VMM_ENVS_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	140
[407]	141	vseg_envs = vmm_create_vseg( process,
	142	VSEG_TYPE_DATA,
	143	base,
	144	size,
	145	0, // file_offset unused
	146	0, // file_size unused
	147	XPTR_NULL, // mapper_xp unused
	148	local_cxy );
[204]	149
[415]	150	if( vseg_envs == NULL )
	151	{
	152	printk("\n[ERROR] in %s : cannot register envs vseg\n", __FUNCTION__ );
	153	return -1;
	154	}
[204]	155
[406]	156	vmm->envs_vpn_base = base;
[1]	157
[409]	158	// create GPT (empty)
[1]	159	error = hal_gpt_create( &vmm->gpt );
	160
[415]	161	if( error )
	162	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
[204]	163
[415]	164	// initialize GPT (architecture specic)
[409]	165	// (For TSAR, identity map the kentry_vseg)
	166	error = hal_vmm_init( vmm );
	167
[415]	168	if( error )
	169	printk("\n[ERROR] in %s : cannot initialize GPT\n", __FUNCTION__ );
[409]	170
[1]	171	// initialize STACK allocator
	172	vmm->stack_mgr.bitmap = 0;
	173	vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
[567]	174	busylock_init( &vmm->stack_mgr.lock , LOCK_VMM_STACK );
[1]	175
	176	// initialize MMAP allocator
[407]	177	vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
	178	vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
	179	vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
[567]	180	busylock_init( &vmm->mmap_mgr.lock , LOCK_VMM_MMAP );
[457]	181
[1]	182	uint32_t i;
	183	for( i = 0 ; i < 32 ; i++ ) list_root_init( &vmm->mmap_mgr.zombi_list[i] );
	184
[21]	185	// initialize instrumentation counters
[409]	186	vmm->pgfault_nr = 0;
[1]	187
[124]	188	hal_fence();
[1]	189
[438]	190	#if DEBUG_VMM_INIT
[433]	191	cycle = (uint32_t)hal_get_cycles();
[438]	192	if( DEBUG_VMM_INIT )
[567]	193	printk("\n[DBG] %s : thread %x in process %x exit / process %x / entry_point = %x / cycle %d\n",
	194	__FUNCTION__, this->trdid, this->process->pid, process->pid , process->vmm.entry_point , cycle );
[433]	195	#endif
[204]	196
[415]	197	return 0;
	198
[204]	199	} // end vmm_init()
	200
[407]	201	//////////////////////////////////////
	202	void vmm_display( process_t * process,
	203	bool_t mapping )
	204	{
	205	vmm_t * vmm = &process->vmm;
	206	gpt_t * gpt = &vmm->gpt;
	207
[457]	208	printk("\n***** VSL and GPT(%x) for process %x in cluster %x\n\n",
	209	process->vmm.gpt.ptr , process->pid , local_cxy );
[407]	210
	211	// get lock protecting the vseg list
[567]	212	remote_rwlock_rd_acquire( XPTR( local_cxy , &vmm->vsegs_lock ) );
[407]	213
	214	// scan the list of vsegs
[408]	215	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	216	xptr_t iter_xp;
	217	xptr_t vseg_xp;
[407]	218	vseg_t * vseg;
[408]	219	XLIST_FOREACH( root_xp , iter_xp )
[407]	220	{
[408]	221	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	222	vseg = GET_PTR( vseg_xp );
[408]	223
[407]	224	printk(" - %s : base = %X / size = %X / npages = %d\n",
	225	vseg_type_str( vseg->type ) , vseg->min , vseg->max - vseg->min , vseg->vpn_size );
	226
	227	if( mapping )
	228	{
	229	vpn_t vpn;
	230	ppn_t ppn;
	231	uint32_t attr;
	232	vpn_t base = vseg->vpn_base;
	233	vpn_t size = vseg->vpn_size;
	234	for( vpn = base ; vpn < (base+size) ; vpn++ )
	235	{
	236	hal_gpt_get_pte( gpt , vpn , &attr , &ppn );
	237	if( attr & GPT_MAPPED )
	238	{
	239	printk(" . vpn = %X / attr = %X / ppn = %X\n", vpn , attr , ppn );
	240	}
	241	}
	242	}
	243	}
	244
	245	// release the lock
[567]	246	remote_rwlock_rd_release( XPTR( local_cxy , &vmm->vsegs_lock ) );
[407]	247
[408]	248	} // vmm_display()
	249
[567]	250	///////////////////////////////////
	251	void vmm_vseg_attach( vmm_t * vmm,
	252	vseg_t * vseg )
	253	{
	254	// build extended pointer on rwlock protecting VSL
	255	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	256
	257	// get rwlock in write mode
	258	remote_rwlock_wr_acquire( lock_xp );
	259
	260	// update vseg descriptor
	261	vseg->vmm = vmm;
	262
	263	// add vseg in vmm list
	264	xlist_add_last( XPTR( local_cxy , &vmm->vsegs_root ),
	265	XPTR( local_cxy , &vseg->xlist ) );
	266
	267	// release rwlock in write mode
	268	remote_rwlock_wr_release( lock_xp );
	269	}
	270
	271	///////////////////////////////////
	272	void vmm_vseg_detach( vmm_t * vmm,
	273	vseg_t * vseg )
	274	{
	275	// build extended pointer on rwlock protecting VSL
	276	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	277
	278	// get rwlock in write mode
	279	remote_rwlock_wr_acquire( lock_xp );
	280
	281	// update vseg descriptor
	282	vseg->vmm = NULL;
	283
	284	// remove vseg from vmm list
	285	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
	286
	287	// release rwlock in write mode
	288	remote_rwlock_wr_release( lock_xp );
	289	}
	290
[433]	291	/////////////////////i//////////////////////////
	292	void vmm_global_update_pte( process_t * process,
	293	vpn_t vpn,
	294	uint32_t attr,
	295	ppn_t ppn )
[23]	296	{
	297
[408]	298	xlist_entry_t * process_root_ptr;
	299	xptr_t process_root_xp;
	300	xptr_t process_iter_xp;
[23]	301
[408]	302	xptr_t remote_process_xp;
	303	cxy_t remote_process_cxy;
	304	process_t * remote_process_ptr;
	305	xptr_t remote_gpt_xp;
[23]	306
[408]	307	pid_t pid;
	308	cxy_t owner_cxy;
	309	lpid_t owner_lpid;
[23]	310
[438]	311	#if DEBUG_VMM_UPDATE_PTE
[433]	312	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	313	if( DEBUG_VMM_UPDATE_PTE < cycle )
[433]	314	printk("\n[DBG] %s : thread %x enter for process %x / vpn %x / cycle %d\n",
	315	__FUNCTION__ , CURRENT_THREAD , process->pid , vpn , cycle );
	316	#endif
	317
[567]	318	// check cluster is reference
	319	assert( (GET_CXY( process->ref_xp ) == local_cxy) ,
	320	"not called in reference cluster\n");
[433]	321
[408]	322	// get extended pointer on root of process copies xlist in owner cluster
	323	pid = process->pid;
	324	owner_cxy = CXY_FROM_PID( pid );
	325	owner_lpid = LPID_FROM_PID( pid );
	326	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
	327	process_root_xp = XPTR( owner_cxy , process_root_ptr );
[23]	328
[408]	329	// loop on destination process copies
	330	XLIST_FOREACH( process_root_xp , process_iter_xp )
	331	{
	332	// get cluster and local pointer on remote process
	333	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
[433]	334	remote_process_ptr = GET_PTR( remote_process_xp );
[408]	335	remote_process_cxy = GET_CXY( remote_process_xp );
[407]	336
[438]	337	#if (DEBUG_VMM_UPDATE_PTE & 0x1)
	338	if( DEBUG_VMM_UPDATE_PTE < cycle )
[433]	339	printk("\n[DBG] %s : thread %x handling process %x in cluster %x\n",
	340	__FUNCTION__ , CURRENT_THREAD , process->pid , remote_process_cxy );
	341	#endif
	342
[408]	343	// get extended pointer on remote gpt
	344	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
	345
[433]	346	// update remote GPT
	347	hal_gpt_update_pte( remote_gpt_xp, vpn, attr, ppn );
[408]	348	}
	349
[438]	350	#if DEBUG_VMM_UPDATE_PTE
[433]	351	cycle = (uint32_t)hal_get_cycles();
[438]	352	if( DEBUG_VMM_UPDATE_PTE < cycle )
[433]	353	printk("\n[DBG] %s : thread %x exit for process %x / vpn %x / cycle %d\n",
	354	__FUNCTION__ , CURRENT_THREAD , process->pid , vpn , cycle );
	355	#endif
	356
	357	} // end vmm_global_update_pte()
	358
[408]	359	///////////////////////////////////////
	360	void vmm_set_cow( process_t * process )
	361	{
	362	vmm_t * vmm;
	363
	364	xlist_entry_t * process_root_ptr;
	365	xptr_t process_root_xp;
	366	xptr_t process_iter_xp;
	367
	368	xptr_t remote_process_xp;
	369	cxy_t remote_process_cxy;
	370	process_t * remote_process_ptr;
	371	xptr_t remote_gpt_xp;
	372
	373	xptr_t vseg_root_xp;
	374	xptr_t vseg_iter_xp;
	375
	376	xptr_t vseg_xp;
	377	vseg_t * vseg;
	378
	379	pid_t pid;
	380	cxy_t owner_cxy;
	381	lpid_t owner_lpid;
	382
[438]	383	#if DEBUG_VMM_SET_COW
[433]	384	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	385	if( DEBUG_VMM_SET_COW < cycle )
[433]	386	printk("\n[DBG] %s : thread %x enter for process %x / cycle %d\n",
	387	__FUNCTION__ , CURRENT_THREAD , process->pid , cycle );
	388	#endif
[408]	389
[567]	390	// check cluster is reference
	391	assert( (GET_CXY( process->ref_xp ) == local_cxy) ,
	392	"local cluster is not process reference cluster\n");
[408]	393
	394	// get pointer on reference VMM
	395	vmm = &process->vmm;
	396
	397	// get extended pointer on root of process copies xlist in owner cluster
	398	pid = process->pid;
	399	owner_cxy = CXY_FROM_PID( pid );
	400	owner_lpid = LPID_FROM_PID( pid );
	401	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
	402	process_root_xp = XPTR( owner_cxy , process_root_ptr );
	403
	404	// get extended pointer on root of vsegs xlist from reference VMM
	405	vseg_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	406
	407	// loop on destination process copies
	408	XLIST_FOREACH( process_root_xp , process_iter_xp )
	409	{
	410	// get cluster and local pointer on remote process
	411	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
[433]	412	remote_process_ptr = GET_PTR( remote_process_xp );
[408]	413	remote_process_cxy = GET_CXY( remote_process_xp );
	414
[438]	415	#if (DEBUG_VMM_SET_COW &0x1)
	416	if( DEBUG_VMM_SET_COW < cycle )
[433]	417	printk("\n[DBG] %s : thread %x handling process %x in cluster %x\n",
	418	__FUNCTION__ , CURRENT_THREAD , process->pid , remote_process_cxy );
	419	#endif
[408]	420
	421	// get extended pointer on remote gpt
	422	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
	423
	424	// loop on vsegs in (local) reference process VSL
	425	XLIST_FOREACH( vseg_root_xp , vseg_iter_xp )
	426	{
	427	// get pointer on vseg
	428	vseg_xp = XLIST_ELEMENT( vseg_iter_xp , vseg_t , xlist );
[433]	429	vseg = GET_PTR( vseg_xp );
[408]	430
[567]	431	assert( (GET_CXY( vseg_xp ) == local_cxy) ,
	432	"all vsegs in reference VSL must be local\n" );
[408]	433
	434	// get vseg type, base and size
	435	uint32_t type = vseg->type;
	436	vpn_t vpn_base = vseg->vpn_base;
	437	vpn_t vpn_size = vseg->vpn_size;
	438
[438]	439	#if (DEBUG_VMM_SET_COW & 0x1)
	440	if( DEBUG_VMM_SET_COW < cycle )
[433]	441	printk("\n[DBG] %s : thread %x handling vseg %s / vpn_base = %x / vpn_size = %x\n",
	442	__FUNCTION__, CURRENT_THREAD , vseg_type_str(type), vpn_base, vpn_size );
	443	#endif
	444	// only DATA, ANON and REMOTE vsegs
[408]	445	if( (type == VSEG_TYPE_DATA) \|\|
	446	(type == VSEG_TYPE_ANON) \|\|
	447	(type == VSEG_TYPE_REMOTE) )
	448	{
[433]	449	vpn_t vpn;
	450	uint32_t attr;
	451	ppn_t ppn;
	452	xptr_t page_xp;
	453	cxy_t page_cxy;
	454	page_t * page_ptr;
	455	xptr_t forks_xp;
[469]	456	xptr_t lock_xp;
[433]	457
	458	// update flags in remote GPT
	459	hal_gpt_set_cow( remote_gpt_xp,
	460	vpn_base,
	461	vpn_size );
	462
	463	// atomically increment pending forks counter in physical pages,
	464	// for all vseg pages that are mapped in reference cluster
	465	if( remote_process_cxy == local_cxy )
	466	{
	467	// the reference GPT is the local GPT
	468	gpt_t * gpt = GET_PTR( remote_gpt_xp );
	469
	470	// scan all pages in vseg
	471	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
	472	{
	473	// get page attributes and PPN from reference GPT
	474	hal_gpt_get_pte( gpt , vpn , &attr , &ppn );
	475
	476	// atomically update pending forks counter if page is mapped
	477	if( attr & GPT_MAPPED )
	478	{
[469]	479	// get pointers and cluster on page descriptor
[433]	480	page_xp = ppm_ppn2page( ppn );
	481	page_cxy = GET_CXY( page_xp );
	482	page_ptr = GET_PTR( page_xp );
[469]	483
	484	// get extended pointers on "forks" and "lock"
[433]	485	forks_xp = XPTR( page_cxy , &page_ptr->forks );
[469]	486	lock_xp = XPTR( page_cxy , &page_ptr->lock );
	487
[567]	488	// take lock protecting "forks" counter
	489	remote_busylock_acquire( lock_xp );
	490
[469]	491	// increment "forks"
[433]	492	hal_remote_atomic_add( forks_xp , 1 );
[567]	493
	494	// release lock protecting "forks" counter
	495	remote_busylock_release( lock_xp );
[433]	496	}
	497	} // end loop on vpn
	498	} // end if local
	499	} // end if vseg type
	500	} // end loop on vsegs
[408]	501	} // end loop on process copies
	502
[438]	503	#if DEBUG_VMM_SET_COW
[433]	504	cycle = (uint32_t)hal_get_cycles();
[438]	505	if( DEBUG_VMM_SET_COW < cycle )
[433]	506	printk("\n[DBG] %s : thread %x exit for process %x / cycle %d\n",
	507	__FUNCTION__ , CURRENT_THREAD , process->pid , cycle );
	508	#endif
[408]	509
	510	} // end vmm_set-cow()
	511
	512	/////////////////////////////////////////////////
	513	error_t vmm_fork_copy( process_t * child_process,
	514	xptr_t parent_process_xp )
	515	{
	516	error_t error;
	517	cxy_t parent_cxy;
	518	process_t * parent_process;
	519	vmm_t * parent_vmm;
	520	xptr_t parent_lock_xp;
	521	vmm_t * child_vmm;
	522	xptr_t iter_xp;
	523	xptr_t parent_vseg_xp;
	524	vseg_t * parent_vseg;
	525	vseg_t * child_vseg;
	526	uint32_t type;
	527	bool_t cow;
	528	vpn_t vpn;
	529	vpn_t vpn_base;
	530	vpn_t vpn_size;
[469]	531	xptr_t page_xp; // extended pointer on page descriptor
[408]	532	page_t * page_ptr;
	533	cxy_t page_cxy;
[469]	534	xptr_t forks_xp; // extended pointer on forks counter in page descriptor
	535	xptr_t lock_xp; // extended pointer on lock protecting the forks counter
[408]	536	xptr_t parent_root_xp;
	537	bool_t mapped;
	538	ppn_t ppn;
	539
[438]	540	#if DEBUG_VMM_FORK_COPY
[433]	541	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	542	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	543	printk("\n[DBG] %s : thread %x enter / cycle %d\n",
	544	__FUNCTION__ , CURRENT_THREAD, cycle );
	545	#endif
[408]	546
	547	// get parent process cluster and local pointer
	548	parent_cxy = GET_CXY( parent_process_xp );
[433]	549	parent_process = GET_PTR( parent_process_xp );
[408]	550
	551	// get local pointers on parent and child VMM
	552	parent_vmm = &parent_process->vmm;
	553	child_vmm = &child_process->vmm;
	554
	555	// get extended pointer on lock protecting the parent VSL
	556	parent_lock_xp = XPTR( parent_cxy , &parent_vmm->vsegs_lock );
	557
	558	// initialize the lock protecting the child VSL
[567]	559	remote_rwlock_init( XPTR( local_cxy , &child_vmm->vsegs_lock ), LOCK_VMM_STACK );
[408]	560
	561	// initialize the child VSL as empty
	562	xlist_root_init( XPTR( local_cxy, &child_vmm->vsegs_root ) );
	563	child_vmm->vsegs_nr = 0;
	564
[415]	565	// create child GPT
[408]	566	error = hal_gpt_create( &child_vmm->gpt );
[415]	567
[407]	568	if( error )
	569	{
[408]	570	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
	571	return -1;
[407]	572	}
	573
[408]	574	// build extended pointer on parent VSL
	575	parent_root_xp = XPTR( parent_cxy , &parent_vmm->vsegs_root );
	576
[567]	577	// take the lock protecting the parent VSL in read mode
	578	remote_rwlock_rd_acquire( parent_lock_xp );
[415]	579
[408]	580	// loop on parent VSL xlist
	581	XLIST_FOREACH( parent_root_xp , iter_xp )
[23]	582	{
[408]	583	// get local and extended pointers on current parent vseg
	584	parent_vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	585	parent_vseg = GET_PTR( parent_vseg_xp );
[23]	586
[408]	587	// get vseg type
[567]	588	type = hal_remote_l32( XPTR( parent_cxy , &parent_vseg->type ) );
[408]	589
[438]	590	#if DEBUG_VMM_FORK_COPY
[433]	591	cycle = (uint32_t)hal_get_cycles();
[438]	592	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	593	printk("\n[DBG] %s : thread %x found parent vseg %s / vpn_base = %x / cycle %d\n",
	594	__FUNCTION__ , CURRENT_THREAD, vseg_type_str(type),
[567]	595	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
[433]	596	#endif
[23]	597
[408]	598	// all parent vsegs - but STACK - must be copied in child VSL
	599	if( type != VSEG_TYPE_STACK )
[23]	600	{
[408]	601	// allocate memory for a new child vseg
	602	child_vseg = vseg_alloc();
	603	if( child_vseg == NULL ) // release all allocated vsegs
[23]	604	{
[408]	605	vmm_destroy( child_process );
	606	printk("\n[ERROR] in %s : cannot create vseg for child\n", __FUNCTION__ );
	607	return -1;
[23]	608	}
	609
[408]	610	// copy parent vseg to child vseg
	611	vseg_init_from_ref( child_vseg , parent_vseg_xp );
[23]	612
[408]	613	// register child vseg in child VSL
[567]	614	vmm_vseg_attach( child_vmm , child_vseg );
[407]	615
[438]	616	#if DEBUG_VMM_FORK_COPY
[433]	617	cycle = (uint32_t)hal_get_cycles();
[438]	618	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	619	printk("\n[DBG] %s : thread %x copied vseg %s / vpn_base = %x to child VSL / cycle %d\n",
	620	__FUNCTION__ , CURRENT_THREAD , vseg_type_str(type),
[567]	621	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
[433]	622	#endif
[23]	623
[408]	624	// copy DATA, MMAP, REMOTE, FILE parent GPT entries to child GPT
	625	if( type != VSEG_TYPE_CODE )
	626	{
	627	// activate the COW for DATA, MMAP, REMOTE vsegs only
	628	cow = ( type != VSEG_TYPE_FILE );
[23]	629
[408]	630	vpn_base = child_vseg->vpn_base;
	631	vpn_size = child_vseg->vpn_size;
[23]	632
[408]	633	// scan pages in parent vseg
	634	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
	635	{
	636	error = hal_gpt_pte_copy( &child_vmm->gpt,
	637	XPTR( parent_cxy , &parent_vmm->gpt ),
	638	vpn,
	639	cow,
	640	&ppn,
	641	&mapped );
	642	if( error )
	643	{
	644	vmm_destroy( child_process );
	645	printk("\n[ERROR] in %s : cannot copy GPT\n", __FUNCTION__ );
	646	return -1;
	647	}
	648
[433]	649	// increment pending forks counter in page if mapped
[408]	650	if( mapped )
	651	{
[469]	652	// get pointers and cluster on page descriptor
	653	page_xp = ppm_ppn2page( ppn );
[408]	654	page_cxy = GET_CXY( page_xp );
[433]	655	page_ptr = GET_PTR( page_xp );
[408]	656
[469]	657	// get extended pointers on "forks" and "lock"
	658	forks_xp = XPTR( page_cxy , &page_ptr->forks );
	659	lock_xp = XPTR( page_cxy , &page_ptr->lock );
	660
[567]	661	// get lock protecting "forks" counter
	662	remote_busylock_acquire( lock_xp );
	663
[469]	664	// increment "forks"
	665	hal_remote_atomic_add( forks_xp , 1 );
	666
[567]	667	// release lock protecting "forks" counter
	668	remote_busylock_release( lock_xp );
	669
[438]	670	#if DEBUG_VMM_FORK_COPY
[433]	671	cycle = (uint32_t)hal_get_cycles();
[438]	672	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	673	printk("\n[DBG] %s : thread %x copied vpn %x to child GPT / cycle %d\n",
	674	__FUNCTION__ , CURRENT_THREAD , vpn , cycle );
	675	#endif
[408]	676	}
	677	}
	678	} // end if no code & no stack
	679	} // end if no stack
	680	} // end loop on vsegs
	681
[567]	682	// release the parent VSL lock in read mode
	683	remote_rwlock_rd_release( parent_lock_xp );
[408]	684
[415]	685	// initialize child GPT (architecture specic)
	686	// => For TSAR, identity map the kentry_vseg
	687	error = hal_vmm_init( child_vmm );
	688
	689	if( error )
	690	{
	691	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
	692	return -1;
	693	}
	694
[408]	695	// initialize the child VMM STACK allocator
	696	child_vmm->stack_mgr.bitmap = 0;
	697	child_vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
	698
	699	// initialize the child VMM MMAP allocator
[23]	700	uint32_t i;
[408]	701	child_vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
	702	child_vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
	703	child_vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
	704	for( i = 0 ; i < 32 ; i++ ) list_root_init( &child_vmm->mmap_mgr.zombi_list[i] );
[23]	705
[178]	706	// initialize instrumentation counters
[408]	707	child_vmm->pgfault_nr = 0;
[23]	708
[408]	709	// copy base addresses from parent VMM to child VMM
	710	child_vmm->kent_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->kent_vpn_base));
	711	child_vmm->args_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->args_vpn_base));
	712	child_vmm->envs_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->envs_vpn_base));
	713	child_vmm->heap_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->heap_vpn_base));
	714	child_vmm->code_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->code_vpn_base));
	715	child_vmm->data_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->data_vpn_base));
[23]	716
[408]	717	child_vmm->entry_point = (intptr_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->entry_point));
[23]	718
[124]	719	hal_fence();
[23]	720
[438]	721	#if DEBUG_VMM_FORK_COPY
[433]	722	cycle = (uint32_t)hal_get_cycles();
[438]	723	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	724	printk("\n[DBG] %s : thread %x exit successfully / cycle %d\n",
	725	__FUNCTION__ , CURRENT_THREAD , cycle );
	726	#endif
	727
[23]	728	return 0;
	729
[408]	730	} // vmm_fork_copy()
[204]	731
[1]	732	///////////////////////////////////////
	733	void vmm_destroy( process_t * process )
	734	{
[408]	735	xptr_t vseg_xp;
[1]	736	vseg_t * vseg;
	737
[438]	738	#if DEBUG_VMM_DESTROY
[433]	739	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	740	if( DEBUG_VMM_DESTROY < cycle )
[443]	741	printk("\n[DBG] %s : thread %x enter for process %x in cluster %x / cycle %d\n",
[469]	742	__FUNCTION__, CURRENT_THREAD->trdid, process->pid, local_cxy, cycle );
[433]	743	#endif
[416]	744
[438]	745	#if (DEBUG_VMM_DESTROY & 1 )
[443]	746	if( DEBUG_VMM_DESTROY < cycle )
[437]	747	vmm_display( process , true );
	748	#endif
	749
[433]	750	// get pointer on local VMM
[1]	751	vmm_t * vmm = &process->vmm;
	752
[408]	753	// get extended pointer on VSL root and VSL lock
	754	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	755
[409]	756	// remove all user vsegs registered in VSL
[408]	757	while( !xlist_is_empty( root_xp ) )
[1]	758	{
[409]	759	// get pointer on first vseg in VSL
[567]	760	vseg_xp = XLIST_FIRST( root_xp , vseg_t , xlist );
[433]	761	vseg = GET_PTR( vseg_xp );
[409]	762
[437]	763	// unmap and release physical pages
[409]	764	vmm_unmap_vseg( process , vseg );
	765
	766	// remove vseg from VSL
[567]	767	vmm_vseg_detach( vmm , vseg );
[409]	768
	769	// release memory allocated to vseg descriptor
[1]	770	vseg_free( vseg );
[443]	771
	772	#if( DEBUG_VMM_DESTROY & 1 )
	773	if( DEBUG_VMM_DESTROY < cycle )
	774	printk("\n[DBG] %s : %s vseg released / vpn_base %x / vpn_size %d\n",
	775	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	776	#endif
	777
[1]	778	}
	779
	780	// remove all vsegs from zombi_lists in MMAP allocator
	781	uint32_t i;
	782	for( i = 0 ; i<32 ; i++ )
	783	{
	784	while( !list_is_empty( &vmm->mmap_mgr.zombi_list[i] ) )
	785	{
[408]	786	vseg = LIST_FIRST( &vmm->mmap_mgr.zombi_list[i] , vseg_t , zlist );
[443]	787
	788	#if( DEBUG_VMM_DESTROY & 1 )
	789	if( DEBUG_VMM_DESTROY < cycle )
	790	printk("\n[DBG] %s : found zombi vseg / vpn_base %x / vpn_size %d\n",
	791	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	792	#endif
[567]	793	vmm_vseg_detach( vmm , vseg );
[1]	794	vseg_free( vseg );
[443]	795
	796	#if( DEBUG_VMM_DESTROY & 1 )
	797	if( DEBUG_VMM_DESTROY < cycle )
	798	printk("\n[DBG] %s : zombi vseg released / vpn_base %x / vpn_size %d\n",
	799	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	800	#endif
[1]	801	}
	802	}
	803
[409]	804	// release memory allocated to the GPT itself
[1]	805	hal_gpt_destroy( &vmm->gpt );
	806
[438]	807	#if DEBUG_VMM_DESTROY
[433]	808	cycle = (uint32_t)hal_get_cycles();
[438]	809	if( DEBUG_VMM_DESTROY < cycle )
[443]	810	printk("\n[DBG] %s : thread %x exit for process %x in cluster %x / cycle %d\n",
[469]	811	__FUNCTION__, CURRENT_THREAD->trdid, process->pid, local_cxy , cycle );
[433]	812	#endif
[416]	813
[204]	814	} // end vmm_destroy()
	815
[1]	816	/////////////////////////////////////////////////
	817	vseg_t * vmm_check_conflict( process_t * process,
[21]	818	vpn_t vpn_base,
[1]	819	vpn_t vpn_size )
	820	{
	821	vmm_t * vmm = &process->vmm;
[408]	822
	823	// scan the VSL
[1]	824	vseg_t * vseg;
[408]	825	xptr_t iter_xp;
	826	xptr_t vseg_xp;
	827	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
[1]	828
[408]	829	XLIST_FOREACH( root_xp , iter_xp )
[1]	830	{
[408]	831	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	832	vseg = GET_PTR( vseg_xp );
[204]	833
[21]	834	if( ((vpn_base + vpn_size) > vseg->vpn_base) &&
	835	(vpn_base < (vseg->vpn_base + vseg->vpn_size)) ) return vseg;
[1]	836	}
	837	return NULL;
	838
[204]	839	} // end vmm_check_conflict()
	840
[1]	841	////////////////////////////////////////////////////////////////////////////////////////////
	842	// This static function is called by the vmm_create_vseg() function, and implements
	843	// the VMM stack_vseg specific allocator.
	844	////////////////////////////////////////////////////////////////////////////////////////////
	845	// @ vmm : pointer on VMM.
[21]	846	// @ vpn_base : (return value) first allocated page
[1]	847	// @ vpn_size : (return value) number of allocated pages
	848	////////////////////////////////////////////////////////////////////////////////////////////
	849	static error_t vmm_stack_alloc( vmm_t * vmm,
	850	vpn_t * vpn_base,
	851	vpn_t * vpn_size )
	852	{
	853	// get stack allocator pointer
	854	stack_mgr_t * mgr = &vmm->stack_mgr;
	855
	856	// get lock on stack allocator
[567]	857	busylock_acquire( &mgr->lock );
[1]	858
	859	// get first free slot index in bitmap
	860	int32_t index = bitmap_ffc( &mgr->bitmap , 4 );
[179]	861	if( (index < 0) \|\| (index > 31) )
	862	{
[567]	863	busylock_release( &mgr->lock );
	864	return 0xFFFFFFFF;
[179]	865	}
[1]	866
	867	// update bitmap
	868	bitmap_set( &mgr->bitmap , index );
[21]	869
[1]	870	// release lock on stack allocator
[567]	871	busylock_release( &mgr->lock );
[1]	872
[21]	873	// returns vpn_base, vpn_size (one page non allocated)
[1]	874	vpn_base = mgr->vpn_base + index CONFIG_VMM_STACK_SIZE + 1;
	875	*vpn_size = CONFIG_VMM_STACK_SIZE - 1;
	876	return 0;
	877
[204]	878	} // end vmm_stack_alloc()
	879
[1]	880	////////////////////////////////////////////////////////////////////////////////////////////
	881	// This static function is called by the vmm_create_vseg() function, and implements
	882	// the VMM MMAP specific allocator.
	883	////////////////////////////////////////////////////////////////////////////////////////////
	884	// @ vmm : [in] pointer on VMM.
	885	// @ npages : [in] requested number of pages.
[21]	886	// @ vpn_base : [out] first allocated page.
[1]	887	// @ vpn_size : [out] actual number of allocated pages.
	888	////////////////////////////////////////////////////////////////////////////////////////////
	889	static error_t vmm_mmap_alloc( vmm_t * vmm,
	890	vpn_t npages,
	891	vpn_t * vpn_base,
	892	vpn_t * vpn_size )
	893	{
	894	uint32_t index;
	895	vseg_t * vseg;
	896	vpn_t base;
	897	vpn_t size;
[21]	898	vpn_t free;
[1]	899
[21]	900	// mmap vseg size must be power of 2
[1]	901	// compute actual size and index in zombi_list array
	902	size = POW2_ROUNDUP( npages );
	903	index = bits_log2( size );
	904
	905	// get mmap allocator pointer
	906	mmap_mgr_t * mgr = &vmm->mmap_mgr;
	907
	908	// get lock on mmap allocator
[567]	909	busylock_acquire( &mgr->lock );
[1]	910
	911	// get vseg from zombi_list or from mmap zone
	912	if( list_is_empty( &mgr->zombi_list[index] ) ) // from mmap zone
	913	{
	914	// check overflow
	915	free = mgr->first_free_vpn;
	916	if( (free + size) > mgr->vpn_size ) return ENOMEM;
	917
	918	// update STACK allocator
	919	mgr->first_free_vpn += size;
	920
	921	// compute base
	922	base = free;
	923	}
	924	else // from zombi_list
	925	{
	926	// get pointer on zombi vseg from zombi_list
[408]	927	vseg = LIST_FIRST( &mgr->zombi_list[index] , vseg_t , zlist );
[1]	928
	929	// remove vseg from free-list
[408]	930	list_unlink( &vseg->zlist );
[1]	931
	932	// compute base
	933	base = vseg->vpn_base;
[21]	934	}
	935
[1]	936	// release lock on mmap allocator
[567]	937	busylock_release( &mgr->lock );
[1]	938
	939	// returns vpn_base, vpn_size
	940	*vpn_base = base;
	941	*vpn_size = size;
	942	return 0;
	943
[204]	944	} // end vmm_mmap_alloc()
	945
[407]	946	////////////////////////////////////////////////
	947	vseg_t * vmm_create_vseg( process_t * process,
	948	vseg_type_t type,
	949	intptr_t base,
	950	uint32_t size,
	951	uint32_t file_offset,
	952	uint32_t file_size,
	953	xptr_t mapper_xp,
	954	cxy_t cxy )
[1]	955	{
	956	vseg_t * vseg; // created vseg pointer
[204]	957	vpn_t vpn_base; // first page index
[1]	958	vpn_t vpn_size; // number of pages
	959	error_t error;
	960
[438]	961	#if DEBUG_VMM_CREATE_VSEG
[433]	962	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	963	if( DEBUG_VMM_CREATE_VSEG < cycle )
[433]	964	printk("\n[DBG] %s : thread %x enter / process %x / base %x / size %x / %s / cxy %x / cycle %d\n",
	965	__FUNCTION__, CURRENT_THREAD, process->pid, base, size, vseg_type_str(type), cxy, cycle );
	966	#endif
[21]	967
[407]	968	// get pointer on VMM
	969	vmm_t * vmm = &process->vmm;
[21]	970
[204]	971	// compute base, size, vpn_base, vpn_size, depending on vseg type
[407]	972	// we use the VMM specific allocators for "stack", "file", "anon", & "remote" vsegs
[1]	973	if( type == VSEG_TYPE_STACK )
	974	{
	975	// get vpn_base and vpn_size from STACK allocator
	976	error = vmm_stack_alloc( vmm , &vpn_base , &vpn_size );
	977	if( error )
	978	{
[407]	979	printk("\n[ERROR] in %s : no space for stack vseg / process %x in cluster %x\n",
	980	__FUNCTION__ , process->pid , local_cxy );
[1]	981	return NULL;
	982	}
	983
	984	// compute vseg base and size from vpn_base and vpn_size
	985	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
	986	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
	987	}
[21]	988	else if( (type == VSEG_TYPE_ANON) \|\|
	989	(type == VSEG_TYPE_FILE) \|\|
[1]	990	(type == VSEG_TYPE_REMOTE) )
	991	{
	992	// get vpn_base and vpn_size from MMAP allocator
	993	vpn_t npages = size >> CONFIG_PPM_PAGE_SHIFT;
	994	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
	995	if( error )
	996	{
	997	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
	998	__FUNCTION__ , process->pid , local_cxy );
	999	return NULL;
	1000	}
	1001
	1002	// compute vseg base and size from vpn_base and vpn_size
	1003	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
	1004	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
	1005	}
	1006	else
	1007	{
[204]	1008	uint32_t vpn_min = base >> CONFIG_PPM_PAGE_SHIFT;
	1009	uint32_t vpn_max = (base + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1010
	1011	vpn_base = vpn_min;
	1012	vpn_size = vpn_max - vpn_min + 1;
[1]	1013	}
	1014
	1015	// check collisions
	1016	vseg = vmm_check_conflict( process , vpn_base , vpn_size );
	1017	if( vseg != NULL )
	1018	{
[21]	1019	printk("\n[ERROR] in %s for process %x : new vseg [vpn_base = %x / vpn_size = %x]\n"
[1]	1020	" overlap existing vseg [vpn_base = %x / vpn_size = %x]\n",
[407]	1021	__FUNCTION__ , process->pid, vpn_base, vpn_size, vseg->vpn_base, vseg->vpn_size );
[1]	1022	return NULL;
	1023	}
	1024
	1025	// allocate physical memory for vseg descriptor
	1026	vseg = vseg_alloc();
	1027	if( vseg == NULL )
	1028	{
	1029	printk("\n[ERROR] in %s for process %x : cannot allocate memory for vseg\n",
[407]	1030	__FUNCTION__ , process->pid );
[1]	1031	return NULL;
	1032	}
	1033
	1034	// initialize vseg descriptor
[407]	1035	vseg_init( vseg,
	1036	type,
	1037	base,
	1038	size,
	1039	vpn_base,
	1040	vpn_size,
	1041	file_offset,
	1042	file_size,
	1043	mapper_xp,
	1044	cxy );
[1]	1045
[408]	1046	// attach vseg to VSL
[567]	1047	vmm_vseg_attach( vmm , vseg );
[1]	1048
[438]	1049	#if DEBUG_VMM_CREATE_VSEG
[433]	1050	cycle = (uint32_t)hal_get_cycles();
[438]	1051	if( DEBUG_VMM_CREATE_VSEG < cycle )
[433]	1052	printk("\n[DBG] %s : thread %x exit / process %x / %s / cxy %x / cycle %d\n",
	1053	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str(type), cxy, cycle );
	1054	#endif
[21]	1055
[1]	1056	return vseg;
	1057
[406]	1058	} // vmm_create_vseg()
	1059
[1]	1060	/////////////////////////////////////
	1061	void vmm_remove_vseg( vseg_t * vseg )
	1062	{
	1063	// get pointers on calling process and VMM
	1064	thread_t * this = CURRENT_THREAD;
	1065	vmm_t * vmm = &this->process->vmm;
	1066	uint32_t type = vseg->type;
	1067
[408]	1068	// detach vseg from VSL
[567]	1069	vmm_vseg_detach( vmm , vseg );
[1]	1070
	1071	// release the stack slot to VMM stack allocator if STACK type
	1072	if( type == VSEG_TYPE_STACK )
	1073	{
	1074	// get pointer on stack allocator
	1075	stack_mgr_t * mgr = &vmm->stack_mgr;
	1076
	1077	// compute slot index
	1078	uint32_t index = ((vseg->vpn_base - mgr->vpn_base - 1) / CONFIG_VMM_STACK_SIZE);
	1079
	1080	// update stacks_bitmap
[567]	1081	busylock_acquire( &mgr->lock );
[1]	1082	bitmap_clear( &mgr->bitmap , index );
[567]	1083	busylock_release( &mgr->lock );
[1]	1084	}
	1085
	1086	// release the vseg to VMM mmap allocator if MMAP type
	1087	if( (type == VSEG_TYPE_ANON) \|\| (type == VSEG_TYPE_FILE) \|\| (type == VSEG_TYPE_REMOTE) )
	1088	{
	1089	// get pointer on mmap allocator
	1090	mmap_mgr_t * mgr = &vmm->mmap_mgr;
	1091
	1092	// compute zombi_list index
	1093	uint32_t index = bits_log2( vseg->vpn_size );
	1094
	1095	// update zombi_list
[567]	1096	busylock_acquire( &mgr->lock );
[408]	1097	list_add_first( &mgr->zombi_list[index] , &vseg->zlist );
[567]	1098	busylock_release( &mgr->lock );
[1]	1099	}
	1100
	1101	// release physical memory allocated for vseg descriptor if no MMAP type
	1102	if( (type != VSEG_TYPE_ANON) && (type != VSEG_TYPE_FILE) && (type != VSEG_TYPE_REMOTE) )
	1103	{
	1104	vseg_free( vseg );
	1105	}
[407]	1106	} // end vmm_remove_vseg()
[1]	1107
	1108	/////////////////////////////////////////
	1109	void vmm_unmap_vseg( process_t * process,
	1110	vseg_t * vseg )
	1111	{
[21]	1112	vpn_t vpn; // VPN of current PTE
	1113	vpn_t vpn_min; // VPN of first PTE
[1]	1114	vpn_t vpn_max; // VPN of last PTE (excluded)
[409]	1115	ppn_t ppn; // current PTE ppn value
	1116	uint32_t attr; // current PTE attributes
	1117	kmem_req_t req; // request to release memory
	1118	xptr_t page_xp; // extended pointer on page descriptor
	1119	cxy_t page_cxy; // page descriptor cluster
	1120	page_t * page_ptr; // page descriptor pointer
[433]	1121	xptr_t forks_xp; // extended pointer on pending forks counter
[469]	1122	xptr_t lock_xp; // extended pointer on lock protecting forks counter
	1123	uint32_t forks; // actual number of pendinf forks
[1]	1124
[438]	1125	#if DEBUG_VMM_UNMAP_VSEG
[433]	1126	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	1127	if( DEBUG_VMM_UNMAP_VSEG < cycle )
[433]	1128	printk("\n[DBG] %s : thread %x enter / process %x / vseg %s / base %x / cycle %d\n",
	1129	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str( vseg->type ), vseg->vpn_base, cycle );
	1130	#endif
[409]	1131
[433]	1132	// get pointer on local GPT
[1]	1133	gpt_t * gpt = &process->vmm.gpt;
	1134
	1135	// loop on pages in vseg
	1136	vpn_min = vseg->vpn_base;
	1137	vpn_max = vpn_min + vseg->vpn_size;
	1138	for( vpn = vpn_min ; vpn < vpn_max ; vpn++ )
	1139	{
[409]	1140	// get GPT entry
	1141	hal_gpt_get_pte( gpt , vpn , &attr , &ppn );
	1142
	1143	if( attr & GPT_MAPPED ) // entry is mapped
	1144	{
[437]	1145
[438]	1146	#if( DEBUG_VMM_UNMAP_VSEG & 1 )
	1147	if( DEBUG_VMM_UNMAP_VSEG < cycle )
[437]	1148	printk("- vpn %x / ppn %x\n" , vpn , ppn );
	1149	#endif
	1150
[567]	1151	// check small page
	1152	assert( (attr & GPT_SMALL) ,
	1153	"an user vseg must use small pages" );
[409]	1154
[433]	1155	// unmap GPT entry in all GPT copies
[409]	1156	hal_gpt_reset_pte( gpt , vpn );
	1157
[433]	1158	// handle pending forks counter if
	1159	// 1) not identity mapped
[567]	1160	// 2) reference cluster
[433]	1161	if( ((vseg->flags & VSEG_IDENT) == 0) &&
	1162	(GET_CXY( process->ref_xp ) == local_cxy) )
[409]	1163	{
[433]	1164	// get extended pointer on physical page descriptor
[409]	1165	page_xp = ppm_ppn2page( ppn );
	1166	page_cxy = GET_CXY( page_xp );
[433]	1167	page_ptr = GET_PTR( page_xp );
[409]	1168
[469]	1169	// get extended pointers on forks and lock fields
	1170	forks_xp = XPTR( page_cxy , &page_ptr->forks );
	1171	lock_xp = XPTR( page_cxy , &page_ptr->lock );
[433]	1172
	1173	// get pending forks counter
[567]	1174	forks = hal_remote_l32( forks_xp );
[433]	1175
[469]	1176	if( forks ) // decrement pending forks counter
[409]	1177	{
[433]	1178	hal_remote_atomic_add( forks_xp , -1 );
	1179	}
	1180	else // release physical page to relevant cluster
[409]	1181	{
[433]	1182	if( page_cxy == local_cxy ) // local cluster
	1183	{
	1184	req.type = KMEM_PAGE;
	1185	req.ptr = page_ptr;
	1186	kmem_free( &req );
	1187	}
	1188	else // remote cluster
	1189	{
	1190	rpc_pmem_release_pages_client( page_cxy , page_ptr );
	1191	}
[409]	1192	}
	1193	}
	1194	}
[1]	1195	}
[433]	1196
[438]	1197	#if DEBUG_VMM_UNMAP_VSEG
[433]	1198	cycle = (uint32_t)hal_get_cycles();
[438]	1199	if( DEBUG_VMM_UNMAP_VSEG < cycle )
[433]	1200	printk("\n[DBG] %s : thread %x exit / process %x / vseg %s / base %x / cycle %d\n",
	1201	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str( vseg->type ), vseg->vpn_base, cycle );
	1202	#endif
	1203
[409]	1204	} // end vmm_unmap_vseg()
[1]	1205
[407]	1206	//////////////////////////////////////////////////////////////////////////////////////////
[440]	1207	// This low-level static function is called by the vmm_get_vseg(), vmm_get_pte(),
	1208	// and vmm_resize_vseg() functions. It scan the local VSL to find the unique vseg
	1209	// containing a given virtual address.
[407]	1210	//////////////////////////////////////////////////////////////////////////////////////////
[406]	1211	// @ vmm : pointer on the process VMM.
	1212	// @ vaddr : virtual address.
	1213	// @ return vseg pointer if success / return NULL if not found.
[407]	1214	//////////////////////////////////////////////////////////////////////////////////////////
[406]	1215	static vseg_t * vseg_from_vaddr( vmm_t * vmm,
	1216	intptr_t vaddr )
	1217	{
[408]	1218	xptr_t iter_xp;
	1219	xptr_t vseg_xp;
	1220	vseg_t * vseg;
[406]	1221
[408]	1222	// get extended pointers on VSL lock and root
	1223	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	1224	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
[406]	1225
[408]	1226	// get lock protecting the VSL
[567]	1227	remote_rwlock_rd_acquire( lock_xp );
[408]	1228
	1229	// scan the list of vsegs in VSL
	1230	XLIST_FOREACH( root_xp , iter_xp )
[406]	1231	{
[408]	1232	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	1233	vseg = GET_PTR( vseg_xp );
[408]	1234	if( (vaddr >= vseg->min) && (vaddr < vseg->max) )
	1235	{
	1236	// return success
[567]	1237	remote_rwlock_rd_release( lock_xp );
[408]	1238	return vseg;
	1239	}
[406]	1240	}
	1241
[408]	1242	// return failure
[567]	1243	remote_rwlock_rd_release( lock_xp );
[408]	1244	return NULL;
[406]	1245
[408]	1246	} // end vseg_from_vaddr()
[406]	1247
[1]	1248	/////////////////////////////////////////////
	1249	error_t vmm_resize_vseg( process_t * process,
	1250	intptr_t base,
	1251	intptr_t size )
	1252	{
[406]	1253	error_t error;
	1254	vseg_t * new;
	1255	vpn_t vpn_min;
	1256	vpn_t vpn_max;
[1]	1257
	1258	// get pointer on process VMM
	1259	vmm_t * vmm = &process->vmm;
	1260
	1261	intptr_t addr_min = base;
	1262	intptr_t addr_max = base + size;
	1263
	1264	// get pointer on vseg
[406]	1265	vseg_t * vseg = vseg_from_vaddr( vmm , base );
[1]	1266
	1267	if( vseg == NULL) return EINVAL;
[21]	1268
[408]	1269	// get extended pointer on VSL lock
	1270	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
[21]	1271
[408]	1272	// get lock protecting VSL
[567]	1273	remote_rwlock_wr_acquire( lock_xp );
[408]	1274
[1]	1275	if( (vseg->min > addr_min) \|\| (vseg->max < addr_max) ) // region not included in vseg
	1276	{
	1277	error = EINVAL;
	1278	}
	1279	else if( (vseg->min == addr_min) && (vseg->max == addr_max) ) // vseg must be removed
	1280	{
	1281	vmm_remove_vseg( vseg );
	1282	error = 0;
	1283	}
[406]	1284	else if( vseg->min == addr_min ) // vseg must be resized
[1]	1285	{
[406]	1286	// update vseg base address
	1287	vseg->min = addr_max;
	1288
	1289	// update vpn_base and vpn_size
	1290	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1291	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1292	vseg->vpn_base = vpn_min;
	1293	vseg->vpn_size = vpn_max - vpn_min + 1;
	1294	error = 0;
[1]	1295	}
[406]	1296	else if( vseg->max == addr_max ) // vseg must be resized
[1]	1297	{
[406]	1298	// update vseg max address
	1299	vseg->max = addr_min;
	1300
	1301	// update vpn_base and vpn_size
	1302	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1303	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1304	vseg->vpn_base = vpn_min;
	1305	vseg->vpn_size = vpn_max - vpn_min + 1;
	1306	error = 0;
[1]	1307	}
[406]	1308	else // vseg cut in three regions
[1]	1309	{
[406]	1310	// resize existing vseg
	1311	vseg->max = addr_min;
	1312
	1313	// update vpn_base and vpn_size
	1314	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1315	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1316	vseg->vpn_base = vpn_min;
	1317	vseg->vpn_size = vpn_max - vpn_min + 1;
	1318
	1319	// create new vseg
[407]	1320	new = vmm_create_vseg( process,
	1321	vseg->type,
	1322	addr_min,
	1323	(vseg->max - addr_max),
	1324	vseg->file_offset,
	1325	vseg->file_size,
	1326	vseg->mapper_xp,
	1327	vseg->cxy );
	1328
[406]	1329	if( new == NULL ) error = EINVAL;
	1330	else error = 0;
[1]	1331	}
	1332
	1333	// release VMM lock
[567]	1334	remote_rwlock_wr_release( lock_xp );
[1]	1335
	1336	return error;
	1337
[406]	1338	} // vmm_resize_vseg()
	1339
[1]	1340	///////////////////////////////////////////
[388]	1341	error_t vmm_get_vseg( process_t * process,
[394]	1342	intptr_t vaddr,
[388]	1343	vseg_t ** found_vseg )
[1]	1344	{
[440]	1345	xptr_t vseg_xp;
	1346	error_t error;
	1347	vseg_t * vseg;
	1348	vmm_t * vmm;
[1]	1349
[440]	1350	// get pointer on local VMM
	1351	vmm = &process->vmm;
[1]	1352
[440]	1353	// try to get vseg from local VMM
	1354	vseg = vseg_from_vaddr( vmm , vaddr );
	1355
[388]	1356	if( vseg == NULL ) // vseg not found in local cluster => try to get it from ref
	1357	{
	1358	// get extended pointer on reference process
	1359	xptr_t ref_xp = process->ref_xp;
[1]	1360
[388]	1361	// get cluster and local pointer on reference process
	1362	cxy_t ref_cxy = GET_CXY( ref_xp );
[433]	1363	process_t * ref_ptr = GET_PTR( ref_xp );
[388]	1364
	1365	if( local_cxy == ref_cxy ) return -1; // local cluster is the reference
	1366
	1367	// get extended pointer on reference vseg
[394]	1368	rpc_vmm_get_vseg_client( ref_cxy , ref_ptr , vaddr , &vseg_xp , &error );
[388]	1369
[440]	1370	if( error ) return -1; // vseg not found => illegal user vaddr
[388]	1371
	1372	// allocate a vseg in local cluster
	1373	vseg = vseg_alloc();
	1374
[440]	1375	if( vseg == NULL ) return -1; // cannot allocate a local vseg
[388]	1376
	1377	// initialise local vseg from reference
	1378	vseg_init_from_ref( vseg , vseg_xp );
	1379
	1380	// register local vseg in local VMM
[567]	1381	vmm_vseg_attach( &process->vmm , vseg );
[388]	1382	}
	1383
	1384	// success
	1385	*found_vseg = vseg;
[394]	1386	return 0;
[388]	1387
	1388	} // end vmm_get_vseg()
	1389
[407]	1390	//////////////////////////////////////////////////////////////////////////////////////
	1391	// This static function compute the target cluster to allocate a physical page
	1392	// for a given <vpn> in a given <vseg>, allocates the page (with an RPC if required)
	1393	// and returns an extended pointer on the allocated page descriptor.
	1394	// The vseg cannot have the FILE type.
	1395	//////////////////////////////////////////////////////////////////////////////////////
	1396	static xptr_t vmm_page_allocate( vseg_t * vseg,
	1397	vpn_t vpn )
	1398	{
[433]	1399
[438]	1400	#if DEBUG_VMM_ALLOCATE_PAGE
	1401	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
[433]	1402	printk("\n[DBG] in %s : thread %x enter for vpn %x\n",
	1403	__FUNCTION__ , CURRENT_THREAD, vpn );
	1404	#endif
	1405
[407]	1406	// compute target cluster
	1407	page_t * page_ptr;
	1408	cxy_t page_cxy;
	1409	kmem_req_t req;
[577]	1410	uint32_t index;
[407]	1411
[577]	1412	uint32_t type = vseg->type;
	1413	uint32_t flags = vseg->flags;
	1414	uint32_t x_size = LOCAL_CLUSTER->x_size;
	1415	uint32_t y_size = LOCAL_CLUSTER->y_size;
[407]	1416
[567]	1417	// check vseg type
	1418	assert( ( type != VSEG_TYPE_FILE ) , "illegal vseg type\n" );
[407]	1419
	1420	if( flags & VSEG_DISTRIB ) // distributed => cxy depends on vpn LSB
	1421	{
[577]	1422	index = vpn & ((x_size * y_size) - 1);
	1423	page_cxy = HAL_CXY_FROM_XY( (index / y_size) , (index % y_size) );
[561]	1424
[577]	1425	// If the cluster selected from VPN's LSBs is empty, we select one randomly
	1426	if ( cluster_is_active( page_cxy ) == false )
	1427	{
	1428	page_cxy = cluster_random_select();
[561]	1429	}
[407]	1430	}
	1431	else // other cases => cxy specified in vseg
	1432	{
[561]	1433	page_cxy = vseg->cxy;
[407]	1434	}
	1435
	1436	// allocate a physical page from target cluster
	1437	if( page_cxy == local_cxy ) // target cluster is the local cluster
	1438	{
	1439	req.type = KMEM_PAGE;
	1440	req.size = 0;
	1441	req.flags = AF_NONE;
	1442	page_ptr = (page_t *)kmem_alloc( &req );
	1443	}
	1444	else // target cluster is not the local cluster
	1445	{
	1446	rpc_pmem_get_pages_client( page_cxy , 0 , &page_ptr );
	1447	}
	1448
[438]	1449	#if DEBUG_VMM_ALLOCATE_PAGE
	1450	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
[433]	1451	printk("\n[DBG] in %s : thread %x exit for vpn = %d / ppn = %x\n",
	1452	__FUNCTION__ , CURRENT_THREAD, vpn, ppm_page2ppn( XPTR( page_cxy , page_ptr ) ) );
	1453	#endif
	1454
[407]	1455	if( page_ptr == NULL ) return XPTR_NULL;
	1456	else return XPTR( page_cxy , page_ptr );
	1457
	1458	} // end vmm_page_allocate()
	1459
[313]	1460	////////////////////////////////////////
	1461	error_t vmm_get_one_ppn( vseg_t * vseg,
	1462	vpn_t vpn,
	1463	ppn_t * ppn )
	1464	{
	1465	error_t error;
[407]	1466	xptr_t page_xp; // extended pointer on physical page descriptor
[313]	1467	page_t * page_ptr; // local pointer on physical page descriptor
[406]	1468	uint32_t index; // missing page index in vseg mapper
	1469	uint32_t type; // vseg type;
[313]	1470
[406]	1471	type = vseg->type;
	1472	index = vpn - vseg->vpn_base;
[313]	1473
[438]	1474	#if DEBUG_VMM_GET_ONE_PPN
[441]	1475	thread_t * this = CURRENT_THREAD;
[469]	1476	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1477	printk("\n[DBG] %s : thread %x enter for vpn = %x / type = %s / index = %d\n",
[441]	1478	__FUNCTION__, this, vpn, vseg_type_str(type), index );
[433]	1479	#endif
[313]	1480
[406]	1481	// FILE type : get the physical page from the file mapper
[313]	1482	if( type == VSEG_TYPE_FILE )
	1483	{
[406]	1484	// get extended pointer on mapper
[407]	1485	xptr_t mapper_xp = vseg->mapper_xp;
[313]	1486
[567]	1487	assert( (mapper_xp != XPTR_NULL),
	1488	"mapper not defined for a FILE vseg\n" );
[406]	1489
	1490	// get mapper cluster and local pointer
	1491	cxy_t mapper_cxy = GET_CXY( mapper_xp );
[433]	1492	mapper_t * mapper_ptr = GET_PTR( mapper_xp );
[406]	1493
[313]	1494	// get page descriptor from mapper
	1495	if( mapper_cxy == local_cxy ) // mapper is local
	1496	{
	1497	page_ptr = mapper_get_page( mapper_ptr , index );
	1498	}
	1499	else // mapper is remote
	1500	{
	1501	rpc_mapper_get_page_client( mapper_cxy , mapper_ptr , index , &page_ptr );
	1502	}
	1503
	1504	if ( page_ptr == NULL ) return EINVAL;
	1505
[407]	1506	page_xp = XPTR( mapper_cxy , page_ptr );
[313]	1507	}
	1508
[406]	1509	// Other types : allocate a physical page from target cluster,
[407]	1510	// as defined by vseg type and vpn value
[313]	1511	else
	1512	{
[433]	1513	// allocate one physical page
[407]	1514	page_xp = vmm_page_allocate( vseg , vpn );
[406]	1515
[407]	1516	if( page_xp == XPTR_NULL ) return ENOMEM;
[313]	1517
[406]	1518	// initialise missing page from .elf file mapper for DATA and CODE types
[440]	1519	// the vseg->mapper_xp field is an extended pointer on the .elf file mapper
[313]	1520	if( (type == VSEG_TYPE_CODE) \|\| (type == VSEG_TYPE_DATA) )
	1521	{
[406]	1522	// get extended pointer on mapper
	1523	xptr_t mapper_xp = vseg->mapper_xp;
[313]	1524
[567]	1525	assert( (mapper_xp != XPTR_NULL),
	1526	"mapper not defined for a CODE or DATA vseg\n" );
[406]	1527
	1528	// get mapper cluster and local pointer
	1529	cxy_t mapper_cxy = GET_CXY( mapper_xp );
[433]	1530	mapper_t * mapper_ptr = GET_PTR( mapper_xp );
[406]	1531
	1532	// compute missing page offset in vseg
	1533	uint32_t offset = index << CONFIG_PPM_PAGE_SHIFT;
	1534
[313]	1535	// compute missing page offset in .elf file
[406]	1536	uint32_t elf_offset = vseg->file_offset + offset;
[313]	1537
[438]	1538	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1539	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1540	printk("\n[DBG] %s : thread %x for vpn = %x / elf_offset = %x\n",
[441]	1541	__FUNCTION__, this, vpn, elf_offset );
[433]	1542	#endif
[406]	1543	// compute extended pointer on page base
[407]	1544	xptr_t base_xp = ppm_page2base( page_xp );
[313]	1545
[406]	1546	// file_size (in .elf mapper) can be smaller than vseg_size (BSS)
	1547	uint32_t file_size = vseg->file_size;
	1548
	1549	if( file_size < offset ) // missing page fully in BSS
[313]	1550	{
[406]	1551
[438]	1552	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1553	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1554	printk("\n[DBG] %s : thread%x for vpn = %x / fully in BSS\n",
[441]	1555	__FUNCTION__, this, vpn );
[433]	1556	#endif
[407]	1557	if( GET_CXY( page_xp ) == local_cxy )
[313]	1558	{
[315]	1559	memset( GET_PTR( base_xp ) , 0 , CONFIG_PPM_PAGE_SIZE );
[313]	1560	}
	1561	else
	1562	{
[315]	1563	hal_remote_memset( base_xp , 0 , CONFIG_PPM_PAGE_SIZE );
[313]	1564	}
	1565	}
[406]	1566	else if( file_size >= (offset + CONFIG_PPM_PAGE_SIZE) ) // fully in mapper
[315]	1567	{
[406]	1568
[438]	1569	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1570	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1571	printk("\n[DBG] %s : thread %x, for vpn = %x / fully in mapper\n",
[441]	1572	__FUNCTION__, this, vpn );
[433]	1573	#endif
[313]	1574	if( mapper_cxy == local_cxy )
	1575	{
	1576	error = mapper_move_kernel( mapper_ptr,
	1577	true, // to_buffer
[406]	1578	elf_offset,
[313]	1579	base_xp,
[315]	1580	CONFIG_PPM_PAGE_SIZE );
[313]	1581	}
	1582	else
	1583	{
	1584	rpc_mapper_move_buffer_client( mapper_cxy,
	1585	mapper_ptr,
	1586	true, // to buffer
	1587	false, // kernel buffer
[406]	1588	elf_offset,
	1589	base_xp,
[315]	1590	CONFIG_PPM_PAGE_SIZE,
[313]	1591	&error );
	1592	}
	1593	if( error ) return EINVAL;
	1594	}
[406]	1595	else // both in mapper and in BSS :
	1596	// - (file_size - offset) bytes from mapper
	1597	// - (page_size + offset - file_size) bytes from BSS
[313]	1598	{
[406]	1599
[438]	1600	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1601	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1602	printk("\n[DBG] %s : thread %x for vpn = %x / both mapper & BSS\n"
	1603	" %d bytes from mapper / %d bytes from BSS\n",
[441]	1604	__FUNCTION__, this, vpn,
[407]	1605	file_size - offset , offset + CONFIG_PPM_PAGE_SIZE - file_size );
[433]	1606	#endif
[313]	1607	// initialize mapper part
[315]	1608	if( mapper_cxy == local_cxy )
[313]	1609	{
	1610	error = mapper_move_kernel( mapper_ptr,
[315]	1611	true, // to buffer
[406]	1612	elf_offset,
[313]	1613	base_xp,
[406]	1614	file_size - offset );
[313]	1615	}
[315]	1616	else
[313]	1617	{
	1618	rpc_mapper_move_buffer_client( mapper_cxy,
	1619	mapper_ptr,
[315]	1620	true, // to buffer
[313]	1621	false, // kernel buffer
[406]	1622	elf_offset,
	1623	base_xp,
	1624	file_size - offset,
[313]	1625	&error );
	1626	}
	1627	if( error ) return EINVAL;
	1628
	1629	// initialize BSS part
[407]	1630	if( GET_CXY( page_xp ) == local_cxy )
[313]	1631	{
[406]	1632	memset( GET_PTR( base_xp ) + file_size - offset , 0 ,
	1633	offset + CONFIG_PPM_PAGE_SIZE - file_size );
[313]	1634	}
	1635	else
	1636	{
[406]	1637	hal_remote_memset( base_xp + file_size - offset , 0 ,
	1638	offset + CONFIG_PPM_PAGE_SIZE - file_size );
[313]	1639	}
	1640	}
	1641	} // end initialisation for CODE or DATA types
	1642	}
	1643
	1644	// return ppn
[407]	1645	*ppn = ppm_page2ppn( page_xp );
[406]	1646
[438]	1647	#if DEBUG_VMM_GET_ONE_PPN
[469]	1648	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1649	printk("\n[DBG] %s : thread %x exit for vpn = %x / ppn = %x\n",
[441]	1650	__FUNCTION__ , this , vpn , *ppn );
[433]	1651	#endif
[406]	1652
[313]	1653	return 0;
	1654
	1655	} // end vmm_get_one_ppn()
	1656
[1]	1657	/////////////////////////////////////////
	1658	error_t vmm_get_pte( process_t * process,
	1659	vpn_t vpn,
[407]	1660	bool_t cow,
	1661	uint32_t * attr,
	1662	ppn_t * ppn )
[1]	1663	{
[440]	1664	ppn_t old_ppn; // current PTE_PPN
	1665	uint32_t old_attr; // current PTE_ATTR
	1666	ppn_t new_ppn; // new PTE_PPN
	1667	uint32_t new_attr; // new PTE_ATTR
	1668	vmm_t * vmm;
	1669	vseg_t * vseg;
	1670	error_t error;
[1]	1671
[441]	1672
[438]	1673	#if DEBUG_VMM_GET_PTE
[567]	1674	thread_t * this = CURRENT_THREAD;
[441]	1675	uint32_t cycle = (uint32_t)hal_get_cycles();
[469]	1676	if( DEBUG_VMM_GET_PTE < cycle )
	1677	printk("\n[DBG] %s : thread %x in process %x enter / vpn %x / cow %d / cycle %d\n",
	1678	__FUNCTION__, this->trdid, process->pid, vpn, cow, cycle );
[433]	1679	#endif
[406]	1680
[1]	1681	// get VMM pointer
[440]	1682	vmm = &process->vmm;
[1]	1683
[440]	1684	// get local vseg descriptor
	1685	error = vmm_get_vseg( process,
	1686	((intptr_t)vpn << CONFIG_PPM_PAGE_SHIFT),
	1687	&vseg );
[1]	1688
[567]	1689	// vseg has been checked by the vmm_handle_page_fault() function
	1690	assert( (vseg != NULL) , "vseg undefined / vpn %x\n");
[406]	1691
[438]	1692	if( cow ) //////////////// copy_on_write request //////////////////////
[440]	1693	// get PTE from local GPT
[438]	1694	// allocate a new physical page if there is pending forks,
	1695	// initialize it from old physical page content,
	1696	// update PTE in all GPT copies,
	1697	{
[440]	1698	// access local GPT to get current PTE attributes and PPN
[438]	1699	hal_gpt_get_pte( &vmm->gpt , vpn , &old_attr , &old_ppn );
[407]	1700
[567]	1701	assert( (old_attr & GPT_MAPPED),
	1702	"PTE unmapped for a COW exception / vpn %x\n" );
[407]	1703
[438]	1704	#if( DEBUG_VMM_GET_PTE & 1 )
[469]	1705	if( DEBUG_VMM_GET_PTE < cycle )
	1706	printk("\n[DBG] %s : thread %x in process %x handling COW for vpn %x\n",
	1707	__FUNCTION__, this->trdid, process->pid, vpn );
[433]	1708	#endif
[407]	1709
[433]	1710	// get extended pointer, cluster and local pointer on physical page descriptor
[408]	1711	xptr_t page_xp = ppm_ppn2page( old_ppn );
	1712	cxy_t page_cxy = GET_CXY( page_xp );
[433]	1713	page_t * page_ptr = GET_PTR( page_xp );
[407]	1714
[469]	1715	// get extended pointers on forks and lock field in page descriptor
	1716	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
	1717	xptr_t lock_xp = XPTR( page_cxy , &page_ptr->lock );
	1718
[567]	1719	// take lock protecting "forks" counter
	1720	remote_busylock_acquire( lock_xp );
[469]	1721
[408]	1722	// get number of pending forks in page descriptor
[567]	1723	uint32_t forks = hal_remote_l32( forks_xp );
[408]	1724
[433]	1725	if( forks ) // pending fork => allocate a new page, copy old to new
[1]	1726	{
[408]	1727	// allocate a new physical page
	1728	page_xp = vmm_page_allocate( vseg , vpn );
	1729	if( page_xp == XPTR_NULL )
	1730	{
	1731	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
	1732	__FUNCTION__ , process->pid , vpn );
	1733	return -1;
	1734	}
[1]	1735
[408]	1736	// compute allocated page PPN
	1737	new_ppn = ppm_page2ppn( page_xp );
[406]	1738
[408]	1739	// copy old page content to new page
	1740	xptr_t old_base_xp = ppm_ppn2base( old_ppn );
	1741	xptr_t new_base_xp = ppm_ppn2base( new_ppn );
	1742	memcpy( GET_PTR( new_base_xp ),
	1743	GET_PTR( old_base_xp ),
	1744	CONFIG_PPM_PAGE_SIZE );
[469]	1745
	1746	// decrement pending forks counter in page descriptor
	1747	hal_remote_atomic_add( forks_xp , -1 );
[408]	1748	}
[469]	1749	else // no pending fork => keep the existing page
[408]	1750	{
	1751	new_ppn = old_ppn;
	1752	}
[1]	1753
[567]	1754	// release lock protecting "forks" counter
	1755	remote_busylock_release( lock_xp );
[469]	1756
[408]	1757	// build new_attr : reset COW and set WRITABLE,
	1758	new_attr = (old_attr \| GPT_WRITABLE) & (~GPT_COW);
[407]	1759
[408]	1760	// update GPT[vpn] for all GPT copies
[433]	1761	vmm_global_update_pte( process, vpn, new_attr, new_ppn );
[407]	1762	}
[438]	1763	else //////////// page_fault request ///////////////////////////
[440]	1764	// get PTE from local GPT
[438]	1765	// allocate a physical page if it is a true page fault,
[440]	1766	// initialize it if type is FILE, CODE, or DATA,
[438]	1767	// register in reference GPT, but don't update GPT copies
[407]	1768	{
[440]	1769	// access local GPT to get current PTE
[438]	1770	hal_gpt_get_pte( &vmm->gpt , vpn , &old_attr , &old_ppn );
	1771
[408]	1772	if( (old_attr & GPT_MAPPED) == 0 ) // true page_fault => map it
[407]	1773	{
[1]	1774
[438]	1775	#if( DEBUG_VMM_GET_PTE & 1 )
[469]	1776	if( DEBUG_VMM_GET_PTE < cycle )
	1777	printk("\n[DBG] %s : thread %x in process %x handling page fault for vpn %x\n",
	1778	__FUNCTION__, this->trdid, process->pid, vpn );
[433]	1779	#endif
[440]	1780	// allocate new_ppn, and initialize the new page
[407]	1781	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
	1782	if( error )
	1783	{
	1784	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
	1785	__FUNCTION__ , process->pid , vpn );
[408]	1786	return -1;
[407]	1787	}
	1788
[408]	1789	// define new_attr from vseg flags
[407]	1790	new_attr = GPT_MAPPED \| GPT_SMALL;
	1791	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
	1792	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
	1793	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
	1794	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
	1795
[408]	1796	// register new PTE in reference GPT
	1797	// on demand policy => no update of GPT copies
	1798	error = hal_gpt_set_pte( &vmm->gpt,
	1799	vpn,
	1800	new_attr,
	1801	new_ppn );
[407]	1802	if( error )
	1803	{
	1804	printk("\n[ERROR] in %s : cannot update GPT / process = %x / vpn = %x\n",
	1805	__FUNCTION__ , process->pid , vpn );
[408]	1806	return -1;
[407]	1807	}
	1808	}
[408]	1809	else // mapped in reference GPT => get it
[1]	1810	{
[408]	1811	new_ppn = old_ppn;
[407]	1812	new_attr = old_attr;
[1]	1813	}
[407]	1814	}
[1]	1815
[438]	1816	#if DEBUG_VMM_GET_PTE
[433]	1817	cycle = (uint32_t)hal_get_cycles();
[469]	1818	if( DEBUG_VMM_GET_PTE < cycle )
	1819	printk("\n[DBG] %s : thread %x in process %x exit / vpn %x / ppn %x / attr %x / cycle %d\n",
	1820	__FUNCTION__, this->trdid, process->pid, vpn, new_ppn, new_attr, cycle );
[433]	1821	#endif
[406]	1822
[438]	1823	// return PPN and flags
[407]	1824	*ppn = new_ppn;
	1825	*attr = new_attr;
[1]	1826	return 0;
	1827
[313]	1828	} // end vmm_get_pte()
	1829
[1]	1830	///////////////////////////////////////////////////
	1831	error_t vmm_handle_page_fault( process_t * process,
[440]	1832	vpn_t vpn,
	1833	bool_t is_cow )
[1]	1834	{
	1835	uint32_t attr; // missing page attributes
[21]	1836	ppn_t ppn; // missing page PPN
[440]	1837	vseg_t * vseg; // vseg containing vpn
	1838	uint32_t type; // vseg type
	1839	cxy_t ref_cxy; // reference cluster for missing vpn
	1840	process_t * ref_ptr; // reference process for missing vpn
[407]	1841	error_t error;
[1]	1842
[440]	1843	thread_t * this = CURRENT_THREAD;
	1844
	1845	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[435]	1846	uint32_t cycle = (uint32_t)hal_get_cycles();
[469]	1847	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
[441]	1848	printk("\n[DBG] %s : thread %x in process %x enter for vpn %x / core[%x,%d] / cycle %d\n",
	1849	__FUNCTION__, this, process->pid, vpn, local_cxy, this->core->lid, cycle );
[435]	1850	#endif
	1851
[440]	1852	// get local vseg (access reference VSL if required)
	1853	error = vmm_get_vseg( process , vpn<<CONFIG_PPM_PAGE_SHIFT , &vseg );
[1]	1854
[440]	1855	if( error )
[1]	1856	{
[440]	1857	printk("\n[ERROR] in %s : vpn %x / process %x / thread %x / core[%x,%d] / cycle %d\n",
	1858	__FUNCTION__, vpn, process->pid, this->trdid, local_cxy, this->core->lid,
	1859	(uint32_t)hal_get_cycles() );
	1860	return error;
	1861	}
[407]	1862
[440]	1863	// get segment type
	1864	type = vseg->type;
[407]	1865
[567]	1866	// get relevant process cluster and local pointer
[440]	1867	// for private vsegs (CODE and DATA type),
[567]	1868	// the relevant process descriptor is local.
[440]	1869	if( (type == VSEG_TYPE_STACK) \|\| (type == VSEG_TYPE_CODE) )
	1870	{
	1871	ref_cxy = local_cxy;
	1872	ref_ptr = process;
[1]	1873	}
[440]	1874	else
[1]	1875	{
[440]	1876	ref_cxy = GET_CXY( process->ref_xp );
	1877	ref_ptr = GET_PTR( process->ref_xp );
[1]	1878	}
	1879
[440]	1880	// get missing PTE attributes and PPN
	1881	if( local_cxy != ref_cxy )
[407]	1882	{
[441]	1883
	1884	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[469]	1885	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
[441]	1886	printk("\n[DBG] %s : thread %x in process %x call RPC_VMM_GET_PTE\n",
	1887	__FUNCTION__, this, process->pid );
	1888	#endif
	1889
[407]	1890	rpc_vmm_get_pte_client( ref_cxy,
	1891	ref_ptr,
	1892	vpn,
[440]	1893	is_cow,
[407]	1894	&attr,
	1895	&ppn,
	1896	&error );
	1897
	1898	// get local VMM pointer
	1899	vmm_t * vmm = &process->vmm;
	1900
	1901	// update local GPT
[408]	1902	error \|= hal_gpt_set_pte( &vmm->gpt,
	1903	vpn,
	1904	attr,
	1905	ppn );
[407]	1906	}
	1907	else // local cluster is the reference cluster
	1908	{
	1909	error = vmm_get_pte( process,
	1910	vpn,
[440]	1911	is_cow,
[407]	1912	&attr,
	1913	&ppn );
	1914	}
	1915
[440]	1916	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[435]	1917	cycle = (uint32_t)hal_get_cycles();
[469]	1918	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
[441]	1919	printk("\n[DBG] %s : thread %x in process %x exit for vpn %x / core[%x,%d] / cycle %d\n",
	1920	__FUNCTION__, this, process->pid, vpn, local_cxy, this->core->lid, cycle );
[435]	1921	#endif
	1922
[407]	1923	return error;
	1924
[441]	1925	} // end vmm_handle_page_fault()
[407]	1926
[441]	1927
	1928
	1929
	1930
	1931
	1932
	1933
	1934
[440]	1935	/* deprecated April 2018 [AG]
	1936
	1937	error_t vmm_v2p_translate( process_t * process,
	1938	void * ptr,
	1939	paddr_t * paddr )
[1]	1940	{
[21]	1941	// access page table
[1]	1942	error_t error;
	1943	vpn_t vpn;
	1944	uint32_t attr;
	1945	ppn_t ppn;
	1946	uint32_t offset;
	1947
[23]	1948	vpn = (vpn_t)( (intptr_t)ptr >> CONFIG_PPM_PAGE_SHIFT );
	1949	offset = (uint32_t)( ((intptr_t)ptr) & CONFIG_PPM_PAGE_MASK );
[1]	1950
[440]	1951	if( local_cxy == GET_CXY( process->ref_xp) ) // local process is reference process
[1]	1952	{
[407]	1953	error = vmm_get_pte( process, vpn , false , &attr , &ppn );
[1]	1954	}
[50]	1955	else // calling process is not reference process
[1]	1956	{
	1957	cxy_t ref_cxy = GET_CXY( process->ref_xp );
[433]	1958	process_t * ref_ptr = GET_PTR( process->ref_xp );
[407]	1959	rpc_vmm_get_pte_client( ref_cxy , ref_ptr , vpn , false , &attr , &ppn , &error );
[1]	1960	}
	1961
[23]	1962	// set paddr
[1]	1963	*paddr = (((paddr_t)ppn) << CONFIG_PPM_PAGE_SHIFT) \| offset;
[21]	1964
[23]	1965	return error;
[313]	1966
[315]	1967	} // end vmm_v2p_translate()
[1]	1968
[440]	1969	*/

Note: See TracBrowser for help on using the repository browser.

Download in other formats: