Context Navigation

source: trunk/kernel/mm/vmm.c @ 633

Last change on this file since 633 was 633, checked in by alain, 5 years ago
cosmetic
File size: 81.4 KB

Rev	Line
[1]	1	/*
[611]	2	* vmm.c - virtual memory manager related operations definition.
[1]	3	*
	4	* Authors Ghassan Almaless (2008,2009,2010,2011, 2012)
	5	* Mohamed Lamine Karaoui (2015)
[625]	6	* Alain Greiner (2016,2017,2018,2019)
[21]	7	*
[1]	8	* Copyright (c) UPMC Sorbonne Universites
	9	*
	10	* This file is part of ALMOS-MKH.
	11	*
	12	* ALMOS-MKH is free software; you can redistribute it and/or modify it
	13	* under the terms of the GNU General Public License as published by
	14	* the Free Software Foundation; version 2.0 of the License.
	15	*
	16	* ALMOS-MKH is distributed in the hope that it will be useful, but
	17	* WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU General Public License
	22	* along with ALMOS-MKH; if not, write to the Free Software Foundation,
	23	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	24	*/
	25
[14]	26	#include <kernel_config.h>
[457]	27	#include <hal_kernel_types.h>
[1]	28	#include <hal_special.h>
	29	#include <hal_gpt.h>
[409]	30	#include <hal_vmm.h>
[577]	31	#include <hal_macros.h>
[1]	32	#include <printk.h>
[23]	33	#include <memcpy.h>
[567]	34	#include <remote_rwlock.h>
	35	#include <remote_queuelock.h>
[1]	36	#include <list.h>
[408]	37	#include <xlist.h>
[1]	38	#include <bits.h>
	39	#include <process.h>
	40	#include <thread.h>
	41	#include <vseg.h>
	42	#include <cluster.h>
	43	#include <scheduler.h>
	44	#include <vfs.h>
	45	#include <mapper.h>
	46	#include <page.h>
	47	#include <kmem.h>
	48	#include <vmm.h>
[585]	49	#include <hal_exception.h>
[1]	50
	51	//////////////////////////////////////////////////////////////////////////////////
	52	// Extern global variables
	53	//////////////////////////////////////////////////////////////////////////////////
	54
[567]	55	extern process_t process_zero; // allocated in cluster.c
[1]	56
[625]	57	////////////////////////////////////////////////////////////////////////////////////////////
	58	// This static function is called by the vmm_create_vseg() function, and implements
	59	// the VMM STACK specific allocator.
	60	////////////////////////////////////////////////////////////////////////////////////////////
	61	// @ vmm : [in] pointer on VMM.
	62	// @ ltid : [in] requested slot == local user thread identifier.
	63	// @ vpn_base : [out] first allocated page
	64	// @ vpn_size : [out] number of allocated pages
	65	////////////////////////////////////////////////////////////////////////////////////////////
	66	static void vmm_stack_alloc( vmm_t * vmm,
	67	ltid_t ltid,
	68	vpn_t * vpn_base,
	69	vpn_t * vpn_size )
[21]	70	{
[625]	71
	72	// check ltid argument
	73	assert( (ltid <= ((CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE) / CONFIG_VMM_STACK_SIZE)),
	74	"slot index %d too large for an user stack vseg", ltid );
	75
	76	// get stack allocator pointer
	77	stack_mgr_t * mgr = &vmm->stack_mgr;
	78
	79	// get lock on stack allocator
	80	busylock_acquire( &mgr->lock );
	81
	82	// check requested slot is available
	83	assert( (bitmap_state( &mgr->bitmap , ltid ) == false),
	84	"slot index %d already allocated", ltid );
	85
	86	// update bitmap
	87	bitmap_set( &mgr->bitmap , ltid );
	88
	89	// release lock on stack allocator
	90	busylock_release( &mgr->lock );
	91
	92	// returns vpn_base, vpn_size (first page non allocated)
	93	vpn_base = mgr->vpn_base + ltid CONFIG_VMM_STACK_SIZE + 1;
	94	*vpn_size = CONFIG_VMM_STACK_SIZE - 1;
	95
	96	} // end vmm_stack_alloc()
	97
	98	////////////////////////////////////////////////////////////////////////////////////////////
	99	// This static function is called by the vmm_remove_vseg() function, and implements
	100	// the VMM STACK specific desallocator.
	101	////////////////////////////////////////////////////////////////////////////////////////////
	102	// @ vmm : [in] pointer on VMM.
	103	// @ vseg : [in] pointer on released vseg.
	104	////////////////////////////////////////////////////////////////////////////////////////////
	105	static void vmm_stack_free( vmm_t * vmm,
	106	vseg_t * vseg )
	107	{
	108	// get stack allocator pointer
	109	stack_mgr_t * mgr = &vmm->stack_mgr;
	110
	111	// compute slot index
	112	uint32_t index = (vseg->vpn_base - 1 - mgr->vpn_base) / CONFIG_VMM_STACK_SIZE;
	113
	114	// check index
	115	assert( (index <= ((CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE) / CONFIG_VMM_STACK_SIZE)),
	116	"slot index %d too large for an user stack vseg", index );
	117
	118	// check released slot is allocated
	119	assert( (bitmap_state( &mgr->bitmap , index ) == true),
	120	"released slot index %d non allocated", index );
	121
	122	// get lock on stack allocator
	123	busylock_acquire( &mgr->lock );
	124
	125	// update stacks_bitmap
	126	bitmap_clear( &mgr->bitmap , index );
	127
	128	// release lock on stack allocator
	129	busylock_release( &mgr->lock );
	130
	131	} // end vmm_stack_free()
	132
	133	////////////////////////////////////////////////////////////////////////////////////////////
	134	// This static function is called by the vmm_create_vseg() function, and implements
	135	// the VMM MMAP specific allocator.
	136	////////////////////////////////////////////////////////////////////////////////////////////
	137	// @ vmm : [in] pointer on VMM.
	138	// @ npages : [in] requested number of pages.
	139	// @ vpn_base : [out] first allocated page.
	140	// @ vpn_size : [out] actual number of allocated pages.
	141	////////////////////////////////////////////////////////////////////////////////////////////
	142	static error_t vmm_mmap_alloc( vmm_t * vmm,
	143	vpn_t npages,
	144	vpn_t * vpn_base,
	145	vpn_t * vpn_size )
	146	{
	147	uint32_t order;
	148	xptr_t vseg_xp;
	149	vseg_t * vseg;
	150	vpn_t base;
	151	vpn_t size;
	152	vpn_t free;
	153
	154	#if DEBUG_VMM_MMAP_ALLOC
	155	thread_t * this = CURRENT_THREAD;
	156	uint32_t cycle = (uint32_t)hal_get_cycles();
	157	if( DEBUG_VMM_MMAP_ALLOC < cycle )
	158	printk("\n[%s] thread[%x,%x] enter / cycle %d\n",
	159	__FUNCTION__, this->process->pid, this->trdid, cycle );
	160	#endif
	161
	162	// number of allocated pages must be power of 2
	163	// compute actual size and order
	164	size = POW2_ROUNDUP( npages );
	165	order = bits_log2( size );
	166
	167	// get mmap allocator pointer
	168	mmap_mgr_t * mgr = &vmm->mmap_mgr;
	169
	170	// build extended pointer on root of zombi_list[order]
	171	xptr_t root_xp = XPTR( local_cxy , &mgr->zombi_list[order] );
	172
	173	// take lock protecting zombi_lists
	174	busylock_acquire( &mgr->lock );
	175
	176	// get vseg from zombi_list or from mmap zone
	177	if( xlist_is_empty( root_xp ) ) // from mmap zone
	178	{
	179	// check overflow
	180	free = mgr->first_free_vpn;
	181	if( (free + size) > mgr->vpn_size ) return -1;
	182
	183	// update MMAP allocator
	184	mgr->first_free_vpn += size;
	185
	186	// compute base
	187	base = free;
	188	}
	189	else // from zombi_list
	190	{
	191	// get pointer on zombi vseg from zombi_list
	192	vseg_xp = XLIST_FIRST( root_xp , vseg_t , xlist );
	193	vseg = GET_PTR( vseg_xp );
	194
	195	// remove vseg from free-list
	196	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
	197
	198	// compute base
	199	base = vseg->vpn_base;
	200	}
	201
	202	// release lock
	203	busylock_release( &mgr->lock );
	204
	205	#if DEBUG_VMM_MMAP_ALLOC
	206	cycle = (uint32_t)hal_get_cycles();
	207	if( DEBUG_VMM_DESTROY < cycle )
	208	printk("\n[%s] thread[%x,%x] exit / vpn_base %x / vpn_size %x / cycle %d\n",
	209	__FUNCTION__, this->process->pid, this->trdid, base, size, cycle );
	210	#endif
	211
	212	// returns vpn_base, vpn_size
	213	*vpn_base = base;
	214	*vpn_size = size;
	215	return 0;
	216
	217	} // end vmm_mmap_alloc()
	218
	219	////////////////////////////////////////////////////////////////////////////////////////////
	220	// This static function is called by the vmm_remove_vseg() function, and implements
	221	// the VMM MMAP specific desallocator.
	222	////////////////////////////////////////////////////////////////////////////////////////////
	223	// @ vmm : [in] pointer on VMM.
	224	// @ vseg : [in] pointer on released vseg.
	225	////////////////////////////////////////////////////////////////////////////////////////////
	226	static void vmm_mmap_free( vmm_t * vmm,
	227	vseg_t * vseg )
	228	{
	229	// get pointer on mmap allocator
	230	mmap_mgr_t * mgr = &vmm->mmap_mgr;
	231
	232	// compute zombi_list order
	233	uint32_t order = bits_log2( vseg->vpn_size );
	234
	235	// take lock protecting zombi lists
	236	busylock_acquire( &mgr->lock );
	237
	238	// update relevant zombi_list
	239	xlist_add_first( XPTR( local_cxy , &mgr->zombi_list[order] ),
	240	XPTR( local_cxy , &vseg->xlist ) );
	241
	242	// release lock
	243	busylock_release( &mgr->lock );
	244
	245	} // end of vmm_mmap_free()
	246
	247	////////////////////////////////////////////////////////////////////////////////////////////
	248	// This static function registers one vseg in the VSL of a local process descriptor.
	249	////////////////////////////////////////////////////////////////////////////////////////////
	250	// vmm : [in] pointer on VMM.
	251	// vseg : [in] pointer on vseg.
	252	////////////////////////////////////////////////////////////////////////////////////////////
	253	void vmm_attach_vseg_to_vsl( vmm_t * vmm,
	254	vseg_t * vseg )
	255	{
	256	// update vseg descriptor
	257	vseg->vmm = vmm;
	258
	259	// increment vsegs number
	260	vmm->vsegs_nr++;
	261
	262	// add vseg in vmm list
	263	xlist_add_last( XPTR( local_cxy , &vmm->vsegs_root ),
	264	XPTR( local_cxy , &vseg->xlist ) );
	265
	266	} // end vmm_attach_vseg_from_vsl()
	267
	268	////////////////////////////////////////////////////////////////////////////////////////////
	269	// This static function removes one vseg from the VSL of a local process descriptor.
	270	////////////////////////////////////////////////////////////////////////////////////////////
	271	// vmm : [in] pointer on VMM.
	272	// vseg : [in] pointer on vseg.
	273	////////////////////////////////////////////////////////////////////////////////////////////
	274	void vmm_detach_vseg_from_vsl( vmm_t * vmm,
	275	vseg_t * vseg )
	276	{
	277	// update vseg descriptor
	278	vseg->vmm = NULL;
	279
	280	// decrement vsegs number
	281	vmm->vsegs_nr--;
	282
	283	// remove vseg from VSL
	284	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
	285
	286	} // end vmm_detach_from_vsl()
	287
	288
	289
	290
	291	////////////////////////////////////////////
	292	error_t vmm_user_init( process_t * process )
	293	{
[1]	294	vseg_t * vseg_args;
	295	vseg_t * vseg_envs;
	296	intptr_t base;
	297	intptr_t size;
[614]	298	uint32_t i;
[1]	299
[625]	300	#if DEBUG_VMM_USER_INIT
[567]	301	thread_t * this = CURRENT_THREAD;
[433]	302	uint32_t cycle = (uint32_t)hal_get_cycles();
[625]	303	if( DEBUG_VMM_USER_INIT )
[614]	304	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
	305	__FUNCTION__ , this->process->pid, this->trdid, process->pid, local_cxy, cycle );
[433]	306	#endif
[204]	307
[1]	308	// get pointer on VMM
	309	vmm_t * vmm = &process->vmm;
	310
[625]	311	// check UTILS zone
[624]	312	assert( ((CONFIG_VMM_ARGS_SIZE + CONFIG_VMM_ENVS_SIZE) <=
	313	(CONFIG_VMM_ELF_BASE - CONFIG_VMM_UTILS_BASE)) ,
	314	"UTILS zone too small\n" );
[21]	315
[625]	316	// check STACK zone
[567]	317	assert( ((CONFIG_VMM_STACK_SIZE * CONFIG_THREADS_MAX_PER_CLUSTER) <=
	318	(CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE)) ,
	319	"STACK zone too small\n");
[1]	320
[625]	321	// register "args" vseg in VSL
[624]	322	base = CONFIG_VMM_UTILS_BASE << CONFIG_PPM_PAGE_SHIFT;
[1]	323	size = CONFIG_VMM_ARGS_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	324
[407]	325	vseg_args = vmm_create_vseg( process,
	326	VSEG_TYPE_DATA,
	327	base,
	328	size,
	329	0, // file_offset unused
	330	0, // file_size unused
	331	XPTR_NULL, // mapper_xp unused
	332	local_cxy );
[415]	333	if( vseg_args == NULL )
	334	{
	335	printk("\n[ERROR] in %s : cannot register args vseg\n", __FUNCTION__ );
	336	return -1;
	337	}
[204]	338
[406]	339	vmm->args_vpn_base = base;
[1]	340
[625]	341	// register "envs" vseg in VSL
[624]	342	base = (CONFIG_VMM_UTILS_BASE + CONFIG_VMM_ARGS_SIZE) << CONFIG_PPM_PAGE_SHIFT;
[1]	343	size = CONFIG_VMM_ENVS_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	344
[407]	345	vseg_envs = vmm_create_vseg( process,
	346	VSEG_TYPE_DATA,
	347	base,
	348	size,
	349	0, // file_offset unused
	350	0, // file_size unused
	351	XPTR_NULL, // mapper_xp unused
	352	local_cxy );
[415]	353	if( vseg_envs == NULL )
	354	{
	355	printk("\n[ERROR] in %s : cannot register envs vseg\n", __FUNCTION__ );
	356	return -1;
	357	}
[204]	358
[406]	359	vmm->envs_vpn_base = base;
[1]	360
	361	// initialize STACK allocator
	362	vmm->stack_mgr.bitmap = 0;
	363	vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
[567]	364	busylock_init( &vmm->stack_mgr.lock , LOCK_VMM_STACK );
[1]	365
	366	// initialize MMAP allocator
[407]	367	vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
	368	vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
	369	vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
[567]	370	busylock_init( &vmm->mmap_mgr.lock , LOCK_VMM_MMAP );
[625]	371	for( i = 0 ; i < 32 ; i++ )
	372	{
	373	xlist_root_init( XPTR( local_cxy , &vmm->mmap_mgr.zombi_list[i] ) );
	374	}
[1]	375
[21]	376	// initialize instrumentation counters
[409]	377	vmm->pgfault_nr = 0;
[1]	378
[124]	379	hal_fence();
[1]	380
[625]	381	#if DEBUG_VMM_USER_INIT
[433]	382	cycle = (uint32_t)hal_get_cycles();
[625]	383	if( DEBUG_VMM_USER_INIT )
[614]	384	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
	385	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
[433]	386	#endif
[204]	387
[415]	388	return 0;
	389
[625]	390	} // end vmm_user_init()
[204]	391
[611]	392	//////////////////////////////////////////
[625]	393	void vmm_user_reset( process_t * process )
[567]	394	{
[625]	395	xptr_t vseg_xp;
	396	vseg_t * vseg;
	397	vseg_type_t vseg_type;
[567]	398
[625]	399	#if DEBUG_VMM_USER_RESET
	400	uint32_t cycle = (uint32_t)hal_get_cycles();
	401	thread_t * this = CURRENT_THREAD;
	402	if( DEBUG_VMM_USER_RESET < cycle )
	403	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
	404	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
	405	#endif
[567]	406
[625]	407	#if (DEBUG_VMM_USER_RESET & 1 )
	408	if( DEBUG_VMM_USER_RESET < cycle )
	409	hal_vmm_display( process , true );
	410	#endif
[567]	411
[625]	412	// get pointer on local VMM
	413	vmm_t * vmm = &process->vmm;
[624]	414
[625]	415	// build extended pointer on VSL root and VSL lock
	416	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	417	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsl_lock );
[567]	418
[625]	419	// take the VSL lock
	420	remote_rwlock_wr_acquire( lock_xp );
[567]	421
[625]	422	// scan the VSL to delete all non kernel vsegs
	423	// (we don't use a FOREACH in case of item deletion)
	424	xptr_t iter_xp;
	425	xptr_t next_xp;
	426	for( iter_xp = hal_remote_l64( root_xp ) ;
	427	iter_xp != root_xp ;
	428	iter_xp = next_xp )
	429	{
	430	// save extended pointer on next item in xlist
	431	next_xp = hal_remote_l64( iter_xp );
[611]	432
[625]	433	// get pointers on current vseg in VSL
	434	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
	435	vseg = GET_PTR( vseg_xp );
	436	vseg_type = vseg->type;
[567]	437
[625]	438	#if( DEBUG_VMM_USER_RESET & 1 )
	439	if( DEBUG_VMM_USER_RESET < cycle )
	440	printk("\n[%s] found %s vseg / vpn_base %x / vpn_size %d\n",
	441	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	442	#endif
	443	// delete non kernel vseg
	444	if( (vseg_type != VSEG_TYPE_KCODE) &&
	445	(vseg_type != VSEG_TYPE_KDATA) &&
	446	(vseg_type != VSEG_TYPE_KDEV ) )
	447	{
	448	// remove vseg from VSL
	449	vmm_remove_vseg( process , vseg );
[567]	450
[625]	451	#if( DEBUG_VMM_USER_RESET & 1 )
	452	if( DEBUG_VMM_USER_RESET < cycle )
	453	printk("\n[%s] %s vseg deleted / vpn_base %x / vpn_size %d\n",
	454	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	455	#endif
	456	}
	457	else
	458	{
[567]	459
[625]	460	#if( DEBUG_VMM_USER_RESET & 1 )
	461	if( DEBUG_VMM_USER_RESET < cycle )
	462	printk("\n[%s] keep %s vseg / vpn_base %x / vpn_size %d\n",
	463	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	464	#endif
	465	}
	466	} // end loop on vsegs in VSL
[567]	467
[625]	468	// release the VSL lock
	469	remote_rwlock_wr_release( lock_xp );
[567]	470
[625]	471	// FIXME il faut gérer les process copies...
[611]	472
[625]	473	#if DEBUG_VMM_USER_RESET
	474	cycle = (uint32_t)hal_get_cycles();
	475	if( DEBUG_VMM_USER_RESET < cycle )
	476	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
	477	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy , cycle );
	478	#endif
[611]	479
[625]	480	} // end vmm_user_reset()
[611]	481
[595]	482	////////////////////////////////////////////////
[433]	483	void vmm_global_update_pte( process_t * process,
	484	vpn_t vpn,
	485	uint32_t attr,
	486	ppn_t ppn )
[23]	487	{
[408]	488	xlist_entry_t * process_root_ptr;
	489	xptr_t process_root_xp;
	490	xptr_t process_iter_xp;
[23]	491
[408]	492	xptr_t remote_process_xp;
	493	cxy_t remote_process_cxy;
	494	process_t * remote_process_ptr;
	495	xptr_t remote_gpt_xp;
[23]	496
[408]	497	pid_t pid;
	498	cxy_t owner_cxy;
	499	lpid_t owner_lpid;
[23]	500
[438]	501	#if DEBUG_VMM_UPDATE_PTE
[433]	502	uint32_t cycle = (uint32_t)hal_get_cycles();
[595]	503	thread_t * this = CURRENT_THREAD;
[438]	504	if( DEBUG_VMM_UPDATE_PTE < cycle )
[595]	505	printk("\n[%s] thread[%x,%x] enter for process %x / vpn %x / cycle %d\n",
	506	__FUNCTION__, this->process->pid, this->trdid, process->pid , vpn , cycle );
[433]	507	#endif
	508
[567]	509	// check cluster is reference
[585]	510	assert( (GET_CXY( process->ref_xp ) == local_cxy) , "not called in reference cluster\n");
[433]	511
[408]	512	// get extended pointer on root of process copies xlist in owner cluster
	513	pid = process->pid;
	514	owner_cxy = CXY_FROM_PID( pid );
	515	owner_lpid = LPID_FROM_PID( pid );
	516	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
	517	process_root_xp = XPTR( owner_cxy , process_root_ptr );
[23]	518
[408]	519	// loop on destination process copies
	520	XLIST_FOREACH( process_root_xp , process_iter_xp )
	521	{
	522	// get cluster and local pointer on remote process
	523	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
[433]	524	remote_process_ptr = GET_PTR( remote_process_xp );
[408]	525	remote_process_cxy = GET_CXY( remote_process_xp );
[407]	526
[438]	527	#if (DEBUG_VMM_UPDATE_PTE & 0x1)
	528	if( DEBUG_VMM_UPDATE_PTE < cycle )
[595]	529	printk("\n[%s] threadr[%x,%x] handling vpn %x for process %x in cluster %x\n",
	530	__FUNCTION__, this->process->pid, this->trdid, vpn, process->pid, remote_process_cxy );
[433]	531	#endif
	532
[408]	533	// get extended pointer on remote gpt
	534	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
	535
[433]	536	// update remote GPT
	537	hal_gpt_update_pte( remote_gpt_xp, vpn, attr, ppn );
[408]	538	}
	539
[438]	540	#if DEBUG_VMM_UPDATE_PTE
[433]	541	cycle = (uint32_t)hal_get_cycles();
[438]	542	if( DEBUG_VMM_UPDATE_PTE < cycle )
[595]	543	printk("\n[%s] thread[%x,%x] exit for process %x / vpn %x / cycle %d\n",
	544	__FUNCTION__, this->process->pid, this->trdid, process->pid , vpn , cycle );
[433]	545	#endif
	546
	547	} // end vmm_global_update_pte()
	548
[408]	549	///////////////////////////////////////
	550	void vmm_set_cow( process_t * process )
	551	{
	552	vmm_t * vmm;
	553
	554	xlist_entry_t * process_root_ptr;
	555	xptr_t process_root_xp;
	556	xptr_t process_iter_xp;
	557
	558	xptr_t remote_process_xp;
	559	cxy_t remote_process_cxy;
	560	process_t * remote_process_ptr;
	561	xptr_t remote_gpt_xp;
	562
	563	xptr_t vseg_root_xp;
	564	xptr_t vseg_iter_xp;
	565
	566	xptr_t vseg_xp;
	567	vseg_t * vseg;
	568
	569	pid_t pid;
	570	cxy_t owner_cxy;
	571	lpid_t owner_lpid;
	572
[438]	573	#if DEBUG_VMM_SET_COW
[595]	574	uint32_t cycle = (uint32_t)hal_get_cycles();
	575	thread_t * this = CURRENT_THREAD;
[438]	576	if( DEBUG_VMM_SET_COW < cycle )
[595]	577	printk("\n[%s] thread[%x,%x] enter for process %x / cycle %d\n",
	578	__FUNCTION__, this->process->pid, this->trdid, process->pid , cycle );
[433]	579	#endif
[408]	580
[567]	581	// check cluster is reference
	582	assert( (GET_CXY( process->ref_xp ) == local_cxy) ,
	583	"local cluster is not process reference cluster\n");
[408]	584
	585	// get pointer on reference VMM
	586	vmm = &process->vmm;
	587
	588	// get extended pointer on root of process copies xlist in owner cluster
	589	pid = process->pid;
	590	owner_cxy = CXY_FROM_PID( pid );
	591	owner_lpid = LPID_FROM_PID( pid );
	592	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
	593	process_root_xp = XPTR( owner_cxy , process_root_ptr );
	594
	595	// get extended pointer on root of vsegs xlist from reference VMM
	596	vseg_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	597
	598	// loop on destination process copies
	599	XLIST_FOREACH( process_root_xp , process_iter_xp )
	600	{
	601	// get cluster and local pointer on remote process
	602	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
[433]	603	remote_process_ptr = GET_PTR( remote_process_xp );
[408]	604	remote_process_cxy = GET_CXY( remote_process_xp );
	605
[595]	606	#if (DEBUG_VMM_SET_COW & 1)
[438]	607	if( DEBUG_VMM_SET_COW < cycle )
[595]	608	printk("\n[%s] thread[%x,%x] handling process %x in cluster %x\n",
	609	__FUNCTION__, this->process->pid, this->trdid, process->pid , remote_process_cxy );
[433]	610	#endif
[408]	611
	612	// get extended pointer on remote gpt
	613	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
	614
	615	// loop on vsegs in (local) reference process VSL
	616	XLIST_FOREACH( vseg_root_xp , vseg_iter_xp )
	617	{
	618	// get pointer on vseg
	619	vseg_xp = XLIST_ELEMENT( vseg_iter_xp , vseg_t , xlist );
[433]	620	vseg = GET_PTR( vseg_xp );
[408]	621
[567]	622	assert( (GET_CXY( vseg_xp ) == local_cxy) ,
	623	"all vsegs in reference VSL must be local\n" );
[408]	624
	625	// get vseg type, base and size
	626	uint32_t type = vseg->type;
	627	vpn_t vpn_base = vseg->vpn_base;
	628	vpn_t vpn_size = vseg->vpn_size;
	629
[595]	630	#if (DEBUG_VMM_SET_COW & 1)
[438]	631	if( DEBUG_VMM_SET_COW < cycle )
[595]	632	printk("\n[%s] thread[%x,%x] handling vseg %s / vpn_base = %x / vpn_size = %x\n",
	633	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), vpn_base, vpn_size );
[433]	634	#endif
	635	// only DATA, ANON and REMOTE vsegs
[408]	636	if( (type == VSEG_TYPE_DATA) \|\|
	637	(type == VSEG_TYPE_ANON) \|\|
	638	(type == VSEG_TYPE_REMOTE) )
	639	{
[433]	640	vpn_t vpn;
	641	uint32_t attr;
	642	ppn_t ppn;
	643	xptr_t page_xp;
	644	cxy_t page_cxy;
	645	page_t * page_ptr;
	646	xptr_t forks_xp;
[469]	647	xptr_t lock_xp;
[433]	648
	649	// update flags in remote GPT
	650	hal_gpt_set_cow( remote_gpt_xp,
	651	vpn_base,
	652	vpn_size );
	653
	654	// atomically increment pending forks counter in physical pages,
	655	// for all vseg pages that are mapped in reference cluster
	656	if( remote_process_cxy == local_cxy )
	657	{
	658	// scan all pages in vseg
	659	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
	660	{
	661	// get page attributes and PPN from reference GPT
[585]	662	hal_gpt_get_pte( remote_gpt_xp , vpn , &attr , &ppn );
[433]	663
	664	// atomically update pending forks counter if page is mapped
	665	if( attr & GPT_MAPPED )
	666	{
[469]	667	// get pointers and cluster on page descriptor
[433]	668	page_xp = ppm_ppn2page( ppn );
	669	page_cxy = GET_CXY( page_xp );
	670	page_ptr = GET_PTR( page_xp );
[469]	671
	672	// get extended pointers on "forks" and "lock"
[433]	673	forks_xp = XPTR( page_cxy , &page_ptr->forks );
[469]	674	lock_xp = XPTR( page_cxy , &page_ptr->lock );
	675
[567]	676	// take lock protecting "forks" counter
	677	remote_busylock_acquire( lock_xp );
	678
[469]	679	// increment "forks"
[433]	680	hal_remote_atomic_add( forks_xp , 1 );
[567]	681
	682	// release lock protecting "forks" counter
	683	remote_busylock_release( lock_xp );
[433]	684	}
	685	} // end loop on vpn
	686	} // end if local
	687	} // end if vseg type
	688	} // end loop on vsegs
[408]	689	} // end loop on process copies
	690
[438]	691	#if DEBUG_VMM_SET_COW
[433]	692	cycle = (uint32_t)hal_get_cycles();
[438]	693	if( DEBUG_VMM_SET_COW < cycle )
[595]	694	printk("\n[%s] thread[%x,%x] exit for process %x / cycle %d\n",
	695	__FUNCTION__, this->process->pid, this->trdid, process->pid , cycle );
[433]	696	#endif
[408]	697
	698	} // end vmm_set-cow()
	699
	700	/////////////////////////////////////////////////
	701	error_t vmm_fork_copy( process_t * child_process,
	702	xptr_t parent_process_xp )
	703	{
	704	error_t error;
	705	cxy_t parent_cxy;
	706	process_t * parent_process;
	707	vmm_t * parent_vmm;
	708	xptr_t parent_lock_xp;
	709	vmm_t * child_vmm;
	710	xptr_t iter_xp;
	711	xptr_t parent_vseg_xp;
	712	vseg_t * parent_vseg;
	713	vseg_t * child_vseg;
	714	uint32_t type;
	715	bool_t cow;
	716	vpn_t vpn;
	717	vpn_t vpn_base;
	718	vpn_t vpn_size;
[469]	719	xptr_t page_xp; // extended pointer on page descriptor
[408]	720	page_t * page_ptr;
	721	cxy_t page_cxy;
[469]	722	xptr_t forks_xp; // extended pointer on forks counter in page descriptor
[408]	723	xptr_t parent_root_xp;
	724	bool_t mapped;
	725	ppn_t ppn;
	726
[438]	727	#if DEBUG_VMM_FORK_COPY
[433]	728	uint32_t cycle = (uint32_t)hal_get_cycles();
[595]	729	thread_t * this = CURRENT_THREAD;
[438]	730	if( DEBUG_VMM_FORK_COPY < cycle )
[595]	731	printk("\n[%s] thread %x enter / cycle %d\n",
	732	__FUNCTION__ , this->process->pid, this->trdid, cycle );
[433]	733	#endif
[408]	734
	735	// get parent process cluster and local pointer
	736	parent_cxy = GET_CXY( parent_process_xp );
[433]	737	parent_process = GET_PTR( parent_process_xp );
[408]	738
	739	// get local pointers on parent and child VMM
	740	parent_vmm = &parent_process->vmm;
	741	child_vmm = &child_process->vmm;
	742
[629]	743	// initialize the lock protecting the child VSL
[625]	744	remote_rwlock_init( XPTR( local_cxy , &child_vmm->vsl_lock ) , LOCK_VMM_VSL );
[408]	745
	746	// initialize the child VSL as empty
	747	xlist_root_init( XPTR( local_cxy, &child_vmm->vsegs_root ) );
	748	child_vmm->vsegs_nr = 0;
	749
[625]	750	// create an empty child GPT
[408]	751	error = hal_gpt_create( &child_vmm->gpt );
[407]	752	if( error )
	753	{
[408]	754	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
	755	return -1;
[407]	756	}
	757
[625]	758	// build extended pointer on parent VSL root and lock
[408]	759	parent_root_xp = XPTR( parent_cxy , &parent_vmm->vsegs_root );
[625]	760	parent_lock_xp = XPTR( parent_cxy , &parent_vmm->vsl_lock );
[408]	761
[567]	762	// take the lock protecting the parent VSL in read mode
	763	remote_rwlock_rd_acquire( parent_lock_xp );
[415]	764
[408]	765	// loop on parent VSL xlist
	766	XLIST_FOREACH( parent_root_xp , iter_xp )
[23]	767	{
[625]	768	// get pointers on current parent vseg
[408]	769	parent_vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	770	parent_vseg = GET_PTR( parent_vseg_xp );
[23]	771
[408]	772	// get vseg type
[567]	773	type = hal_remote_l32( XPTR( parent_cxy , &parent_vseg->type ) );
[408]	774
[438]	775	#if DEBUG_VMM_FORK_COPY
[433]	776	cycle = (uint32_t)hal_get_cycles();
[438]	777	if( DEBUG_VMM_FORK_COPY < cycle )
[595]	778	printk("\n[%s] thread[%x,%x] found parent vseg %s / vpn_base = %x / cycle %d\n",
	779	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
[567]	780	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
[433]	781	#endif
[23]	782
[623]	783	// all parent vsegs - but STACK and kernel vsegs - must be copied in child VSL
	784	if( (type != VSEG_TYPE_STACK) && (type != VSEG_TYPE_KCODE) &&
	785	(type != VSEG_TYPE_KDATA) && (type != VSEG_TYPE_KDEV) )
[23]	786	{
[408]	787	// allocate memory for a new child vseg
	788	child_vseg = vseg_alloc();
	789	if( child_vseg == NULL ) // release all allocated vsegs
[23]	790	{
[408]	791	vmm_destroy( child_process );
	792	printk("\n[ERROR] in %s : cannot create vseg for child\n", __FUNCTION__ );
	793	return -1;
[23]	794	}
	795
[408]	796	// copy parent vseg to child vseg
	797	vseg_init_from_ref( child_vseg , parent_vseg_xp );
[23]	798
[625]	799	// build extended pointer on VSL lock
	800	xptr_t lock_xp = XPTR( local_cxy , &child_vmm->vsl_lock );
	801
	802	// take the VSL lock in write mode
	803	remote_rwlock_wr_acquire( lock_xp );
	804
[408]	805	// register child vseg in child VSL
[611]	806	vmm_attach_vseg_to_vsl( child_vmm , child_vseg );
[407]	807
[625]	808	// release the VSL lock
	809	remote_rwlock_wr_release( lock_xp );
	810
[438]	811	#if DEBUG_VMM_FORK_COPY
[433]	812	cycle = (uint32_t)hal_get_cycles();
[438]	813	if( DEBUG_VMM_FORK_COPY < cycle )
[595]	814	printk("\n[%s] thread[%x,%x] copied vseg %s / vpn_base = %x to child VSL / cycle %d\n",
	815	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
[567]	816	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
[433]	817	#endif
[625]	818	// copy DATA, ANON, REMOTE, FILE parent GPT entries to child GPT
[408]	819	if( type != VSEG_TYPE_CODE )
	820	{
[625]	821	// activate the COW for DATA, ANON, REMOTE vsegs only
[408]	822	cow = ( type != VSEG_TYPE_FILE );
[23]	823
[408]	824	vpn_base = child_vseg->vpn_base;
	825	vpn_size = child_vseg->vpn_size;
[23]	826
[408]	827	// scan pages in parent vseg
	828	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
	829	{
	830	error = hal_gpt_pte_copy( &child_vmm->gpt,
[625]	831	vpn,
[408]	832	XPTR( parent_cxy , &parent_vmm->gpt ),
	833	vpn,
	834	cow,
	835	&ppn,
	836	&mapped );
	837	if( error )
	838	{
	839	vmm_destroy( child_process );
	840	printk("\n[ERROR] in %s : cannot copy GPT\n", __FUNCTION__ );
	841	return -1;
	842	}
	843
[433]	844	// increment pending forks counter in page if mapped
[408]	845	if( mapped )
	846	{
[469]	847	// get pointers and cluster on page descriptor
	848	page_xp = ppm_ppn2page( ppn );
[408]	849	page_cxy = GET_CXY( page_xp );
[433]	850	page_ptr = GET_PTR( page_xp );
[408]	851
[469]	852	// get extended pointers on "forks" and "lock"
	853	forks_xp = XPTR( page_cxy , &page_ptr->forks );
	854	lock_xp = XPTR( page_cxy , &page_ptr->lock );
	855
[567]	856	// get lock protecting "forks" counter
	857	remote_busylock_acquire( lock_xp );
	858
[469]	859	// increment "forks"
	860	hal_remote_atomic_add( forks_xp , 1 );
	861
[567]	862	// release lock protecting "forks" counter
	863	remote_busylock_release( lock_xp );
	864
[438]	865	#if DEBUG_VMM_FORK_COPY
[433]	866	cycle = (uint32_t)hal_get_cycles();
[438]	867	if( DEBUG_VMM_FORK_COPY < cycle )
[595]	868	printk("\n[%s] thread[%x,%x] copied vpn %x to child GPT / cycle %d\n",
	869	__FUNCTION__ , this->process->pid, this->trdid , vpn , cycle );
[433]	870	#endif
[408]	871	}
	872	}
	873	} // end if no code & no stack
	874	} // end if no stack
	875	} // end loop on vsegs
	876
[567]	877	// release the parent VSL lock in read mode
	878	remote_rwlock_rd_release( parent_lock_xp );
[408]	879
[623]	880	// update child VMM with kernel vsegs
	881	error = hal_vmm_kernel_update( child_process );
[415]	882
	883	if( error )
	884	{
[623]	885	printk("\n[ERROR] in %s : cannot update child VMM\n", __FUNCTION__ );
[415]	886	return -1;
	887	}
	888
[408]	889	// initialize the child VMM STACK allocator
	890	child_vmm->stack_mgr.bitmap = 0;
	891	child_vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
	892
	893	// initialize the child VMM MMAP allocator
[23]	894	uint32_t i;
[408]	895	child_vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
	896	child_vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
	897	child_vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
[625]	898	for( i = 0 ; i < 32 ; i++ )
	899	{
	900	xlist_root_init( XPTR( local_cxy , &child_vmm->mmap_mgr.zombi_list[i] ) );
	901	}
[23]	902
[178]	903	// initialize instrumentation counters
[408]	904	child_vmm->pgfault_nr = 0;
[23]	905
[408]	906	// copy base addresses from parent VMM to child VMM
	907	child_vmm->args_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->args_vpn_base));
	908	child_vmm->envs_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->envs_vpn_base));
	909	child_vmm->heap_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->heap_vpn_base));
	910	child_vmm->code_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->code_vpn_base));
	911	child_vmm->data_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->data_vpn_base));
[23]	912
[408]	913	child_vmm->entry_point = (intptr_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->entry_point));
[23]	914
[124]	915	hal_fence();
[23]	916
[438]	917	#if DEBUG_VMM_FORK_COPY
[433]	918	cycle = (uint32_t)hal_get_cycles();
[438]	919	if( DEBUG_VMM_FORK_COPY < cycle )
[595]	920	printk("\n[%s] thread[%x,%x] exit successfully / cycle %d\n",
	921	__FUNCTION__ , this->process->pid, this->trdid , cycle );
[433]	922	#endif
	923
[23]	924	return 0;
	925
[408]	926	} // vmm_fork_copy()
[204]	927
[1]	928	///////////////////////////////////////
	929	void vmm_destroy( process_t * process )
	930	{
[408]	931	xptr_t vseg_xp;
[1]	932	vseg_t * vseg;
	933
[438]	934	#if DEBUG_VMM_DESTROY
[433]	935	uint32_t cycle = (uint32_t)hal_get_cycles();
[595]	936	thread_t * this = CURRENT_THREAD;
[438]	937	if( DEBUG_VMM_DESTROY < cycle )
[595]	938	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
	939	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
[433]	940	#endif
[416]	941
[438]	942	#if (DEBUG_VMM_DESTROY & 1 )
[443]	943	if( DEBUG_VMM_DESTROY < cycle )
[624]	944	hal_vmm_display( process , true );
[437]	945	#endif
	946
[433]	947	// get pointer on local VMM
[1]	948	vmm_t * vmm = &process->vmm;
	949
[625]	950	// build extended pointer on VSL root, VSL lock and GPT lock
	951	xptr_t vsl_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	952	xptr_t vsl_lock_xp = XPTR( local_cxy , &vmm->vsl_lock );
[408]	953
[625]	954	// take the VSL lock
	955	remote_rwlock_wr_acquire( vsl_lock_xp );
	956
[611]	957	// scan the VSL to delete all registered vsegs
[625]	958	// (we don't use a FOREACH in case of item deletion)
	959	xptr_t iter_xp;
	960	xptr_t next_xp;
	961	for( iter_xp = hal_remote_l64( vsl_root_xp ) ;
	962	iter_xp != vsl_root_xp ;
	963	iter_xp = next_xp )
[1]	964	{
[625]	965	// save extended pointer on next item in xlist
	966	next_xp = hal_remote_l64( iter_xp );
[409]	967
[625]	968	// get pointers on current vseg in VSL
	969	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
	970	vseg = GET_PTR( vseg_xp );
	971
[611]	972	// delete vseg and release physical pages
[625]	973	vmm_remove_vseg( process , vseg );
[409]	974
[443]	975	#if( DEBUG_VMM_DESTROY & 1 )
	976	if( DEBUG_VMM_DESTROY < cycle )
[611]	977	printk("\n[%s] %s vseg deleted / vpn_base %x / vpn_size %d\n",
[443]	978	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	979	#endif
	980
[1]	981	}
	982
[625]	983	// release the VSL lock
	984	remote_rwlock_wr_release( vsl_lock_xp );
	985
	986	// remove all registered MMAP vsegs
	987	// from zombi_lists in MMAP allocator
[1]	988	uint32_t i;
	989	for( i = 0 ; i<32 ; i++ )
	990	{
[625]	991	// build extended pointer on zombi_list[i]
	992	xptr_t root_xp = XPTR( local_cxy , &vmm->mmap_mgr.zombi_list[i] );
	993
	994	// scan zombi_list[i]
	995	while( !xlist_is_empty( root_xp ) )
[1]	996	{
[625]	997	vseg_xp = XLIST_FIRST( root_xp , vseg_t , xlist );
	998	vseg = GET_PTR( vseg_xp );
[443]	999
	1000	#if( DEBUG_VMM_DESTROY & 1 )
	1001	if( DEBUG_VMM_DESTROY < cycle )
[595]	1002	printk("\n[%s] found zombi vseg / vpn_base %x / vpn_size %d\n",
[443]	1003	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	1004	#endif
[611]	1005	// clean vseg descriptor
	1006	vseg->vmm = NULL;
	1007
[625]	1008	// remove vseg from zombi_list
[611]	1009	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
	1010
	1011	// release vseg descriptor
[1]	1012	vseg_free( vseg );
[443]	1013
	1014	#if( DEBUG_VMM_DESTROY & 1 )
	1015	if( DEBUG_VMM_DESTROY < cycle )
[595]	1016	printk("\n[%s] zombi vseg released / vpn_base %x / vpn_size %d\n",
[443]	1017	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	1018	#endif
[1]	1019	}
	1020	}
	1021
[409]	1022	// release memory allocated to the GPT itself
[1]	1023	hal_gpt_destroy( &vmm->gpt );
	1024
[438]	1025	#if DEBUG_VMM_DESTROY
[433]	1026	cycle = (uint32_t)hal_get_cycles();
[438]	1027	if( DEBUG_VMM_DESTROY < cycle )
[595]	1028	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
	1029	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy , cycle );
[433]	1030	#endif
[416]	1031
[204]	1032	} // end vmm_destroy()
	1033
[1]	1034	/////////////////////////////////////////////////
	1035	vseg_t * vmm_check_conflict( process_t * process,
[21]	1036	vpn_t vpn_base,
[1]	1037	vpn_t vpn_size )
	1038	{
	1039	vmm_t * vmm = &process->vmm;
[408]	1040
	1041	// scan the VSL
[1]	1042	vseg_t * vseg;
[408]	1043	xptr_t iter_xp;
	1044	xptr_t vseg_xp;
	1045	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
[1]	1046
[408]	1047	XLIST_FOREACH( root_xp , iter_xp )
[1]	1048	{
[408]	1049	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	1050	vseg = GET_PTR( vseg_xp );
[204]	1051
[21]	1052	if( ((vpn_base + vpn_size) > vseg->vpn_base) &&
	1053	(vpn_base < (vseg->vpn_base + vseg->vpn_size)) ) return vseg;
[1]	1054	}
	1055	return NULL;
	1056
[204]	1057	} // end vmm_check_conflict()
	1058
[1]	1059
	1060
[407]	1061	////////////////////////////////////////////////
	1062	vseg_t * vmm_create_vseg( process_t * process,
	1063	vseg_type_t type,
	1064	intptr_t base,
	1065	uint32_t size,
	1066	uint32_t file_offset,
	1067	uint32_t file_size,
	1068	xptr_t mapper_xp,
	1069	cxy_t cxy )
[1]	1070	{
	1071	vseg_t * vseg; // created vseg pointer
[204]	1072	vpn_t vpn_base; // first page index
[595]	1073	vpn_t vpn_size; // number of pages covered by vseg
[1]	1074	error_t error;
	1075
[438]	1076	#if DEBUG_VMM_CREATE_VSEG
[595]	1077	thread_t * this = CURRENT_THREAD;
	1078	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	1079	if( DEBUG_VMM_CREATE_VSEG < cycle )
[614]	1080	printk("\n[%s] thread[%x,%x] enter for process %x / %s / cxy %x / cycle %d\n",
	1081	__FUNCTION__, this->process->pid, this->trdid, process->pid, vseg_type_str(type), cxy, cycle );
[433]	1082	#endif
[21]	1083
[407]	1084	// get pointer on VMM
	1085	vmm_t * vmm = &process->vmm;
[21]	1086
[204]	1087	// compute base, size, vpn_base, vpn_size, depending on vseg type
[407]	1088	// we use the VMM specific allocators for "stack", "file", "anon", & "remote" vsegs
[595]	1089
[1]	1090	if( type == VSEG_TYPE_STACK )
	1091	{
	1092	// get vpn_base and vpn_size from STACK allocator
[625]	1093	vmm_stack_alloc( vmm , base , &vpn_base , &vpn_size );
[1]	1094
	1095	// compute vseg base and size from vpn_base and vpn_size
	1096	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
	1097	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
	1098	}
[595]	1099	else if( type == VSEG_TYPE_FILE )
	1100	{
	1101	// compute page index (in mapper) for first byte
	1102	vpn_t vpn_min = file_offset >> CONFIG_PPM_PAGE_SHIFT;
	1103
	1104	// compute page index (in mapper) for last byte
	1105	vpn_t vpn_max = (file_offset + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1106
	1107	// compute offset in first page
	1108	uint32_t offset = file_offset & CONFIG_PPM_PAGE_MASK;
	1109
	1110	// compute number of pages required in virtual space
	1111	vpn_t npages = vpn_max - vpn_min + 1;
	1112
	1113	// get vpn_base and vpn_size from MMAP allocator
	1114	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
	1115	if( error )
	1116	{
	1117	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
	1118	__FUNCTION__ , process->pid , local_cxy );
	1119	return NULL;
	1120	}
	1121
	1122	// set the vseg base (not always aligned for FILE)
	1123	base = (vpn_base << CONFIG_PPM_PAGE_SHIFT) + offset;
	1124	}
[21]	1125	else if( (type == VSEG_TYPE_ANON) \|\|
[1]	1126	(type == VSEG_TYPE_REMOTE) )
	1127	{
[595]	1128	// compute number of required pages in virtual space
	1129	vpn_t npages = size >> CONFIG_PPM_PAGE_SHIFT;
	1130	if( size & CONFIG_PPM_PAGE_MASK) npages++;
	1131
[1]	1132	// get vpn_base and vpn_size from MMAP allocator
	1133	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
	1134	if( error )
	1135	{
	1136	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
	1137	__FUNCTION__ , process->pid , local_cxy );
	1138	return NULL;
	1139	}
	1140
[595]	1141	// set vseg base (always aligned for ANON or REMOTE)
[1]	1142	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
	1143	}
[623]	1144	else // VSEG_TYPE_DATA, VSEG_TYPE_CODE or KERNEL vseg
[1]	1145	{
[204]	1146	uint32_t vpn_min = base >> CONFIG_PPM_PAGE_SHIFT;
	1147	uint32_t vpn_max = (base + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1148
	1149	vpn_base = vpn_min;
	1150	vpn_size = vpn_max - vpn_min + 1;
[1]	1151	}
	1152
	1153	// check collisions
	1154	vseg = vmm_check_conflict( process , vpn_base , vpn_size );
[624]	1155
[1]	1156	if( vseg != NULL )
	1157	{
[614]	1158	printk("\n[ERROR] in %s for process %x : new vseg [vpn_base %x / vpn_size %x]\n"
	1159	" overlap existing vseg [vpn_base %x / vpn_size %x]\n",
[407]	1160	__FUNCTION__ , process->pid, vpn_base, vpn_size, vseg->vpn_base, vseg->vpn_size );
[1]	1161	return NULL;
	1162	}
	1163
	1164	// allocate physical memory for vseg descriptor
	1165	vseg = vseg_alloc();
	1166	if( vseg == NULL )
	1167	{
	1168	printk("\n[ERROR] in %s for process %x : cannot allocate memory for vseg\n",
[407]	1169	__FUNCTION__ , process->pid );
[1]	1170	return NULL;
	1171	}
	1172
[614]	1173	#if DEBUG_VMM_CREATE_VSEG
	1174	if( DEBUG_VMM_CREATE_VSEG < cycle )
	1175	printk("\n[%s] thread[%x,%x] : base %x / size %x / vpn_base %x / vpn_size %x\n",
	1176	__FUNCTION__, this->process->pid, this->trdid, base, size, vpn_base, vpn_size );
	1177	#endif
	1178
[1]	1179	// initialize vseg descriptor
[407]	1180	vseg_init( vseg,
	1181	type,
	1182	base,
	1183	size,
	1184	vpn_base,
	1185	vpn_size,
	1186	file_offset,
	1187	file_size,
	1188	mapper_xp,
	1189	cxy );
[1]	1190
[625]	1191	// build extended pointer on VSL lock
	1192	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsl_lock );
	1193
	1194	// take the VSL lock in write mode
	1195	remote_rwlock_wr_acquire( lock_xp );
	1196
[408]	1197	// attach vseg to VSL
[611]	1198	vmm_attach_vseg_to_vsl( vmm , vseg );
[1]	1199
[625]	1200	// release the VSL lock
	1201	remote_rwlock_wr_release( lock_xp );
	1202
[438]	1203	#if DEBUG_VMM_CREATE_VSEG
[433]	1204	cycle = (uint32_t)hal_get_cycles();
[438]	1205	if( DEBUG_VMM_CREATE_VSEG < cycle )
[595]	1206	printk("\n[%s] thread[%x,%x] exit / %s / cxy %x / cycle %d\n",
	1207	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), cxy, cycle );
[433]	1208	#endif
[21]	1209
[1]	1210	return vseg;
	1211
[406]	1212	} // vmm_create_vseg()
	1213
[625]	1214
	1215	//////////////////////////////////////////
	1216	void vmm_remove_vseg( process_t * process,
	1217	vseg_t * vseg )
[1]	1218	{
[625]	1219	vmm_t * vmm; // local pointer on process VMM
[629]	1220	xptr_t gpt_xp; // extended pointer on GPT
[625]	1221	bool_t is_ref; // local process is reference process
	1222	uint32_t vseg_type; // vseg type
[21]	1223	vpn_t vpn; // VPN of current PTE
	1224	vpn_t vpn_min; // VPN of first PTE
[1]	1225	vpn_t vpn_max; // VPN of last PTE (excluded)
[409]	1226	ppn_t ppn; // current PTE ppn value
	1227	uint32_t attr; // current PTE attributes
	1228	xptr_t page_xp; // extended pointer on page descriptor
	1229	cxy_t page_cxy; // page descriptor cluster
	1230	page_t * page_ptr; // page descriptor pointer
[625]	1231	xptr_t count_xp; // extended pointer on page refcount
[1]	1232
[625]	1233	// check arguments
	1234	assert( (process != NULL), "process argument is NULL" );
	1235	assert( (vseg != NULL), "vseg argument is NULL" );
[409]	1236
[625]	1237	// compute is_ref
	1238	is_ref = (GET_CXY( process->ref_xp ) == local_cxy);
[1]	1239
[625]	1240	// get pointers on local process VMM
[611]	1241	vmm = &process->vmm;
	1242
[629]	1243	// build extended pointer on GPT
	1244	gpt_xp = XPTR( local_cxy , &vmm->gpt );
	1245
[623]	1246	// get relevant vseg infos
[624]	1247	vseg_type = vseg->type;
	1248	vpn_min = vseg->vpn_base;
	1249	vpn_max = vpn_min + vseg->vpn_size;
[623]	1250
[625]	1251	#if DEBUG_VMM_REMOVE_VSEG
	1252	uint32_t cycle = (uint32_t)hal_get_cycles();
	1253	thread_t * this = CURRENT_THREAD;
	1254	if( DEBUG_VMM_REMOVE_VSEG < cycle )
	1255	printk("\n[%s] thread[%x,%x] enter / process %x / %s / base %x / cycle %d\n",
	1256	__FUNCTION__, this->process->pid, this->trdid,
	1257	process->pid, vseg_type_str(vseg->type), vseg->min, cycle );
	1258	#endif
	1259
	1260	// loop on PTEs in GPT
[1]	1261	for( vpn = vpn_min ; vpn < vpn_max ; vpn++ )
	1262	{
[625]	1263	// get ppn and attr
[629]	1264	hal_gpt_get_pte( gpt_xp , vpn , &attr , &ppn );
[409]	1265
[625]	1266	if( attr & GPT_MAPPED ) // PTE is mapped
[409]	1267	{
[437]	1268
[625]	1269	#if( DEBUG_VMM_REMOVE_VSEG & 1 )
	1270	if( DEBUG_VMM_REMOVE_VSEG < cycle )
	1271	printk("- unmap vpn %x / ppn %x / %s" , vpn , ppn, vseg_type_str(vseg_type) );
[437]	1272	#endif
[585]	1273	// unmap GPT entry in local GPT
[629]	1274	hal_gpt_reset_pte( gpt_xp , vpn );
[409]	1275
[625]	1276	// get pointers on physical page descriptor
	1277	page_xp = ppm_ppn2page( ppn );
	1278	page_cxy = GET_CXY( page_xp );
	1279	page_ptr = GET_PTR( page_xp );
[409]	1280
[625]	1281	// decrement page refcount
	1282	count_xp = XPTR( page_cxy , &page_ptr->refcount );
[633]	1283	hal_remote_atomic_add( count_xp , -1 );
[624]	1284
[625]	1285	// compute the ppn_release condition depending on vseg type
	1286	bool_t ppn_release;
	1287	if( (vseg_type == VSEG_TYPE_FILE) \|\|
	1288	(vseg_type == VSEG_TYPE_KCODE) \|\|
	1289	(vseg_type == VSEG_TYPE_KDATA) \|\|
	1290	(vseg_type == VSEG_TYPE_KDEV) )
	1291	{
	1292	// no physical page release for FILE and KERNEL
	1293	ppn_release = false;
	1294	}
	1295	else if( (vseg_type == VSEG_TYPE_CODE) \|\|
	1296	(vseg_type == VSEG_TYPE_STACK) )
	1297	{
	1298	// always release physical page for private vsegs
	1299	ppn_release = true;
	1300	}
	1301	else if( (vseg_type == VSEG_TYPE_ANON) \|\|
	1302	(vseg_type == VSEG_TYPE_REMOTE) )
	1303	{
	1304	// release physical page if reference cluster
	1305	ppn_release = is_ref;
	1306	}
	1307	else if( is_ref ) // vseg_type == DATA in reference cluster
	1308	{
	1309	// get extended pointers on forks and lock field in page descriptor
	1310	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
	1311	xptr_t lock_xp = XPTR( page_cxy , &page_ptr->lock );
[433]	1312
[625]	1313	// take lock protecting "forks" counter
[621]	1314	remote_busylock_acquire( lock_xp );
[623]	1315
[625]	1316	// get number of pending forks from page descriptor
	1317	uint32_t forks = hal_remote_l32( forks_xp );
[623]	1318
[625]	1319	// decrement pending forks counter if required
	1320	if( forks ) hal_remote_atomic_add( forks_xp , -1 );
[624]	1321
[625]	1322	// release lock protecting "forks" counter
	1323	remote_busylock_release( lock_xp );
[624]	1324
[625]	1325	// release physical page if forks == 0
	1326	ppn_release = (forks == 0);
	1327	}
	1328	else // vseg_type == DATA not in reference cluster
	1329	{
	1330	// no physical page release if not in reference cluster
	1331	ppn_release = false;
	1332	}
[611]	1333
[625]	1334	// release physical page to relevant kmem when required
[632]	1335	if( ppn_release ) ppm_remote_free_pages( page_cxy , page_ptr );
[623]	1336
[625]	1337	#if( DEBUG_VMM_REMOVE_VSEG & 1 )
	1338	if( DEBUG_VMM_REMOVE_VSEG < cycle )
	1339	{
	1340	if( ppn_release ) printk(" / released to kmem\n" );
	1341	else printk("\n");
	1342	}
	1343	#endif
[409]	1344	}
[1]	1345	}
[433]	1346
[625]	1347	// remove vseg from VSL
[611]	1348	vmm_detach_vseg_from_vsl( vmm , vseg );
	1349
[625]	1350	// release vseg descriptor depending on vseg type
	1351	if( vseg_type == VSEG_TYPE_STACK )
	1352	{
	1353	// release slot to local stack allocator
	1354	vmm_stack_free( vmm , vseg );
	1355
	1356	// release vseg descriptor to local kmem
	1357	vseg_free( vseg );
	1358	}
	1359	else if( (vseg_type == VSEG_TYPE_ANON) \|\|
	1360	(vseg_type == VSEG_TYPE_FILE) \|\|
	1361	(vseg_type == VSEG_TYPE_REMOTE) )
	1362	{
	1363	// release vseg to local mmap allocator
	1364	vmm_mmap_free( vmm , vseg );
	1365	}
	1366	else
	1367	{
	1368	// release vseg descriptor to local kmem
	1369	vseg_free( vseg );
	1370	}
	1371
	1372	#if DEBUG_VMM_REMOVE_VSEG
[433]	1373	cycle = (uint32_t)hal_get_cycles();
[625]	1374	if( DEBUG_VMM_REMOVE_VSEG < cycle )
	1375	printk("[%s] thread[%x,%x] exit / process %x / %s / base %x / cycle %d\n",
	1376	__FUNCTION__, this->process->pid, this->trdid,
	1377	process->pid, vseg_type_str(vseg->type), vseg->min, cycle );
[433]	1378	#endif
	1379
[625]	1380	} // end vmm_remove_vseg()
[1]	1381
[625]	1382
	1383	///////////////////////////////////
	1384	void vmm_delete_vseg( pid_t pid,
	1385	intptr_t vaddr )
	1386	{
	1387	process_t * process; // local pointer on local process
	1388	vseg_t * vseg; // local pointer on local vseg containing vaddr
	1389
	1390	// get local pointer on local process descriptor
	1391	process = cluster_get_local_process_from_pid( pid );
	1392
	1393	if( process == NULL )
	1394	{
	1395	printk("\n[WARNING] in %s : cannot get local process descriptor\n",
	1396	__FUNCTION__ );
	1397	return;
	1398	}
	1399
	1400	// get local pointer on local vseg containing vaddr
	1401	vseg = vmm_vseg_from_vaddr( &process->vmm , vaddr );
	1402
	1403	if( vseg == NULL )
	1404	{
	1405	printk("\n[WARNING] in %s : cannot get vseg descriptor\n",
	1406	__FUNCTION__ );
	1407	return;
	1408	}
	1409
	1410	// call relevant function
	1411	vmm_remove_vseg( process , vseg );
	1412
	1413	} // end vmm_delete_vseg
	1414
	1415
[611]	1416	/////////////////////////////////////////////
	1417	vseg_t * vmm_vseg_from_vaddr( vmm_t * vmm,
	1418	intptr_t vaddr )
[406]	1419	{
[408]	1420	xptr_t vseg_xp;
	1421	vseg_t * vseg;
[625]	1422	xptr_t iter_xp;
[406]	1423
[408]	1424	// get extended pointers on VSL lock and root
[625]	1425	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsl_lock );
[408]	1426	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
[406]	1427
[408]	1428	// get lock protecting the VSL
[567]	1429	remote_rwlock_rd_acquire( lock_xp );
[408]	1430
	1431	// scan the list of vsegs in VSL
	1432	XLIST_FOREACH( root_xp , iter_xp )
[406]	1433	{
[625]	1434	// get pointers on vseg
[408]	1435	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	1436	vseg = GET_PTR( vseg_xp );
[595]	1437
[625]	1438	// return success when match
[408]	1439	if( (vaddr >= vseg->min) && (vaddr < vseg->max) )
[595]	1440	{
[408]	1441	// return success
[567]	1442	remote_rwlock_rd_release( lock_xp );
[408]	1443	return vseg;
	1444	}
[406]	1445	}
	1446
[408]	1447	// return failure
[567]	1448	remote_rwlock_rd_release( lock_xp );
[408]	1449	return NULL;
[406]	1450
[595]	1451	} // end vmm_vseg_from_vaddr()
[406]	1452
[1]	1453	/////////////////////////////////////////////
	1454	error_t vmm_resize_vseg( process_t * process,
	1455	intptr_t base,
	1456	intptr_t size )
	1457	{
[406]	1458	error_t error;
	1459	vseg_t * new;
	1460	vpn_t vpn_min;
	1461	vpn_t vpn_max;
[1]	1462
[623]	1463	#if DEBUG_VMM_RESIZE_VSEG
	1464	uint32_t cycle = (uint32_t)hal_get_cycles();
	1465	thread_t * this = CURRENT_THREAD;
	1466	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1467	printk("\n[%s] thread[%x,%x] enter / process %x / base %x / size %d / cycle %d\n",
	1468	__FUNCTION__, this->process->pid, this->trdid, process->pid, base, size, cycle );
	1469	#endif
	1470
[1]	1471	// get pointer on process VMM
	1472	vmm_t * vmm = &process->vmm;
	1473
	1474	intptr_t addr_min = base;
	1475	intptr_t addr_max = base + size;
	1476
	1477	// get pointer on vseg
[595]	1478	vseg_t * vseg = vmm_vseg_from_vaddr( vmm , base );
[1]	1479
[623]	1480	if( vseg == NULL)
	1481	{
	1482	printk("\n[ERROR] in %s : vseg(%x,%d) not found\n",
	1483	__FUNCTION__, base , size );
	1484	return -1;
	1485	}
[21]	1486
[623]	1487	// resize depends on unmapped region base and size
[611]	1488	if( (vseg->min > addr_min) \|\| (vseg->max < addr_max) ) // not included in vseg
[1]	1489	{
[623]	1490	printk("\n[ERROR] in %s : unmapped region[%x->%x[ not included in vseg[%x->%x[\n",
	1491	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
	1492
[611]	1493	error = -1;
[1]	1494	}
[611]	1495	else if( (vseg->min == addr_min) && (vseg->max == addr_max) ) // vseg must be deleted
[1]	1496	{
[623]	1497
	1498	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1499	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1500	printk("\n[%s] unmapped region[%x->%x[ equal vseg[%x->%x[\n",
	1501	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
	1502	#endif
[611]	1503	vmm_delete_vseg( process->pid , vseg->min );
[623]	1504
	1505	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1506	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1507	printk("\n[%s] thread[%x,%x] deleted vseg\n",
	1508	__FUNCTION__, this->process->pid, this->trdid );
	1509	#endif
[1]	1510	error = 0;
	1511	}
[611]	1512	else if( vseg->min == addr_min ) // vseg must be resized
[1]	1513	{
[623]	1514
	1515	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1516	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1517	printk("\n[%s] unmapped region[%x->%x[ included in vseg[%x->%x[\n",
	1518	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
	1519	#endif
	1520	// update vseg min address
[406]	1521	vseg->min = addr_max;
	1522
	1523	// update vpn_base and vpn_size
	1524	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1525	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1526	vseg->vpn_base = vpn_min;
	1527	vseg->vpn_size = vpn_max - vpn_min + 1;
[623]	1528
	1529	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1530	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1531	printk("\n[%s] thread[%x,%x] changed vseg_min\n",
	1532	__FUNCTION__, this->process->pid, this->trdid );
	1533	#endif
[406]	1534	error = 0;
[1]	1535	}
[611]	1536	else if( vseg->max == addr_max ) // vseg must be resized
[1]	1537	{
[623]	1538
	1539	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1540	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1541	printk("\n[%s] unmapped region[%x->%x[ included in vseg[%x->%x[\n",
	1542	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
	1543	#endif
[406]	1544	// update vseg max address
	1545	vseg->max = addr_min;
	1546
	1547	// update vpn_base and vpn_size
	1548	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1549	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1550	vseg->vpn_base = vpn_min;
	1551	vseg->vpn_size = vpn_max - vpn_min + 1;
[623]	1552
	1553	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1554	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1555	printk("\n[%s] thread[%x,%x] changed vseg_max\n",
	1556	__FUNCTION__, this->process->pid, this->trdid );
	1557	#endif
[406]	1558	error = 0;
[623]	1559
[1]	1560	}
[611]	1561	else // vseg cut in three regions
[1]	1562	{
[623]	1563
	1564	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1565	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1566	printk("\n[%s] unmapped region[%x->%x[ included in vseg[%x->%x[\n",
	1567	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
	1568	#endif
[406]	1569	// resize existing vseg
	1570	vseg->max = addr_min;
	1571
	1572	// update vpn_base and vpn_size
	1573	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1574	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1575	vseg->vpn_base = vpn_min;
	1576	vseg->vpn_size = vpn_max - vpn_min + 1;
	1577
	1578	// create new vseg
[407]	1579	new = vmm_create_vseg( process,
	1580	vseg->type,
	1581	addr_min,
	1582	(vseg->max - addr_max),
	1583	vseg->file_offset,
	1584	vseg->file_size,
	1585	vseg->mapper_xp,
	1586	vseg->cxy );
	1587
[623]	1588	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1589	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1590	printk("\n[%s] thread[%x,%x] replaced vseg by two smal vsegs\n",
	1591	__FUNCTION__, this->process->pid, this->trdid );
	1592	#endif
	1593
	1594	if( new == NULL ) error = -1;
[406]	1595	else error = 0;
[1]	1596	}
	1597
[623]	1598	#if DEBUG_VMM_RESIZE_VSEG
	1599	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1600	printk("\n[%s] thread[%x,%x] exit / process %x / base %x / size %d / cycle %d\n",
	1601	__FUNCTION__, this->process->pid, this->trdid, process->pid, base, size, cycle );
	1602	#endif
[1]	1603
	1604	return error;
	1605
[406]	1606	} // vmm_resize_vseg()
	1607
[1]	1608	///////////////////////////////////////////
[388]	1609	error_t vmm_get_vseg( process_t * process,
[394]	1610	intptr_t vaddr,
[388]	1611	vseg_t ** found_vseg )
[1]	1612	{
[595]	1613	xptr_t vseg_xp;
	1614	vseg_t * vseg;
	1615	vmm_t * vmm;
	1616	error_t error;
[1]	1617
[440]	1618	// get pointer on local VMM
	1619	vmm = &process->vmm;
[1]	1620
[440]	1621	// try to get vseg from local VMM
[595]	1622	vseg = vmm_vseg_from_vaddr( vmm , vaddr );
[440]	1623
[388]	1624	if( vseg == NULL ) // vseg not found in local cluster => try to get it from ref
	1625	{
	1626	// get extended pointer on reference process
	1627	xptr_t ref_xp = process->ref_xp;
[1]	1628
[388]	1629	// get cluster and local pointer on reference process
	1630	cxy_t ref_cxy = GET_CXY( ref_xp );
[433]	1631	process_t * ref_ptr = GET_PTR( ref_xp );
[388]	1632
	1633	if( local_cxy == ref_cxy ) return -1; // local cluster is the reference
	1634
	1635	// get extended pointer on reference vseg
[394]	1636	rpc_vmm_get_vseg_client( ref_cxy , ref_ptr , vaddr , &vseg_xp , &error );
[388]	1637
[440]	1638	if( error ) return -1; // vseg not found => illegal user vaddr
[388]	1639
	1640	// allocate a vseg in local cluster
	1641	vseg = vseg_alloc();
	1642
[440]	1643	if( vseg == NULL ) return -1; // cannot allocate a local vseg
[388]	1644
	1645	// initialise local vseg from reference
	1646	vseg_init_from_ref( vseg , vseg_xp );
	1647
[625]	1648	// build extended pointer on VSL lock
	1649	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsl_lock );
	1650
	1651	// take the VSL lock in write mode
	1652	remote_rwlock_wr_acquire( lock_xp );
	1653
[611]	1654	// register local vseg in local VSL
	1655	vmm_attach_vseg_to_vsl( vmm , vseg );
[625]	1656
	1657	// release the VSL lock
	1658	remote_rwlock_wr_release( lock_xp );
[388]	1659	}
[595]	1660
[388]	1661	// success
	1662	*found_vseg = vseg;
[394]	1663	return 0;
[388]	1664
	1665	} // end vmm_get_vseg()
	1666
[407]	1667	//////////////////////////////////////////////////////////////////////////////////////
	1668	// This static function compute the target cluster to allocate a physical page
[632]	1669	// for a given <vpn> in a given <vseg>, allocates the page and returns an extended
	1670	// pointer on the allocated page descriptor.
[407]	1671	// The vseg cannot have the FILE type.
	1672	//////////////////////////////////////////////////////////////////////////////////////
	1673	static xptr_t vmm_page_allocate( vseg_t * vseg,
	1674	vpn_t vpn )
	1675	{
[433]	1676
[632]	1677	#if DEBUG_VMM_PAGE_ALLOCATE
[619]	1678	uint32_t cycle = (uint32_t)hal_get_cycles();
	1679	thread_t * this = CURRENT_THREAD;
[632]	1680	if( DEBUG_VMM_PAGE_ALLOCATE < cycle )
[595]	1681	printk("\n[%s] thread[%x,%x] enter for vpn %x / cycle %d\n",
	1682	__FUNCTION__ , this->process->pid, this->trdid, vpn, cycle );
[433]	1683	#endif
	1684
[632]	1685	xptr_t page_xp;
[407]	1686	cxy_t page_cxy;
[577]	1687	uint32_t index;
[407]	1688
[577]	1689	uint32_t type = vseg->type;
	1690	uint32_t flags = vseg->flags;
	1691	uint32_t x_size = LOCAL_CLUSTER->x_size;
	1692	uint32_t y_size = LOCAL_CLUSTER->y_size;
[407]	1693
[567]	1694	// check vseg type
	1695	assert( ( type != VSEG_TYPE_FILE ) , "illegal vseg type\n" );
[407]	1696
	1697	if( flags & VSEG_DISTRIB ) // distributed => cxy depends on vpn LSB
	1698	{
[577]	1699	index = vpn & ((x_size * y_size) - 1);
	1700	page_cxy = HAL_CXY_FROM_XY( (index / y_size) , (index % y_size) );
[561]	1701
[577]	1702	// If the cluster selected from VPN's LSBs is empty, we select one randomly
	1703	if ( cluster_is_active( page_cxy ) == false )
	1704	{
	1705	page_cxy = cluster_random_select();
[561]	1706	}
[407]	1707	}
	1708	else // other cases => cxy specified in vseg
	1709	{
[561]	1710	page_cxy = vseg->cxy;
[407]	1711	}
	1712
[632]	1713	// allocate a 4 Kbytes physical page from target cluster
	1714	page_xp = ppm_remote_alloc_pages( page_cxy , 0 );
[407]	1715
[632]	1716	#if DEBUG_VMM_PAGE_ALLOCATE
[595]	1717	cycle = (uint32_t)hal_get_cycles();
[632]	1718	if( DEBUG_VMM_PAGE_ALLOCATE < cycle )
	1719	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cluster %x / cycle %d\n",
	1720	__FUNCTION__ , this->process->pid, this->trdid, vpn, ppm_page2ppn(page_xp), page_cxy, cycle );
[433]	1721	#endif
	1722
[632]	1723	return page_xp;
[407]	1724
	1725	} // end vmm_page_allocate()
	1726
[313]	1727	////////////////////////////////////////
	1728	error_t vmm_get_one_ppn( vseg_t * vseg,
	1729	vpn_t vpn,
	1730	ppn_t * ppn )
	1731	{
	1732	error_t error;
[407]	1733	xptr_t page_xp; // extended pointer on physical page descriptor
[606]	1734	uint32_t page_id; // missing page index in vseg mapper
[406]	1735	uint32_t type; // vseg type;
[313]	1736
[406]	1737	type = vseg->type;
[606]	1738	page_id = vpn - vseg->vpn_base;
[313]	1739
[438]	1740	#if DEBUG_VMM_GET_ONE_PPN
[595]	1741	uint32_t cycle = (uint32_t)hal_get_cycles();
	1742	thread_t * this = CURRENT_THREAD;
[632]	1743	// if( DEBUG_VMM_GET_ONE_PPN < cycle )
	1744	if( vpn == 0x40B )
[606]	1745	printk("\n[%s] thread[%x,%x] enter for vpn %x / type %s / page_id %d / cycle %d\n",
	1746	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(type), page_id, cycle );
[433]	1747	#endif
[313]	1748
[406]	1749	// FILE type : get the physical page from the file mapper
[313]	1750	if( type == VSEG_TYPE_FILE )
	1751	{
[406]	1752	// get extended pointer on mapper
[407]	1753	xptr_t mapper_xp = vseg->mapper_xp;
[313]	1754
[567]	1755	assert( (mapper_xp != XPTR_NULL),
	1756	"mapper not defined for a FILE vseg\n" );
[406]	1757
[606]	1758	// get extended pointer on page descriptor
	1759	page_xp = mapper_remote_get_page( mapper_xp , page_id );
[406]	1760
[606]	1761	if ( page_xp == XPTR_NULL ) return EINVAL;
[313]	1762	}
	1763
[406]	1764	// Other types : allocate a physical page from target cluster,
[407]	1765	// as defined by vseg type and vpn value
[313]	1766	else
	1767	{
[433]	1768	// allocate one physical page
[407]	1769	page_xp = vmm_page_allocate( vseg , vpn );
[406]	1770
[407]	1771	if( page_xp == XPTR_NULL ) return ENOMEM;
[313]	1772
[406]	1773	// initialise missing page from .elf file mapper for DATA and CODE types
[440]	1774	// the vseg->mapper_xp field is an extended pointer on the .elf file mapper
[313]	1775	if( (type == VSEG_TYPE_CODE) \|\| (type == VSEG_TYPE_DATA) )
	1776	{
[406]	1777	// get extended pointer on mapper
	1778	xptr_t mapper_xp = vseg->mapper_xp;
[313]	1779
[567]	1780	assert( (mapper_xp != XPTR_NULL),
	1781	"mapper not defined for a CODE or DATA vseg\n" );
[406]	1782
	1783	// compute missing page offset in vseg
[606]	1784	uint32_t offset = page_id << CONFIG_PPM_PAGE_SHIFT;
[406]	1785
[313]	1786	// compute missing page offset in .elf file
[406]	1787	uint32_t elf_offset = vseg->file_offset + offset;
[313]	1788
[438]	1789	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[632]	1790	// if( DEBUG_VMM_GET_ONE_PPN < cycle )
	1791	if( vpn == 0x40B )
[595]	1792	printk("\n[%s] thread[%x,%x] for vpn = %x / elf_offset = %x\n",
	1793	__FUNCTION__, this->process->pid, this->trdid, vpn, elf_offset );
[433]	1794	#endif
[406]	1795	// compute extended pointer on page base
[407]	1796	xptr_t base_xp = ppm_page2base( page_xp );
[313]	1797
[406]	1798	// file_size (in .elf mapper) can be smaller than vseg_size (BSS)
	1799	uint32_t file_size = vseg->file_size;
	1800
	1801	if( file_size < offset ) // missing page fully in BSS
[313]	1802	{
[406]	1803
[438]	1804	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[632]	1805	// if( DEBUG_VMM_GET_ONE_PPN < cycle )
	1806	if( vpn == 0x40B )
[595]	1807	printk("\n[%s] thread[%x,%x] for vpn %x / fully in BSS\n",
	1808	__FUNCTION__, this->process->pid, this->trdid, vpn );
[433]	1809	#endif
[407]	1810	if( GET_CXY( page_xp ) == local_cxy )
[313]	1811	{
[315]	1812	memset( GET_PTR( base_xp ) , 0 , CONFIG_PPM_PAGE_SIZE );
[313]	1813	}
	1814	else
	1815	{
[315]	1816	hal_remote_memset( base_xp , 0 , CONFIG_PPM_PAGE_SIZE );
[313]	1817	}
	1818	}
[406]	1819	else if( file_size >= (offset + CONFIG_PPM_PAGE_SIZE) ) // fully in mapper
[315]	1820	{
[406]	1821
[438]	1822	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[632]	1823	// if( DEBUG_VMM_GET_ONE_PPN < cycle )
	1824	if( vpn == 0x40B )
[595]	1825	printk("\n[%s] thread[%x,%x] for vpn %x / fully in mapper\n",
	1826	__FUNCTION__, this->process->pid, this->trdid, vpn );
[433]	1827	#endif
[606]	1828	error = mapper_move_kernel( mapper_xp,
	1829	true, // to_buffer
	1830	elf_offset,
	1831	base_xp,
	1832	CONFIG_PPM_PAGE_SIZE );
[313]	1833	if( error ) return EINVAL;
	1834	}
[406]	1835	else // both in mapper and in BSS :
	1836	// - (file_size - offset) bytes from mapper
	1837	// - (page_size + offset - file_size) bytes from BSS
[313]	1838	{
[406]	1839
[438]	1840	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[632]	1841	// if( DEBUG_VMM_GET_ONE_PPN < cycle )
	1842	if( vpn == 0x40B )
[610]	1843	printk("\n[%s] thread[%x,%x] for vpn %x / both mapper & BSS\n"
[433]	1844	" %d bytes from mapper / %d bytes from BSS\n",
[595]	1845	__FUNCTION__, this->process->pid, this->trdid, vpn,
[407]	1846	file_size - offset , offset + CONFIG_PPM_PAGE_SIZE - file_size );
[433]	1847	#endif
[313]	1848	// initialize mapper part
[606]	1849	error = mapper_move_kernel( mapper_xp,
	1850	true, // to buffer
	1851	elf_offset,
	1852	base_xp,
	1853	file_size - offset );
[313]	1854	if( error ) return EINVAL;
	1855
	1856	// initialize BSS part
[407]	1857	if( GET_CXY( page_xp ) == local_cxy )
[313]	1858	{
[406]	1859	memset( GET_PTR( base_xp ) + file_size - offset , 0 ,
	1860	offset + CONFIG_PPM_PAGE_SIZE - file_size );
[313]	1861	}
	1862	else
	1863	{
[406]	1864	hal_remote_memset( base_xp + file_size - offset , 0 ,
	1865	offset + CONFIG_PPM_PAGE_SIZE - file_size );
[313]	1866	}
	1867	}
	1868	} // end initialisation for CODE or DATA types
	1869	}
	1870
	1871	// return ppn
[407]	1872	*ppn = ppm_page2ppn( page_xp );
[406]	1873
[438]	1874	#if DEBUG_VMM_GET_ONE_PPN
[595]	1875	cycle = (uint32_t)hal_get_cycles();
[632]	1876	// if( DEBUG_VMM_GET_ONE_PPN < cycle )
	1877	if( vpn == 0x40B )
[595]	1878	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle\n",
	1879	__FUNCTION__ , this->process->pid, this->trdid , vpn , *ppn, cycle );
[433]	1880	#endif
[406]	1881
[313]	1882	return 0;
	1883
	1884	} // end vmm_get_one_ppn()
	1885
[585]	1886	///////////////////////////////////////////////////
	1887	error_t vmm_handle_page_fault( process_t * process,
	1888	vpn_t vpn )
[1]	1889	{
[585]	1890	vseg_t * vseg; // vseg containing vpn
[629]	1891	uint32_t attr; // PTE_ATTR value
	1892	ppn_t ppn; // PTE_PPN value
[585]	1893	uint32_t ref_attr; // PTE_ATTR value in reference GPT
	1894	ppn_t ref_ppn; // PTE_PPN value in reference GPT
	1895	cxy_t ref_cxy; // reference cluster for missing vpn
	1896	process_t * ref_ptr; // reference process for missing vpn
	1897	xptr_t local_gpt_xp; // extended pointer on local GPT
	1898	xptr_t ref_gpt_xp; // extended pointer on reference GPT
	1899	error_t error; // value returned by called functions
[1]	1900
[629]	1901	thread_t * this = CURRENT_THREAD;
	1902
	1903	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
	1904	uint32_t start_cycle = (uint32_t)hal_get_cycles();
	1905	#endif
	1906
[625]	1907	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[632]	1908	if( vpn == 0x40b )
[625]	1909	printk("\n[%s] thread[%x,%x] enter for vpn %x / cycle %d\n",
[629]	1910	__FUNCTION__, this->process->pid, this->trdid, vpn, start_cycle );
[625]	1911	#endif
	1912
[629]	1913	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
	1914	hal_vmm_display( this->process , false );
	1915	#endif
	1916
[585]	1917	// get local vseg (access to reference VSL can be required)
	1918	error = vmm_get_vseg( process,
	1919	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
	1920	&vseg );
	1921	if( error )
	1922	{
[629]	1923	printk("\n[ERROR] in %s : vpn %x in thread[%x,%x] not in registered vseg\n",
	1924	__FUNCTION__ , vpn , process->pid, this->trdid );
[585]	1925
	1926	return EXCP_USER_ERROR;
	1927	}
	1928
[625]	1929	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[632]	1930	uint32_t cycle = (uint32_t)hal_get_cycles();
	1931	if( vpn == 0x40b )
	1932	printk("\n[%s] thread[%x,%x] found vseg %s / cycle %d\n",
	1933	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(vseg->type), cycle );
[433]	1934	#endif
[406]	1935
[629]	1936	// build extended pointer on local GPT
	1937	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
	1938
[632]	1939	// lock PTE in local GPT and get current PPN and attributes
[629]	1940	error = hal_gpt_lock_pte( local_gpt_xp,
	1941	vpn,
	1942	&attr,
	1943	&ppn );
	1944	if( error )
[438]	1945	{
[629]	1946	printk("\n[PANIC] in %s : cannot lock PTE in local GPT / vpn %x / process %x\n",
	1947	__FUNCTION__ , vpn , process->pid );
	1948
	1949	return EXCP_KERNEL_PANIC;
	1950	}
[407]	1951
[632]	1952	#if DEBUG_VMM_HANDLE_PAGE_FAULT
	1953	cycle = (uint32_t)hal_get_cycles();
	1954	if( vpn == 0x40b )
	1955	printk("\n[%s] thread[%x,%x] locked vpn %x in cluster %x / cycle %d\n",
	1956	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy, cycle );
	1957	#endif
	1958
	1959	// handle page fault only if local PTE still unmapped after lock
[629]	1960	if( (attr & GPT_MAPPED) == 0 )
	1961	{
	1962	// get reference process cluster and local pointer
	1963	ref_cxy = GET_CXY( process->ref_xp );
	1964	ref_ptr = GET_PTR( process->ref_xp );
[407]	1965
[630]	1966	/////////////// private vseg or (local == reference)
	1967	/////////////// => access only the local GPT
[629]	1968	if( (vseg->type == VSEG_TYPE_STACK) \|\|
	1969	(vseg->type == VSEG_TYPE_CODE) \|\|
	1970	(ref_cxy == local_cxy ) )
	1971	{
[632]	1972
	1973	#if DEBUG_VMM_HANDLE_PAGE_FAULT
	1974	if( vpn == 0x40b )
	1975	printk("\n[%s] thread[%x,%x] : access local gpt : local_cxy %x / ref_cxy %x / type %s\n",
	1976	__FUNCTION__, this->process->pid, this->trdid, local_cxy, ref_cxy, vseg_type_str(vseg->type) );
	1977	#endif
	1978	// allocate and initialise a physical page
[629]	1979	error = vmm_get_one_ppn( vseg , vpn , &ppn );
[407]	1980
[585]	1981	if( error )
[408]	1982	{
[629]	1983	printk("\n[ERROR] in %s : no physical page / process = %x / vpn = %x\n",
[408]	1984	__FUNCTION__ , process->pid , vpn );
[1]	1985
[629]	1986	// unlock PTE in local GPT
	1987	hal_gpt_unlock_pte( local_gpt_xp , vpn );
[406]	1988
[585]	1989	return EXCP_KERNEL_PANIC;
[407]	1990	}
	1991
[629]	1992	// define attr from vseg flags
[632]	1993	attr = GPT_MAPPED \| GPT_SMALL \| GPT_READABLE;
[629]	1994	if( vseg->flags & VSEG_USER ) attr \|= GPT_USER;
	1995	if( vseg->flags & VSEG_WRITE ) attr \|= GPT_WRITABLE;
	1996	if( vseg->flags & VSEG_EXEC ) attr \|= GPT_EXECUTABLE;
	1997	if( vseg->flags & VSEG_CACHE ) attr \|= GPT_CACHABLE;
[407]	1998
[629]	1999	// set PTE to local GPT
[632]	2000	// it unlocks this PTE
[629]	2001	hal_gpt_set_pte( local_gpt_xp,
	2002	vpn,
	2003	attr,
	2004	ppn );
[585]	2005
[629]	2006	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
	2007	uint32_t end_cycle = (uint32_t)hal_get_cycles();
	2008	#endif
[585]	2009
	2010	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[632]	2011	if( vpn == 0x40b )
	2012	printk("\n[%s] thread[%x,%x] handled local pgfault / ppn %x / attr %x / cycle %d\n",
	2013	__FUNCTION__, this->process->pid, this->trdid, ppn, attr, end_cycle );
[585]	2014	#endif
	2015
[629]	2016	#if CONFIG_INSTRUMENTATION_PGFAULTS
	2017	this->info.local_pgfault_nr++;
	2018	this->info.local_pgfault_cost += (end_cycle - start_cycle);
	2019	#endif
	2020	return EXCP_NON_FATAL;
[585]	2021
[629]	2022	} // end local GPT access
[585]	2023
[630]	2024	/////////////////// public vseg and (local != reference)
	2025	/////////////////// => access ref GPT to update local GPT
[629]	2026	else
	2027	{
[632]	2028
	2029	#if DEBUG_VMM_HANDLE_PAGE_FAULT
	2030	if( vpn == 0x40b )
	2031	printk("\n[%s] thread[%x,%x] access ref gpt : local_cxy %x / ref_cxy %x / type %s\n",
	2032	__FUNCTION__, this->process->pid, this->trdid, local_cxy, ref_cxy, vseg_type_str(vseg->type) );
	2033	#endif
[629]	2034	// build extended pointer on reference GPT
	2035	ref_gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
[585]	2036
[632]	2037	// lock PTE in reference GPT and get current PPN and attributes
	2038	error = hal_gpt_lock_pte( ref_gpt_xp,
	2039	vpn,
	2040	&ref_attr,
	2041	&ref_ppn );
	2042	if( error )
	2043	{
	2044	printk("\n[PANIC] in %s : cannot lock PTE in ref GPT / vpn %x / process %x\n",
	2045	__FUNCTION__ , vpn , process->pid );
	2046
	2047	// unlock PTE in local GPT
	2048	hal_gpt_unlock_pte( local_gpt_xp , vpn );
	2049
	2050	return EXCP_KERNEL_PANIC;
	2051	}
[1]	2052
[632]	2053	#if DEBUG_VMM_HANDLE_PAGE_FAULT
	2054	if( vpn == 0x40b )
	2055	printk("\n[%s] thread[%x,%x] get pte from ref gpt / attr %x / ppn %x\n",
	2056	__FUNCTION__, this->process->pid, this->trdid, ref_attr, ref_ppn );
	2057	#endif
	2058
	2059	if( ref_attr & GPT_MAPPED ) // false page fault
[585]	2060	{
[629]	2061	// update local GPT from reference GPT values
[632]	2062	// this unlocks the PTE in local GPT
[629]	2063	hal_gpt_set_pte( local_gpt_xp,
	2064	vpn,
	2065	ref_attr,
	2066	ref_ppn );
[585]	2067
[632]	2068	#if DEBUG_VMM_HANDLE_PAGE_FAULT
	2069	if( vpn == 0x40b )
	2070	printk("\n[%s] thread[%x,%x] updated local gpt for a false pgfault\n",
	2071	__FUNCTION__, this->process->pid, this->trdid );
	2072	#endif
	2073
	2074	// unlock the PTE in reference GPT
	2075	hal_gpt_unlock_pte( ref_gpt_xp, vpn );
	2076
	2077	#if DEBUG_VMM_HANDLE_PAGE_FAULT
	2078	if( vpn == 0x40b )
	2079	printk("\n[%s] thread[%x,%x] unlock the ref gpt after a false pgfault\n",
	2080	__FUNCTION__, this->process->pid, this->trdid );
	2081	#endif
	2082
[629]	2083	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
	2084	uint32_t end_cycle = (uint32_t)hal_get_cycles();
	2085	#endif
	2086
[585]	2087	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[632]	2088	if( vpn == 0x40b )
	2089	printk("\n[%s] thread[%x,%x] handled false pgfault / ppn %x / attr %x / cycle %d\n",
	2090	__FUNCTION__, this->process->pid, this->trdid, ref_ppn, ref_attr, end_cycle );
[433]	2091	#endif
[406]	2092
[629]	2093	#if CONFIG_INSTRUMENTATION_PGFAULTS
	2094	this->info.false_pgfault_nr++;
	2095	this->info.false_pgfault_cost += (end_cycle - start_cycle);
	2096	#endif
	2097	return EXCP_NON_FATAL;
	2098	}
[632]	2099	else // true page fault
[629]	2100	{
[585]	2101	// allocate and initialise a physical page depending on the vseg type
[629]	2102	error = vmm_get_one_ppn( vseg , vpn , &ppn );
[1]	2103
[585]	2104	if( error )
	2105	{
	2106	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
	2107	__FUNCTION__ , process->pid , vpn );
[313]	2108
[632]	2109	// unlock PTE in local GPT and in reference GPT
[629]	2110	hal_gpt_unlock_pte( local_gpt_xp , vpn );
[632]	2111	hal_gpt_unlock_pte( ref_gpt_xp , vpn );
[585]	2112
[629]	2113	return EXCP_KERNEL_PANIC;
[585]	2114	}
[1]	2115
[629]	2116	// define attr from vseg flags
[632]	2117	attr = GPT_MAPPED \| GPT_SMALL \| GPT_READABLE;
[629]	2118	if( vseg->flags & VSEG_USER ) attr \|= GPT_USER;
	2119	if( vseg->flags & VSEG_WRITE ) attr \|= GPT_WRITABLE;
	2120	if( vseg->flags & VSEG_EXEC ) attr \|= GPT_EXECUTABLE;
	2121	if( vseg->flags & VSEG_CACHE ) attr \|= GPT_CACHABLE;
[585]	2122
[632]	2123	#if DEBUG_VMM_HANDLE_PAGE_FAULT
	2124	if( vpn == 0x40b )
	2125	printk("\n[%s] thread[%x,%x] build a new PTE for a true pgfault\n",
	2126	__FUNCTION__, this->process->pid, this->trdid );
	2127	#endif
[629]	2128	// set PTE in reference GPT
[632]	2129	// this unlock the PTE
[629]	2130	hal_gpt_set_pte( ref_gpt_xp,
	2131	vpn,
	2132	attr,
	2133	ppn );
	2134
[632]	2135	#if DEBUG_VMM_HANDLE_PAGE_FAULT
	2136	if( vpn == 0x40b )
	2137	printk("\n[%s] thread[%x,%x] set new PTE in ref gpt for a true page fault\n",
	2138	__FUNCTION__, this->process->pid, this->trdid );
	2139	#endif
	2140
[629]	2141	// set PTE in local GPT
[632]	2142	// this unlock the PTE
[629]	2143	hal_gpt_set_pte( local_gpt_xp,
	2144	vpn,
	2145	attr,
	2146	ppn );
	2147
	2148	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
	2149	uint32_t end_cycle = (uint32_t)hal_get_cycles();
	2150	#endif
	2151
[440]	2152	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[632]	2153	if( vpn == 0x40b )
	2154	printk("\n[%s] thread[%x,%x] handled global pgfault / ppn %x / attr %x / cycle %d\n",
	2155	__FUNCTION__, this->process->pid, this->trdid, ppn, attr, end_cycle );
[435]	2156	#endif
[629]	2157
	2158	#if CONFIG_INSTRUMENTATION_PGFAULTS
	2159	this->info.global_pgfault_nr++;
	2160	this->info.global_pgfault_cost += (end_cycle - start_cycle);
	2161	#endif
	2162	return EXCP_NON_FATAL;
	2163	}
[585]	2164	}
	2165	}
[629]	2166	else // page has been locally mapped by another concurrent thread
	2167	{
[632]	2168	// unlock the PTE in local GPT
[629]	2169	hal_gpt_unlock_pte( local_gpt_xp , vpn );
	2170
[632]	2171	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
	2172	uint32_t end_cycle = (uint32_t)hal_get_cycles();
	2173	#endif
	2174
	2175	#if DEBUG_VMM_HANDLE_PAGE_FAULT
	2176	if( vpn == 0x40b )
	2177	printk("\n[%s] handled by another thread / vpn %x / ppn %x / attr %x / cycle %d\n",
	2178	__FUNCTION__, vpn, ppn, attr, end_cycle );
	2179	#endif
	2180
	2181	#if CONFIG_INSTRUMENTATION_PGFAULTS
	2182	this->info.false_pgfault_nr++;
	2183	this->info.false_pgfault_cost += (end_cycle - start_cycle);
	2184	#endif
[629]	2185	return EXCP_NON_FATAL;
	2186	}
	2187
[585]	2188	} // end vmm_handle_page_fault()
[435]	2189
[585]	2190	////////////////////////////////////////////
	2191	error_t vmm_handle_cow( process_t * process,
	2192	vpn_t vpn )
	2193	{
	2194	vseg_t * vseg; // vseg containing vpn
[629]	2195	xptr_t gpt_xp; // extended pointer on GPT (local or reference)
	2196	gpt_t * gpt_ptr; // local pointer on GPT (local or reference)
	2197	cxy_t gpt_cxy; // GPT cluster identifier
[585]	2198	uint32_t old_attr; // current PTE_ATTR value
	2199	ppn_t old_ppn; // current PTE_PPN value
	2200	uint32_t new_attr; // new PTE_ATTR value
	2201	ppn_t new_ppn; // new PTE_PPN value
[629]	2202	cxy_t ref_cxy; // reference process cluster
	2203	process_t * ref_ptr; // local pointer on reference process
[585]	2204	error_t error;
[1]	2205
[629]	2206	thread_t * this = CURRENT_THREAD;
[625]	2207
[585]	2208	#if DEBUG_VMM_HANDLE_COW
[629]	2209	uint32_t cycle = (uint32_t)hal_get_cycles();
[585]	2210	if( DEBUG_VMM_HANDLE_COW < cycle )
[595]	2211	printk("\n[%s] thread[%x,%x] enter for vpn %x / core[%x,%d] / cycle %d\n",
[619]	2212	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
[629]	2213	#endif
	2214
	2215	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
[625]	2216	hal_vmm_display( process , true );
[585]	2217	#endif
	2218
	2219	// get local vseg
	2220	error = vmm_get_vseg( process,
	2221	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
	2222	&vseg );
[440]	2223	if( error )
[1]	2224	{
[629]	2225	printk("\n[ERROR] in %s : vpn %x in thread[%x,%x] not in a registered vseg\n",
[625]	2226	__FUNCTION__, vpn, process->pid, this->trdid );
[585]	2227
[629]	2228	return EXCP_USER_ERROR;
[440]	2229	}
[407]	2230
[629]	2231	#if DEBUG_VMM_HANDLE_COW
[619]	2232	if( DEBUG_VMM_HANDLE_COW < cycle )
[629]	2233	printk("\n[%s] thread[%x,%x] get vseg %s\n",
	2234	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(vseg->type) );
[619]	2235	#endif
	2236
[629]	2237	// get reference process cluster and local pointer
[585]	2238	ref_cxy = GET_CXY( process->ref_xp );
	2239	ref_ptr = GET_PTR( process->ref_xp );
[407]	2240
[629]	2241	// build pointers on relevant GPT
	2242	// - access only local GPT for a private vseg
	2243	// - access reference GPT and all copies for a public vseg
[585]	2244	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
[440]	2245	{
[629]	2246	gpt_cxy = local_cxy;
	2247	gpt_ptr = &process->vmm.gpt;
	2248	gpt_xp = XPTR( gpt_cxy , gpt_ptr );
[1]	2249	}
[440]	2250	else
[1]	2251	{
[629]	2252	gpt_cxy = ref_cxy;
	2253	gpt_ptr = &ref_ptr->vmm.gpt;
	2254	gpt_xp = XPTR( gpt_cxy , gpt_ptr );
[1]	2255	}
	2256
[629]	2257	// lock target PTE in relevant GPT (local or reference)
[632]	2258	// and get current PTE value
[629]	2259	error = hal_gpt_lock_pte( gpt_xp,
	2260	vpn,
	2261	&old_attr,
	2262	&old_ppn );
	2263	if( error )
	2264	{
	2265	printk("\n[PANIC] in %s : cannot lock PTE in GPT / cxy %x / vpn %x / process %x\n",
	2266	__FUNCTION__ , gpt_cxy, vpn , process->pid );
	2267
	2268	return EXCP_KERNEL_PANIC;
	2269	}
[441]	2270
[629]	2271	#if DEBUG_VMM_HANDLE_COW
[619]	2272	if( DEBUG_VMM_HANDLE_COW < cycle )
	2273	printk("\n[%s] thread[%x,%x] get pte for vpn %x : ppn %x / attr %x\n",
	2274	__FUNCTION__, this->process->pid, this->trdid, vpn, old_ppn, old_attr );
	2275	#endif
	2276
[629]	2277	// return user error if COW attribute not set or PTE2 unmapped
	2278	if( ((old_attr & GPT_COW) == 0) \|\| ((old_attr & GPT_MAPPED) == 0) )
[585]	2279	{
[629]	2280	hal_gpt_unlock_pte( gpt_xp , vpn );
[407]	2281
[629]	2282	return EXCP_USER_ERROR;
[407]	2283	}
	2284
[619]	2285	// get pointers on physical page descriptor
[585]	2286	xptr_t page_xp = ppm_ppn2page( old_ppn );
	2287	cxy_t page_cxy = GET_CXY( page_xp );
	2288	page_t * page_ptr = GET_PTR( page_xp );
[435]	2289
[585]	2290	// get extended pointers on forks and lock field in page descriptor
	2291	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
	2292	xptr_t forks_lock_xp = XPTR( page_cxy , &page_ptr->lock );
[407]	2293
[585]	2294	// take lock protecting "forks" counter
	2295	remote_busylock_acquire( forks_lock_xp );
[407]	2296
[585]	2297	// get number of pending forks from page descriptor
	2298	uint32_t forks = hal_remote_l32( forks_xp );
[441]	2299
[629]	2300	#if DEBUG_VMM_HANDLE_COW
[619]	2301	if( DEBUG_VMM_HANDLE_COW < cycle )
	2302	printk("\n[%s] thread[%x,%x] get forks = %d for vpn %x\n",
	2303	__FUNCTION__, this->process->pid, this->trdid, forks, vpn );
	2304	#endif
	2305
[585]	2306	if( forks ) // pending fork => allocate a new page, and copy old to new
	2307	{
[619]	2308	// decrement pending forks counter in page descriptor
	2309	hal_remote_atomic_add( forks_xp , -1 );
	2310
	2311	// release lock protecting "forks" counter
	2312	remote_busylock_release( forks_lock_xp );
	2313
[629]	2314	// allocate a new physical page depending on vseg type
[585]	2315	page_xp = vmm_page_allocate( vseg , vpn );
[619]	2316
[585]	2317	if( page_xp == XPTR_NULL )
	2318	{
	2319	printk("\n[PANIC] in %s : no memory for vpn %x in process %x\n",
	2320	__FUNCTION__ , vpn, process->pid );
[441]	2321
[629]	2322	hal_gpt_unlock_pte( gpt_xp , vpn );
[441]	2323
[585]	2324	return EXCP_KERNEL_PANIC;
	2325	}
[441]	2326
[585]	2327	// compute allocated page PPN
	2328	new_ppn = ppm_page2ppn( page_xp );
[441]	2329
[629]	2330	#if DEBUG_VMM_HANDLE_COW
[619]	2331	if( DEBUG_VMM_HANDLE_COW < cycle )
	2332	printk("\n[%s] thread[%x,%x] get new ppn %x for vpn %x\n",
	2333	__FUNCTION__, this->process->pid, this->trdid, new_ppn, vpn );
	2334	#endif
	2335
[585]	2336	// copy old page content to new page
[619]	2337	hal_remote_memcpy( ppm_ppn2base( new_ppn ),
	2338	ppm_ppn2base( old_ppn ),
	2339	CONFIG_PPM_PAGE_SIZE );
[441]	2340
[629]	2341	#if DEBUG_VMM_HANDLE_COW
[585]	2342	if( DEBUG_VMM_HANDLE_COW < cycle )
[619]	2343	printk("\n[%s] thread[%x,%x] copied old page to new page\n",
	2344	__FUNCTION__, this->process->pid, this->trdid );
[585]	2345	#endif
[440]	2346
[585]	2347	}
	2348	else // no pending fork => keep the existing page
	2349	{
[619]	2350	// release lock protecting "forks" counter
	2351	remote_busylock_release( forks_lock_xp );
[1]	2352
[585]	2353	#if(DEBUG_VMM_HANDLE_COW & 1)
	2354	if( DEBUG_VMM_HANDLE_COW < cycle )
[619]	2355	printk("\n[%s] thread[%x,%x] no pending forks / keep existing PPN %x\n",
	2356	__FUNCTION__, this->process->pid, this->trdid, old_ppn );
[585]	2357	#endif
	2358	new_ppn = old_ppn;
	2359	}
[1]	2360
[629]	2361	// build new_attr : set WRITABLE, reset COW, reset LOCKED
	2362	new_attr = (((old_attr \| GPT_WRITABLE) & (~GPT_COW)) & (~GPT_LOCKED));
[585]	2363
[629]	2364	// update the relevant GPT(s)
	2365	// - private vseg => update only the local GPT
	2366	// - public vseg => update the reference GPT AND all the GPT copies
[585]	2367	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
[1]	2368	{
[629]	2369	// set the new PTE2
[585]	2370	hal_gpt_set_pte( gpt_xp,
	2371	vpn,
	2372	new_attr,
	2373	new_ppn );
[1]	2374	}
[585]	2375	else
[1]	2376	{
[585]	2377	if( ref_cxy == local_cxy ) // reference cluster is local
	2378	{
	2379	vmm_global_update_pte( process,
	2380	vpn,
	2381	new_attr,
	2382	new_ppn );
	2383	}
	2384	else // reference cluster is remote
	2385	{
	2386	rpc_vmm_global_update_pte_client( ref_cxy,
	2387	ref_ptr,
	2388	vpn,
	2389	new_attr,
	2390	new_ppn );
	2391	}
[1]	2392	}
	2393
[585]	2394	#if DEBUG_VMM_HANDLE_COW
	2395	cycle = (uint32_t)hal_get_cycles();
	2396	if( DEBUG_VMM_HANDLE_COW < cycle )
[595]	2397	printk("\n[%s] thread[%x,%x] exit for vpn %x / core[%x,%d] / cycle %d\n",
[619]	2398	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
[585]	2399	#endif
[313]	2400
[585]	2401	return EXCP_NON_FATAL;
[1]	2402
[585]	2403	} // end vmm_handle_cow()
	2404

Note: See TracBrowser for help on using the repository browser.

Download in other formats: