Context Navigation

source: trunk/kernel/mm/vmm.c @ 640

Last change on this file since 640 was 640, checked in by alain, 5 years ago
Remove all RPCs in page-fault handling.
File size: 89.7 KB

Line
1	/*
2	* vmm.c - virtual memory manager related operations definition.
3	*
4	* Authors Ghassan Almaless (2008,2009,2010,2011, 2012)
5	* Mohamed Lamine Karaoui (2015)
6	* Alain Greiner (2016,2017,2018,2019)
7	*
8	* Copyright (c) UPMC Sorbonne Universites
9	*
10	* This file is part of ALMOS-MKH.
11	*
12	* ALMOS-MKH is free software; you can redistribute it and/or modify it
13	* under the terms of the GNU General Public License as published by
14	* the Free Software Foundation; version 2.0 of the License.
15	*
16	* ALMOS-MKH is distributed in the hope that it will be useful, but
17	* WITHOUT ANY WARRANTY; without even the implied warranty of
18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19	* General Public License for more details.
20	*
21	* You should have received a copy of the GNU General Public License
22	* along with ALMOS-MKH; if not, write to the Free Software Foundation,
23	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
24	*/
25
26	#include <kernel_config.h>
27	#include <hal_kernel_types.h>
28	#include <hal_special.h>
29	#include <hal_gpt.h>
30	#include <hal_vmm.h>
31	#include <hal_macros.h>
32	#include <printk.h>
33	#include <memcpy.h>
34	#include <remote_queuelock.h>
35	#include <list.h>
36	#include <xlist.h>
37	#include <bits.h>
38	#include <process.h>
39	#include <thread.h>
40	#include <vseg.h>
41	#include <cluster.h>
42	#include <scheduler.h>
43	#include <vfs.h>
44	#include <mapper.h>
45	#include <page.h>
46	#include <kmem.h>
47	#include <vmm.h>
48	#include <hal_exception.h>
49
50	////////////////////////////////////////////////////////////////////////////////////////////
51	// Extern global variables
52	////////////////////////////////////////////////////////////////////////////////////////////
53
54	extern process_t process_zero; // allocated in cluster.c
55
56	////////////////////////////////////////////////////////////////////////////////////////////
57	// This static function is called by the vmm_create_vseg() function, and implements
58	// the VMM STACK specific allocator.
59	////////////////////////////////////////////////////////////////////////////////////////////
60	// @ vmm : [in] pointer on VMM.
61	// @ ltid : [in] requested slot == local user thread identifier.
62	// @ vpn_base : [out] first allocated page
63	// @ vpn_size : [out] number of allocated pages
64	////////////////////////////////////////////////////////////////////////////////////////////
65	static void vmm_stack_alloc( vmm_t * vmm,
66	ltid_t ltid,
67	vpn_t * vpn_base,
68	vpn_t * vpn_size )
69	{
70
71	// check ltid argument
72	assert( (ltid <= ((CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE) / CONFIG_VMM_STACK_SIZE)),
73	"slot index %d too large for an user stack vseg", ltid );
74
75	// get stack allocator pointer
76	stack_mgr_t * mgr = &vmm->stack_mgr;
77
78	// get lock on stack allocator
79	busylock_acquire( &mgr->lock );
80
81	// check requested slot is available
82	assert( (bitmap_state( &mgr->bitmap , ltid ) == false),
83	"slot index %d already allocated", ltid );
84
85	// update bitmap
86	bitmap_set( &mgr->bitmap , ltid );
87
88	// release lock on stack allocator
89	busylock_release( &mgr->lock );
90
91	// returns vpn_base, vpn_size (first page non allocated)
92	vpn_base = mgr->vpn_base + ltid CONFIG_VMM_STACK_SIZE + 1;
93	*vpn_size = CONFIG_VMM_STACK_SIZE - 1;
94
95	} // end vmm_stack_alloc()
96
97	////////////////////////////////////////////////////////////////////////////////////////////
98	// This static function is called by the vmm_remove_vseg() function, and implements
99	// the VMM STACK specific desallocator.
100	////////////////////////////////////////////////////////////////////////////////////////////
101	// @ vmm : [in] pointer on VMM.
102	// @ vseg : [in] pointer on released vseg.
103	////////////////////////////////////////////////////////////////////////////////////////////
104	static void vmm_stack_free( vmm_t * vmm,
105	vseg_t * vseg )
106	{
107	// get stack allocator pointer
108	stack_mgr_t * mgr = &vmm->stack_mgr;
109
110	// compute slot index
111	uint32_t index = (vseg->vpn_base - 1 - mgr->vpn_base) / CONFIG_VMM_STACK_SIZE;
112
113	// check index
114	assert( (index <= ((CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE) / CONFIG_VMM_STACK_SIZE)),
115	"slot index %d too large for an user stack vseg", index );
116
117	// check released slot is allocated
118	assert( (bitmap_state( &mgr->bitmap , index ) == true),
119	"released slot index %d non allocated", index );
120
121	// get lock on stack allocator
122	busylock_acquire( &mgr->lock );
123
124	// update stacks_bitmap
125	bitmap_clear( &mgr->bitmap , index );
126
127	// release lock on stack allocator
128	busylock_release( &mgr->lock );
129
130	} // end vmm_stack_free()
131
132	////////////////////////////////////////////////////////////////////////////////////////////
133	// This static function is called by the vmm_create_vseg() function, and implements
134	// the VMM MMAP specific allocator.
135	////////////////////////////////////////////////////////////////////////////////////////////
136	// @ vmm : [in] pointer on VMM.
137	// @ npages : [in] requested number of pages.
138	// @ vpn_base : [out] first allocated page.
139	// @ vpn_size : [out] actual number of allocated pages.
140	////////////////////////////////////////////////////////////////////////////////////////////
141	static error_t vmm_mmap_alloc( vmm_t * vmm,
142	vpn_t npages,
143	vpn_t * vpn_base,
144	vpn_t * vpn_size )
145	{
146	uint32_t order;
147	xptr_t vseg_xp;
148	vseg_t * vseg;
149	vpn_t base;
150	vpn_t size;
151	vpn_t free;
152
153	#if DEBUG_VMM_MMAP_ALLOC
154	thread_t * this = CURRENT_THREAD;
155	uint32_t cycle = (uint32_t)hal_get_cycles();
156	if( DEBUG_VMM_MMAP_ALLOC < cycle )
157	printk("\n[%s] thread[%x,%x] enter / cycle %d\n",
158	__FUNCTION__, this->process->pid, this->trdid, cycle );
159	#endif
160
161	// number of allocated pages must be power of 2
162	// compute actual size and order
163	size = POW2_ROUNDUP( npages );
164	order = bits_log2( size );
165
166	// get mmap allocator pointer
167	mmap_mgr_t * mgr = &vmm->mmap_mgr;
168
169	// build extended pointer on root of zombi_list[order]
170	xptr_t root_xp = XPTR( local_cxy , &mgr->zombi_list[order] );
171
172	// take lock protecting zombi_lists
173	busylock_acquire( &mgr->lock );
174
175	// get vseg from zombi_list or from mmap zone
176	if( xlist_is_empty( root_xp ) ) // from mmap zone
177	{
178	// check overflow
179	free = mgr->first_free_vpn;
180	if( (free + size) > mgr->vpn_size ) return -1;
181
182	// update MMAP allocator
183	mgr->first_free_vpn += size;
184
185	// compute base
186	base = free;
187	}
188	else // from zombi_list
189	{
190	// get pointer on zombi vseg from zombi_list
191	vseg_xp = XLIST_FIRST( root_xp , vseg_t , xlist );
192	vseg = GET_PTR( vseg_xp );
193
194	// remove vseg from free-list
195	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
196
197	// compute base
198	base = vseg->vpn_base;
199	}
200
201	// release lock
202	busylock_release( &mgr->lock );
203
204	#if DEBUG_VMM_MMAP_ALLOC
205	cycle = (uint32_t)hal_get_cycles();
206	if( DEBUG_VMM_DESTROY < cycle )
207	printk("\n[%s] thread[%x,%x] exit / vpn_base %x / vpn_size %x / cycle %d\n",
208	__FUNCTION__, this->process->pid, this->trdid, base, size, cycle );
209	#endif
210
211	// returns vpn_base, vpn_size
212	*vpn_base = base;
213	*vpn_size = size;
214	return 0;
215
216	} // end vmm_mmap_alloc()
217
218	////////////////////////////////////////////////////////////////////////////////////////////
219	// This static function is called by the vmm_remove_vseg() function, and implements
220	// the VMM MMAP specific desallocator.
221	////////////////////////////////////////////////////////////////////////////////////////////
222	// @ vmm : [in] pointer on VMM.
223	// @ vseg : [in] pointer on released vseg.
224	////////////////////////////////////////////////////////////////////////////////////////////
225	static void vmm_mmap_free( vmm_t * vmm,
226	vseg_t * vseg )
227	{
228	// get pointer on mmap allocator
229	mmap_mgr_t * mgr = &vmm->mmap_mgr;
230
231	// compute zombi_list order
232	uint32_t order = bits_log2( vseg->vpn_size );
233
234	// take lock protecting zombi lists
235	busylock_acquire( &mgr->lock );
236
237	// update relevant zombi_list
238	xlist_add_first( XPTR( local_cxy , &mgr->zombi_list[order] ),
239	XPTR( local_cxy , &vseg->xlist ) );
240
241	// release lock
242	busylock_release( &mgr->lock );
243
244	} // end of vmm_mmap_free()
245
246	////////////////////////////////////////////////////////////////////////////////////////////
247	// This static function registers one vseg in the VSL of a local process descriptor.
248	////////////////////////////////////////////////////////////////////////////////////////////
249	// vmm : [in] pointer on VMM.
250	// vseg : [in] pointer on vseg.
251	////////////////////////////////////////////////////////////////////////////////////////////
252	void vmm_attach_vseg_to_vsl( vmm_t * vmm,
253	vseg_t * vseg )
254	{
255	// update vseg descriptor
256	vseg->vmm = vmm;
257
258	// increment vsegs number
259	vmm->vsegs_nr++;
260
261	// add vseg in vmm list
262	xlist_add_last( XPTR( local_cxy , &vmm->vsegs_root ),
263	XPTR( local_cxy , &vseg->xlist ) );
264
265	} // end vmm_attach_vseg_from_vsl()
266
267	////////////////////////////////////////////////////////////////////////////////////////////
268	// This static function removes one vseg from the VSL of a local process descriptor.
269	////////////////////////////////////////////////////////////////////////////////////////////
270	// vmm : [in] pointer on VMM.
271	// vseg : [in] pointer on vseg.
272	////////////////////////////////////////////////////////////////////////////////////////////
273	void vmm_detach_vseg_from_vsl( vmm_t * vmm,
274	vseg_t * vseg )
275	{
276	// update vseg descriptor
277	vseg->vmm = NULL;
278
279	// decrement vsegs number
280	vmm->vsegs_nr--;
281
282	// remove vseg from VSL
283	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
284
285	} // end vmm_detach_from_vsl()
286
287	////////////////////////////////////////////
288	error_t vmm_user_init( process_t * process )
289	{
290	uint32_t i;
291
292	#if DEBUG_VMM_USER_INIT
293	thread_t * this = CURRENT_THREAD;
294	uint32_t cycle = (uint32_t)hal_get_cycles();
295	if( DEBUG_VMM_USER_INIT )
296	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
297	__FUNCTION__ , this->process->pid, this->trdid, process->pid, local_cxy, cycle );
298	#endif
299
300	// get pointer on VMM
301	vmm_t * vmm = &process->vmm;
302
303	// check UTILS zone
304	assert( ((CONFIG_VMM_ARGS_SIZE + CONFIG_VMM_ENVS_SIZE) <=
305	(CONFIG_VMM_ELF_BASE - CONFIG_VMM_UTILS_BASE)) ,
306	"UTILS zone too small\n" );
307
308	// check STACK zone
309	assert( ((CONFIG_VMM_STACK_SIZE * CONFIG_THREADS_MAX_PER_CLUSTER) <=
310	(CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE)) ,
311	"STACK zone too small\n");
312
313	// initialize the lock protecting the VSL
314	remote_queuelock_init( XPTR( local_cxy , &vmm->vsl_lock ) , LOCK_VMM_VSL );
315
316
317	/*
318	// register "args" vseg in VSL
319	base = CONFIG_VMM_UTILS_BASE << CONFIG_PPM_PAGE_SHIFT;
320	size = CONFIG_VMM_ARGS_SIZE << CONFIG_PPM_PAGE_SHIFT;
321
322	vseg_args = vmm_create_vseg( process,
323	VSEG_TYPE_DATA,
324	base,
325	size,
326	0, // file_offset unused
327	0, // file_size unused
328	XPTR_NULL, // mapper_xp unused
329	local_cxy );
330	if( vseg_args == NULL )
331	{
332	printk("\n[ERROR] in %s : cannot register args vseg\n", __FUNCTION__ );
333	return -1;
334	}
335
336	vmm->args_vpn_base = base;
337
338	// register "envs" vseg in VSL
339	base = (CONFIG_VMM_UTILS_BASE + CONFIG_VMM_ARGS_SIZE) << CONFIG_PPM_PAGE_SHIFT;
340	size = CONFIG_VMM_ENVS_SIZE << CONFIG_PPM_PAGE_SHIFT;
341
342	vseg_envs = vmm_create_vseg( process,
343	VSEG_TYPE_DATA,
344	base,
345	size,
346	0, // file_offset unused
347	0, // file_size unused
348	XPTR_NULL, // mapper_xp unused
349	local_cxy );
350	if( vseg_envs == NULL )
351	{
352	printk("\n[ERROR] in %s : cannot register envs vseg\n", __FUNCTION__ );
353	return -1;
354	}
355
356	vmm->envs_vpn_base = base;
357	*/
358	// initialize STACK allocator
359	vmm->stack_mgr.bitmap = 0;
360	vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
361	busylock_init( &vmm->stack_mgr.lock , LOCK_VMM_STACK );
362
363	// initialize MMAP allocator
364	vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
365	vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
366	vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
367	busylock_init( &vmm->mmap_mgr.lock , LOCK_VMM_MMAP );
368	for( i = 0 ; i < 32 ; i++ )
369	{
370	xlist_root_init( XPTR( local_cxy , &vmm->mmap_mgr.zombi_list[i] ) );
371	}
372
373	// initialize instrumentation counters
374	vmm->false_pgfault_nr = 0;
375	vmm->local_pgfault_nr = 0;
376	vmm->global_pgfault_nr = 0;
377	vmm->false_pgfault_cost = 0;
378	vmm->local_pgfault_cost = 0;
379	vmm->global_pgfault_cost = 0;
380
381	hal_fence();
382
383	#if DEBUG_VMM_USER_INIT
384	cycle = (uint32_t)hal_get_cycles();
385	if( DEBUG_VMM_USER_INIT )
386	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
387	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
388	#endif
389
390	return 0;
391
392	} // end vmm_user_init()
393
394	//////////////////////////////////////////
395	void vmm_user_reset( process_t * process )
396	{
397	xptr_t vseg_xp;
398	vseg_t * vseg;
399	vseg_type_t vseg_type;
400
401	#if DEBUG_VMM_USER_RESET
402	uint32_t cycle;
403	thread_t * this = CURRENT_THREAD;
404	#endif
405
406	#if (DEBUG_VMM_USER_RESET & 1 )
407	cycle = (uint32_t)hal_get_cycles();
408	if( DEBUG_VMM_USER_RESET < cycle )
409	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
410	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
411	#endif
412
413	#if (DEBUG_VMM_USER_RESET & 1 )
414	if( DEBUG_VMM_USER_RESET < cycle )
415	hal_vmm_display( XPTR( local_cxy , process ) , true );
416	#endif
417
418	// get pointer on local VMM
419	vmm_t * vmm = &process->vmm;
420
421	// build extended pointer on VSL root and VSL lock
422	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
423	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsl_lock );
424
425	// take the VSL lock
426	remote_queuelock_acquire( lock_xp );
427
428	// scan the VSL to delete all non kernel vsegs
429	// (we don't use a FOREACH in case of item deletion)
430	xptr_t iter_xp;
431	xptr_t next_xp;
432	for( iter_xp = hal_remote_l64( root_xp ) ;
433	iter_xp != root_xp ;
434	iter_xp = next_xp )
435	{
436	// save extended pointer on next item in xlist
437	next_xp = hal_remote_l64( iter_xp );
438
439	// get pointers on current vseg in VSL
440	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
441	vseg = GET_PTR( vseg_xp );
442	vseg_type = vseg->type;
443
444	#if( DEBUG_VMM_USER_RESET & 1 )
445	if( DEBUG_VMM_USER_RESET < cycle )
446	printk("\n[%s] found %s vseg / vpn_base %x / vpn_size %d\n",
447	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
448	#endif
449	// delete non kernel vseg
450	if( (vseg_type != VSEG_TYPE_KCODE) &&
451	(vseg_type != VSEG_TYPE_KDATA) &&
452	(vseg_type != VSEG_TYPE_KDEV ) )
453	{
454	// remove vseg from VSL
455	vmm_remove_vseg( process , vseg );
456
457	#if( DEBUG_VMM_USER_RESET & 1 )
458	if( DEBUG_VMM_USER_RESET < cycle )
459	printk("\n[%s] %s vseg deleted / vpn_base %x / vpn_size %d\n",
460	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
461	#endif
462	}
463	else
464	{
465
466	#if( DEBUG_VMM_USER_RESET & 1 )
467	if( DEBUG_VMM_USER_RESET < cycle )
468	printk("\n[%s] keep %s vseg / vpn_base %x / vpn_size %d\n",
469	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
470	#endif
471	}
472	} // end loop on vsegs in VSL
473
474	// release the VSL lock
475	remote_queuelock_release( lock_xp );
476
477	// FIXME il faut gérer les process copies...
478
479	#if DEBUG_VMM_USER_RESET
480	cycle = (uint32_t)hal_get_cycles();
481	if( DEBUG_VMM_USER_RESET < cycle )
482	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
483	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy , cycle );
484	#endif
485
486	#if (DEBUG_VMM_USER_RESET & 1 )
487	if( DEBUG_VMM_USER_RESET < cycle )
488	hal_vmm_display( XPTR( local_cxy , process ) , true );
489	#endif
490
491	} // end vmm_user_reset()
492
493	/////////////////////////////////////////////////
494	void vmm_global_delete_vseg( process_t * process,
495	intptr_t base )
496	{
497	pid_t pid;
498	cxy_t owner_cxy;
499	lpid_t owner_lpid;
500
501	xlist_entry_t * process_root_ptr;
502	xptr_t process_root_xp;
503	xptr_t process_iter_xp;
504
505	xptr_t remote_process_xp;
506	cxy_t remote_process_cxy;
507	process_t * remote_process_ptr;
508
509	xptr_t vsl_root_xp;
510	xptr_t vsl_lock_xp;
511	xptr_t vsl_iter_xp;
512
513	#if DEBUG_VMM_GLOBAL_DELETE_VSEG
514	uint32_t cycle = (uint32_t)hal_get_cycles();
515	thread_t * this = CURRENT_THREAD;
516	#endif
517
518	#if (DEBUG_VMM_GLOBAL_DELETE_VSEG & 1)
519	if( DEBUG_VMM_GLOBAL_DELETE_VSEG < cycle )
520	printk("\n[%s] thread[%x,%x] : process %x / base %x / cycle %d\n",
521	__FUNCTION__, this->process->pid, this->trdid, process->pid, base, cycle );
522	#endif
523
524	// get owner process cluster and local index
525	pid = process->pid;
526	owner_cxy = CXY_FROM_PID( pid );
527	owner_lpid = LPID_FROM_PID( pid );
528
529	// get extended pointer on root of process copies xlist in owner cluster
530	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
531	process_root_xp = XPTR( owner_cxy , process_root_ptr );
532
533	// loop on process copies
534	XLIST_FOREACH( process_root_xp , process_iter_xp )
535	{
536	// get cluster and local pointer on remote process
537	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
538	remote_process_ptr = GET_PTR( remote_process_xp );
539	remote_process_cxy = GET_CXY( remote_process_xp );
540
541	// build extended pointers on remote VSL root and lock
542	vsl_root_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.vsegs_root );
543	vsl_lock_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.vsl_lock );
544
545	// get lock on remote VSL
546	remote_queuelock_acquire( vsl_lock_xp );
547
548	// loop on vsegs in remote process VSL
549	XLIST_FOREACH( vsl_root_xp , vsl_iter_xp )
550	{
551	// get pointers on current vseg
552	xptr_t vseg_xp = XLIST_ELEMENT( vsl_iter_xp , vseg_t , xlist );
553	vseg_t * vseg_ptr = GET_PTR( vseg_xp );
554
555	// get current vseg base address
556	intptr_t vseg_base = (intptr_t)hal_remote_lpt( XPTR( remote_process_cxy,
557	&vseg_ptr->min ) );
558
559	if( vseg_base == base ) // found searched vseg
560	{
561	if( remote_process_cxy == local_cxy )
562	{
563	vmm_remove_vseg( process,
564	vseg_ptr );
565	}
566	else
567	{
568	rpc_vmm_remove_vseg_client( remote_process_cxy,
569	remote_process_ptr,
570	vseg_ptr );
571	}
572
573	#if (DEBUG_VMM_GLOBAL_DELETE_VSEG & 1)
574	if( DEBUG_VMM_GLOBAL_DELETE_VSEG < cycle )
575	printk("\n[%s] thread[%x,%x] deleted vseg %x for process %x in cluster %x\n",
576	__FUNCTION__, this->process->pid, this->trdid, base, process->pid, remote_process_cxy );
577	#endif
578
579	}
580	} // end of loop on vsegs
581
582	#if (DEBUG_VMM_GLOBAL_DELETE_VSEG & 1)
583	if( DEBUG_VMM_GLOBAL_DELETE_VSEG < cycle )
584	hal_vmm_display( remote_process_xp , false );
585	#endif
586
587	// release lock on remote VSL
588	remote_queuelock_release( vsl_lock_xp );
589
590	} // end of loop on process copies
591
592	#if DEBUG_VMM_GLOBAL_DELETE_VSEG
593	cycle = (uint32_t)hal_get_cycles();
594	if( DEBUG_VMM_GLOBAL_DELETE_VSEG < cycle )
595	printk("\n[%s] thread[%x,%x] exit for process %x / base %x / cycle %d\n",
596	__FUNCTION__, this->process->pid, this->trdid, process->pid , base, cycle );
597	#endif
598
599	} // end vmm_global_delete_vseg()
600
601	////////////////////////////////////////////////
602	void vmm_global_resize_vseg( process_t * process,
603	intptr_t base,
604	intptr_t new_base,
605	intptr_t new_size )
606	{
607	pid_t pid;
608	cxy_t owner_cxy;
609	lpid_t owner_lpid;
610
611	xlist_entry_t * process_root_ptr;
612	xptr_t process_root_xp;
613	xptr_t process_iter_xp;
614
615	xptr_t remote_process_xp;
616	cxy_t remote_process_cxy;
617	process_t * remote_process_ptr;
618
619	xptr_t vsl_root_xp;
620	xptr_t vsl_lock_xp;
621	xptr_t vsl_iter_xp;
622
623	#if DEBUG_VMM_GLOBAL_RESIZE_VSEG
624	uint32_t cycle = (uint32_t)hal_get_cycles();
625	thread_t * this = CURRENT_THREAD;
626	#endif
627
628	#if (DEBUG_VMM_GLOBAL_RESIZE_VSEG & 1)
629	if( DEBUG_VMM_GLOBAL_RESIZE_VSEG < cycle )
630	printk("\n[%s] thread[%x,%x] : process %x / base %x / new_base %x / new_size %x / cycle %d\n",
631	__FUNCTION__, this->process->pid, this->trdid, process->pid, base, new_base, new_size, cycle );
632	#endif
633
634	// get owner process cluster and local index
635	pid = process->pid;
636	owner_cxy = CXY_FROM_PID( pid );
637	owner_lpid = LPID_FROM_PID( pid );
638
639	// get extended pointer on root of process copies xlist in owner cluster
640	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
641	process_root_xp = XPTR( owner_cxy , process_root_ptr );
642
643	// loop on process copies
644	XLIST_FOREACH( process_root_xp , process_iter_xp )
645	{
646	// get cluster and local pointer on remote process
647	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
648	remote_process_ptr = GET_PTR( remote_process_xp );
649	remote_process_cxy = GET_CXY( remote_process_xp );
650
651	// build extended pointers on remote VSL root and lock
652	vsl_root_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.vsegs_root );
653	vsl_lock_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.vsl_lock );
654
655	// get lock on remote VSL
656	remote_queuelock_acquire( vsl_lock_xp );
657
658	// loop on vsegs in remote process VSL
659	XLIST_FOREACH( vsl_root_xp , vsl_iter_xp )
660	{
661	// get pointers on current vseg
662	xptr_t vseg_xp = XLIST_ELEMENT( vsl_iter_xp , vseg_t , xlist );
663	vseg_t * vseg_ptr = GET_PTR( vseg_xp );
664
665	// get current vseg base address
666	intptr_t vseg_base = (intptr_t)hal_remote_lpt( XPTR( remote_process_cxy,
667	&vseg_ptr->min ) );
668
669	if( vseg_base == base ) // found searched vseg
670	{
671	if( remote_process_cxy == local_cxy )
672	{
673	vmm_resize_vseg( remote_process_ptr,
674	vseg_ptr,
675	new_base,
676	new_size );
677	}
678	else
679	{
680	rpc_vmm_resize_vseg_client( remote_process_cxy,
681	remote_process_ptr,
682	vseg_ptr,
683	new_base,
684	new_size );
685	}
686
687	#if (DEBUG_VMM_GLOBAL_RESIZE_VSEG & 1)
688	if( DEBUG_VMM_GLOBAL_RESIZE_VSEG < cycle )
689	printk("\n[%s] thread[%x,%x] resized vseg %x for process %x in cluster %x\n",
690	__FUNCTION__, this->process->pid, this->trdid, base, process->pid, remote_process_cxy );
691	#endif
692
693	}
694	} // end of loop on vsegs
695
696	#if (DEBUG_VMM_GLOBAL_RESIZE_VSEG & 1)
697	if( DEBUG_VMM_GLOBAL_RESIZE_VSEG < cycle )
698	hal_vmm_display( remote_process_xp , false );
699	#endif
700
701	// release lock on remote VSL
702	remote_queuelock_release( vsl_lock_xp );
703	} // end of loop on process copies
704
705	#if DEBUG_VMM_GLOBAL_RESIZE_VSEG
706	cycle = (uint32_t)hal_get_cycles();
707	if( DEBUG_VMM_GLOBAL_RESIZE_VSEG < cycle )
708	printk("\n[%s] thread[%x,%x] exit for process %x / base %x / cycle %d\n",
709	__FUNCTION__, this->process->pid, this->trdid, process->pid , base, cycle );
710	#endif
711
712	} // end vmm_global_resize_vseg()
713
714	////////////////////////////////////////////////
715	void vmm_global_update_pte( process_t * process,
716	vpn_t vpn,
717	uint32_t attr,
718	ppn_t ppn )
719	{
720	pid_t pid;
721	cxy_t owner_cxy;
722	lpid_t owner_lpid;
723
724	xlist_entry_t * process_root_ptr;
725	xptr_t process_root_xp;
726	xptr_t process_iter_xp;
727
728	xptr_t remote_process_xp;
729	cxy_t remote_process_cxy;
730	process_t * remote_process_ptr;
731	xptr_t remote_gpt_xp;
732
733	#if DEBUG_VMM_GLOBAL_UPDATE_PTE
734	uint32_t cycle = (uint32_t)hal_get_cycles();
735	thread_t * this = CURRENT_THREAD;
736	#endif
737
738
739	#if (DEBUG_VMM_GLOBAL_UPDATE_PTE & 1)
740	if( DEBUG_VMM_GLOBAL_UPDATE_PTE < cycle )
741	printk("\n[%s] thread[%x,%x] enter for process %x / vpn %x / attr %x / ppn %x / ycle %d\n",
742	__FUNCTION__, this->process->pid, this->trdid, process->pid, vpn, attr, ppn, cycle );
743	#endif
744
745	// get owner process cluster and local index
746	pid = process->pid;
747	owner_cxy = CXY_FROM_PID( pid );
748	owner_lpid = LPID_FROM_PID( pid );
749
750	// get extended pointer on root of process copies xlist in owner cluster
751	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
752	process_root_xp = XPTR( owner_cxy , process_root_ptr );
753
754	// loop on process copies
755	XLIST_FOREACH( process_root_xp , process_iter_xp )
756	{
757	// get cluster and local pointer on remote process
758	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
759	remote_process_ptr = GET_PTR( remote_process_xp );
760	remote_process_cxy = GET_CXY( remote_process_xp );
761
762	#if (DEBUG_VMM_GLOBAL_UPDATE_PTE & 1)
763	if( DEBUG_VMM_GLOBAL_UPDATE_PTE < cycle )
764	printk("\n[%s] thread[%x,%x] handling vpn %x for process %x in cluster %x\n",
765	__FUNCTION__, this->process->pid, this->trdid, vpn, process->pid, remote_process_cxy );
766	#endif
767
768	// get extended pointer on remote gpt
769	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
770
771	// update remote GPT
772	hal_gpt_update_pte( remote_gpt_xp, vpn, attr, ppn );
773	}
774
775	#if DEBUG_VMM_GLOBAL_UPDATE_PTE
776	cycle = (uint32_t)hal_get_cycles();
777	if( DEBUG_VMM_GLOBAL_UPDATE_PTE < cycle )
778	printk("\n[%s] thread[%x,%x] exit for process %x / vpn %x / cycle %d\n",
779	__FUNCTION__, this->process->pid, this->trdid, process->pid , vpn , cycle );
780	#endif
781
782	#if (DEBUG_VMM_GLOBAL_UPDATE_PTE & 1)
783	hal_vmm_display( process , true );
784	#endif
785
786	} // end vmm_global_update_pte()
787
788	///////////////////////////////////////
789	void vmm_set_cow( process_t * process )
790	{
791	vmm_t * vmm;
792
793	xlist_entry_t * process_root_ptr;
794	xptr_t process_root_xp;
795	xptr_t process_iter_xp;
796
797	xptr_t remote_process_xp;
798	cxy_t remote_process_cxy;
799	process_t * remote_process_ptr;
800	xptr_t remote_gpt_xp;
801
802	xptr_t vseg_root_xp;
803	xptr_t vseg_iter_xp;
804
805	xptr_t vseg_xp;
806	vseg_t * vseg;
807
808	pid_t pid;
809	cxy_t owner_cxy;
810	lpid_t owner_lpid;
811
812	// get target process PID
813	pid = process->pid;
814
815	#if DEBUG_VMM_SET_COW
816	uint32_t cycle = (uint32_t)hal_get_cycles();
817	thread_t * this = CURRENT_THREAD;
818	if( DEBUG_VMM_SET_COW < cycle )
819	printk("\n[%s] thread[%x,%x] enter for process %x / cycle %d\n",
820	__FUNCTION__, this->process->pid, this->trdid, pid , cycle );
821	#endif
822
823	#if (DEBUG_VMM_SET_COW & 1)
824	if( DEBUG_VMM_SET_COW < cycle )
825	hal_vmm_display( process , true );
826	#endif
827
828	// check cluster is reference
829	assert( (XPTR( local_cxy , process ) == process->ref_xp),
830	"local cluster must be process reference cluster\n");
831
832	// get pointer on reference VMM
833	vmm = &process->vmm;
834
835	// get extended pointer on root of process copies xlist in owner cluster
836	owner_cxy = CXY_FROM_PID( pid );
837	owner_lpid = LPID_FROM_PID( pid );
838	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
839	process_root_xp = XPTR( owner_cxy , process_root_ptr );
840
841	// get extended pointer on root of vsegs xlist from reference VMM
842	vseg_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
843
844	// loop on target process copies
845	XLIST_FOREACH( process_root_xp , process_iter_xp )
846	{
847	// get cluster and local pointer on remote process copy
848	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
849	remote_process_ptr = GET_PTR( remote_process_xp );
850	remote_process_cxy = GET_CXY( remote_process_xp );
851
852	#if (DEBUG_VMM_SET_COW & 1)
853	if( DEBUG_VMM_SET_COW < cycle )
854	printk("\n[%s] thread[%x,%x] (%x) handles process %x in cluster %x\n",
855	__FUNCTION__, this->process->pid, this->trdid, this, pid, remote_process_cxy );
856	#endif
857
858	// get extended pointer on remote gpt
859	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
860
861	// loop on vsegs in (local) reference process VSL
862	XLIST_FOREACH( vseg_root_xp , vseg_iter_xp )
863	{
864	// get pointer on vseg
865	vseg_xp = XLIST_ELEMENT( vseg_iter_xp , vseg_t , xlist );
866	vseg = GET_PTR( vseg_xp );
867
868	// get vseg type, base and size
869	uint32_t type = vseg->type;
870	vpn_t vpn_base = vseg->vpn_base;
871	vpn_t vpn_size = vseg->vpn_size;
872
873	#if (DEBUG_VMM_SET_COW & 1)
874	if( DEBUG_VMM_SET_COW < cycle )
875	printk("\n[%s] thread[%x,%x] found vseg %s / vpn_base = %x / vpn_size = %x\n",
876	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), vpn_base, vpn_size );
877	#endif
878	// only DATA, ANON and REMOTE vsegs
879	if( (type == VSEG_TYPE_DATA) \|\|
880	(type == VSEG_TYPE_ANON) \|\|
881	(type == VSEG_TYPE_REMOTE) )
882	{
883	vpn_t vpn;
884	uint32_t attr;
885	ppn_t ppn;
886	xptr_t page_xp;
887	cxy_t page_cxy;
888	page_t * page_ptr;
889	xptr_t forks_xp;
890	xptr_t lock_xp;
891
892	// update flags in remote GPT
893	hal_gpt_set_cow( remote_gpt_xp,
894	vpn_base,
895	vpn_size );
896
897	// atomically increment pending forks counter in physical pages,
898	// this is only done once, when handling the reference copy
899	if( remote_process_cxy == local_cxy )
900	{
901
902	#if (DEBUG_VMM_SET_COW & 1)
903	if( DEBUG_VMM_SET_COW < cycle )
904	printk("\n[%s] thread[%x,%x] handles vseg %s / vpn_base = %x / vpn_size = %x\n",
905	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), vpn_base, vpn_size );
906	#endif
907	// scan all pages in vseg
908	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
909	{
910	// get page attributes and PPN from reference GPT
911	hal_gpt_get_pte( remote_gpt_xp , vpn , &attr , &ppn );
912
913	// atomically update pending forks counter if page is mapped
914	if( attr & GPT_MAPPED )
915	{
916	// get pointers and cluster on page descriptor
917	page_xp = ppm_ppn2page( ppn );
918	page_cxy = GET_CXY( page_xp );
919	page_ptr = GET_PTR( page_xp );
920
921	// get extended pointers on "forks" and "lock"
922	forks_xp = XPTR( page_cxy , &page_ptr->forks );
923	lock_xp = XPTR( page_cxy , &page_ptr->lock );
924
925	// take lock protecting "forks" counter
926	remote_busylock_acquire( lock_xp );
927
928	// increment "forks"
929	hal_remote_atomic_add( forks_xp , 1 );
930
931	// release lock protecting "forks" counter
932	remote_busylock_release( lock_xp );
933	}
934	} // end loop on vpn
935
936	#if (DEBUG_VMM_SET_COW & 1)
937	if( DEBUG_VMM_SET_COW < cycle )
938	printk("\n[%s] thread[%x,%x] completes vseg %s / vpn_base = %x / vpn_size = %x\n",
939	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), vpn_base, vpn_size );
940	#endif
941	} // end if local
942	} // end if vseg type
943	} // end loop on vsegs
944	} // end loop on process copies
945
946	#if DEBUG_VMM_SET_COW
947	cycle = (uint32_t)hal_get_cycles();
948	if( DEBUG_VMM_SET_COW < cycle )
949	printk("\n[%s] thread[%x,%x] exit for process %x / cycle %d\n",
950	__FUNCTION__, this->process->pid, this->trdid, process->pid , cycle );
951	#endif
952
953	} // end vmm_set-cow()
954
955	/////////////////////////////////////////////////
956	error_t vmm_fork_copy( process_t * child_process,
957	xptr_t parent_process_xp )
958	{
959	error_t error;
960	cxy_t parent_cxy;
961	process_t * parent_process;
962	vmm_t * parent_vmm;
963	xptr_t parent_lock_xp;
964	vmm_t * child_vmm;
965	xptr_t iter_xp;
966	xptr_t parent_vseg_xp;
967	vseg_t * parent_vseg;
968	vseg_t * child_vseg;
969	uint32_t type;
970	vpn_t vpn;
971	vpn_t vpn_base;
972	vpn_t vpn_size;
973	xptr_t parent_root_xp;
974	bool_t mapped;
975	ppn_t ppn;
976
977	#if DEBUG_VMM_FORK_COPY
978	uint32_t cycle = (uint32_t)hal_get_cycles();
979	thread_t * this = CURRENT_THREAD;
980	if( DEBUG_VMM_FORK_COPY < cycle )
981	printk("\n[%s] thread %x enter / cycle %d\n",
982	__FUNCTION__ , this->process->pid, this->trdid, cycle );
983	#endif
984
985	// get parent process cluster and local pointer
986	parent_cxy = GET_CXY( parent_process_xp );
987	parent_process = GET_PTR( parent_process_xp );
988
989	// get local pointers on parent and child VMM
990	parent_vmm = &parent_process->vmm;
991	child_vmm = &child_process->vmm;
992
993	// build extended pointer on parent VSL root and lock
994	parent_root_xp = XPTR( parent_cxy , &parent_vmm->vsegs_root );
995	parent_lock_xp = XPTR( parent_cxy , &parent_vmm->vsl_lock );
996
997	// take the lock protecting the parent VSL
998	remote_queuelock_acquire( parent_lock_xp );
999
1000	// loop on parent VSL xlist
1001	XLIST_FOREACH( parent_root_xp , iter_xp )
1002	{
1003	// get pointers on current parent vseg
1004	parent_vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
1005	parent_vseg = GET_PTR( parent_vseg_xp );
1006
1007	// get vseg type
1008	type = hal_remote_l32( XPTR( parent_cxy , &parent_vseg->type ) );
1009
1010	#if DEBUG_VMM_FORK_COPY
1011	cycle = (uint32_t)hal_get_cycles();
1012	if( DEBUG_VMM_FORK_COPY < cycle )
1013	printk("\n[%s] thread[%x,%x] found parent vseg %s / vpn_base = %x / cycle %d\n",
1014	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
1015	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
1016	#endif
1017
1018	// all parent vsegs - but STACK and kernel vsegs - must be copied in child VSL
1019	if( (type != VSEG_TYPE_STACK) && (type != VSEG_TYPE_KCODE) &&
1020	(type != VSEG_TYPE_KDATA) && (type != VSEG_TYPE_KDEV) )
1021	{
1022	// allocate memory for a new child vseg
1023	child_vseg = vseg_alloc();
1024	if( child_vseg == NULL ) // release all allocated vsegs
1025	{
1026	vmm_destroy( child_process );
1027	printk("\n[ERROR] in %s : cannot create vseg for child\n", __FUNCTION__ );
1028	return -1;
1029	}
1030
1031	// copy parent vseg to child vseg
1032	vseg_init_from_ref( child_vseg , parent_vseg_xp );
1033
1034	// build extended pointer on child VSL lock
1035	xptr_t child_lock_xp = XPTR( local_cxy , &child_vmm->vsl_lock );
1036
1037	// take the child VSL lock
1038	remote_queuelock_acquire( child_lock_xp );
1039
1040	// register child vseg in child VSL
1041	vmm_attach_vseg_to_vsl( child_vmm , child_vseg );
1042
1043	// release the child VSL lock
1044	remote_queuelock_release( child_lock_xp );
1045
1046	#if DEBUG_VMM_FORK_COPY
1047	cycle = (uint32_t)hal_get_cycles();
1048	if( DEBUG_VMM_FORK_COPY < cycle )
1049	printk("\n[%s] thread[%x,%x] copied vseg %s / vpn_base = %x to child VSL / cycle %d\n",
1050	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
1051	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
1052	#endif
1053	// copy DATA, ANON, REMOTE, FILE parent GPT entries to child GPT
1054	if( type != VSEG_TYPE_CODE )
1055	{
1056	// activate the COW for DATA, ANON, REMOTE vsegs only
1057	// cow = ( type != VSEG_TYPE_FILE );
1058
1059	vpn_base = child_vseg->vpn_base;
1060	vpn_size = child_vseg->vpn_size;
1061
1062	// scan pages in parent vseg
1063	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
1064	{
1065	error = hal_gpt_pte_copy( &child_vmm->gpt,
1066	vpn,
1067	XPTR( parent_cxy , &parent_vmm->gpt ),
1068	vpn,
1069	false, // does not handle COW flag
1070	&ppn, // unused
1071	&mapped ); // unused
1072	if( error )
1073	{
1074	vmm_destroy( child_process );
1075	printk("\n[ERROR] in %s : cannot copy GPT\n", __FUNCTION__ );
1076	return -1;
1077	}
1078
1079	#if DEBUG_VMM_FORK_COPY
1080	cycle = (uint32_t)hal_get_cycles();
1081	if( DEBUG_VMM_FORK_COPY < cycle )
1082	printk("\n[%s] thread[%x,%x] copied vpn %x to child GPT / cycle %d\n",
1083	__FUNCTION__ , this->process->pid, this->trdid , vpn , cycle );
1084	#endif
1085	}
1086	} // end if no code & no stack
1087	} // end if no stack
1088	} // end loop on vsegs
1089
1090	// release the parent VSL lock in read mode
1091	remote_queuelock_release( parent_lock_xp );
1092
1093	// initialize the child VMM STACK allocator
1094	child_vmm->stack_mgr.bitmap = 0;
1095	child_vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
1096
1097	// initialize the child VMM MMAP allocator
1098	uint32_t i;
1099	child_vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
1100	child_vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
1101	child_vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
1102	for( i = 0 ; i < 32 ; i++ )
1103	{
1104	xlist_root_init( XPTR( local_cxy , &child_vmm->mmap_mgr.zombi_list[i] ) );
1105	}
1106
1107	// initialize instrumentation counters
1108	child_vmm->false_pgfault_nr = 0;
1109	child_vmm->local_pgfault_nr = 0;
1110	child_vmm->global_pgfault_nr = 0;
1111	child_vmm->false_pgfault_cost = 0;
1112	child_vmm->local_pgfault_cost = 0;
1113	child_vmm->global_pgfault_cost = 0;
1114
1115	// copy base addresses from parent VMM to child VMM
1116	child_vmm->args_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->args_vpn_base));
1117	child_vmm->envs_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->envs_vpn_base));
1118	child_vmm->heap_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->heap_vpn_base));
1119	child_vmm->code_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->code_vpn_base));
1120	child_vmm->data_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->data_vpn_base));
1121
1122	child_vmm->entry_point = (intptr_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->entry_point));
1123
1124	hal_fence();
1125
1126	#if DEBUG_VMM_FORK_COPY
1127	cycle = (uint32_t)hal_get_cycles();
1128	if( DEBUG_VMM_FORK_COPY < cycle )
1129	printk("\n[%s] thread[%x,%x] exit successfully / cycle %d\n",
1130	__FUNCTION__ , this->process->pid, this->trdid , cycle );
1131	#endif
1132
1133	return 0;
1134
1135	} // vmm_fork_copy()
1136
1137	///////////////////////////////////////
1138	void vmm_destroy( process_t * process )
1139	{
1140	xptr_t vseg_xp;
1141	vseg_t * vseg;
1142
1143	#if DEBUG_VMM_DESTROY
1144	uint32_t cycle = (uint32_t)hal_get_cycles();
1145	thread_t * this = CURRENT_THREAD;
1146	if( DEBUG_VMM_DESTROY < cycle )
1147	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
1148	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
1149	#endif
1150
1151	#if (DEBUG_VMM_DESTROY & 1 )
1152	if( DEBUG_VMM_DESTROY < cycle )
1153	hal_vmm_display( XPTR( local_cxy, process ) , true );
1154	#endif
1155
1156	// get pointer on local VMM
1157	vmm_t * vmm = &process->vmm;
1158
1159	// build extended pointer on VSL root, VSL lock and GPT lock
1160	xptr_t vsl_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
1161	xptr_t vsl_lock_xp = XPTR( local_cxy , &vmm->vsl_lock );
1162
1163	// take the VSL lock
1164	remote_queuelock_acquire( vsl_lock_xp );
1165
1166	// scan the VSL to delete all registered vsegs
1167	// (we don't use a FOREACH in case of item deletion)
1168	xptr_t iter_xp;
1169	xptr_t next_xp;
1170	for( iter_xp = hal_remote_l64( vsl_root_xp ) ;
1171	iter_xp != vsl_root_xp ;
1172	iter_xp = next_xp )
1173	{
1174	// save extended pointer on next item in xlist
1175	next_xp = hal_remote_l64( iter_xp );
1176
1177	// get pointers on current vseg in VSL
1178	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
1179	vseg = GET_PTR( vseg_xp );
1180
1181	// delete vseg and release physical pages
1182	vmm_remove_vseg( process , vseg );
1183
1184	#if( DEBUG_VMM_DESTROY & 1 )
1185	if( DEBUG_VMM_DESTROY < cycle )
1186	printk("\n[%s] %s vseg deleted / vpn_base %x / vpn_size %d\n",
1187	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
1188	#endif
1189
1190	}
1191
1192	// release the VSL lock
1193	remote_queuelock_release( vsl_lock_xp );
1194
1195	// remove all registered MMAP vsegs
1196	// from zombi_lists in MMAP allocator
1197	uint32_t i;
1198	for( i = 0 ; i<32 ; i++ )
1199	{
1200	// build extended pointer on zombi_list[i]
1201	xptr_t root_xp = XPTR( local_cxy , &vmm->mmap_mgr.zombi_list[i] );
1202
1203	// scan zombi_list[i]
1204	while( !xlist_is_empty( root_xp ) )
1205	{
1206	vseg_xp = XLIST_FIRST( root_xp , vseg_t , xlist );
1207	vseg = GET_PTR( vseg_xp );
1208
1209	#if( DEBUG_VMM_DESTROY & 1 )
1210	if( DEBUG_VMM_DESTROY < cycle )
1211	printk("\n[%s] found zombi vseg / vpn_base %x / vpn_size %d\n",
1212	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
1213	#endif
1214	// clean vseg descriptor
1215	vseg->vmm = NULL;
1216
1217	// remove vseg from zombi_list
1218	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
1219
1220	// release vseg descriptor
1221	vseg_free( vseg );
1222
1223	#if( DEBUG_VMM_DESTROY & 1 )
1224	if( DEBUG_VMM_DESTROY < cycle )
1225	printk("\n[%s] zombi vseg released / vpn_base %x / vpn_size %d\n",
1226	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
1227	#endif
1228	}
1229	}
1230
1231	// release memory allocated to the GPT itself
1232	hal_gpt_destroy( &vmm->gpt );
1233
1234	#if DEBUG_VMM_DESTROY
1235	cycle = (uint32_t)hal_get_cycles();
1236	if( DEBUG_VMM_DESTROY < cycle )
1237	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
1238	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy , cycle );
1239	#endif
1240
1241	} // end vmm_destroy()
1242
1243	/////////////////////////////////////////////////
1244	vseg_t * vmm_check_conflict( process_t * process,
1245	vpn_t vpn_base,
1246	vpn_t vpn_size )
1247	{
1248	vmm_t * vmm = &process->vmm;
1249
1250	// scan the VSL
1251	vseg_t * vseg;
1252	xptr_t iter_xp;
1253	xptr_t vseg_xp;
1254	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
1255
1256	XLIST_FOREACH( root_xp , iter_xp )
1257	{
1258	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
1259	vseg = GET_PTR( vseg_xp );
1260
1261	if( ((vpn_base + vpn_size) > vseg->vpn_base) &&
1262	(vpn_base < (vseg->vpn_base + vseg->vpn_size)) ) return vseg;
1263	}
1264	return NULL;
1265
1266	} // end vmm_check_conflict()
1267
1268	////////////////////////////////////////////////
1269	vseg_t * vmm_create_vseg( process_t * process,
1270	vseg_type_t type,
1271	intptr_t base, // ltid for VSEG_TYPE_STACK
1272	uint32_t size,
1273	uint32_t file_offset,
1274	uint32_t file_size,
1275	xptr_t mapper_xp,
1276	cxy_t cxy )
1277	{
1278	vseg_t * vseg; // created vseg pointer
1279	vpn_t vpn_base; // first page index
1280	vpn_t vpn_size; // number of pages covered by vseg
1281	error_t error;
1282
1283	#if DEBUG_VMM_CREATE_VSEG
1284	thread_t * this = CURRENT_THREAD;
1285	uint32_t cycle;
1286	#endif
1287
1288	#if (DEBUG_VMM_CREATE_VSEG & 1)
1289	cycle = (uint32_t)hal_get_cycles();
1290	if( DEBUG_VMM_CREATE_VSEG < cycle )
1291	printk("\n[%s] thread[%x,%x] enter / process %x / %s / base %x / cxy %x / cycle %d\n",
1292	__FUNCTION__, this->process->pid, this->trdid,
1293	process->pid, vseg_type_str(type), base, cxy, cycle );
1294	#endif
1295
1296	// get pointer on VMM
1297	vmm_t * vmm = &process->vmm;
1298
1299	// compute base, size, vpn_base, vpn_size, depending on vseg type
1300	// we use the VMM specific allocators for "stack", "file", "anon", & "remote" vsegs
1301
1302	if( type == VSEG_TYPE_STACK )
1303	{
1304	// get vpn_base and vpn_size from STACK allocator
1305	vmm_stack_alloc( vmm , base , &vpn_base , &vpn_size );
1306
1307	// compute vseg base and size from vpn_base and vpn_size
1308	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
1309	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
1310	}
1311	else if( type == VSEG_TYPE_FILE )
1312	{
1313	// compute page index (in mapper) for first byte
1314	vpn_t vpn_min = file_offset >> CONFIG_PPM_PAGE_SHIFT;
1315
1316	// compute page index (in mapper) for last byte
1317	vpn_t vpn_max = (file_offset + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
1318
1319	// compute offset in first page
1320	uint32_t offset = file_offset & CONFIG_PPM_PAGE_MASK;
1321
1322	// compute number of pages required in virtual space
1323	vpn_t npages = vpn_max - vpn_min + 1;
1324
1325	// get vpn_base and vpn_size from MMAP allocator
1326	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
1327	if( error )
1328	{
1329	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
1330	__FUNCTION__ , process->pid , local_cxy );
1331	return NULL;
1332	}
1333
1334	// set the vseg base (not always aligned for FILE)
1335	base = (vpn_base << CONFIG_PPM_PAGE_SHIFT) + offset;
1336	}
1337	else if( (type == VSEG_TYPE_ANON) \|\|
1338	(type == VSEG_TYPE_REMOTE) )
1339	{
1340	// compute number of required pages in virtual space
1341	vpn_t npages = size >> CONFIG_PPM_PAGE_SHIFT;
1342	if( size & CONFIG_PPM_PAGE_MASK) npages++;
1343
1344	// get vpn_base and vpn_size from MMAP allocator
1345	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
1346	if( error )
1347	{
1348	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
1349	__FUNCTION__ , process->pid , local_cxy );
1350	return NULL;
1351	}
1352
1353	// set vseg base (always aligned for ANON or REMOTE)
1354	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
1355	}
1356	else // VSEG_TYPE_DATA, VSEG_TYPE_CODE or KERNEL vseg
1357	{
1358	uint32_t vpn_min = base >> CONFIG_PPM_PAGE_SHIFT;
1359	uint32_t vpn_max = (base + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
1360
1361	vpn_base = vpn_min;
1362	vpn_size = vpn_max - vpn_min + 1;
1363	}
1364
1365	// check collisions
1366	vseg = vmm_check_conflict( process , vpn_base , vpn_size );
1367
1368	if( vseg != NULL )
1369	{
1370	printk("\n[ERROR] in %s for process %x : new vseg [vpn_base %x / vpn_size %x]\n"
1371	" overlap existing vseg [vpn_base %x / vpn_size %x]\n",
1372	__FUNCTION__ , process->pid, vpn_base, vpn_size, vseg->vpn_base, vseg->vpn_size );
1373	return NULL;
1374	}
1375
1376	// allocate physical memory for vseg descriptor
1377	vseg = vseg_alloc();
1378	if( vseg == NULL )
1379	{
1380	printk("\n[ERROR] in %s for process %x : cannot allocate memory for vseg\n",
1381	__FUNCTION__ , process->pid );
1382	return NULL;
1383	}
1384
1385	#if (DEBUG_VMM_CREATE_VSEG & 1)
1386	if( DEBUG_VMM_CREATE_VSEG < cycle )
1387	printk("\n[%s] thread[%x,%x] : base %x / size %x / vpn_base %x / vpn_size %x\n",
1388	__FUNCTION__, this->process->pid, this->trdid, base, size, vpn_base, vpn_size );
1389	#endif
1390
1391	// initialize vseg descriptor
1392	vseg_init( vseg,
1393	type,
1394	base,
1395	size,
1396	vpn_base,
1397	vpn_size,
1398	file_offset,
1399	file_size,
1400	mapper_xp,
1401	cxy );
1402
1403	// build extended pointer on VSL lock
1404	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsl_lock );
1405
1406	// take the VSL lock in write mode
1407	remote_queuelock_acquire( lock_xp );
1408
1409	// attach vseg to VSL
1410	vmm_attach_vseg_to_vsl( vmm , vseg );
1411
1412	// release the VSL lock
1413	remote_queuelock_release( lock_xp );
1414
1415	#if DEBUG_VMM_CREATE_VSEG
1416	cycle = (uint32_t)hal_get_cycles();
1417	// if( DEBUG_VMM_CREATE_VSEG < cycle )
1418	if( type == VSEG_TYPE_REMOTE )
1419	printk("\n[%s] thread[%x,%x] exit / process %x / %s / base %x / cxy %x / cycle %d\n",
1420	__FUNCTION__, this->process->pid, this->trdid,
1421	process->pid, vseg_type_str(type), base, cxy, cycle );
1422	#endif
1423
1424	return vseg;
1425
1426	} // vmm_create_vseg()
1427
1428	////////////////////////////////////////////////////////////////////////////////////////////
1429	// This static function is called by the vmm_remove_vseg() and vmm_resize_vseg() functions.
1430	// Depending on the vseg <type>, it decrements the physical page refcount, and
1431	// conditionnally release to the relevant kmem the physical page identified by <ppn>.
1432	////////////////////////////////////////////////////////////////////////////////////////////
1433	// @ process : local pointer on process.
1434	// @ vseg : local pointer on vseg.
1435	// @ ppn : released pysical page index.
1436	////////////////////////////////////////////////////////////////////////////////////////////
1437	static void vmm_ppn_release( process_t * process,
1438	vseg_t * vseg,
1439	ppn_t ppn )
1440	{
1441	bool_t do_release;
1442
1443	// get vseg type
1444	vseg_type_t type = vseg->type;
1445
1446	// compute is_ref
1447	bool_t is_ref = (GET_CXY( process->ref_xp ) == local_cxy);
1448
1449	// get pointers on physical page descriptor
1450	xptr_t page_xp = ppm_ppn2page( ppn );
1451	cxy_t page_cxy = GET_CXY( page_xp );
1452	page_t * page_ptr = GET_PTR( page_xp );
1453
1454	// decrement page refcount
1455	xptr_t count_xp = XPTR( page_cxy , &page_ptr->refcount );
1456	hal_remote_atomic_add( count_xp , -1 );
1457
1458	// compute the do_release condition depending on vseg type
1459	if( (type == VSEG_TYPE_FILE) \|\|
1460	(type == VSEG_TYPE_KCODE) \|\|
1461	(type == VSEG_TYPE_KDATA) \|\|
1462	(type == VSEG_TYPE_KDEV) )
1463	{
1464	// no physical page release for FILE and KERNEL
1465	do_release = false;
1466	}
1467	else if( (type == VSEG_TYPE_CODE) \|\|
1468	(type == VSEG_TYPE_STACK) )
1469	{
1470	// always release physical page for private vsegs
1471	do_release = true;
1472	}
1473	else if( (type == VSEG_TYPE_ANON) \|\|
1474	(type == VSEG_TYPE_REMOTE) )
1475	{
1476	// release physical page if reference cluster
1477	do_release = is_ref;
1478	}
1479	else if( is_ref ) // vseg_type == DATA in reference cluster
1480	{
1481	// get extended pointers on forks and lock field in page descriptor
1482	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
1483	xptr_t lock_xp = XPTR( page_cxy , &page_ptr->lock );
1484
1485	// take lock protecting "forks" counter
1486	remote_busylock_acquire( lock_xp );
1487
1488	// get number of pending forks from page descriptor
1489	uint32_t forks = hal_remote_l32( forks_xp );
1490
1491	// decrement pending forks counter if required
1492	if( forks ) hal_remote_atomic_add( forks_xp , -1 );
1493
1494	// release lock protecting "forks" counter
1495	remote_busylock_release( lock_xp );
1496
1497	// release physical page if forks == 0
1498	do_release = (forks == 0);
1499	}
1500	else // vseg_type == DATA not in reference cluster
1501	{
1502	// no physical page release if not in reference cluster
1503	do_release = false;
1504	}
1505
1506	// release physical page to relevant kmem when required
1507	if( do_release )
1508	{
1509	ppm_remote_free_pages( page_cxy , page_ptr );
1510
1511	#if DEBUG_VMM_PPN_RELEASE
1512	thread_t * this = CURRENT_THREAD;
1513	if( DEBUG_VMM_PPN_RELEASE < cycle )
1514	printk("\n[%s] thread[%x,%x] released ppn %x to kmem\n",
1515	__FUNCTION__, this->process->pid, this->trdid, ppn );
1516	#endif
1517
1518	}
1519	} // end vmm_ppn_release()
1520
1521	//////////////////////////////////////////
1522	void vmm_remove_vseg( process_t * process,
1523	vseg_t * vseg )
1524	{
1525	uint32_t vseg_type; // vseg type
1526	vpn_t vpn; // VPN of current PTE
1527	vpn_t vpn_min; // VPN of first PTE
1528	vpn_t vpn_max; // VPN of last PTE (excluded)
1529	ppn_t ppn; // current PTE ppn value
1530	uint32_t attr; // current PTE attributes
1531
1532	// check arguments
1533	assert( (process != NULL), "process argument is NULL" );
1534	assert( (vseg != NULL), "vseg argument is NULL" );
1535
1536	// get pointers on local process VMM
1537	vmm_t * vmm = &process->vmm;
1538
1539	// build extended pointer on GPT
1540	xptr_t gpt_xp = XPTR( local_cxy , &vmm->gpt );
1541
1542	// get relevant vseg infos
1543	vseg_type = vseg->type;
1544	vpn_min = vseg->vpn_base;
1545	vpn_max = vpn_min + vseg->vpn_size;
1546
1547	#if DEBUG_VMM_REMOVE_VSEG
1548	uint32_t cycle = (uint32_t)hal_get_cycles();
1549	thread_t * this = CURRENT_THREAD;
1550	#endif
1551
1552	#if (DEBUG_VMM_REMOVE_VSEG & 1 )
1553	if( DEBUG_VMM_REMOVE_VSEG < cycle )
1554	printk("\n[%s] thread[%x,%x] enter / process %x / %s / base %x / cycle %d\n",
1555	__FUNCTION__, this->process->pid, this->trdid,
1556	process->pid, vseg_type_str(vseg->type), vseg->min, cycle );
1557	#endif
1558
1559	// loop on PTEs in GPT to unmap all mapped PTE
1560	for( vpn = vpn_min ; vpn < vpn_max ; vpn++ )
1561	{
1562	// get ppn and attr
1563	hal_gpt_get_pte( gpt_xp , vpn , &attr , &ppn );
1564
1565	if( attr & GPT_MAPPED ) // PTE is mapped
1566	{
1567
1568	#if( DEBUG_VMM_REMOVE_VSEG & 1 )
1569	if( DEBUG_VMM_REMOVE_VSEG < cycle )
1570	printk("\n[%s] thread[%x,%x] unmap vpn %x / ppn %x / %s",
1571	__FUNCTION__, this->process->pid, this->trdid, vpn , ppn, vseg_type_str(vseg_type) );
1572	#endif
1573	// unmap GPT entry in local GPT
1574	hal_gpt_reset_pte( gpt_xp , vpn );
1575
1576	// release physical page when required
1577	vmm_ppn_release( process , vseg , ppn );
1578	}
1579	}
1580
1581	// remove vseg from VSL
1582	vmm_detach_vseg_from_vsl( vmm , vseg );
1583
1584	// release vseg descriptor depending on vseg type
1585	if( vseg_type == VSEG_TYPE_STACK )
1586	{
1587	// release slot to local stack allocator
1588	vmm_stack_free( vmm , vseg );
1589
1590	// release vseg descriptor to local kmem
1591	vseg_free( vseg );
1592	}
1593	else if( (vseg_type == VSEG_TYPE_ANON) \|\|
1594	(vseg_type == VSEG_TYPE_FILE) \|\|
1595	(vseg_type == VSEG_TYPE_REMOTE) )
1596	{
1597	// release vseg to local mmap allocator
1598	vmm_mmap_free( vmm , vseg );
1599	}
1600	else
1601	{
1602	// release vseg descriptor to local kmem
1603	vseg_free( vseg );
1604	}
1605
1606	#if DEBUG_VMM_REMOVE_VSEG
1607	cycle = (uint32_t)hal_get_cycles();
1608	if( DEBUG_VMM_REMOVE_VSEG < cycle )
1609	printk("[%s] thread[%x,%x] exit / process %x / %s / base %x / cycle %d\n",
1610	__FUNCTION__, this->process->pid, this->trdid,
1611	process->pid, vseg_type_str(vseg->type), vseg->min, cycle );
1612	#endif
1613
1614	} // end vmm_remove_vseg()
1615
1616	/////////////////////////////////////////////
1617	void vmm_resize_vseg( process_t * process,
1618	vseg_t * vseg,
1619	intptr_t new_base,
1620	intptr_t new_size )
1621	{
1622	vpn_t vpn;
1623	ppn_t ppn;
1624	uint32_t attr;
1625
1626	// check arguments
1627	assert( (process != NULL), "process argument is NULL" );
1628	assert( (vseg != NULL), "vseg argument is NULL" );
1629
1630	#if DEBUG_VMM_RESIZE_VSEG
1631	uint32_t cycle = (uint32_t)hal_get_cycles();
1632	thread_t * this = CURRENT_THREAD;
1633	#endif
1634
1635	#if (DEBUG_VMM_RESIZE_VSEG & 1)
1636	if( DEBUG_VMM_RESIZE_VSEG < cycle )
1637	printk("\n[%s] thread[%x,%x] enter / process %x / %s / base %x / cycle %d\n",
1638	__FUNCTION__, this->process->pid, this->trdid,
1639	process->pid, vseg_type_str(vseg->type), old_base, cycle );
1640	#endif
1641
1642	// get existing vseg vpn_min and vpn_max
1643	vpn_t old_vpn_min = vseg->vpn_base;
1644	vpn_t old_vpn_max = old_vpn_min + vseg->vpn_size - 1;
1645
1646	// compute new vseg vpn_min & vpn_max
1647	intptr_t min = new_base;
1648	intptr_t max = new_base + new_size;
1649	vpn_t new_vpn_min = min >> CONFIG_PPM_PAGE_SHIFT;
1650	vpn_t new_vpn_max = (max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1651
1652	// build extended pointer on GPT
1653	xptr_t gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1654
1655	// loop on PTEs in GPT to unmap PTE if (oldd_vpn_min <= vpn < new_vpn_min)
1656	for( vpn = old_vpn_min ; vpn < new_vpn_min ; vpn++ )
1657	{
1658	// get ppn and attr
1659	hal_gpt_get_pte( gpt_xp , vpn , &attr , &ppn );
1660
1661	if( attr & GPT_MAPPED ) // PTE is mapped
1662	{
1663
1664	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
1665	if( DEBUG_VMM_RESIZE_VSEG < cycle )
1666	printk("\n[%s] thread[%x,%x] unmap vpn %x / ppn %x / %s",
1667	__FUNCTION__, this->process->pid, this->trdid, vpn , ppn, vseg_type_str(vseg_type) );
1668	#endif
1669	// unmap GPT entry
1670	hal_gpt_reset_pte( gpt_xp , vpn );
1671
1672	// release physical page when required
1673	vmm_ppn_release( process , vseg , ppn );
1674	}
1675	}
1676
1677	// loop on PTEs in GPT to unmap PTE if (new vpn_max <= vpn < old_vpn_max)
1678	for( vpn = new_vpn_max ; vpn < old_vpn_max ; vpn++ )
1679	{
1680	// get ppn and attr
1681	hal_gpt_get_pte( gpt_xp , vpn , &attr , &ppn );
1682
1683	if( attr & GPT_MAPPED ) // PTE is mapped
1684	{
1685
1686	#if( DEBUG_VMM_REMOVE_VSEG & 1 )
1687	if( DEBUG_VMM_RESIZE_VSEG < cycle )
1688	printk("\n[%s] thread[%x,%x] unmap vpn %x / ppn %x / %s",
1689	__FUNCTION__, this->process->pid, this->trdid, vpn , ppn, vseg_type_str(vseg_type) );
1690	#endif
1691	// unmap GPT entry in local GPT
1692	hal_gpt_reset_pte( gpt_xp , vpn );
1693
1694	// release physical page when required
1695	vmm_ppn_release( process , vseg , ppn );
1696	}
1697	}
1698
1699	// resize vseg in VSL
1700	vseg->min = min;
1701	vseg->max = max;
1702	vseg->vpn_base = new_vpn_min;
1703	vseg->vpn_size = new_vpn_max - new_vpn_min + 1;
1704
1705	#if DEBUG_VMM_RESIZE_VSEG
1706	cycle = (uint32_t)hal_get_cycles();
1707	if( DEBUG_VMM_RESIZE_VSEG < cycle )
1708	printk("[%s] thread[%x,%x] exit / process %x / %s / base %x / cycle %d\n",
1709	__FUNCTION__, this->process->pid, this->trdid,
1710	process->pid, vseg_type_str(vseg->type), vseg->min, cycle );
1711	#endif
1712
1713	} // end vmm_resize_vseg
1714
1715	/////////////////////////////////////////////////////////////////////////////////////////////
1716	// This static function is called twice by the vmm_get_vseg() function.
1717	// It scan the - possibly remote - VSL defined by the <vmm_xp> argument to find the vseg
1718	// containing a given virtual address <vaddr>. It uses remote accesses to access the remote
1719	// VSL if required. The VSL lock protecting the VSL must be taken by the caller.
1720	/////////////////////////////////////////////////////////////////////////////////////////////
1721	// @ vmm_xp : extended pointer on the process VMM.
1722	// @ vaddr : virtual address.
1723	// @ return local pointer on remote vseg if success / return NULL if not found.
1724	/////////////////////////////////////////////////////////////////////////////////////////////
1725	static vseg_t * vmm_vseg_from_vaddr( xptr_t vmm_xp,
1726	intptr_t vaddr )
1727	{
1728	xptr_t iter_xp;
1729	xptr_t vseg_xp;
1730	vseg_t * vseg;
1731	intptr_t min;
1732	intptr_t max;
1733
1734	// get cluster and local pointer on target VMM
1735	vmm_t * vmm_ptr = GET_PTR( vmm_xp );
1736	cxy_t vmm_cxy = GET_CXY( vmm_xp );
1737
1738	// build extended pointer on VSL root
1739	xptr_t root_xp = XPTR( vmm_cxy , &vmm_ptr->vsegs_root );
1740
1741	// scan the list of vsegs in VSL
1742	XLIST_FOREACH( root_xp , iter_xp )
1743	{
1744	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
1745	vseg = GET_PTR( vseg_xp );
1746
1747	min = hal_remote_l32( XPTR( vmm_cxy , &vseg->min ) );
1748	max = hal_remote_l32( XPTR( vmm_cxy , &vseg->max ) );
1749
1750	// return success when match
1751	if( (vaddr >= min) && (vaddr < max) ) return vseg;
1752	}
1753
1754	// return failure
1755	return NULL;
1756
1757	} // end vmm_vseg_from_vaddr()
1758
1759	///////////////////////////////////////////
1760	error_t vmm_get_vseg( process_t * process,
1761	intptr_t vaddr,
1762	vseg_t ** found_vseg )
1763	{
1764	xptr_t loc_lock_xp; // extended pointer on local VSL lock
1765	xptr_t ref_lock_xp; // extended pointer on reference VSL lock
1766	vseg_t * loc_vseg; // local pointer on local vseg
1767	vseg_t * ref_vseg; // local pointer on reference vseg
1768
1769	// build extended pointer on local VSL lock
1770	loc_lock_xp = XPTR( local_cxy , &process->vmm.vsl_lock );
1771
1772	// get local VSL lock
1773	remote_queuelock_acquire( loc_lock_xp );
1774
1775	// try to get vseg from local VMM
1776	loc_vseg = vmm_vseg_from_vaddr( XPTR( local_cxy, &process->vmm ) , vaddr );
1777
1778	if (loc_vseg == NULL) // vseg not found => access reference VSL
1779	{
1780	// get extended pointer on reference process
1781	xptr_t ref_xp = process->ref_xp;
1782
1783	// get cluster and local pointer on reference process
1784	cxy_t ref_cxy = GET_CXY( ref_xp );
1785	process_t * ref_ptr = GET_PTR( ref_xp );
1786
1787	// build extended pointer on reference VSL lock
1788	ref_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.vsl_lock );
1789
1790	// get reference VSL lock
1791	remote_queuelock_acquire( ref_lock_xp );
1792
1793	// try to get vseg from reference VMM
1794	ref_vseg = vmm_vseg_from_vaddr( XPTR( ref_cxy , &ref_ptr->vmm ) , vaddr );
1795
1796	if( ref_vseg == NULL ) // vseg not found => return error
1797	{
1798	printk("\n[ERROR] in %s : vaddr %x in process %x out of segment\n",
1799	__FUNCTION__, vaddr, process->pid );
1800
1801	// release reference VSL lock
1802	remote_queuelock_release( ref_lock_xp );
1803
1804	return -1;
1805	}
1806	else // vseg found => try to update local VSL
1807	{
1808	// allocate a local vseg descriptor
1809	loc_vseg = vseg_alloc();
1810
1811	if( loc_vseg == NULL ) // no memory => return error
1812	{
1813	printk("\n[ERROR] in %s : vaddr %x in process %x / no memory for local vseg\n",
1814	__FUNCTION__, vaddr, process->pid );
1815
1816	// release reference VSL & local VSL locks
1817	remote_queuelock_release( ref_lock_xp );
1818	remote_queuelock_release( loc_lock_xp );
1819
1820	return -1;
1821	}
1822	else // update local VSL and return success
1823	{
1824	// initialize local vseg
1825	vseg_init_from_ref( loc_vseg , XPTR( ref_cxy , ref_vseg ) );
1826
1827	// register local vseg in local VSL
1828	vmm_attach_vseg_to_vsl( &process->vmm , loc_vseg );
1829
1830	// release reference VSL & local VSL locks
1831	remote_queuelock_release( ref_lock_xp );
1832	remote_queuelock_release( loc_lock_xp );
1833
1834	*found_vseg = loc_vseg;
1835	return 0;
1836	}
1837	}
1838	}
1839	else // vseg found in local VSL => return success
1840	{
1841	// release local VSL lock
1842	remote_queuelock_release( loc_lock_xp );
1843
1844	*found_vseg = loc_vseg;
1845	return 0;
1846	}
1847	} // end vmm_get_vseg()
1848
1849	//////////////////////////////////////////////////////////////////////////////////////
1850	// This static function compute the target cluster to allocate a physical page
1851	// for a given <vpn> in a given <vseg>, allocates the page and returns an extended
1852	// pointer on the allocated page descriptor.
1853	// The vseg cannot have the FILE type.
1854	//////////////////////////////////////////////////////////////////////////////////////
1855	// @ vseg : local pointer on vseg.
1856	// @ vpn : unmapped vpn.
1857	// @ return an extended pointer on the allocated page
1858	//////////////////////////////////////////////////////////////////////////////////////
1859	static xptr_t vmm_page_allocate( vseg_t * vseg,
1860	vpn_t vpn )
1861	{
1862
1863	#if DEBUG_VMM_PAGE_ALLOCATE
1864	uint32_t cycle = (uint32_t)hal_get_cycles();
1865	thread_t * this = CURRENT_THREAD;
1866	if( DEBUG_VMM_PAGE_ALLOCATE < cycle )
1867	printk("\n[%s] thread[%x,%x] enter for vpn %x / cycle %d\n",
1868	__FUNCTION__ , this->process->pid, this->trdid, vpn, cycle );
1869	#endif
1870
1871	xptr_t page_xp;
1872	cxy_t page_cxy;
1873	page_t * page_ptr;
1874	uint32_t index;
1875
1876	uint32_t type = vseg->type;
1877	uint32_t flags = vseg->flags;
1878	uint32_t x_size = LOCAL_CLUSTER->x_size;
1879	uint32_t y_size = LOCAL_CLUSTER->y_size;
1880
1881	// check vseg type
1882	assert( ( type != VSEG_TYPE_FILE ) , "illegal vseg type\n" );
1883
1884	if( flags & VSEG_DISTRIB ) // distributed => cxy depends on vpn LSB
1885	{
1886	index = vpn & ((x_size * y_size) - 1);
1887	page_cxy = HAL_CXY_FROM_XY( (index / y_size) , (index % y_size) );
1888
1889	// If the cluster selected from VPN's LSBs is empty, we select one randomly
1890	if ( cluster_is_active( page_cxy ) == false )
1891	{
1892	page_cxy = cluster_random_select();
1893	}
1894	}
1895	else // other cases => cxy specified in vseg
1896	{
1897	page_cxy = vseg->cxy;
1898	}
1899
1900	// allocate one small physical page from target cluster
1901	page_ptr = ppm_remote_alloc_pages( page_cxy , 0 );
1902
1903	page_xp = XPTR( page_cxy , page_ptr );
1904
1905	#if DEBUG_VMM_PAGE_ALLOCATE
1906	cycle = (uint32_t)hal_get_cycles();
1907	if( DEBUG_VMM_PAGE_ALLOCATE < cycle )
1908	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle %d\n",
1909	__FUNCTION__ , this->process->pid, this->trdid, vpn, ppm_page2ppn(page_xp), cycle );
1910	#endif
1911
1912	return page_xp;
1913
1914	} // end vmm_page_allocate()
1915
1916	////////////////////////////////////////
1917	error_t vmm_get_one_ppn( vseg_t * vseg,
1918	vpn_t vpn,
1919	ppn_t * ppn )
1920	{
1921	error_t error;
1922	xptr_t page_xp; // extended pointer on physical page descriptor
1923	uint32_t page_id; // missing page index in vseg mapper
1924	uint32_t type; // vseg type;
1925
1926	type = vseg->type;
1927	page_id = vpn - vseg->vpn_base;
1928
1929	#if DEBUG_VMM_GET_ONE_PPN
1930	uint32_t cycle = (uint32_t)hal_get_cycles();
1931	thread_t * this = CURRENT_THREAD;
1932	if( (DEBUG_VMM_GET_ONE_PPN < cycle) && (vpn == 0x40b) )
1933	printk("\n[%s] thread[%x,%x] enter for vpn %x / type %s / page_id %d / cycle %d\n",
1934	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(type), page_id, cycle );
1935	#endif
1936
1937	// FILE type : get the physical page from the file mapper
1938	if( type == VSEG_TYPE_FILE )
1939	{
1940	// get extended pointer on mapper
1941	xptr_t mapper_xp = vseg->mapper_xp;
1942
1943	assert( (mapper_xp != XPTR_NULL),
1944	"mapper not defined for a FILE vseg\n" );
1945
1946	// get extended pointer on page descriptor
1947	page_xp = mapper_remote_get_page( mapper_xp , page_id );
1948
1949	if ( page_xp == XPTR_NULL ) return EINVAL;
1950	}
1951
1952	// Other types : allocate a physical page from target cluster,
1953	// as defined by vseg type and vpn value
1954	else
1955	{
1956	// allocate one physical page
1957	page_xp = vmm_page_allocate( vseg , vpn );
1958
1959	if( page_xp == XPTR_NULL ) return -1;
1960
1961	// initialise missing page from .elf file mapper for DATA and CODE types
1962	// the vseg->mapper_xp field is an extended pointer on the .elf file mapper
1963	if( (type == VSEG_TYPE_CODE) \|\| (type == VSEG_TYPE_DATA) )
1964	{
1965	// get extended pointer on mapper
1966	xptr_t mapper_xp = vseg->mapper_xp;
1967
1968	assert( (mapper_xp != XPTR_NULL),
1969	"mapper not defined for a CODE or DATA vseg\n" );
1970
1971	// compute missing page offset in vseg
1972	uint32_t offset = page_id << CONFIG_PPM_PAGE_SHIFT;
1973
1974	// compute missing page offset in .elf file
1975	uint32_t elf_offset = vseg->file_offset + offset;
1976
1977	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1978	if( (DEBUG_VMM_GET_ONE_PPN < cycle) && (vpn == 0x40b) )
1979	printk("\n[%s] thread[%x,%x] for vpn = %x / elf_offset = %x\n",
1980	__FUNCTION__, this->process->pid, this->trdid, vpn, elf_offset );
1981	#endif
1982	// compute extended pointer on page base
1983	xptr_t base_xp = ppm_page2base( page_xp );
1984
1985	// file_size (in .elf mapper) can be smaller than vseg_size (BSS)
1986	uint32_t file_size = vseg->file_size;
1987
1988	if( file_size < offset ) // missing page fully in BSS
1989	{
1990
1991	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1992	if( (DEBUG_VMM_GET_ONE_PPN < cycle) && (vpn == 0x40b) )
1993	printk("\n[%s] thread[%x,%x] for vpn %x / fully in BSS\n",
1994	__FUNCTION__, this->process->pid, this->trdid, vpn );
1995	#endif
1996	if( GET_CXY( page_xp ) == local_cxy )
1997	{
1998	memset( GET_PTR( base_xp ) , 0 , CONFIG_PPM_PAGE_SIZE );
1999	}
2000	else
2001	{
2002	hal_remote_memset( base_xp , 0 , CONFIG_PPM_PAGE_SIZE );
2003	}
2004	}
2005	else if( file_size >= (offset + CONFIG_PPM_PAGE_SIZE) ) // fully in mapper
2006	{
2007
2008	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
2009	if( (DEBUG_VMM_GET_ONE_PPN < cycle) && (vpn == 0x40b) )
2010	printk("\n[%s] thread[%x,%x] for vpn %x / fully in mapper\n",
2011	__FUNCTION__, this->process->pid, this->trdid, vpn );
2012	#endif
2013	error = mapper_move_kernel( mapper_xp,
2014	true, // to_buffer
2015	elf_offset,
2016	base_xp,
2017	CONFIG_PPM_PAGE_SIZE );
2018	if( error ) return EINVAL;
2019	}
2020	else // both in mapper and in BSS :
2021	// - (file_size - offset) bytes from mapper
2022	// - (page_size + offset - file_size) bytes from BSS
2023	{
2024
2025	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
2026	if( (DEBUG_VMM_GET_ONE_PPN < cycle) && (vpn == 0x40b) )
2027	printk("\n[%s] thread[%x,%x] for vpn %x / both mapper & BSS\n"
2028	" %d bytes from mapper / %d bytes from BSS\n",
2029	__FUNCTION__, this->process->pid, this->trdid, vpn,
2030	file_size - offset , offset + CONFIG_PPM_PAGE_SIZE - file_size );
2031	#endif
2032	// initialize mapper part
2033	error = mapper_move_kernel( mapper_xp,
2034	true, // to buffer
2035	elf_offset,
2036	base_xp,
2037	file_size - offset );
2038	if( error ) return EINVAL;
2039
2040	// initialize BSS part
2041	if( GET_CXY( page_xp ) == local_cxy )
2042	{
2043	memset( GET_PTR( base_xp ) + file_size - offset , 0 ,
2044	offset + CONFIG_PPM_PAGE_SIZE - file_size );
2045	}
2046	else
2047	{
2048	hal_remote_memset( base_xp + file_size - offset , 0 ,
2049	offset + CONFIG_PPM_PAGE_SIZE - file_size );
2050	}
2051	}
2052	} // end initialisation for CODE or DATA types
2053	}
2054
2055	// return ppn
2056	*ppn = ppm_page2ppn( page_xp );
2057
2058	#if DEBUG_VMM_GET_ONE_PPN
2059	cycle = (uint32_t)hal_get_cycles();
2060	if( (DEBUG_VMM_GET_ONE_PPN < cycle) && (vpn == 0x40b) )
2061	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle %d\n",
2062	__FUNCTION__ , this->process->pid, this->trdid , vpn , *ppn, cycle );
2063	#endif
2064
2065	return 0;
2066
2067	} // end vmm_get_one_ppn()
2068
2069	///////////////////////////////////////////////////
2070	error_t vmm_handle_page_fault( process_t * process,
2071	vpn_t vpn )
2072	{
2073	vseg_t * vseg; // vseg containing vpn
2074	uint32_t attr; // PTE_ATTR value
2075	ppn_t ppn; // PTE_PPN value
2076	uint32_t ref_attr; // PTE_ATTR value in reference GPT
2077	ppn_t ref_ppn; // PTE_PPN value in reference GPT
2078	cxy_t ref_cxy; // reference cluster for missing vpn
2079	process_t * ref_ptr; // reference process for missing vpn
2080	xptr_t local_gpt_xp; // extended pointer on local GPT
2081	xptr_t ref_gpt_xp; // extended pointer on reference GPT
2082	error_t error; // value returned by called functions
2083
2084	thread_t * this = CURRENT_THREAD;
2085
2086	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
2087	uint32_t start_cycle = (uint32_t)hal_get_cycles();
2088	#endif
2089
2090	#if DEBUG_VMM_HANDLE_PAGE_FAULT
2091	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2092	printk("\n[%s] thread[%x,%x] enter for vpn %x / cycle %d\n",
2093	__FUNCTION__, this->process->pid, this->trdid, vpn, start_cycle );
2094	#endif
2095
2096	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
2097	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2098	hal_vmm_display( this->process , true );
2099	#endif
2100
2101	// get local vseg (access to reference VSL can be required)
2102	error = vmm_get_vseg( process,
2103	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
2104	&vseg );
2105	if( error )
2106	{
2107	printk("\n[ERROR] in %s : vpn %x in thread[%x,%x] not in registered vseg\n",
2108	__FUNCTION__ , vpn , process->pid, this->trdid );
2109
2110	return EXCP_USER_ERROR;
2111	}
2112
2113	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
2114	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2115	printk("\n[%s] thread[%x,%x] found vseg %s\n",
2116	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(vseg->type) );
2117	#endif
2118
2119	// build extended pointer on local GPT
2120	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
2121
2122	// lock PTE in local GPT and get current PPN and attributes
2123	error = hal_gpt_lock_pte( local_gpt_xp,
2124	vpn,
2125	&attr,
2126	&ppn );
2127	if( error )
2128	{
2129	printk("\n[PANIC] in %s : cannot lock PTE in local GPT / vpn %x / process %x\n",
2130	__FUNCTION__ , vpn , process->pid );
2131
2132	return EXCP_KERNEL_PANIC;
2133	}
2134
2135	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
2136	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2137	printk("\n[%s] thread[%x,%x] locked vpn %x in cluster %x\n",
2138	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy );
2139	#endif
2140
2141	// handle page fault only if local PTE still unmapped after lock
2142	if( (attr & GPT_MAPPED) == 0 )
2143	{
2144	// get reference process cluster and local pointer
2145	ref_cxy = GET_CXY( process->ref_xp );
2146	ref_ptr = GET_PTR( process->ref_xp );
2147
2148	/////////////// private vseg or (local == reference)
2149	/////////////// => access only the local GPT
2150	if( (vseg->type == VSEG_TYPE_STACK) \|\|
2151	(vseg->type == VSEG_TYPE_CODE) \|\|
2152	(ref_cxy == local_cxy ) )
2153	{
2154
2155	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
2156	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2157	printk("\n[%s] thread[%x,%x] access local gpt : cxy %x / ref_cxy %x / type %s / cycle %d\n",
2158	__FUNCTION__, this->process->pid, this->trdid,
2159	local_cxy, ref_cxy, vseg_type_str(vseg->type), (uint32_t)hal_get_cycles() );
2160	#endif
2161	// allocate and initialise a physical page
2162	error = vmm_get_one_ppn( vseg , vpn , &ppn );
2163
2164	if( error )
2165	{
2166	printk("\n[ERROR] in %s : no physical page / process = %x / vpn = %x\n",
2167	__FUNCTION__ , process->pid , vpn );
2168
2169	// unlock PTE in local GPT
2170	hal_gpt_unlock_pte( local_gpt_xp , vpn );
2171
2172	return EXCP_KERNEL_PANIC;
2173	}
2174
2175	// define attr from vseg flags
2176	attr = GPT_MAPPED \| GPT_SMALL \| GPT_READABLE;
2177	if( vseg->flags & VSEG_USER ) attr \|= GPT_USER;
2178	if( vseg->flags & VSEG_WRITE ) attr \|= GPT_WRITABLE;
2179	if( vseg->flags & VSEG_EXEC ) attr \|= GPT_EXECUTABLE;
2180	if( vseg->flags & VSEG_CACHE ) attr \|= GPT_CACHABLE;
2181
2182	// set PTE to local GPT
2183	// it unlocks this PTE
2184	hal_gpt_set_pte( local_gpt_xp,
2185	vpn,
2186	attr,
2187	ppn );
2188
2189	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
2190	uint32_t end_cycle = (uint32_t)hal_get_cycles();
2191	#endif
2192
2193	#if DEBUG_VMM_HANDLE_PAGE_FAULT
2194	if( (end_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2195	printk("\n[%s] thread[%x,%x] handled local pgfault / ppn %x / attr %x / cycle %d\n",
2196	__FUNCTION__, this->process->pid, this->trdid, ppn, attr, end_cycle );
2197	#endif
2198
2199	#if CONFIG_INSTRUMENTATION_PGFAULTS
2200	this->info.local_pgfault_nr++;
2201	this->info.local_pgfault_cost += (end_cycle - start_cycle);
2202	#endif
2203	return EXCP_NON_FATAL;
2204
2205	} // end local GPT access
2206
2207	/////////////////// public vseg and (local != reference)
2208	/////////////////// => access ref GPT to update local GPT
2209	else
2210	{
2211
2212	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
2213	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2214	printk("\n[%s] thread[%x,%x] access ref gpt : cxy %x / ref_cxy %x / type %s / cycle %d\n",
2215	__FUNCTION__, this->process->pid, this->trdid,
2216	local_cxy, ref_cxy, vseg_type_str(vseg->type), (uint32_t)hal_get_cycles() );
2217	#endif
2218	// build extended pointer on reference GPT
2219	ref_gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
2220
2221	// lock PTE in reference GPT and get current PPN and attributes
2222	error = hal_gpt_lock_pte( ref_gpt_xp,
2223	vpn,
2224	&ref_attr,
2225	&ref_ppn );
2226	if( error )
2227	{
2228	printk("\n[PANIC] in %s : cannot lock PTE in ref GPT / vpn %x / process %x\n",
2229	__FUNCTION__ , vpn , process->pid );
2230
2231	// unlock PTE in local GPT
2232	hal_gpt_unlock_pte( local_gpt_xp , vpn );
2233
2234	return EXCP_KERNEL_PANIC;
2235	}
2236
2237	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
2238	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2239	printk("\n[%s] thread[%x,%x] get pte from ref gpt / attr %x / ppn %x\n",
2240	__FUNCTION__, this->process->pid, this->trdid, ref_attr, ref_ppn );
2241	#endif
2242
2243	if( ref_attr & GPT_MAPPED ) // false page fault
2244	{
2245	// update local GPT from reference GPT values
2246	// this unlocks the PTE in local GPT
2247	hal_gpt_set_pte( local_gpt_xp,
2248	vpn,
2249	ref_attr,
2250	ref_ppn );
2251
2252	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
2253	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2254	printk("\n[%s] thread[%x,%x] updated local gpt for a false pgfault\n",
2255	__FUNCTION__, this->process->pid, this->trdid );
2256	#endif
2257
2258	// unlock the PTE in reference GPT
2259	hal_gpt_unlock_pte( ref_gpt_xp, vpn );
2260
2261	#if (DEBUG_VMM_HANDLE_PAGE_FAULT &1)
2262	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2263	printk("\n[%s] thread[%x,%x] unlock the ref gpt after a false pgfault\n",
2264	__FUNCTION__, this->process->pid, this->trdid );
2265	#endif
2266
2267	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
2268	uint32_t end_cycle = (uint32_t)hal_get_cycles();
2269	#endif
2270
2271	#if DEBUG_VMM_HANDLE_PAGE_FAULT
2272	if( (end_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2273	printk("\n[%s] thread[%x,%x] handled false pgfault / ppn %x / attr %x / cycle %d\n",
2274	__FUNCTION__, this->process->pid, this->trdid, ref_ppn, ref_attr, end_cycle );
2275	#endif
2276
2277	#if CONFIG_INSTRUMENTATION_PGFAULTS
2278	this->info.false_pgfault_nr++;
2279	this->info.false_pgfault_cost += (end_cycle - start_cycle);
2280	#endif
2281	return EXCP_NON_FATAL;
2282	}
2283	else // true page fault
2284	{
2285	// allocate and initialise a physical page depending on the vseg type
2286	error = vmm_get_one_ppn( vseg , vpn , &ppn );
2287
2288	if( error )
2289	{
2290	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
2291	__FUNCTION__ , process->pid , vpn );
2292
2293	// unlock PTE in local GPT and in reference GPT
2294	hal_gpt_unlock_pte( local_gpt_xp , vpn );
2295	hal_gpt_unlock_pte( ref_gpt_xp , vpn );
2296
2297	return EXCP_KERNEL_PANIC;
2298	}
2299
2300	// define attr from vseg flags
2301	attr = GPT_MAPPED \| GPT_SMALL \| GPT_READABLE;
2302	if( vseg->flags & VSEG_USER ) attr \|= GPT_USER;
2303	if( vseg->flags & VSEG_WRITE ) attr \|= GPT_WRITABLE;
2304	if( vseg->flags & VSEG_EXEC ) attr \|= GPT_EXECUTABLE;
2305	if( vseg->flags & VSEG_CACHE ) attr \|= GPT_CACHABLE;
2306
2307	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
2308	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2309	printk("\n[%s] thread[%x,%x] build a new PTE for a true pgfault\n",
2310	__FUNCTION__, this->process->pid, this->trdid );
2311	#endif
2312	// set PTE in reference GPT
2313	// this unlock the PTE
2314	hal_gpt_set_pte( ref_gpt_xp,
2315	vpn,
2316	attr,
2317	ppn );
2318
2319	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
2320	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2321	printk("\n[%s] thread[%x,%x] set new PTE in ref gpt for a true page fault\n",
2322	__FUNCTION__, this->process->pid, this->trdid );
2323	#endif
2324
2325	// set PTE in local GPT
2326	// this unlock the PTE
2327	hal_gpt_set_pte( local_gpt_xp,
2328	vpn,
2329	attr,
2330	ppn );
2331
2332	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
2333	uint32_t end_cycle = (uint32_t)hal_get_cycles();
2334	#endif
2335
2336	#if DEBUG_VMM_HANDLE_PAGE_FAULT
2337	if( (end_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2338	printk("\n[%s] thread[%x,%x] handled global pgfault / ppn %x / attr %x / cycle %d\n",
2339	__FUNCTION__, this->process->pid, this->trdid, ppn, attr, end_cycle );
2340	#endif
2341
2342	#if CONFIG_INSTRUMENTATION_PGFAULTS
2343	this->info.global_pgfault_nr++;
2344	this->info.global_pgfault_cost += (end_cycle - start_cycle);
2345	#endif
2346	return EXCP_NON_FATAL;
2347	}
2348	}
2349	}
2350	else // page has been locally mapped by another concurrent thread
2351	{
2352	// unlock the PTE in local GPT
2353	hal_gpt_unlock_pte( local_gpt_xp , vpn );
2354
2355	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
2356	uint32_t end_cycle = (uint32_t)hal_get_cycles();
2357	#endif
2358
2359	#if DEBUG_VMM_HANDLE_PAGE_FAULT
2360	if( (end_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2361	printk("\n[%s] handled by another thread / vpn %x / ppn %x / attr %x / cycle %d\n",
2362	__FUNCTION__, vpn, ppn, attr, end_cycle );
2363	#endif
2364
2365	#if CONFIG_INSTRUMENTATION_PGFAULTS
2366	this->info.false_pgfault_nr++;
2367	this->info.false_pgfault_cost += (end_cycle - start_cycle);
2368	#endif
2369	return EXCP_NON_FATAL;
2370	}
2371
2372	} // end vmm_handle_page_fault()
2373
2374	////////////////////////////////////////////
2375	error_t vmm_handle_cow( process_t * process,
2376	vpn_t vpn )
2377	{
2378	vseg_t * vseg; // vseg containing vpn
2379	xptr_t gpt_xp; // extended pointer on GPT (local or reference)
2380	gpt_t * gpt_ptr; // local pointer on GPT (local or reference)
2381	cxy_t gpt_cxy; // GPT cluster identifier
2382	uint32_t old_attr; // current PTE_ATTR value
2383	ppn_t old_ppn; // current PTE_PPN value
2384	uint32_t new_attr; // new PTE_ATTR value
2385	ppn_t new_ppn; // new PTE_PPN value
2386	cxy_t ref_cxy; // reference process cluster
2387	process_t * ref_ptr; // local pointer on reference process
2388	error_t error;
2389
2390	thread_t * this = CURRENT_THREAD;
2391
2392	#if DEBUG_VMM_HANDLE_COW
2393	uint32_t cycle = (uint32_t)hal_get_cycles();
2394	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2395	printk("\n[%s] thread[%x,%x] enter for vpn %x / core[%x,%d] / cycle %d\n",
2396	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
2397	#endif
2398
2399	#if ((DEBUG_VMM_HANDLE_COW & 3) == 3 )
2400	hal_vmm_display( XPTR( local_cxy , process ) , true );
2401	#endif
2402
2403	// get local vseg
2404	error = vmm_get_vseg( process,
2405	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
2406	&vseg );
2407	if( error )
2408	{
2409	printk("\n[ERROR] in %s : vpn %x in thread[%x,%x] not in a registered vseg\n",
2410	__FUNCTION__, vpn, process->pid, this->trdid );
2411
2412	return EXCP_USER_ERROR;
2413	}
2414
2415	#if DEBUG_VMM_HANDLE_COW
2416	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2417	printk("\n[%s] thread[%x,%x] get vseg %s\n",
2418	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(vseg->type) );
2419	#endif
2420
2421	// get reference process cluster and local pointer
2422	ref_cxy = GET_CXY( process->ref_xp );
2423	ref_ptr = GET_PTR( process->ref_xp );
2424
2425	// build pointers on relevant GPT
2426	// - access only local GPT for a private vseg
2427	// - access reference GPT and all copies for a public vseg
2428	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
2429	{
2430	gpt_cxy = local_cxy;
2431	gpt_ptr = &process->vmm.gpt;
2432	gpt_xp = XPTR( gpt_cxy , gpt_ptr );
2433	}
2434	else
2435	{
2436	gpt_cxy = ref_cxy;
2437	gpt_ptr = &ref_ptr->vmm.gpt;
2438	gpt_xp = XPTR( gpt_cxy , gpt_ptr );
2439	}
2440
2441	// lock target PTE in relevant GPT (local or reference)
2442	// and get current PTE value
2443	error = hal_gpt_lock_pte( gpt_xp,
2444	vpn,
2445	&old_attr,
2446	&old_ppn );
2447	if( error )
2448	{
2449	printk("\n[PANIC] in %s : cannot lock PTE in GPT / cxy %x / vpn %x / process %x\n",
2450	__FUNCTION__ , gpt_cxy, vpn , process->pid );
2451
2452	return EXCP_KERNEL_PANIC;
2453	}
2454
2455	#if DEBUG_VMM_HANDLE_COW
2456	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2457	printk("\n[%s] thread[%x,%x] get pte for vpn %x : ppn %x / attr %x\n",
2458	__FUNCTION__, this->process->pid, this->trdid, vpn, old_ppn, old_attr );
2459	#endif
2460
2461	// return user error if COW attribute not set or PTE2 unmapped
2462	if( ((old_attr & GPT_COW) == 0) \|\| ((old_attr & GPT_MAPPED) == 0) )
2463	{
2464	hal_gpt_unlock_pte( gpt_xp , vpn );
2465
2466	return EXCP_USER_ERROR;
2467	}
2468
2469	// get pointers on physical page descriptor
2470	xptr_t page_xp = ppm_ppn2page( old_ppn );
2471	cxy_t page_cxy = GET_CXY( page_xp );
2472	page_t * page_ptr = GET_PTR( page_xp );
2473
2474	// get extended pointers on forks and lock field in page descriptor
2475	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
2476	xptr_t forks_lock_xp = XPTR( page_cxy , &page_ptr->lock );
2477
2478	// take lock protecting "forks" counter
2479	remote_busylock_acquire( forks_lock_xp );
2480
2481	// get number of pending forks from page descriptor
2482	uint32_t forks = hal_remote_l32( forks_xp );
2483
2484	#if DEBUG_VMM_HANDLE_COW
2485	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2486	printk("\n[%s] thread[%x,%x] get forks = %d for vpn %x\n",
2487	__FUNCTION__, this->process->pid, this->trdid, forks, vpn );
2488	#endif
2489
2490	if( forks ) // pending fork => allocate a new page, and copy old to new
2491	{
2492	// decrement pending forks counter in page descriptor
2493	hal_remote_atomic_add( forks_xp , -1 );
2494
2495	// release lock protecting "forks" counter
2496	remote_busylock_release( forks_lock_xp );
2497
2498	// allocate a new physical page depending on vseg type
2499	page_xp = vmm_page_allocate( vseg , vpn );
2500
2501	if( page_xp == XPTR_NULL )
2502	{
2503	printk("\n[PANIC] in %s : no memory for vpn %x in process %x\n",
2504	__FUNCTION__ , vpn, process->pid );
2505
2506	hal_gpt_unlock_pte( gpt_xp , vpn );
2507
2508	return EXCP_KERNEL_PANIC;
2509	}
2510
2511	// compute allocated page PPN
2512	new_ppn = ppm_page2ppn( page_xp );
2513
2514	#if DEBUG_VMM_HANDLE_COW
2515	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2516	printk("\n[%s] thread[%x,%x] get new ppn %x for vpn %x\n",
2517	__FUNCTION__, this->process->pid, this->trdid, new_ppn, vpn );
2518	#endif
2519
2520	// copy old page content to new page
2521	hal_remote_memcpy( ppm_ppn2base( new_ppn ),
2522	ppm_ppn2base( old_ppn ),
2523	CONFIG_PPM_PAGE_SIZE );
2524
2525	#if DEBUG_VMM_HANDLE_COW
2526	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2527	printk("\n[%s] thread[%x,%x] copied old page to new page\n",
2528	__FUNCTION__, this->process->pid, this->trdid );
2529	#endif
2530
2531	}
2532	else // no pending fork => keep the existing page
2533	{
2534	// release lock protecting "forks" counter
2535	remote_busylock_release( forks_lock_xp );
2536
2537	#if(DEBUG_VMM_HANDLE_COW & 1)
2538	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2539	printk("\n[%s] thread[%x,%x] no pending forks / keep existing PPN %x\n",
2540	__FUNCTION__, this->process->pid, this->trdid, old_ppn );
2541	#endif
2542	new_ppn = old_ppn;
2543	}
2544
2545	// build new_attr : set WRITABLE, reset COW, reset LOCKED
2546	new_attr = (((old_attr \| GPT_WRITABLE) & (~GPT_COW)) & (~GPT_LOCKED));
2547
2548	#if(DEBUG_VMM_HANDLE_COW & 1)
2549	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2550	printk("\n[%s] thread[%x,%x] new_attr %x / new_ppn %x\n",
2551	__FUNCTION__, this->process->pid, this->trdid, new_attr, new_ppn );
2552	#endif
2553
2554	// update the relevant GPT(s)
2555	// - private vseg => update only the local GPT
2556	// - public vseg => update the reference GPT AND all the GPT copies
2557	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
2558	{
2559	// set new PTE in local gpt
2560	hal_gpt_set_pte( gpt_xp,
2561	vpn,
2562	new_attr,
2563	new_ppn );
2564	}
2565	else
2566	{
2567	// set new PTE in all GPT copies
2568	vmm_global_update_pte( process,
2569	vpn,
2570	new_attr,
2571	new_ppn );
2572	}
2573
2574	#if DEBUG_VMM_HANDLE_COW
2575	cycle = (uint32_t)hal_get_cycles();
2576	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2577	printk("\n[%s] thread[%x,%x] exit for vpn %x / core[%x,%d] / cycle %d\n",
2578	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
2579	#endif
2580
2581	#if ((DEBUG_VMM_HANDLE_COW & 3) == 3)
2582	hal_vmm_display( XPTR( local_cxy , process ) , true );
2583	#endif
2584
2585	return EXCP_NON_FATAL;
2586
2587	} // end vmm_handle_cow()
2588

Note: See TracBrowser for help on using the repository browser.

Download in other formats: