Context Navigation

source: trunk/kernel/mm/vmm.c @ 621

Last change on this file since 621 was 621, checked in by viala@…, 5 years ago

Fix lock untaken in vmm_delete_vseg.

Catched thanks to gcc warning.

File size: 74.3 KB

Line
1	/*
2	* vmm.c - virtual memory manager related operations definition.
3	*
4	* Authors Ghassan Almaless (2008,2009,2010,2011, 2012)
5	* Mohamed Lamine Karaoui (2015)
6	* Alain Greiner (2016,2017,2018)
7	*
8	* Copyright (c) UPMC Sorbonne Universites
9	*
10	* This file is part of ALMOS-MKH.
11	*
12	* ALMOS-MKH is free software; you can redistribute it and/or modify it
13	* under the terms of the GNU General Public License as published by
14	* the Free Software Foundation; version 2.0 of the License.
15	*
16	* ALMOS-MKH is distributed in the hope that it will be useful, but
17	* WITHOUT ANY WARRANTY; without even the implied warranty of
18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19	* General Public License for more details.
20	*
21	* You should have received a copy of the GNU General Public License
22	* along with ALMOS-MKH; if not, write to the Free Software Foundation,
23	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
24	*/
25
26	#include <kernel_config.h>
27	#include <hal_kernel_types.h>
28	#include <hal_special.h>
29	#include <hal_gpt.h>
30	#include <hal_vmm.h>
31	#include <hal_macros.h>
32	#include <printk.h>
33	#include <memcpy.h>
34	#include <remote_rwlock.h>
35	#include <remote_queuelock.h>
36	#include <list.h>
37	#include <xlist.h>
38	#include <bits.h>
39	#include <process.h>
40	#include <thread.h>
41	#include <vseg.h>
42	#include <cluster.h>
43	#include <scheduler.h>
44	#include <vfs.h>
45	#include <mapper.h>
46	#include <page.h>
47	#include <kmem.h>
48	#include <vmm.h>
49	#include <hal_exception.h>
50
51	//////////////////////////////////////////////////////////////////////////////////
52	// Extern global variables
53	//////////////////////////////////////////////////////////////////////////////////
54
55	extern process_t process_zero; // allocated in cluster.c
56
57	///////////////////////////////////////
58	error_t vmm_init( process_t * process )
59	{
60	error_t error;
61	vseg_t * vseg_kentry;
62	vseg_t * vseg_args;
63	vseg_t * vseg_envs;
64	intptr_t base;
65	intptr_t size;
66	uint32_t i;
67
68	#if DEBUG_VMM_INIT
69	thread_t * this = CURRENT_THREAD;
70	uint32_t cycle = (uint32_t)hal_get_cycles();
71	if( DEBUG_VMM_INIT )
72	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
73	__FUNCTION__ , this->process->pid, this->trdid, process->pid, local_cxy, cycle );
74	#endif
75
76	// get pointer on VMM
77	vmm_t * vmm = &process->vmm;
78
79	// initialize local list of vsegs
80	vmm->vsegs_nr = 0;
81	xlist_root_init( XPTR( local_cxy , &vmm->vsegs_root ) );
82	remote_rwlock_init( XPTR( local_cxy , &vmm->vsegs_lock ) , LOCK_VMM_VSL );
83
84	assert( ((CONFIG_VMM_KENTRY_SIZE + CONFIG_VMM_ARGS_SIZE + CONFIG_VMM_ENVS_SIZE)
85	<= CONFIG_VMM_ELF_BASE) , "UTILS zone too small\n" );
86
87	assert( (CONFIG_THREADS_MAX_PER_CLUSTER <= 32) ,
88	"no more than 32 threads per cluster for a single process\n");
89
90	assert( ((CONFIG_VMM_STACK_SIZE * CONFIG_THREADS_MAX_PER_CLUSTER) <=
91	(CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE)) ,
92	"STACK zone too small\n");
93
94	// register kentry vseg in VSL
95	base = CONFIG_VMM_KENTRY_BASE << CONFIG_PPM_PAGE_SHIFT;
96	size = CONFIG_VMM_KENTRY_SIZE << CONFIG_PPM_PAGE_SHIFT;
97
98	vseg_kentry = vmm_create_vseg( process,
99	VSEG_TYPE_CODE,
100	base,
101	size,
102	0, // file_offset unused
103	0, // file_size unused
104	XPTR_NULL, // mapper_xp unused
105	local_cxy );
106
107	if( vseg_kentry == NULL )
108	{
109	printk("\n[ERROR] in %s : cannot register kentry vseg\n", __FUNCTION__ );
110	return -1;
111	}
112
113	vmm->kent_vpn_base = base;
114
115	// register args vseg in VSL
116	base = (CONFIG_VMM_KENTRY_BASE +
117	CONFIG_VMM_KENTRY_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
118	size = CONFIG_VMM_ARGS_SIZE << CONFIG_PPM_PAGE_SHIFT;
119
120	vseg_args = vmm_create_vseg( process,
121	VSEG_TYPE_DATA,
122	base,
123	size,
124	0, // file_offset unused
125	0, // file_size unused
126	XPTR_NULL, // mapper_xp unused
127	local_cxy );
128
129	if( vseg_args == NULL )
130	{
131	printk("\n[ERROR] in %s : cannot register args vseg\n", __FUNCTION__ );
132	return -1;
133	}
134
135	vmm->args_vpn_base = base;
136
137	// register the envs vseg in VSL
138	base = (CONFIG_VMM_KENTRY_BASE +
139	CONFIG_VMM_KENTRY_SIZE +
140	CONFIG_VMM_ARGS_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
141	size = CONFIG_VMM_ENVS_SIZE << CONFIG_PPM_PAGE_SHIFT;
142
143	vseg_envs = vmm_create_vseg( process,
144	VSEG_TYPE_DATA,
145	base,
146	size,
147	0, // file_offset unused
148	0, // file_size unused
149	XPTR_NULL, // mapper_xp unused
150	local_cxy );
151
152	if( vseg_envs == NULL )
153	{
154	printk("\n[ERROR] in %s : cannot register envs vseg\n", __FUNCTION__ );
155	return -1;
156	}
157
158	vmm->envs_vpn_base = base;
159
160	// create GPT (empty)
161	error = hal_gpt_create( &vmm->gpt );
162
163	if( error )
164	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
165
166	// initialize GPT lock
167	remote_rwlock_init( XPTR( local_cxy , &vmm->gpt_lock ) , LOCK_VMM_GPT );
168
169	// architecture specic GPT initialisation
170	// (For TSAR, identity map the kentry_vseg)
171	error = hal_vmm_init( vmm );
172
173	if( error )
174	printk("\n[ERROR] in %s : cannot initialize GPT\n", __FUNCTION__ );
175
176	// initialize STACK allocator
177	vmm->stack_mgr.bitmap = 0;
178	vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
179	busylock_init( &vmm->stack_mgr.lock , LOCK_VMM_STACK );
180
181	// initialize MMAP allocator
182	vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
183	vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
184	vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
185	busylock_init( &vmm->mmap_mgr.lock , LOCK_VMM_MMAP );
186	for( i = 0 ; i < 32 ; i++ ) list_root_init( &vmm->mmap_mgr.zombi_list[i] );
187
188	// initialize instrumentation counters
189	vmm->pgfault_nr = 0;
190
191	hal_fence();
192
193	#if DEBUG_VMM_INIT
194	cycle = (uint32_t)hal_get_cycles();
195	if( DEBUG_VMM_INIT )
196	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
197	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
198	#endif
199
200	return 0;
201
202	} // end vmm_init()
203
204	//////////////////////////////////////
205	void vmm_display( process_t * process,
206	bool_t mapping )
207	{
208	vmm_t * vmm = &process->vmm;
209	gpt_t * gpt = &vmm->gpt;
210
211	printk("\n***** VSL and GPT(%x) for process %x in cluster %x\n\n",
212	process->vmm.gpt.ptr , process->pid , local_cxy );
213
214	// get lock protecting the VSL and the GPT
215	remote_rwlock_rd_acquire( XPTR( local_cxy , &vmm->vsegs_lock ) );
216	remote_rwlock_rd_acquire( XPTR( local_cxy , &vmm->gpt_lock ) );
217
218	// scan the list of vsegs
219	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
220	xptr_t iter_xp;
221	xptr_t vseg_xp;
222	vseg_t * vseg;
223	XLIST_FOREACH( root_xp , iter_xp )
224	{
225	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
226	vseg = GET_PTR( vseg_xp );
227
228	printk(" - %s : base = %X / size = %X / npages = %d\n",
229	vseg_type_str( vseg->type ) , vseg->min , vseg->max - vseg->min , vseg->vpn_size );
230
231	if( mapping )
232	{
233	vpn_t vpn;
234	ppn_t ppn;
235	uint32_t attr;
236	vpn_t base = vseg->vpn_base;
237	vpn_t size = vseg->vpn_size;
238	for( vpn = base ; vpn < (base+size) ; vpn++ )
239	{
240	hal_gpt_get_pte( XPTR( local_cxy , gpt ) , vpn , &attr , &ppn );
241	if( attr & GPT_MAPPED )
242	{
243	printk(" . vpn = %X / attr = %X / ppn = %X\n", vpn , attr , ppn );
244	}
245	}
246	}
247	}
248
249	// release the locks
250	remote_rwlock_rd_release( XPTR( local_cxy , &vmm->vsegs_lock ) );
251	remote_rwlock_rd_release( XPTR( local_cxy , &vmm->gpt_lock ) );
252
253	} // vmm_display()
254
255	//////////////////////////////////////////
256	void vmm_attach_vseg_to_vsl( vmm_t * vmm,
257	vseg_t * vseg )
258	{
259	// build extended pointer on rwlock protecting VSL
260	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
261
262	// get rwlock in write mode
263	remote_rwlock_wr_acquire( lock_xp );
264
265	// update vseg descriptor
266	vseg->vmm = vmm;
267
268	// add vseg in vmm list
269	xlist_add_last( XPTR( local_cxy , &vmm->vsegs_root ),
270	XPTR( local_cxy , &vseg->xlist ) );
271
272	// release rwlock in write mode
273	remote_rwlock_wr_release( lock_xp );
274	}
275
276	////////////////////////////////////////////
277	void vmm_detach_vseg_from_vsl( vmm_t * vmm,
278	vseg_t * vseg )
279	{
280	// get vseg type
281	uint32_t type = vseg->type;
282
283	// build extended pointer on rwlock protecting VSL
284	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
285
286	// get rwlock in write mode
287	remote_rwlock_wr_acquire( lock_xp );
288
289	// update vseg descriptor
290	vseg->vmm = NULL;
291
292	// remove vseg from VSL
293	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
294
295	// release rwlock in write mode
296	remote_rwlock_wr_release( lock_xp );
297
298	// release the stack slot to VMM stack allocator if STACK type
299	if( type == VSEG_TYPE_STACK )
300	{
301	// get pointer on stack allocator
302	stack_mgr_t * mgr = &vmm->stack_mgr;
303
304	// compute slot index
305	uint32_t index = ((vseg->vpn_base - mgr->vpn_base - 1) / CONFIG_VMM_STACK_SIZE);
306
307	// update stacks_bitmap
308	busylock_acquire( &mgr->lock );
309	bitmap_clear( &mgr->bitmap , index );
310	busylock_release( &mgr->lock );
311	}
312
313	// release the vseg to VMM mmap allocator if MMAP type
314	if( (type == VSEG_TYPE_ANON) \|\| (type == VSEG_TYPE_FILE) \|\| (type == VSEG_TYPE_REMOTE) )
315	{
316	// get pointer on mmap allocator
317	mmap_mgr_t * mgr = &vmm->mmap_mgr;
318
319	// compute zombi_list index
320	uint32_t index = bits_log2( vseg->vpn_size );
321
322	// update zombi_list
323	busylock_acquire( &mgr->lock );
324	list_add_first( &mgr->zombi_list[index] , &vseg->zlist );
325	busylock_release( &mgr->lock );
326	}
327
328	// release physical memory allocated for vseg descriptor if no MMAP type
329	if( (type != VSEG_TYPE_ANON) && (type != VSEG_TYPE_FILE) && (type != VSEG_TYPE_REMOTE) )
330	{
331	vseg_free( vseg );
332	}
333
334	} // end vmm_remove_vseg_from_vsl()
335
336	////////////////////////////////////////////////
337	void vmm_global_update_pte( process_t * process,
338	vpn_t vpn,
339	uint32_t attr,
340	ppn_t ppn )
341	{
342	xlist_entry_t * process_root_ptr;
343	xptr_t process_root_xp;
344	xptr_t process_iter_xp;
345
346	xptr_t remote_process_xp;
347	cxy_t remote_process_cxy;
348	process_t * remote_process_ptr;
349	xptr_t remote_gpt_xp;
350
351	pid_t pid;
352	cxy_t owner_cxy;
353	lpid_t owner_lpid;
354
355	#if DEBUG_VMM_UPDATE_PTE
356	uint32_t cycle = (uint32_t)hal_get_cycles();
357	thread_t * this = CURRENT_THREAD;
358	if( DEBUG_VMM_UPDATE_PTE < cycle )
359	printk("\n[%s] thread[%x,%x] enter for process %x / vpn %x / cycle %d\n",
360	__FUNCTION__, this->process->pid, this->trdid, process->pid , vpn , cycle );
361	#endif
362
363	// check cluster is reference
364	assert( (GET_CXY( process->ref_xp ) == local_cxy) , "not called in reference cluster\n");
365
366	// get extended pointer on root of process copies xlist in owner cluster
367	pid = process->pid;
368	owner_cxy = CXY_FROM_PID( pid );
369	owner_lpid = LPID_FROM_PID( pid );
370	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
371	process_root_xp = XPTR( owner_cxy , process_root_ptr );
372
373	// loop on destination process copies
374	XLIST_FOREACH( process_root_xp , process_iter_xp )
375	{
376	// get cluster and local pointer on remote process
377	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
378	remote_process_ptr = GET_PTR( remote_process_xp );
379	remote_process_cxy = GET_CXY( remote_process_xp );
380
381	#if (DEBUG_VMM_UPDATE_PTE & 0x1)
382	if( DEBUG_VMM_UPDATE_PTE < cycle )
383	printk("\n[%s] threadr[%x,%x] handling vpn %x for process %x in cluster %x\n",
384	__FUNCTION__, this->process->pid, this->trdid, vpn, process->pid, remote_process_cxy );
385	#endif
386
387	// get extended pointer on remote gpt
388	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
389
390	// update remote GPT
391	hal_gpt_update_pte( remote_gpt_xp, vpn, attr, ppn );
392	}
393
394	#if DEBUG_VMM_UPDATE_PTE
395	cycle = (uint32_t)hal_get_cycles();
396	if( DEBUG_VMM_UPDATE_PTE < cycle )
397	printk("\n[%s] thread[%x,%x] exit for process %x / vpn %x / cycle %d\n",
398	__FUNCTION__, this->process->pid, this->trdid, process->pid , vpn , cycle );
399	#endif
400
401	} // end vmm_global_update_pte()
402
403	///////////////////////////////////////
404	void vmm_set_cow( process_t * process )
405	{
406	vmm_t * vmm;
407
408	xlist_entry_t * process_root_ptr;
409	xptr_t process_root_xp;
410	xptr_t process_iter_xp;
411
412	xptr_t remote_process_xp;
413	cxy_t remote_process_cxy;
414	process_t * remote_process_ptr;
415	xptr_t remote_gpt_xp;
416
417	xptr_t vseg_root_xp;
418	xptr_t vseg_iter_xp;
419
420	xptr_t vseg_xp;
421	vseg_t * vseg;
422
423	pid_t pid;
424	cxy_t owner_cxy;
425	lpid_t owner_lpid;
426
427	#if DEBUG_VMM_SET_COW
428	uint32_t cycle = (uint32_t)hal_get_cycles();
429	thread_t * this = CURRENT_THREAD;
430	if( DEBUG_VMM_SET_COW < cycle )
431	printk("\n[%s] thread[%x,%x] enter for process %x / cycle %d\n",
432	__FUNCTION__, this->process->pid, this->trdid, process->pid , cycle );
433	#endif
434
435	// check cluster is reference
436	assert( (GET_CXY( process->ref_xp ) == local_cxy) ,
437	"local cluster is not process reference cluster\n");
438
439	// get pointer on reference VMM
440	vmm = &process->vmm;
441
442	// get extended pointer on root of process copies xlist in owner cluster
443	pid = process->pid;
444	owner_cxy = CXY_FROM_PID( pid );
445	owner_lpid = LPID_FROM_PID( pid );
446	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
447	process_root_xp = XPTR( owner_cxy , process_root_ptr );
448
449	// get extended pointer on root of vsegs xlist from reference VMM
450	vseg_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
451
452	// loop on destination process copies
453	XLIST_FOREACH( process_root_xp , process_iter_xp )
454	{
455	// get cluster and local pointer on remote process
456	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
457	remote_process_ptr = GET_PTR( remote_process_xp );
458	remote_process_cxy = GET_CXY( remote_process_xp );
459
460	#if (DEBUG_VMM_SET_COW & 1)
461	if( DEBUG_VMM_SET_COW < cycle )
462	printk("\n[%s] thread[%x,%x] handling process %x in cluster %x\n",
463	__FUNCTION__, this->process->pid, this->trdid, process->pid , remote_process_cxy );
464	#endif
465
466	// get extended pointer on remote gpt
467	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
468
469	// loop on vsegs in (local) reference process VSL
470	XLIST_FOREACH( vseg_root_xp , vseg_iter_xp )
471	{
472	// get pointer on vseg
473	vseg_xp = XLIST_ELEMENT( vseg_iter_xp , vseg_t , xlist );
474	vseg = GET_PTR( vseg_xp );
475
476	assert( (GET_CXY( vseg_xp ) == local_cxy) ,
477	"all vsegs in reference VSL must be local\n" );
478
479	// get vseg type, base and size
480	uint32_t type = vseg->type;
481	vpn_t vpn_base = vseg->vpn_base;
482	vpn_t vpn_size = vseg->vpn_size;
483
484	#if (DEBUG_VMM_SET_COW & 1)
485	if( DEBUG_VMM_SET_COW < cycle )
486	printk("\n[%s] thread[%x,%x] handling vseg %s / vpn_base = %x / vpn_size = %x\n",
487	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), vpn_base, vpn_size );
488	#endif
489	// only DATA, ANON and REMOTE vsegs
490	if( (type == VSEG_TYPE_DATA) \|\|
491	(type == VSEG_TYPE_ANON) \|\|
492	(type == VSEG_TYPE_REMOTE) )
493	{
494	vpn_t vpn;
495	uint32_t attr;
496	ppn_t ppn;
497	xptr_t page_xp;
498	cxy_t page_cxy;
499	page_t * page_ptr;
500	xptr_t forks_xp;
501	xptr_t lock_xp;
502
503	// update flags in remote GPT
504	hal_gpt_set_cow( remote_gpt_xp,
505	vpn_base,
506	vpn_size );
507
508	// atomically increment pending forks counter in physical pages,
509	// for all vseg pages that are mapped in reference cluster
510	if( remote_process_cxy == local_cxy )
511	{
512	// scan all pages in vseg
513	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
514	{
515	// get page attributes and PPN from reference GPT
516	hal_gpt_get_pte( remote_gpt_xp , vpn , &attr , &ppn );
517
518	// atomically update pending forks counter if page is mapped
519	if( attr & GPT_MAPPED )
520	{
521	// get pointers and cluster on page descriptor
522	page_xp = ppm_ppn2page( ppn );
523	page_cxy = GET_CXY( page_xp );
524	page_ptr = GET_PTR( page_xp );
525
526	// get extended pointers on "forks" and "lock"
527	forks_xp = XPTR( page_cxy , &page_ptr->forks );
528	lock_xp = XPTR( page_cxy , &page_ptr->lock );
529
530	// take lock protecting "forks" counter
531	remote_busylock_acquire( lock_xp );
532
533	// increment "forks"
534	hal_remote_atomic_add( forks_xp , 1 );
535
536	// release lock protecting "forks" counter
537	remote_busylock_release( lock_xp );
538	}
539	} // end loop on vpn
540	} // end if local
541	} // end if vseg type
542	} // end loop on vsegs
543	} // end loop on process copies
544
545	#if DEBUG_VMM_SET_COW
546	cycle = (uint32_t)hal_get_cycles();
547	if( DEBUG_VMM_SET_COW < cycle )
548	printk("\n[%s] thread[%x,%x] exit for process %x / cycle %d\n",
549	__FUNCTION__, this->process->pid, this->trdid, process->pid , cycle );
550	#endif
551
552	} // end vmm_set-cow()
553
554	/////////////////////////////////////////////////
555	error_t vmm_fork_copy( process_t * child_process,
556	xptr_t parent_process_xp )
557	{
558	error_t error;
559	cxy_t parent_cxy;
560	process_t * parent_process;
561	vmm_t * parent_vmm;
562	xptr_t parent_lock_xp;
563	vmm_t * child_vmm;
564	xptr_t iter_xp;
565	xptr_t parent_vseg_xp;
566	vseg_t * parent_vseg;
567	vseg_t * child_vseg;
568	uint32_t type;
569	bool_t cow;
570	vpn_t vpn;
571	vpn_t vpn_base;
572	vpn_t vpn_size;
573	xptr_t page_xp; // extended pointer on page descriptor
574	page_t * page_ptr;
575	cxy_t page_cxy;
576	xptr_t forks_xp; // extended pointer on forks counter in page descriptor
577	xptr_t lock_xp; // extended pointer on lock protecting the forks counter
578	xptr_t parent_root_xp;
579	bool_t mapped;
580	ppn_t ppn;
581
582	#if DEBUG_VMM_FORK_COPY
583	uint32_t cycle = (uint32_t)hal_get_cycles();
584	thread_t * this = CURRENT_THREAD;
585	if( DEBUG_VMM_FORK_COPY < cycle )
586	printk("\n[%s] thread %x enter / cycle %d\n",
587	__FUNCTION__ , this->process->pid, this->trdid, cycle );
588	#endif
589
590	// get parent process cluster and local pointer
591	parent_cxy = GET_CXY( parent_process_xp );
592	parent_process = GET_PTR( parent_process_xp );
593
594	// get local pointers on parent and child VMM
595	parent_vmm = &parent_process->vmm;
596	child_vmm = &child_process->vmm;
597
598	// get extended pointer on lock protecting the parent VSL
599	parent_lock_xp = XPTR( parent_cxy , &parent_vmm->vsegs_lock );
600
601	// initialize the lock protecting the child VSL
602	remote_rwlock_init( XPTR( local_cxy , &child_vmm->vsegs_lock ), LOCK_VMM_STACK );
603
604	// initialize the child VSL as empty
605	xlist_root_init( XPTR( local_cxy, &child_vmm->vsegs_root ) );
606	child_vmm->vsegs_nr = 0;
607
608	// create child GPT
609	error = hal_gpt_create( &child_vmm->gpt );
610
611	if( error )
612	{
613	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
614	return -1;
615	}
616
617	// build extended pointer on parent VSL
618	parent_root_xp = XPTR( parent_cxy , &parent_vmm->vsegs_root );
619
620	// take the lock protecting the parent VSL in read mode
621	remote_rwlock_rd_acquire( parent_lock_xp );
622
623	// loop on parent VSL xlist
624	XLIST_FOREACH( parent_root_xp , iter_xp )
625	{
626	// get local and extended pointers on current parent vseg
627	parent_vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
628	parent_vseg = GET_PTR( parent_vseg_xp );
629
630	// get vseg type
631	type = hal_remote_l32( XPTR( parent_cxy , &parent_vseg->type ) );
632
633	#if DEBUG_VMM_FORK_COPY
634	cycle = (uint32_t)hal_get_cycles();
635	if( DEBUG_VMM_FORK_COPY < cycle )
636	printk("\n[%s] thread[%x,%x] found parent vseg %s / vpn_base = %x / cycle %d\n",
637	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
638	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
639	#endif
640
641	// all parent vsegs - but STACK - must be copied in child VSL
642	if( type != VSEG_TYPE_STACK )
643	{
644	// allocate memory for a new child vseg
645	child_vseg = vseg_alloc();
646	if( child_vseg == NULL ) // release all allocated vsegs
647	{
648	vmm_destroy( child_process );
649	printk("\n[ERROR] in %s : cannot create vseg for child\n", __FUNCTION__ );
650	return -1;
651	}
652
653	// copy parent vseg to child vseg
654	vseg_init_from_ref( child_vseg , parent_vseg_xp );
655
656	// register child vseg in child VSL
657	vmm_attach_vseg_to_vsl( child_vmm , child_vseg );
658
659	#if DEBUG_VMM_FORK_COPY
660	cycle = (uint32_t)hal_get_cycles();
661	if( DEBUG_VMM_FORK_COPY < cycle )
662	printk("\n[%s] thread[%x,%x] copied vseg %s / vpn_base = %x to child VSL / cycle %d\n",
663	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
664	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
665	#endif
666
667	// copy DATA, MMAP, REMOTE, FILE parent GPT entries to child GPT
668	if( type != VSEG_TYPE_CODE )
669	{
670	// activate the COW for DATA, MMAP, REMOTE vsegs only
671	cow = ( type != VSEG_TYPE_FILE );
672
673	vpn_base = child_vseg->vpn_base;
674	vpn_size = child_vseg->vpn_size;
675
676	// scan pages in parent vseg
677	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
678	{
679	error = hal_gpt_pte_copy( &child_vmm->gpt,
680	XPTR( parent_cxy , &parent_vmm->gpt ),
681	vpn,
682	cow,
683	&ppn,
684	&mapped );
685	if( error )
686	{
687	vmm_destroy( child_process );
688	printk("\n[ERROR] in %s : cannot copy GPT\n", __FUNCTION__ );
689	return -1;
690	}
691
692	// increment pending forks counter in page if mapped
693	if( mapped )
694	{
695	// get pointers and cluster on page descriptor
696	page_xp = ppm_ppn2page( ppn );
697	page_cxy = GET_CXY( page_xp );
698	page_ptr = GET_PTR( page_xp );
699
700	// get extended pointers on "forks" and "lock"
701	forks_xp = XPTR( page_cxy , &page_ptr->forks );
702	lock_xp = XPTR( page_cxy , &page_ptr->lock );
703
704	// get lock protecting "forks" counter
705	remote_busylock_acquire( lock_xp );
706
707	// increment "forks"
708	hal_remote_atomic_add( forks_xp , 1 );
709
710	// release lock protecting "forks" counter
711	remote_busylock_release( lock_xp );
712
713	#if DEBUG_VMM_FORK_COPY
714	cycle = (uint32_t)hal_get_cycles();
715	if( DEBUG_VMM_FORK_COPY < cycle )
716	printk("\n[%s] thread[%x,%x] copied vpn %x to child GPT / cycle %d\n",
717	__FUNCTION__ , this->process->pid, this->trdid , vpn , cycle );
718	#endif
719	}
720	}
721	} // end if no code & no stack
722	} // end if no stack
723	} // end loop on vsegs
724
725	// release the parent VSL lock in read mode
726	remote_rwlock_rd_release( parent_lock_xp );
727
728	// initialize child GPT (architecture specic)
729	// => For TSAR, identity map the kentry_vseg
730	error = hal_vmm_init( child_vmm );
731
732	if( error )
733	{
734	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
735	return -1;
736	}
737
738	// initialize the child VMM STACK allocator
739	child_vmm->stack_mgr.bitmap = 0;
740	child_vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
741
742	// initialize the child VMM MMAP allocator
743	uint32_t i;
744	child_vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
745	child_vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
746	child_vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
747	for( i = 0 ; i < 32 ; i++ ) list_root_init( &child_vmm->mmap_mgr.zombi_list[i] );
748
749	// initialize instrumentation counters
750	child_vmm->pgfault_nr = 0;
751
752	// copy base addresses from parent VMM to child VMM
753	child_vmm->kent_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->kent_vpn_base));
754	child_vmm->args_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->args_vpn_base));
755	child_vmm->envs_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->envs_vpn_base));
756	child_vmm->heap_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->heap_vpn_base));
757	child_vmm->code_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->code_vpn_base));
758	child_vmm->data_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->data_vpn_base));
759
760	child_vmm->entry_point = (intptr_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->entry_point));
761
762	hal_fence();
763
764	#if DEBUG_VMM_FORK_COPY
765	cycle = (uint32_t)hal_get_cycles();
766	if( DEBUG_VMM_FORK_COPY < cycle )
767	printk("\n[%s] thread[%x,%x] exit successfully / cycle %d\n",
768	__FUNCTION__ , this->process->pid, this->trdid , cycle );
769	#endif
770
771	return 0;
772
773	} // vmm_fork_copy()
774
775	///////////////////////////////////////
776	void vmm_destroy( process_t * process )
777	{
778	xptr_t vseg_xp;
779	vseg_t * vseg;
780
781	#if DEBUG_VMM_DESTROY
782	uint32_t cycle = (uint32_t)hal_get_cycles();
783	thread_t * this = CURRENT_THREAD;
784	if( DEBUG_VMM_DESTROY < cycle )
785	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
786	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
787	#endif
788
789	#if (DEBUG_VMM_DESTROY & 1 )
790	if( DEBUG_VMM_DESTROY < cycle )
791	vmm_display( process , true );
792	#endif
793
794	// get pointer on local VMM
795	vmm_t * vmm = &process->vmm;
796
797	// get extended pointer on VSL root and VSL lock
798	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
799
800	// scan the VSL to delete all registered vsegs
801	// (don't use a FOREACH for item deletion in xlist)
802
803	uint32_t count = 0;
804
805	while( !xlist_is_empty( root_xp ) && (count < 10 ) )
806	{
807	// get pointer on first vseg in VSL
808	vseg_xp = XLIST_FIRST( root_xp , vseg_t , xlist );
809	vseg = GET_PTR( vseg_xp );
810
811	// delete vseg and release physical pages
812	vmm_delete_vseg( process->pid , vseg->min );
813
814	#if( DEBUG_VMM_DESTROY & 1 )
815	if( DEBUG_VMM_DESTROY < cycle )
816	printk("\n[%s] %s vseg deleted / vpn_base %x / vpn_size %d\n",
817	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
818	#endif
819
820	count++;
821
822	}
823
824	// remove all vsegs from zombi_lists in MMAP allocator
825	uint32_t i;
826	for( i = 0 ; i<32 ; i++ )
827	{
828	while( !list_is_empty( &vmm->mmap_mgr.zombi_list[i] ) )
829	{
830	vseg = LIST_FIRST( &vmm->mmap_mgr.zombi_list[i] , vseg_t , zlist );
831
832	#if( DEBUG_VMM_DESTROY & 1 )
833	if( DEBUG_VMM_DESTROY < cycle )
834	printk("\n[%s] found zombi vseg / vpn_base %x / vpn_size %d\n",
835	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
836	#endif
837	// clean vseg descriptor
838	vseg->vmm = NULL;
839
840	// remove vseg from xlist
841	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
842
843	// release vseg descriptor
844	vseg_free( vseg );
845
846	#if( DEBUG_VMM_DESTROY & 1 )
847	if( DEBUG_VMM_DESTROY < cycle )
848	printk("\n[%s] zombi vseg released / vpn_base %x / vpn_size %d\n",
849	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
850	#endif
851	}
852	}
853
854	// release memory allocated to the GPT itself
855	hal_gpt_destroy( &vmm->gpt );
856
857	#if DEBUG_VMM_DESTROY
858	cycle = (uint32_t)hal_get_cycles();
859	if( DEBUG_VMM_DESTROY < cycle )
860	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
861	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy , cycle );
862	#endif
863
864	} // end vmm_destroy()
865
866	/////////////////////////////////////////////////
867	vseg_t * vmm_check_conflict( process_t * process,
868	vpn_t vpn_base,
869	vpn_t vpn_size )
870	{
871	vmm_t * vmm = &process->vmm;
872
873	// scan the VSL
874	vseg_t * vseg;
875	xptr_t iter_xp;
876	xptr_t vseg_xp;
877	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
878
879	XLIST_FOREACH( root_xp , iter_xp )
880	{
881	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
882	vseg = GET_PTR( vseg_xp );
883
884	if( ((vpn_base + vpn_size) > vseg->vpn_base) &&
885	(vpn_base < (vseg->vpn_base + vseg->vpn_size)) ) return vseg;
886	}
887	return NULL;
888
889	} // end vmm_check_conflict()
890
891	////////////////////////////////////////////////////////////////////////////////////////////
892	// This static function is called by the vmm_create_vseg() function, and implements
893	// the VMM stack_vseg specific allocator.
894	////////////////////////////////////////////////////////////////////////////////////////////
895	// @ vmm : pointer on VMM.
896	// @ vpn_base : (return value) first allocated page
897	// @ vpn_size : (return value) number of allocated pages
898	////////////////////////////////////////////////////////////////////////////////////////////
899	static error_t vmm_stack_alloc( vmm_t * vmm,
900	vpn_t * vpn_base,
901	vpn_t * vpn_size )
902	{
903	// get stack allocator pointer
904	stack_mgr_t * mgr = &vmm->stack_mgr;
905
906	// get lock on stack allocator
907	busylock_acquire( &mgr->lock );
908
909	// get first free slot index in bitmap
910	int32_t index = bitmap_ffc( &mgr->bitmap , 4 );
911	if( (index < 0) \|\| (index > 31) )
912	{
913	busylock_release( &mgr->lock );
914	return 0xFFFFFFFF;
915	}
916
917	// update bitmap
918	bitmap_set( &mgr->bitmap , index );
919
920	// release lock on stack allocator
921	busylock_release( &mgr->lock );
922
923	// returns vpn_base, vpn_size (one page non allocated)
924	vpn_base = mgr->vpn_base + index CONFIG_VMM_STACK_SIZE + 1;
925	*vpn_size = CONFIG_VMM_STACK_SIZE - 1;
926	return 0;
927
928	} // end vmm_stack_alloc()
929
930	////////////////////////////////////////////////////////////////////////////////////////////
931	// This static function is called by the vmm_create_vseg() function, and implements
932	// the VMM MMAP specific allocator.
933	////////////////////////////////////////////////////////////////////////////////////////////
934	// @ vmm : [in] pointer on VMM.
935	// @ npages : [in] requested number of pages.
936	// @ vpn_base : [out] first allocated page.
937	// @ vpn_size : [out] actual number of allocated pages.
938	////////////////////////////////////////////////////////////////////////////////////////////
939	static error_t vmm_mmap_alloc( vmm_t * vmm,
940	vpn_t npages,
941	vpn_t * vpn_base,
942	vpn_t * vpn_size )
943	{
944	uint32_t index;
945	vseg_t * vseg;
946	vpn_t base;
947	vpn_t size;
948	vpn_t free;
949
950	#if DEBUG_VMM_MMAP_ALLOC
951	thread_t * this = CURRENT_THREAD;
952	uint32_t cycle = (uint32_t)hal_get_cycles();
953	if( DEBUG_VMM_MMAP_ALLOC < cycle )
954	printk("\n[%s] thread[%x,%x] enter / cycle %d\n",
955	__FUNCTION__, this->process->pid, this->trdid, cycle );
956	#endif
957
958	// vseg size must be power of 2
959	// compute actual size and index in zombi_list array
960	size = POW2_ROUNDUP( npages );
961	index = bits_log2( size );
962
963	// get mmap allocator pointer
964	mmap_mgr_t * mgr = &vmm->mmap_mgr;
965
966	// get lock on mmap allocator
967	busylock_acquire( &mgr->lock );
968
969	// get vseg from zombi_list or from mmap zone
970	if( list_is_empty( &mgr->zombi_list[index] ) ) // from mmap zone
971	{
972	// check overflow
973	free = mgr->first_free_vpn;
974	if( (free + size) > mgr->vpn_size ) return -1;
975
976	// update MMAP allocator
977	mgr->first_free_vpn += size;
978
979	// compute base
980	base = free;
981	}
982	else // from zombi_list
983	{
984	// get pointer on zombi vseg from zombi_list
985	vseg = LIST_FIRST( &mgr->zombi_list[index] , vseg_t , zlist );
986
987	// remove vseg from free-list
988	list_unlink( &vseg->zlist );
989
990	// compute base
991	base = vseg->vpn_base;
992	}
993
994	// release lock on mmap allocator
995	busylock_release( &mgr->lock );
996
997	#if DEBUG_VMM_MMAP_ALLOC
998	cycle = (uint32_t)hal_get_cycles();
999	if( DEBUG_VMM_DESTROY < cycle )
1000	printk("\n[%s] thread[%x,%x] exit / vpn_base %x / vpn_size %x / cycle %d\n",
1001	__FUNCTION__, this->process->pid, this->trdid, base, size, cycle );
1002	#endif
1003
1004	// returns vpn_base, vpn_size
1005	*vpn_base = base;
1006	*vpn_size = size;
1007	return 0;
1008
1009	} // end vmm_mmap_alloc()
1010
1011	////////////////////////////////////////////////
1012	vseg_t * vmm_create_vseg( process_t * process,
1013	vseg_type_t type,
1014	intptr_t base,
1015	uint32_t size,
1016	uint32_t file_offset,
1017	uint32_t file_size,
1018	xptr_t mapper_xp,
1019	cxy_t cxy )
1020	{
1021	vseg_t * vseg; // created vseg pointer
1022	vpn_t vpn_base; // first page index
1023	vpn_t vpn_size; // number of pages covered by vseg
1024	error_t error;
1025
1026	#if DEBUG_VMM_CREATE_VSEG
1027	thread_t * this = CURRENT_THREAD;
1028	uint32_t cycle = (uint32_t)hal_get_cycles();
1029	if( DEBUG_VMM_CREATE_VSEG < cycle )
1030	printk("\n[%s] thread[%x,%x] enter for process %x / %s / cxy %x / cycle %d\n",
1031	__FUNCTION__, this->process->pid, this->trdid, process->pid, vseg_type_str(type), cxy, cycle );
1032	#endif
1033
1034	// get pointer on VMM
1035	vmm_t * vmm = &process->vmm;
1036
1037	// compute base, size, vpn_base, vpn_size, depending on vseg type
1038	// we use the VMM specific allocators for "stack", "file", "anon", & "remote" vsegs
1039
1040	if( type == VSEG_TYPE_STACK )
1041	{
1042	// get vpn_base and vpn_size from STACK allocator
1043	error = vmm_stack_alloc( vmm , &vpn_base , &vpn_size );
1044	if( error )
1045	{
1046	printk("\n[ERROR] in %s : no space for stack vseg / process %x in cluster %x\n",
1047	__FUNCTION__ , process->pid , local_cxy );
1048	return NULL;
1049	}
1050
1051	// compute vseg base and size from vpn_base and vpn_size
1052	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
1053	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
1054	}
1055	else if( type == VSEG_TYPE_FILE )
1056	{
1057	// compute page index (in mapper) for first byte
1058	vpn_t vpn_min = file_offset >> CONFIG_PPM_PAGE_SHIFT;
1059
1060	// compute page index (in mapper) for last byte
1061	vpn_t vpn_max = (file_offset + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
1062
1063	// compute offset in first page
1064	uint32_t offset = file_offset & CONFIG_PPM_PAGE_MASK;
1065
1066	// compute number of pages required in virtual space
1067	vpn_t npages = vpn_max - vpn_min + 1;
1068
1069	// get vpn_base and vpn_size from MMAP allocator
1070	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
1071	if( error )
1072	{
1073	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
1074	__FUNCTION__ , process->pid , local_cxy );
1075	return NULL;
1076	}
1077
1078	// set the vseg base (not always aligned for FILE)
1079	base = (vpn_base << CONFIG_PPM_PAGE_SHIFT) + offset;
1080	}
1081	else if( (type == VSEG_TYPE_ANON) \|\|
1082	(type == VSEG_TYPE_REMOTE) )
1083	{
1084	// compute number of required pages in virtual space
1085	vpn_t npages = size >> CONFIG_PPM_PAGE_SHIFT;
1086	if( size & CONFIG_PPM_PAGE_MASK) npages++;
1087
1088	// get vpn_base and vpn_size from MMAP allocator
1089	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
1090	if( error )
1091	{
1092	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
1093	__FUNCTION__ , process->pid , local_cxy );
1094	return NULL;
1095	}
1096
1097	// set vseg base (always aligned for ANON or REMOTE)
1098	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
1099	}
1100	else // VSEG_TYPE_DATA or VSEG_TYPE_CODE
1101	{
1102	uint32_t vpn_min = base >> CONFIG_PPM_PAGE_SHIFT;
1103	uint32_t vpn_max = (base + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
1104
1105	vpn_base = vpn_min;
1106	vpn_size = vpn_max - vpn_min + 1;
1107	}
1108
1109	// check collisions
1110	vseg = vmm_check_conflict( process , vpn_base , vpn_size );
1111	if( vseg != NULL )
1112	{
1113	printk("\n[ERROR] in %s for process %x : new vseg [vpn_base %x / vpn_size %x]\n"
1114	" overlap existing vseg [vpn_base %x / vpn_size %x]\n",
1115	__FUNCTION__ , process->pid, vpn_base, vpn_size, vseg->vpn_base, vseg->vpn_size );
1116	return NULL;
1117	}
1118
1119	// allocate physical memory for vseg descriptor
1120	vseg = vseg_alloc();
1121	if( vseg == NULL )
1122	{
1123	printk("\n[ERROR] in %s for process %x : cannot allocate memory for vseg\n",
1124	__FUNCTION__ , process->pid );
1125	return NULL;
1126	}
1127
1128	#if DEBUG_VMM_CREATE_VSEG
1129	if( DEBUG_VMM_CREATE_VSEG < cycle )
1130	printk("\n[%s] thread[%x,%x] : base %x / size %x / vpn_base %x / vpn_size %x\n",
1131	__FUNCTION__, this->process->pid, this->trdid, base, size, vpn_base, vpn_size );
1132	#endif
1133
1134	// initialize vseg descriptor
1135	vseg_init( vseg,
1136	type,
1137	base,
1138	size,
1139	vpn_base,
1140	vpn_size,
1141	file_offset,
1142	file_size,
1143	mapper_xp,
1144	cxy );
1145
1146	// attach vseg to VSL
1147	vmm_attach_vseg_to_vsl( vmm , vseg );
1148
1149	#if DEBUG_VMM_CREATE_VSEG
1150	cycle = (uint32_t)hal_get_cycles();
1151	if( DEBUG_VMM_CREATE_VSEG < cycle )
1152	printk("\n[%s] thread[%x,%x] exit / %s / cxy %x / cycle %d\n",
1153	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), cxy, cycle );
1154	#endif
1155
1156	return vseg;
1157
1158	} // vmm_create_vseg()
1159
1160	///////////////////////////////////
1161	void vmm_delete_vseg( pid_t pid,
1162	intptr_t vaddr )
1163	{
1164	process_t * process; // local pointer on local process
1165	vmm_t * vmm; // local pointer on local process VMM
1166	vseg_t * vseg; // local pointer on local vseg containing vaddr
1167	gpt_t * gpt; // local pointer on local process GPT
1168	vpn_t vpn; // VPN of current PTE
1169	vpn_t vpn_min; // VPN of first PTE
1170	vpn_t vpn_max; // VPN of last PTE (excluded)
1171	ppn_t ppn; // current PTE ppn value
1172	uint32_t attr; // current PTE attributes
1173	kmem_req_t req; // request to release memory
1174	xptr_t page_xp; // extended pointer on page descriptor
1175	cxy_t page_cxy; // page descriptor cluster
1176	page_t * page_ptr; // page descriptor pointer
1177	xptr_t forks_xp; // extended pointer on pending forks counter
1178	xptr_t lock_xp; // extended pointer on lock protecting forks counter
1179	uint32_t forks; // actual number of pendinf forks
1180
1181	#if DEBUG_VMM_DELETE_VSEG
1182	uint32_t cycle = (uint32_t)hal_get_cycles();
1183	thread_t * this = CURRENT_THREAD;
1184	if( DEBUG_VMM_DELETE_VSEG < cycle )
1185	printk("\n[%s] thread[%x,%x] enter / process %x / vaddr %x / cycle %d\n",
1186	__FUNCTION__, this->process->pid, this->trdid, pid, vaddr, cycle );
1187	#endif
1188
1189	// get local pointer on local process descriptor
1190	process = cluster_get_local_process_from_pid( pid );
1191
1192	if( process == NULL ) return;
1193
1194	// get pointers on local process VMM an GPT
1195	vmm = &process->vmm;
1196	gpt = &process->vmm.gpt;
1197
1198	// get local pointer on vseg containing vaddr
1199	vseg = vmm_vseg_from_vaddr( vmm , vaddr );
1200
1201	if( vseg == NULL ) return;
1202
1203	// loop to invalidate all vseg PTEs in GPT
1204	vpn_min = vseg->vpn_base;
1205	vpn_max = vpn_min + vseg->vpn_size;
1206	for( vpn = vpn_min ; vpn < vpn_max ; vpn++ )
1207	{
1208	// get GPT entry
1209	hal_gpt_get_pte( XPTR( local_cxy , gpt ) , vpn , &attr , &ppn );
1210
1211	if( attr & GPT_MAPPED ) // entry is mapped
1212	{
1213
1214	#if( DEBUG_VMM_DELETE_VSEG & 1 )
1215	if( DEBUG_VMM_DELETE_VSEG < cycle )
1216	printk("- unmap vpn %x / ppn %x / vseg %s \n" , vpn , ppn, vseg_type_str(vseg->type) );
1217	#endif
1218
1219	// check small page
1220	assert( (attr & GPT_SMALL) , "an user vseg must use small pages" );
1221
1222	// unmap GPT entry in local GPT
1223	hal_gpt_reset_pte( gpt , vpn );
1224
1225	// handle pending forks counter if
1226	// 1) not identity mapped
1227	// 2) reference cluster
1228	if( ((vseg->flags & VSEG_IDENT) == 0) &&
1229	(GET_CXY( process->ref_xp ) == local_cxy) )
1230	{
1231	// get extended pointer on physical page descriptor
1232	page_xp = ppm_ppn2page( ppn );
1233	page_cxy = GET_CXY( page_xp );
1234	page_ptr = GET_PTR( page_xp );
1235
1236	// get extended pointers on forks and lock fields
1237	forks_xp = XPTR( page_cxy , &page_ptr->forks );
1238	lock_xp = XPTR( page_cxy , &page_ptr->lock );
1239
1240	remote_busylock_acquire( lock_xp );
1241	// get pending forks counter
1242	forks = hal_remote_l32( forks_xp );
1243	if( forks ) // decrement pending forks counter
1244	{
1245	hal_remote_atomic_add( forks_xp , -1 );
1246	}
1247	else // release physical page to relevant cluster
1248	{
1249	if( page_cxy == local_cxy ) // local cluster
1250	{
1251	req.type = KMEM_PAGE;
1252	req.ptr = page_ptr;
1253	kmem_free( &req );
1254	}
1255	else // remote cluster
1256	{
1257	rpc_pmem_release_pages_client( page_cxy , page_ptr );
1258	}
1259
1260	#if( DEBUG_VMM_DELETE_VSEG & 1 )
1261	if( DEBUG_VMM_DELETE_VSEG < cycle )
1262	printk("- release ppn %x\n", ppn );
1263	#endif
1264	}
1265	remote_busylock_release( lock_xp );
1266	}
1267	}
1268	}
1269
1270	// remove vseg from VSL and release vseg descriptor (if not MMAP)
1271	vmm_detach_vseg_from_vsl( vmm , vseg );
1272
1273	#if DEBUG_VMM_DELETE_VSEG
1274	cycle = (uint32_t)hal_get_cycles();
1275	if( DEBUG_VMM_DELETE_VSEG < cycle )
1276	printk("\n[%s] thread[%x,%x] exit / process %x / vseg %s / base %x / cycle %d\n",
1277	__FUNCTION__, this->process->pid, this->trdid, pid, vseg_type_str(vseg->type), vaddr, cycle );
1278	#endif
1279
1280	} // end vmm_delete_vseg()
1281
1282	/////////////////////////////////////////////
1283	vseg_t * vmm_vseg_from_vaddr( vmm_t * vmm,
1284	intptr_t vaddr )
1285	{
1286	xptr_t iter_xp;
1287	xptr_t vseg_xp;
1288	vseg_t * vseg;
1289
1290	// get extended pointers on VSL lock and root
1291	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
1292	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
1293
1294	// get lock protecting the VSL
1295	remote_rwlock_rd_acquire( lock_xp );
1296
1297	// scan the list of vsegs in VSL
1298	XLIST_FOREACH( root_xp , iter_xp )
1299	{
1300	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
1301	vseg = GET_PTR( vseg_xp );
1302
1303	if( (vaddr >= vseg->min) && (vaddr < vseg->max) )
1304	{
1305	// return success
1306	remote_rwlock_rd_release( lock_xp );
1307	return vseg;
1308	}
1309	}
1310
1311	// return failure
1312	remote_rwlock_rd_release( lock_xp );
1313	return NULL;
1314
1315	} // end vmm_vseg_from_vaddr()
1316
1317	/////////////////////////////////////////////
1318	error_t vmm_resize_vseg( process_t * process,
1319	intptr_t base,
1320	intptr_t size )
1321	{
1322	error_t error;
1323	vseg_t * new;
1324	vpn_t vpn_min;
1325	vpn_t vpn_max;
1326
1327	// get pointer on process VMM
1328	vmm_t * vmm = &process->vmm;
1329
1330	intptr_t addr_min = base;
1331	intptr_t addr_max = base + size;
1332
1333	// get pointer on vseg
1334	vseg_t * vseg = vmm_vseg_from_vaddr( vmm , base );
1335
1336	if( vseg == NULL) return EINVAL;
1337
1338	// get extended pointer on VSL lock
1339	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
1340
1341	// get lock protecting VSL
1342	remote_rwlock_wr_acquire( lock_xp );
1343
1344	if( (vseg->min > addr_min) \|\| (vseg->max < addr_max) ) // not included in vseg
1345	{
1346	error = -1;
1347	}
1348	else if( (vseg->min == addr_min) && (vseg->max == addr_max) ) // vseg must be deleted
1349	{
1350	vmm_delete_vseg( process->pid , vseg->min );
1351	error = 0;
1352	}
1353	else if( vseg->min == addr_min ) // vseg must be resized
1354	{
1355	// update vseg base address
1356	vseg->min = addr_max;
1357
1358	// update vpn_base and vpn_size
1359	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1360	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1361	vseg->vpn_base = vpn_min;
1362	vseg->vpn_size = vpn_max - vpn_min + 1;
1363	error = 0;
1364	}
1365	else if( vseg->max == addr_max ) // vseg must be resized
1366	{
1367	// update vseg max address
1368	vseg->max = addr_min;
1369
1370	// update vpn_base and vpn_size
1371	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1372	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1373	vseg->vpn_base = vpn_min;
1374	vseg->vpn_size = vpn_max - vpn_min + 1;
1375	error = 0;
1376	}
1377	else // vseg cut in three regions
1378	{
1379	// resize existing vseg
1380	vseg->max = addr_min;
1381
1382	// update vpn_base and vpn_size
1383	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1384	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1385	vseg->vpn_base = vpn_min;
1386	vseg->vpn_size = vpn_max - vpn_min + 1;
1387
1388	// create new vseg
1389	new = vmm_create_vseg( process,
1390	vseg->type,
1391	addr_min,
1392	(vseg->max - addr_max),
1393	vseg->file_offset,
1394	vseg->file_size,
1395	vseg->mapper_xp,
1396	vseg->cxy );
1397
1398	if( new == NULL ) error = EINVAL;
1399	else error = 0;
1400	}
1401
1402	// release VMM lock
1403	remote_rwlock_wr_release( lock_xp );
1404
1405	return error;
1406
1407	} // vmm_resize_vseg()
1408
1409	///////////////////////////////////////////
1410	error_t vmm_get_vseg( process_t * process,
1411	intptr_t vaddr,
1412	vseg_t ** found_vseg )
1413	{
1414	xptr_t vseg_xp;
1415	vseg_t * vseg;
1416	vmm_t * vmm;
1417	error_t error;
1418
1419	// get pointer on local VMM
1420	vmm = &process->vmm;
1421
1422	// try to get vseg from local VMM
1423	vseg = vmm_vseg_from_vaddr( vmm , vaddr );
1424
1425	if( vseg == NULL ) // vseg not found in local cluster => try to get it from ref
1426	{
1427	// get extended pointer on reference process
1428	xptr_t ref_xp = process->ref_xp;
1429
1430	// get cluster and local pointer on reference process
1431	cxy_t ref_cxy = GET_CXY( ref_xp );
1432	process_t * ref_ptr = GET_PTR( ref_xp );
1433
1434	if( local_cxy == ref_cxy ) return -1; // local cluster is the reference
1435
1436	// get extended pointer on reference vseg
1437	rpc_vmm_get_vseg_client( ref_cxy , ref_ptr , vaddr , &vseg_xp , &error );
1438
1439	if( error ) return -1; // vseg not found => illegal user vaddr
1440
1441	// allocate a vseg in local cluster
1442	vseg = vseg_alloc();
1443
1444	if( vseg == NULL ) return -1; // cannot allocate a local vseg
1445
1446	// initialise local vseg from reference
1447	vseg_init_from_ref( vseg , vseg_xp );
1448
1449	// register local vseg in local VSL
1450	vmm_attach_vseg_to_vsl( vmm , vseg );
1451	}
1452
1453	// success
1454	*found_vseg = vseg;
1455	return 0;
1456
1457	} // end vmm_get_vseg()
1458
1459	//////////////////////////////////////////////////////////////////////////////////////
1460	// This static function compute the target cluster to allocate a physical page
1461	// for a given <vpn> in a given <vseg>, allocates the page (with an RPC if required)
1462	// and returns an extended pointer on the allocated page descriptor.
1463	// It can be called by a thread running in any cluster.
1464	// The vseg cannot have the FILE type.
1465	//////////////////////////////////////////////////////////////////////////////////////
1466	static xptr_t vmm_page_allocate( vseg_t * vseg,
1467	vpn_t vpn )
1468	{
1469
1470	#if DEBUG_VMM_ALLOCATE_PAGE
1471	uint32_t cycle = (uint32_t)hal_get_cycles();
1472	thread_t * this = CURRENT_THREAD;
1473	xptr_t this_xp = XPTR( local_cxy , this );
1474	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
1475	printk("\n[%s] thread[%x,%x] enter for vpn %x / cycle %d\n",
1476	__FUNCTION__ , this->process->pid, this->trdid, vpn, cycle );
1477	#endif
1478
1479	page_t * page_ptr;
1480	cxy_t page_cxy;
1481	kmem_req_t req;
1482	uint32_t index;
1483
1484	uint32_t type = vseg->type;
1485	uint32_t flags = vseg->flags;
1486	uint32_t x_size = LOCAL_CLUSTER->x_size;
1487	uint32_t y_size = LOCAL_CLUSTER->y_size;
1488
1489	// check vseg type
1490	assert( ( type != VSEG_TYPE_FILE ) , "illegal vseg type\n" );
1491
1492	if( flags & VSEG_DISTRIB ) // distributed => cxy depends on vpn LSB
1493	{
1494	index = vpn & ((x_size * y_size) - 1);
1495	page_cxy = HAL_CXY_FROM_XY( (index / y_size) , (index % y_size) );
1496
1497	// If the cluster selected from VPN's LSBs is empty, we select one randomly
1498	if ( cluster_is_active( page_cxy ) == false )
1499	{
1500	page_cxy = cluster_random_select();
1501	}
1502	}
1503	else // other cases => cxy specified in vseg
1504	{
1505	page_cxy = vseg->cxy;
1506	}
1507
1508	// allocate a physical page from target cluster
1509	if( page_cxy == local_cxy ) // target cluster is the local cluster
1510	{
1511	req.type = KMEM_PAGE;
1512	req.size = 0;
1513	req.flags = AF_NONE;
1514	page_ptr = (page_t *)kmem_alloc( &req );
1515	}
1516	else // target cluster is not the local cluster
1517	{
1518	rpc_pmem_get_pages_client( page_cxy , 0 , &page_ptr );
1519	}
1520
1521	#if DEBUG_VMM_ALLOCATE_PAGE
1522	cycle = (uint32_t)hal_get_cycles();
1523	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
1524	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle %d\n",
1525	__FUNCTION__ , this->process->pid, this->trdid, vpn,
1526	ppm_page2ppn( XPTR( page_cxy , page_ptr ) , cycle );
1527	#endif
1528
1529	if( page_ptr == NULL ) return XPTR_NULL;
1530	else return XPTR( page_cxy , page_ptr );
1531
1532	} // end vmm_page_allocate()
1533
1534	////////////////////////////////////////
1535	error_t vmm_get_one_ppn( vseg_t * vseg,
1536	vpn_t vpn,
1537	ppn_t * ppn )
1538	{
1539	error_t error;
1540	xptr_t page_xp; // extended pointer on physical page descriptor
1541	uint32_t page_id; // missing page index in vseg mapper
1542	uint32_t type; // vseg type;
1543
1544	type = vseg->type;
1545	page_id = vpn - vseg->vpn_base;
1546
1547	#if DEBUG_VMM_GET_ONE_PPN
1548	uint32_t cycle = (uint32_t)hal_get_cycles();
1549	thread_t * this = CURRENT_THREAD;
1550	if( DEBUG_VMM_GET_ONE_PPN < cycle )
1551	printk("\n[%s] thread[%x,%x] enter for vpn %x / type %s / page_id %d / cycle %d\n",
1552	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(type), page_id, cycle );
1553	#endif
1554
1555	// FILE type : get the physical page from the file mapper
1556	if( type == VSEG_TYPE_FILE )
1557	{
1558	// get extended pointer on mapper
1559	xptr_t mapper_xp = vseg->mapper_xp;
1560
1561	assert( (mapper_xp != XPTR_NULL),
1562	"mapper not defined for a FILE vseg\n" );
1563
1564	// get extended pointer on page descriptor
1565	page_xp = mapper_remote_get_page( mapper_xp , page_id );
1566
1567	if ( page_xp == XPTR_NULL ) return EINVAL;
1568	}
1569
1570	// Other types : allocate a physical page from target cluster,
1571	// as defined by vseg type and vpn value
1572	else
1573	{
1574	// allocate one physical page
1575	page_xp = vmm_page_allocate( vseg , vpn );
1576
1577	if( page_xp == XPTR_NULL ) return ENOMEM;
1578
1579	// initialise missing page from .elf file mapper for DATA and CODE types
1580	// the vseg->mapper_xp field is an extended pointer on the .elf file mapper
1581	if( (type == VSEG_TYPE_CODE) \|\| (type == VSEG_TYPE_DATA) )
1582	{
1583	// get extended pointer on mapper
1584	xptr_t mapper_xp = vseg->mapper_xp;
1585
1586	assert( (mapper_xp != XPTR_NULL),
1587	"mapper not defined for a CODE or DATA vseg\n" );
1588
1589	// compute missing page offset in vseg
1590	uint32_t offset = page_id << CONFIG_PPM_PAGE_SHIFT;
1591
1592	// compute missing page offset in .elf file
1593	uint32_t elf_offset = vseg->file_offset + offset;
1594
1595	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1596	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1597	printk("\n[%s] thread[%x,%x] for vpn = %x / elf_offset = %x\n",
1598	__FUNCTION__, this->process->pid, this->trdid, vpn, elf_offset );
1599	#endif
1600	// compute extended pointer on page base
1601	xptr_t base_xp = ppm_page2base( page_xp );
1602
1603	// file_size (in .elf mapper) can be smaller than vseg_size (BSS)
1604	uint32_t file_size = vseg->file_size;
1605
1606	if( file_size < offset ) // missing page fully in BSS
1607	{
1608
1609	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1610	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1611	printk("\n[%s] thread[%x,%x] for vpn %x / fully in BSS\n",
1612	__FUNCTION__, this->process->pid, this->trdid, vpn );
1613	#endif
1614	if( GET_CXY( page_xp ) == local_cxy )
1615	{
1616	memset( GET_PTR( base_xp ) , 0 , CONFIG_PPM_PAGE_SIZE );
1617	}
1618	else
1619	{
1620	hal_remote_memset( base_xp , 0 , CONFIG_PPM_PAGE_SIZE );
1621	}
1622	}
1623	else if( file_size >= (offset + CONFIG_PPM_PAGE_SIZE) ) // fully in mapper
1624	{
1625
1626	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1627	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1628	printk("\n[%s] thread[%x,%x] for vpn %x / fully in mapper\n",
1629	__FUNCTION__, this->process->pid, this->trdid, vpn );
1630	#endif
1631	error = mapper_move_kernel( mapper_xp,
1632	true, // to_buffer
1633	elf_offset,
1634	base_xp,
1635	CONFIG_PPM_PAGE_SIZE );
1636	if( error ) return EINVAL;
1637	}
1638	else // both in mapper and in BSS :
1639	// - (file_size - offset) bytes from mapper
1640	// - (page_size + offset - file_size) bytes from BSS
1641	{
1642
1643	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1644	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1645	printk("\n[%s] thread[%x,%x] for vpn %x / both mapper & BSS\n"
1646	" %d bytes from mapper / %d bytes from BSS\n",
1647	__FUNCTION__, this->process->pid, this->trdid, vpn,
1648	file_size - offset , offset + CONFIG_PPM_PAGE_SIZE - file_size );
1649	#endif
1650	// initialize mapper part
1651	error = mapper_move_kernel( mapper_xp,
1652	true, // to buffer
1653	elf_offset,
1654	base_xp,
1655	file_size - offset );
1656	if( error ) return EINVAL;
1657
1658	// initialize BSS part
1659	if( GET_CXY( page_xp ) == local_cxy )
1660	{
1661	memset( GET_PTR( base_xp ) + file_size - offset , 0 ,
1662	offset + CONFIG_PPM_PAGE_SIZE - file_size );
1663	}
1664	else
1665	{
1666	hal_remote_memset( base_xp + file_size - offset , 0 ,
1667	offset + CONFIG_PPM_PAGE_SIZE - file_size );
1668	}
1669	}
1670	} // end initialisation for CODE or DATA types
1671	}
1672
1673	// return ppn
1674	*ppn = ppm_page2ppn( page_xp );
1675
1676	#if DEBUG_VMM_GET_ONE_PPN
1677	cycle = (uint32_t)hal_get_cycles();
1678	if( DEBUG_VMM_GET_ONE_PPN < cycle )
1679	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle\n",
1680	__FUNCTION__ , this->process->pid, this->trdid , vpn , *ppn, cycle );
1681	#endif
1682
1683	return 0;
1684
1685	} // end vmm_get_one_ppn()
1686
1687	///////////////////////////////////////////////////
1688	error_t vmm_handle_page_fault( process_t * process,
1689	vpn_t vpn )
1690	{
1691	vseg_t * vseg; // vseg containing vpn
1692	uint32_t new_attr; // new PTE_ATTR value
1693	ppn_t new_ppn; // new PTE_PPN value
1694	uint32_t ref_attr; // PTE_ATTR value in reference GPT
1695	ppn_t ref_ppn; // PTE_PPN value in reference GPT
1696	cxy_t ref_cxy; // reference cluster for missing vpn
1697	process_t * ref_ptr; // reference process for missing vpn
1698	xptr_t local_gpt_xp; // extended pointer on local GPT
1699	xptr_t local_lock_xp; // extended pointer on local GPT lock
1700	xptr_t ref_gpt_xp; // extended pointer on reference GPT
1701	xptr_t ref_lock_xp; // extended pointer on reference GPT lock
1702	error_t error; // value returned by called functions
1703
1704	// get local vseg (access to reference VSL can be required)
1705	error = vmm_get_vseg( process,
1706	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
1707	&vseg );
1708	if( error )
1709	{
1710	printk("\n[ERROR] in %s : vpn %x in process %x not in a registered vseg\n",
1711	__FUNCTION__ , vpn , process->pid );
1712
1713	return EXCP_USER_ERROR;
1714	}
1715
1716	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1717	uint32_t cycle = (uint32_t)hal_get_cycles();
1718	thread_t * this = CURRENT_THREAD;
1719	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1720	printk("\n[%s] threadr[%x,%x] enter for vpn %x / %s / cycle %d\n",
1721	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(vseg->type), cycle );
1722	#endif
1723
1724	//////////////// private vseg => access only the local GPT
1725	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
1726	{
1727	// build extended pointer on local GPT and local GPT lock
1728	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1729	local_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
1730
1731	// take local GPT lock in write mode
1732	remote_rwlock_wr_acquire( local_lock_xp );
1733
1734	// check VPN still unmapped in local GPT
1735
1736	// do nothing if VPN has been mapped by a a concurrent page_fault
1737	hal_gpt_get_pte( local_gpt_xp,
1738	vpn,
1739	&new_attr,
1740	&new_ppn );
1741
1742	if( (new_attr & GPT_MAPPED) == 0 ) // VPN still unmapped
1743	{
1744	// allocate and initialise a physical page depending on the vseg type
1745	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
1746
1747	if( error )
1748	{
1749	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
1750	__FUNCTION__ , process->pid , vpn );
1751
1752	// release local GPT lock in write mode
1753	remote_rwlock_wr_release( local_lock_xp );
1754
1755	return EXCP_KERNEL_PANIC;
1756	}
1757
1758	// define new_attr from vseg flags
1759	new_attr = GPT_MAPPED \| GPT_SMALL;
1760	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
1761	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
1762	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
1763	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
1764
1765	// set PTE (PPN & attribute) to local GPT
1766	error = hal_gpt_set_pte( local_gpt_xp,
1767	vpn,
1768	new_attr,
1769	new_ppn );
1770	if ( error )
1771	{
1772	printk("\n[ERROR] in %s : cannot update local GPT / process %x / vpn = %x\n",
1773	__FUNCTION__ , process->pid , vpn );
1774
1775	// release local GPT lock in write mode
1776	remote_rwlock_wr_release( local_lock_xp );
1777
1778	return EXCP_KERNEL_PANIC;
1779	}
1780	}
1781
1782	// release local GPT lock in write mode
1783	remote_rwlock_wr_release( local_lock_xp );
1784
1785	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1786	cycle = (uint32_t)hal_get_cycles();
1787	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1788	printk("\n[%s] private page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
1789	__FUNCTION__, vpn, new_ppn, new_attr, cycle );
1790	#endif
1791	return EXCP_NON_FATAL;
1792
1793	} // end local GPT access
1794
1795	//////////// public vseg => access reference GPT
1796	else
1797	{
1798	// get reference process cluster and local pointer
1799	ref_cxy = GET_CXY( process->ref_xp );
1800	ref_ptr = GET_PTR( process->ref_xp );
1801
1802	// build extended pointer on reference GPT and reference GPT lock
1803	ref_gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
1804	ref_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt_lock );
1805
1806	// build extended pointer on local GPT and local GPT lock
1807	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1808	local_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
1809
1810	// take reference GPT lock in read mode
1811	remote_rwlock_rd_acquire( ref_lock_xp );
1812
1813	// get directly PPN & attributes from reference GPT
1814	// this can avoids a costly RPC for a false page fault
1815	hal_gpt_get_pte( ref_gpt_xp,
1816	vpn,
1817	&ref_attr,
1818	&ref_ppn );
1819
1820	// release reference GPT lock in read mode
1821	remote_rwlock_rd_release( ref_lock_xp );
1822
1823	if( ref_attr & GPT_MAPPED ) // false page fault => update local GPT
1824	{
1825	// take local GPT lock in write mode
1826	remote_rwlock_wr_acquire( local_lock_xp );
1827
1828	// check VPN still unmapped in local GPT
1829	hal_gpt_get_pte( local_gpt_xp,
1830	vpn,
1831	&new_attr,
1832	&new_ppn );
1833
1834	if( (new_attr & GPT_MAPPED) == 0 ) // VPN still unmapped
1835	{
1836	// update local GPT from reference GPT
1837	error = hal_gpt_set_pte( local_gpt_xp,
1838	vpn,
1839	ref_attr,
1840	ref_ppn );
1841	if( error )
1842	{
1843	printk("\n[ERROR] in %s : cannot update local GPT / process %x / vpn %x\n",
1844	__FUNCTION__ , process->pid , vpn );
1845
1846	// release local GPT lock in write mode
1847	remote_rwlock_wr_release( local_lock_xp );
1848
1849	return EXCP_KERNEL_PANIC;
1850	}
1851	}
1852	else // VPN has been mapped by a a concurrent page_fault
1853	{
1854	// keep PTE from local GPT
1855	ref_attr = new_attr;
1856	ref_ppn = new_ppn;
1857	}
1858
1859	// release local GPT lock in write mode
1860	remote_rwlock_wr_release( local_lock_xp );
1861
1862	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1863	cycle = (uint32_t)hal_get_cycles();
1864	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1865	printk("\n[%s] false page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
1866	__FUNCTION__, vpn, ref_ppn, ref_attr, cycle );
1867	#endif
1868	return EXCP_NON_FATAL;
1869	}
1870	else // true page fault => update reference GPT
1871	{
1872	// take reference GPT lock in write mode
1873	remote_rwlock_wr_acquire( ref_lock_xp );
1874
1875	// check VPN still unmapped in reference GPT
1876	// do nothing if VPN has been mapped by a a concurrent page_fault
1877	hal_gpt_get_pte( ref_gpt_xp,
1878	vpn,
1879	&ref_attr,
1880	&ref_ppn );
1881
1882	if( (ref_attr & GPT_MAPPED) == 0 ) // VPN actually unmapped
1883	{
1884	// allocate and initialise a physical page depending on the vseg type
1885	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
1886
1887	if( error )
1888	{
1889	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
1890	__FUNCTION__ , process->pid , vpn );
1891
1892	// release reference GPT lock in write mode
1893	remote_rwlock_wr_release( ref_lock_xp );
1894
1895	return EXCP_KERNEL_PANIC;
1896	}
1897
1898	// define new_attr from vseg flags
1899	new_attr = GPT_MAPPED \| GPT_SMALL;
1900	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
1901	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
1902	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
1903	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
1904
1905	// update reference GPT
1906	error = hal_gpt_set_pte( ref_gpt_xp,
1907	vpn,
1908	new_attr,
1909	new_ppn );
1910
1911	// update local GPT (protected by reference GPT lock)
1912	error \|= hal_gpt_set_pte( local_gpt_xp,
1913	vpn,
1914	new_attr,
1915	new_ppn );
1916
1917	if( error )
1918	{
1919	printk("\n[ERROR] in %s : cannot update GPT / process %x / vpn = %x\n",
1920	__FUNCTION__ , process->pid , vpn );
1921
1922	// release reference GPT lock in write mode
1923	remote_rwlock_wr_release( ref_lock_xp );
1924
1925	return EXCP_KERNEL_PANIC;
1926	}
1927	}
1928
1929	// release reference GPT lock in write mode
1930	remote_rwlock_wr_release( ref_lock_xp );
1931
1932	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1933	cycle = (uint32_t)hal_get_cycles();
1934	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1935	printk("\n[%s] true page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
1936	__FUNCTION__, vpn, new_ppn, new_attr, cycle );
1937	#endif
1938	return EXCP_NON_FATAL;
1939	}
1940	}
1941	} // end vmm_handle_page_fault()
1942
1943	////////////////////////////////////////////
1944	error_t vmm_handle_cow( process_t * process,
1945	vpn_t vpn )
1946	{
1947	vseg_t * vseg; // vseg containing vpn
1948	cxy_t ref_cxy; // reference cluster for missing vpn
1949	process_t * ref_ptr; // reference process for missing vpn
1950	xptr_t gpt_xp; // extended pointer on GPT
1951	xptr_t gpt_lock_xp; // extended pointer on GPT lock
1952	uint32_t old_attr; // current PTE_ATTR value
1953	ppn_t old_ppn; // current PTE_PPN value
1954	uint32_t new_attr; // new PTE_ATTR value
1955	ppn_t new_ppn; // new PTE_PPN value
1956	error_t error;
1957
1958	#if DEBUG_VMM_HANDLE_COW
1959	uint32_t cycle = (uint32_t)hal_get_cycles();
1960	thread_t * this = CURRENT_THREAD;
1961	xptr_t this_xp = XPTR( local_cxy , this );
1962	if( DEBUG_VMM_HANDLE_COW < cycle )
1963	printk("\n[%s] thread[%x,%x] enter for vpn %x / core[%x,%d] / cycle %d\n",
1964	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
1965	#endif
1966
1967	// access local GPT to get GPT_COW flag
1968	bool_t cow = hal_gpt_pte_is_cow( &(process->vmm.gpt), vpn );
1969
1970	if( cow == false ) return EXCP_USER_ERROR;
1971
1972	// get local vseg
1973	error = vmm_get_vseg( process,
1974	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
1975	&vseg );
1976	if( error )
1977	{
1978	printk("\n[PANIC] in %s : vpn %x in process %x not in a registered vseg\n",
1979	__FUNCTION__, vpn, process->pid );
1980
1981	return EXCP_KERNEL_PANIC;
1982	}
1983
1984	#if( DEBUG_VMM_HANDLE_COW & 1)
1985	if( DEBUG_VMM_HANDLE_COW < cycle )
1986	printk("\n[%s] thread[%x,%x] get vseg for vpn %x\n",
1987	__FUNCTION__, this->process->pid, this->trdid, vpn );
1988	#endif
1989
1990	// get reference GPT cluster and local pointer
1991	ref_cxy = GET_CXY( process->ref_xp );
1992	ref_ptr = GET_PTR( process->ref_xp );
1993
1994	// build relevant extended pointers on relevant GPT and GPT lock
1995	// - access local GPT for a private vseg
1996	// - access reference GPT for a public vseg
1997	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
1998	{
1999	gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
2000	gpt_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
2001	}
2002	else
2003	{
2004	gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
2005	gpt_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt_lock );
2006	}
2007
2008	// take GPT lock in write mode
2009	remote_rwlock_wr_acquire( gpt_lock_xp );
2010
2011	// get current PTE from reference GPT
2012	hal_gpt_get_pte( gpt_xp,
2013	vpn,
2014	&old_attr,
2015	&old_ppn );
2016
2017	#if( DEBUG_VMM_HANDLE_COW & 1)
2018	if( DEBUG_VMM_HANDLE_COW < cycle )
2019	printk("\n[%s] thread[%x,%x] get pte for vpn %x : ppn %x / attr %x\n",
2020	__FUNCTION__, this->process->pid, this->trdid, vpn, old_ppn, old_attr );
2021	#endif
2022
2023	// the PTE must be mapped for a COW
2024	if( (old_attr & GPT_MAPPED) == 0 )
2025	{
2026	printk("\n[PANIC] in %s : VPN %x in process %x unmapped\n",
2027	__FUNCTION__, vpn, process->pid );
2028
2029	// release GPT lock in write mode
2030	remote_rwlock_wr_release( gpt_lock_xp );
2031
2032	return EXCP_KERNEL_PANIC;
2033	}
2034
2035	// get pointers on physical page descriptor
2036	xptr_t page_xp = ppm_ppn2page( old_ppn );
2037	cxy_t page_cxy = GET_CXY( page_xp );
2038	page_t * page_ptr = GET_PTR( page_xp );
2039
2040	// get extended pointers on forks and lock field in page descriptor
2041	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
2042	xptr_t forks_lock_xp = XPTR( page_cxy , &page_ptr->lock );
2043
2044	// take lock protecting "forks" counter
2045	remote_busylock_acquire( forks_lock_xp );
2046
2047	// get number of pending forks from page descriptor
2048	uint32_t forks = hal_remote_l32( forks_xp );
2049
2050	#if( DEBUG_VMM_HANDLE_COW & 1)
2051	if( DEBUG_VMM_HANDLE_COW < cycle )
2052	printk("\n[%s] thread[%x,%x] get forks = %d for vpn %x\n",
2053	__FUNCTION__, this->process->pid, this->trdid, forks, vpn );
2054	#endif
2055
2056	if( forks ) // pending fork => allocate a new page, and copy old to new
2057	{
2058	// decrement pending forks counter in page descriptor
2059	hal_remote_atomic_add( forks_xp , -1 );
2060
2061	// release lock protecting "forks" counter
2062	remote_busylock_release( forks_lock_xp );
2063
2064	// allocate a new page
2065	page_xp = vmm_page_allocate( vseg , vpn );
2066
2067	if( page_xp == XPTR_NULL )
2068	{
2069	printk("\n[PANIC] in %s : no memory for vpn %x in process %x\n",
2070	__FUNCTION__ , vpn, process->pid );
2071
2072	// release GPT lock in write mode
2073	remote_rwlock_wr_acquire( gpt_lock_xp );
2074
2075	return EXCP_KERNEL_PANIC;
2076	}
2077
2078	// compute allocated page PPN
2079	new_ppn = ppm_page2ppn( page_xp );
2080
2081	#if( DEBUG_VMM_HANDLE_COW & 1)
2082	if( DEBUG_VMM_HANDLE_COW < cycle )
2083	printk("\n[%s] thread[%x,%x] get new ppn %x for vpn %x\n",
2084	__FUNCTION__, this->process->pid, this->trdid, new_ppn, vpn );
2085	#endif
2086
2087	// copy old page content to new page
2088	hal_remote_memcpy( ppm_ppn2base( new_ppn ),
2089	ppm_ppn2base( old_ppn ),
2090	CONFIG_PPM_PAGE_SIZE );
2091
2092	#if(DEBUG_VMM_HANDLE_COW & 1)
2093	if( DEBUG_VMM_HANDLE_COW < cycle )
2094	printk("\n[%s] thread[%x,%x] copied old page to new page\n",
2095	__FUNCTION__, this->process->pid, this->trdid );
2096	#endif
2097
2098	}
2099	else // no pending fork => keep the existing page
2100	{
2101	// release lock protecting "forks" counter
2102	remote_busylock_release( forks_lock_xp );
2103
2104	#if(DEBUG_VMM_HANDLE_COW & 1)
2105	if( DEBUG_VMM_HANDLE_COW < cycle )
2106	printk("\n[%s] thread[%x,%x] no pending forks / keep existing PPN %x\n",
2107	__FUNCTION__, this->process->pid, this->trdid, old_ppn );
2108	#endif
2109	new_ppn = old_ppn;
2110	}
2111
2112	// build new_attr : reset COW and set WRITABLE,
2113	new_attr = (old_attr \| GPT_WRITABLE) & (~GPT_COW);
2114
2115	// update the relevant GPT
2116	// - private vseg => update local GPT
2117	// - public vseg => update all GPT copies
2118	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
2119	{
2120	hal_gpt_set_pte( gpt_xp,
2121	vpn,
2122	new_attr,
2123	new_ppn );
2124	}
2125	else
2126	{
2127	if( ref_cxy == local_cxy ) // reference cluster is local
2128	{
2129	vmm_global_update_pte( process,
2130	vpn,
2131	new_attr,
2132	new_ppn );
2133	}
2134	else // reference cluster is remote
2135	{
2136	rpc_vmm_global_update_pte_client( ref_cxy,
2137	ref_ptr,
2138	vpn,
2139	new_attr,
2140	new_ppn );
2141	}
2142	}
2143
2144	// release GPT lock in write mode
2145	remote_rwlock_wr_release( gpt_lock_xp );
2146
2147	#if DEBUG_VMM_HANDLE_COW
2148	cycle = (uint32_t)hal_get_cycles();
2149	if( DEBUG_VMM_HANDLE_COW < cycle )
2150	printk("\n[%s] thread[%x,%x] exit for vpn %x / core[%x,%d] / cycle %d\n",
2151	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
2152	#endif
2153
2154	return EXCP_NON_FATAL;
2155
2156	} // end vmm_handle_cow()
2157

Note: See TracBrowser for help on using the repository browser.

Download in other formats: