Context Navigation

source: trunk/kernel/mm/vmm.c @ 623

Last change on this file since 623 was 623, checked in by alain, 5 years ago
Introduce three new types of vsegs (KCODE,KDATA,KDEV) to map the kernel vsegs in the process VSL and GPT. This now used by both the TSAR and the I86 architectures.
File size: 76.4 KB

Line
1	/*
2	* vmm.c - virtual memory manager related operations definition.
3	*
4	* Authors Ghassan Almaless (2008,2009,2010,2011, 2012)
5	* Mohamed Lamine Karaoui (2015)
6	* Alain Greiner (2016,2017,2018)
7	*
8	* Copyright (c) UPMC Sorbonne Universites
9	*
10	* This file is part of ALMOS-MKH.
11	*
12	* ALMOS-MKH is free software; you can redistribute it and/or modify it
13	* under the terms of the GNU General Public License as published by
14	* the Free Software Foundation; version 2.0 of the License.
15	*
16	* ALMOS-MKH is distributed in the hope that it will be useful, but
17	* WITHOUT ANY WARRANTY; without even the implied warranty of
18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19	* General Public License for more details.
20	*
21	* You should have received a copy of the GNU General Public License
22	* along with ALMOS-MKH; if not, write to the Free Software Foundation,
23	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
24	*/
25
26	#include <kernel_config.h>
27	#include <hal_kernel_types.h>
28	#include <hal_special.h>
29	#include <hal_gpt.h>
30	#include <hal_vmm.h>
31	#include <hal_macros.h>
32	#include <printk.h>
33	#include <memcpy.h>
34	#include <remote_rwlock.h>
35	#include <remote_queuelock.h>
36	#include <list.h>
37	#include <xlist.h>
38	#include <bits.h>
39	#include <process.h>
40	#include <thread.h>
41	#include <vseg.h>
42	#include <cluster.h>
43	#include <scheduler.h>
44	#include <vfs.h>
45	#include <mapper.h>
46	#include <page.h>
47	#include <kmem.h>
48	#include <vmm.h>
49	#include <hal_exception.h>
50
51	//////////////////////////////////////////////////////////////////////////////////
52	// Extern global variables
53	//////////////////////////////////////////////////////////////////////////////////
54
55	extern process_t process_zero; // allocated in cluster.c
56
57	///////////////////////////////////////
58	error_t vmm_init( process_t * process )
59	{
60	error_t error;
61	vseg_t * vseg_args;
62	vseg_t * vseg_envs;
63	intptr_t base;
64	intptr_t size;
65	uint32_t i;
66
67	#if DEBUG_VMM_INIT
68	thread_t * this = CURRENT_THREAD;
69	uint32_t cycle = (uint32_t)hal_get_cycles();
70	if( DEBUG_VMM_INIT )
71	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
72	__FUNCTION__ , this->process->pid, this->trdid, process->pid, local_cxy, cycle );
73	#endif
74
75	// get pointer on VMM
76	vmm_t * vmm = &process->vmm;
77
78	// initialize local list of vsegs
79	vmm->vsegs_nr = 0;
80	xlist_root_init( XPTR( local_cxy , &vmm->vsegs_root ) );
81	remote_rwlock_init( XPTR( local_cxy , &vmm->vsegs_lock ) , LOCK_VMM_VSL );
82
83	assert( ((CONFIG_VMM_KENTRY_SIZE + CONFIG_VMM_ARGS_SIZE + CONFIG_VMM_ENVS_SIZE)
84	<= CONFIG_VMM_ELF_BASE) , "UTILS zone too small\n" );
85
86	assert( (CONFIG_THREADS_MAX_PER_CLUSTER <= 32) ,
87	"no more than 32 threads per cluster for a single process\n");
88
89	assert( ((CONFIG_VMM_STACK_SIZE * CONFIG_THREADS_MAX_PER_CLUSTER) <=
90	(CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE)) ,
91	"STACK zone too small\n");
92
93	// register args vseg in VSL
94	base = (CONFIG_VMM_KENTRY_BASE +
95	CONFIG_VMM_KENTRY_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
96	size = CONFIG_VMM_ARGS_SIZE << CONFIG_PPM_PAGE_SHIFT;
97
98	vseg_args = vmm_create_vseg( process,
99	VSEG_TYPE_DATA,
100	base,
101	size,
102	0, // file_offset unused
103	0, // file_size unused
104	XPTR_NULL, // mapper_xp unused
105	local_cxy );
106
107	if( vseg_args == NULL )
108	{
109	printk("\n[ERROR] in %s : cannot register args vseg\n", __FUNCTION__ );
110	return -1;
111	}
112
113	vmm->args_vpn_base = base;
114
115	// register the envs vseg in VSL
116	base = (CONFIG_VMM_KENTRY_BASE +
117	CONFIG_VMM_KENTRY_SIZE +
118	CONFIG_VMM_ARGS_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
119	size = CONFIG_VMM_ENVS_SIZE << CONFIG_PPM_PAGE_SHIFT;
120
121	vseg_envs = vmm_create_vseg( process,
122	VSEG_TYPE_DATA,
123	base,
124	size,
125	0, // file_offset unused
126	0, // file_size unused
127	XPTR_NULL, // mapper_xp unused
128	local_cxy );
129
130	if( vseg_envs == NULL )
131	{
132	printk("\n[ERROR] in %s : cannot register envs vseg\n", __FUNCTION__ );
133	return -1;
134	}
135
136	vmm->envs_vpn_base = base;
137
138	// create GPT (empty)
139	error = hal_gpt_create( &vmm->gpt );
140
141	if( error )
142	{
143	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
144	return -1;
145	}
146
147	// initialize GPT lock
148	remote_rwlock_init( XPTR( local_cxy , &vmm->gpt_lock ) , LOCK_VMM_GPT );
149
150	// update process VMM with kernel vsegs
151	error = hal_vmm_kernel_update( process );
152
153	if( error )
154	{
155	printk("\n[ERROR] in %s : cannot update GPT for kernel vsegs\n", __FUNCTION__ );
156	return -1;
157	}
158
159	// initialize STACK allocator
160	vmm->stack_mgr.bitmap = 0;
161	vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
162	busylock_init( &vmm->stack_mgr.lock , LOCK_VMM_STACK );
163
164	// initialize MMAP allocator
165	vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
166	vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
167	vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
168	busylock_init( &vmm->mmap_mgr.lock , LOCK_VMM_MMAP );
169	for( i = 0 ; i < 32 ; i++ ) list_root_init( &vmm->mmap_mgr.zombi_list[i] );
170
171	// initialize instrumentation counters
172	vmm->pgfault_nr = 0;
173
174	hal_fence();
175
176	#if DEBUG_VMM_INIT
177	cycle = (uint32_t)hal_get_cycles();
178	if( DEBUG_VMM_INIT )
179	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
180	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
181	#endif
182
183	return 0;
184
185	} // end vmm_init()
186
187	//////////////////////////////////////
188	void vmm_display( process_t * process,
189	bool_t mapping )
190	{
191	vmm_t * vmm = &process->vmm;
192	gpt_t * gpt = &vmm->gpt;
193
194	printk("\n***** VSL and GPT(%x) for process %x in cluster %x\n\n",
195	process->vmm.gpt.ptr , process->pid , local_cxy );
196
197	// get lock protecting the VSL and the GPT
198	remote_rwlock_rd_acquire( XPTR( local_cxy , &vmm->vsegs_lock ) );
199	remote_rwlock_rd_acquire( XPTR( local_cxy , &vmm->gpt_lock ) );
200
201	// scan the list of vsegs
202	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
203	xptr_t iter_xp;
204	xptr_t vseg_xp;
205	vseg_t * vseg;
206	XLIST_FOREACH( root_xp , iter_xp )
207	{
208	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
209	vseg = GET_PTR( vseg_xp );
210
211	printk(" - %s : base = %X / size = %X / npages = %d\n",
212	vseg_type_str( vseg->type ) , vseg->min , vseg->max - vseg->min , vseg->vpn_size );
213
214	if( mapping )
215	{
216	vpn_t vpn;
217	ppn_t ppn;
218	uint32_t attr;
219	vpn_t base = vseg->vpn_base;
220	vpn_t size = vseg->vpn_size;
221	for( vpn = base ; vpn < (base+size) ; vpn++ )
222	{
223	hal_gpt_get_pte( XPTR( local_cxy , gpt ) , vpn , &attr , &ppn );
224	if( attr & GPT_MAPPED )
225	{
226	printk(" . vpn = %X / attr = %X / ppn = %X\n", vpn , attr , ppn );
227	}
228	}
229	}
230	}
231
232	// release the locks
233	remote_rwlock_rd_release( XPTR( local_cxy , &vmm->vsegs_lock ) );
234	remote_rwlock_rd_release( XPTR( local_cxy , &vmm->gpt_lock ) );
235
236	} // vmm_display()
237
238	//////////////////////////////////////////
239	void vmm_attach_vseg_to_vsl( vmm_t * vmm,
240	vseg_t * vseg )
241	{
242	// build extended pointer on rwlock protecting VSL
243	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
244
245	// get rwlock in write mode
246	remote_rwlock_wr_acquire( lock_xp );
247
248	// update vseg descriptor
249	vseg->vmm = vmm;
250
251	// add vseg in vmm list
252	xlist_add_last( XPTR( local_cxy , &vmm->vsegs_root ),
253	XPTR( local_cxy , &vseg->xlist ) );
254
255	// release rwlock in write mode
256	remote_rwlock_wr_release( lock_xp );
257	}
258
259	////////////////////////////////////////////
260	void vmm_detach_vseg_from_vsl( vmm_t * vmm,
261	vseg_t * vseg )
262	{
263	// get vseg type
264	uint32_t type = vseg->type;
265
266	// build extended pointer on rwlock protecting VSL
267	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
268
269	// get rwlock in write mode
270	remote_rwlock_wr_acquire( lock_xp );
271
272	// update vseg descriptor
273	vseg->vmm = NULL;
274
275	// remove vseg from VSL
276	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
277
278	// release rwlock in write mode
279	remote_rwlock_wr_release( lock_xp );
280
281	// release the stack slot to VMM stack allocator if STACK type
282	if( type == VSEG_TYPE_STACK )
283	{
284	// get pointer on stack allocator
285	stack_mgr_t * mgr = &vmm->stack_mgr;
286
287	// compute slot index
288	uint32_t index = ((vseg->vpn_base - mgr->vpn_base - 1) / CONFIG_VMM_STACK_SIZE);
289
290	// update stacks_bitmap
291	busylock_acquire( &mgr->lock );
292	bitmap_clear( &mgr->bitmap , index );
293	busylock_release( &mgr->lock );
294	}
295
296	// release the vseg to VMM mmap allocator if MMAP type
297	if( (type == VSEG_TYPE_ANON) \|\| (type == VSEG_TYPE_FILE) \|\| (type == VSEG_TYPE_REMOTE) )
298	{
299	// get pointer on mmap allocator
300	mmap_mgr_t * mgr = &vmm->mmap_mgr;
301
302	// compute zombi_list index
303	uint32_t index = bits_log2( vseg->vpn_size );
304
305	// update zombi_list
306	busylock_acquire( &mgr->lock );
307	list_add_first( &mgr->zombi_list[index] , &vseg->zlist );
308	busylock_release( &mgr->lock );
309	}
310
311	// release physical memory allocated for vseg if no MMAP and no kernel type
312	if( (type != VSEG_TYPE_ANON) && (type != VSEG_TYPE_FILE) && (type != VSEG_TYPE_REMOTE) &&
313	(type != VSEG_TYPE_KCODE) && (type != VSEG_TYPE_KDATA) && (type != VSEG_TYPE_KDEV) )
314	{
315	vseg_free( vseg );
316	}
317
318	} // end vmm_remove_vseg_from_vsl()
319
320	////////////////////////////////////////////////
321	void vmm_global_update_pte( process_t * process,
322	vpn_t vpn,
323	uint32_t attr,
324	ppn_t ppn )
325	{
326	xlist_entry_t * process_root_ptr;
327	xptr_t process_root_xp;
328	xptr_t process_iter_xp;
329
330	xptr_t remote_process_xp;
331	cxy_t remote_process_cxy;
332	process_t * remote_process_ptr;
333	xptr_t remote_gpt_xp;
334
335	pid_t pid;
336	cxy_t owner_cxy;
337	lpid_t owner_lpid;
338
339	#if DEBUG_VMM_UPDATE_PTE
340	uint32_t cycle = (uint32_t)hal_get_cycles();
341	thread_t * this = CURRENT_THREAD;
342	if( DEBUG_VMM_UPDATE_PTE < cycle )
343	printk("\n[%s] thread[%x,%x] enter for process %x / vpn %x / cycle %d\n",
344	__FUNCTION__, this->process->pid, this->trdid, process->pid , vpn , cycle );
345	#endif
346
347	// check cluster is reference
348	assert( (GET_CXY( process->ref_xp ) == local_cxy) , "not called in reference cluster\n");
349
350	// get extended pointer on root of process copies xlist in owner cluster
351	pid = process->pid;
352	owner_cxy = CXY_FROM_PID( pid );
353	owner_lpid = LPID_FROM_PID( pid );
354	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
355	process_root_xp = XPTR( owner_cxy , process_root_ptr );
356
357	// loop on destination process copies
358	XLIST_FOREACH( process_root_xp , process_iter_xp )
359	{
360	// get cluster and local pointer on remote process
361	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
362	remote_process_ptr = GET_PTR( remote_process_xp );
363	remote_process_cxy = GET_CXY( remote_process_xp );
364
365	#if (DEBUG_VMM_UPDATE_PTE & 0x1)
366	if( DEBUG_VMM_UPDATE_PTE < cycle )
367	printk("\n[%s] threadr[%x,%x] handling vpn %x for process %x in cluster %x\n",
368	__FUNCTION__, this->process->pid, this->trdid, vpn, process->pid, remote_process_cxy );
369	#endif
370
371	// get extended pointer on remote gpt
372	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
373
374	// update remote GPT
375	hal_gpt_update_pte( remote_gpt_xp, vpn, attr, ppn );
376	}
377
378	#if DEBUG_VMM_UPDATE_PTE
379	cycle = (uint32_t)hal_get_cycles();
380	if( DEBUG_VMM_UPDATE_PTE < cycle )
381	printk("\n[%s] thread[%x,%x] exit for process %x / vpn %x / cycle %d\n",
382	__FUNCTION__, this->process->pid, this->trdid, process->pid , vpn , cycle );
383	#endif
384
385	} // end vmm_global_update_pte()
386
387	///////////////////////////////////////
388	void vmm_set_cow( process_t * process )
389	{
390	vmm_t * vmm;
391
392	xlist_entry_t * process_root_ptr;
393	xptr_t process_root_xp;
394	xptr_t process_iter_xp;
395
396	xptr_t remote_process_xp;
397	cxy_t remote_process_cxy;
398	process_t * remote_process_ptr;
399	xptr_t remote_gpt_xp;
400
401	xptr_t vseg_root_xp;
402	xptr_t vseg_iter_xp;
403
404	xptr_t vseg_xp;
405	vseg_t * vseg;
406
407	pid_t pid;
408	cxy_t owner_cxy;
409	lpid_t owner_lpid;
410
411	#if DEBUG_VMM_SET_COW
412	uint32_t cycle = (uint32_t)hal_get_cycles();
413	thread_t * this = CURRENT_THREAD;
414	if( DEBUG_VMM_SET_COW < cycle )
415	printk("\n[%s] thread[%x,%x] enter for process %x / cycle %d\n",
416	__FUNCTION__, this->process->pid, this->trdid, process->pid , cycle );
417	#endif
418
419	// check cluster is reference
420	assert( (GET_CXY( process->ref_xp ) == local_cxy) ,
421	"local cluster is not process reference cluster\n");
422
423	// get pointer on reference VMM
424	vmm = &process->vmm;
425
426	// get extended pointer on root of process copies xlist in owner cluster
427	pid = process->pid;
428	owner_cxy = CXY_FROM_PID( pid );
429	owner_lpid = LPID_FROM_PID( pid );
430	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
431	process_root_xp = XPTR( owner_cxy , process_root_ptr );
432
433	// get extended pointer on root of vsegs xlist from reference VMM
434	vseg_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
435
436	// loop on destination process copies
437	XLIST_FOREACH( process_root_xp , process_iter_xp )
438	{
439	// get cluster and local pointer on remote process
440	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
441	remote_process_ptr = GET_PTR( remote_process_xp );
442	remote_process_cxy = GET_CXY( remote_process_xp );
443
444	#if (DEBUG_VMM_SET_COW & 1)
445	if( DEBUG_VMM_SET_COW < cycle )
446	printk("\n[%s] thread[%x,%x] handling process %x in cluster %x\n",
447	__FUNCTION__, this->process->pid, this->trdid, process->pid , remote_process_cxy );
448	#endif
449
450	// get extended pointer on remote gpt
451	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
452
453	// loop on vsegs in (local) reference process VSL
454	XLIST_FOREACH( vseg_root_xp , vseg_iter_xp )
455	{
456	// get pointer on vseg
457	vseg_xp = XLIST_ELEMENT( vseg_iter_xp , vseg_t , xlist );
458	vseg = GET_PTR( vseg_xp );
459
460	assert( (GET_CXY( vseg_xp ) == local_cxy) ,
461	"all vsegs in reference VSL must be local\n" );
462
463	// get vseg type, base and size
464	uint32_t type = vseg->type;
465	vpn_t vpn_base = vseg->vpn_base;
466	vpn_t vpn_size = vseg->vpn_size;
467
468	#if (DEBUG_VMM_SET_COW & 1)
469	if( DEBUG_VMM_SET_COW < cycle )
470	printk("\n[%s] thread[%x,%x] handling vseg %s / vpn_base = %x / vpn_size = %x\n",
471	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), vpn_base, vpn_size );
472	#endif
473	// only DATA, ANON and REMOTE vsegs
474	if( (type == VSEG_TYPE_DATA) \|\|
475	(type == VSEG_TYPE_ANON) \|\|
476	(type == VSEG_TYPE_REMOTE) )
477	{
478	vpn_t vpn;
479	uint32_t attr;
480	ppn_t ppn;
481	xptr_t page_xp;
482	cxy_t page_cxy;
483	page_t * page_ptr;
484	xptr_t forks_xp;
485	xptr_t lock_xp;
486
487	// update flags in remote GPT
488	hal_gpt_set_cow( remote_gpt_xp,
489	vpn_base,
490	vpn_size );
491
492	// atomically increment pending forks counter in physical pages,
493	// for all vseg pages that are mapped in reference cluster
494	if( remote_process_cxy == local_cxy )
495	{
496	// scan all pages in vseg
497	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
498	{
499	// get page attributes and PPN from reference GPT
500	hal_gpt_get_pte( remote_gpt_xp , vpn , &attr , &ppn );
501
502	// atomically update pending forks counter if page is mapped
503	if( attr & GPT_MAPPED )
504	{
505	// get pointers and cluster on page descriptor
506	page_xp = ppm_ppn2page( ppn );
507	page_cxy = GET_CXY( page_xp );
508	page_ptr = GET_PTR( page_xp );
509
510	// get extended pointers on "forks" and "lock"
511	forks_xp = XPTR( page_cxy , &page_ptr->forks );
512	lock_xp = XPTR( page_cxy , &page_ptr->lock );
513
514	// take lock protecting "forks" counter
515	remote_busylock_acquire( lock_xp );
516
517	// increment "forks"
518	hal_remote_atomic_add( forks_xp , 1 );
519
520	// release lock protecting "forks" counter
521	remote_busylock_release( lock_xp );
522	}
523	} // end loop on vpn
524	} // end if local
525	} // end if vseg type
526	} // end loop on vsegs
527	} // end loop on process copies
528
529	#if DEBUG_VMM_SET_COW
530	cycle = (uint32_t)hal_get_cycles();
531	if( DEBUG_VMM_SET_COW < cycle )
532	printk("\n[%s] thread[%x,%x] exit for process %x / cycle %d\n",
533	__FUNCTION__, this->process->pid, this->trdid, process->pid , cycle );
534	#endif
535
536	} // end vmm_set-cow()
537
538	/////////////////////////////////////////////////
539	error_t vmm_fork_copy( process_t * child_process,
540	xptr_t parent_process_xp )
541	{
542	error_t error;
543	cxy_t parent_cxy;
544	process_t * parent_process;
545	vmm_t * parent_vmm;
546	xptr_t parent_lock_xp;
547	vmm_t * child_vmm;
548	xptr_t iter_xp;
549	xptr_t parent_vseg_xp;
550	vseg_t * parent_vseg;
551	vseg_t * child_vseg;
552	uint32_t type;
553	bool_t cow;
554	vpn_t vpn;
555	vpn_t vpn_base;
556	vpn_t vpn_size;
557	xptr_t page_xp; // extended pointer on page descriptor
558	page_t * page_ptr;
559	cxy_t page_cxy;
560	xptr_t forks_xp; // extended pointer on forks counter in page descriptor
561	xptr_t lock_xp; // extended pointer on lock protecting the forks counter
562	xptr_t parent_root_xp;
563	bool_t mapped;
564	ppn_t ppn;
565
566	#if DEBUG_VMM_FORK_COPY
567	uint32_t cycle = (uint32_t)hal_get_cycles();
568	thread_t * this = CURRENT_THREAD;
569	if( DEBUG_VMM_FORK_COPY < cycle )
570	printk("\n[%s] thread %x enter / cycle %d\n",
571	__FUNCTION__ , this->process->pid, this->trdid, cycle );
572	#endif
573
574	// get parent process cluster and local pointer
575	parent_cxy = GET_CXY( parent_process_xp );
576	parent_process = GET_PTR( parent_process_xp );
577
578	// get local pointers on parent and child VMM
579	parent_vmm = &parent_process->vmm;
580	child_vmm = &child_process->vmm;
581
582	// get extended pointer on lock protecting the parent VSL
583	parent_lock_xp = XPTR( parent_cxy , &parent_vmm->vsegs_lock );
584
585	// initialize the lock protecting the child VSL
586	remote_rwlock_init( XPTR( local_cxy , &child_vmm->vsegs_lock ), LOCK_VMM_STACK );
587
588	// initialize the child VSL as empty
589	xlist_root_init( XPTR( local_cxy, &child_vmm->vsegs_root ) );
590	child_vmm->vsegs_nr = 0;
591
592	// create the child GPT
593	error = hal_gpt_create( &child_vmm->gpt );
594
595	if( error )
596	{
597	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
598	return -1;
599	}
600
601	// build extended pointer on parent VSL
602	parent_root_xp = XPTR( parent_cxy , &parent_vmm->vsegs_root );
603
604	// take the lock protecting the parent VSL in read mode
605	remote_rwlock_rd_acquire( parent_lock_xp );
606
607	// loop on parent VSL xlist
608	XLIST_FOREACH( parent_root_xp , iter_xp )
609	{
610	// get local and extended pointers on current parent vseg
611	parent_vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
612	parent_vseg = GET_PTR( parent_vseg_xp );
613
614	// get vseg type
615	type = hal_remote_l32( XPTR( parent_cxy , &parent_vseg->type ) );
616
617	#if DEBUG_VMM_FORK_COPY
618	cycle = (uint32_t)hal_get_cycles();
619	if( DEBUG_VMM_FORK_COPY < cycle )
620	printk("\n[%s] thread[%x,%x] found parent vseg %s / vpn_base = %x / cycle %d\n",
621	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
622	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
623	#endif
624
625	// all parent vsegs - but STACK and kernel vsegs - must be copied in child VSL
626	if( (type != VSEG_TYPE_STACK) && (type != VSEG_TYPE_KCODE) &&
627	(type != VSEG_TYPE_KDATA) && (type != VSEG_TYPE_KDEV) )
628	{
629	// allocate memory for a new child vseg
630	child_vseg = vseg_alloc();
631	if( child_vseg == NULL ) // release all allocated vsegs
632	{
633	vmm_destroy( child_process );
634	printk("\n[ERROR] in %s : cannot create vseg for child\n", __FUNCTION__ );
635	return -1;
636	}
637
638	// copy parent vseg to child vseg
639	vseg_init_from_ref( child_vseg , parent_vseg_xp );
640
641	// register child vseg in child VSL
642	vmm_attach_vseg_to_vsl( child_vmm , child_vseg );
643
644	#if DEBUG_VMM_FORK_COPY
645	cycle = (uint32_t)hal_get_cycles();
646	if( DEBUG_VMM_FORK_COPY < cycle )
647	printk("\n[%s] thread[%x,%x] copied vseg %s / vpn_base = %x to child VSL / cycle %d\n",
648	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
649	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
650	#endif
651
652	// copy DATA, MMAP, REMOTE, FILE parent GPT entries to child GPT
653	if( type != VSEG_TYPE_CODE )
654	{
655	// activate the COW for DATA, MMAP, REMOTE vsegs only
656	cow = ( type != VSEG_TYPE_FILE );
657
658	vpn_base = child_vseg->vpn_base;
659	vpn_size = child_vseg->vpn_size;
660
661	// scan pages in parent vseg
662	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
663	{
664	error = hal_gpt_pte_copy( &child_vmm->gpt,
665	XPTR( parent_cxy , &parent_vmm->gpt ),
666	vpn,
667	cow,
668	&ppn,
669	&mapped );
670	if( error )
671	{
672	vmm_destroy( child_process );
673	printk("\n[ERROR] in %s : cannot copy GPT\n", __FUNCTION__ );
674	return -1;
675	}
676
677	// increment pending forks counter in page if mapped
678	if( mapped )
679	{
680	// get pointers and cluster on page descriptor
681	page_xp = ppm_ppn2page( ppn );
682	page_cxy = GET_CXY( page_xp );
683	page_ptr = GET_PTR( page_xp );
684
685	// get extended pointers on "forks" and "lock"
686	forks_xp = XPTR( page_cxy , &page_ptr->forks );
687	lock_xp = XPTR( page_cxy , &page_ptr->lock );
688
689	// get lock protecting "forks" counter
690	remote_busylock_acquire( lock_xp );
691
692	// increment "forks"
693	hal_remote_atomic_add( forks_xp , 1 );
694
695	// release lock protecting "forks" counter
696	remote_busylock_release( lock_xp );
697
698	#if DEBUG_VMM_FORK_COPY
699	cycle = (uint32_t)hal_get_cycles();
700	if( DEBUG_VMM_FORK_COPY < cycle )
701	printk("\n[%s] thread[%x,%x] copied vpn %x to child GPT / cycle %d\n",
702	__FUNCTION__ , this->process->pid, this->trdid , vpn , cycle );
703	#endif
704	}
705	}
706	} // end if no code & no stack
707	} // end if no stack
708	} // end loop on vsegs
709
710	// release the parent VSL lock in read mode
711	remote_rwlock_rd_release( parent_lock_xp );
712
713	// update child VMM with kernel vsegs
714	error = hal_vmm_kernel_update( child_process );
715
716	if( error )
717	{
718	printk("\n[ERROR] in %s : cannot update child VMM\n", __FUNCTION__ );
719	return -1;
720	}
721
722	// initialize the child VMM STACK allocator
723	child_vmm->stack_mgr.bitmap = 0;
724	child_vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
725
726	// initialize the child VMM MMAP allocator
727	uint32_t i;
728	child_vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
729	child_vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
730	child_vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
731	for( i = 0 ; i < 32 ; i++ ) list_root_init( &child_vmm->mmap_mgr.zombi_list[i] );
732
733	// initialize instrumentation counters
734	child_vmm->pgfault_nr = 0;
735
736	// copy base addresses from parent VMM to child VMM
737	child_vmm->kent_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->kent_vpn_base));
738	child_vmm->args_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->args_vpn_base));
739	child_vmm->envs_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->envs_vpn_base));
740	child_vmm->heap_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->heap_vpn_base));
741	child_vmm->code_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->code_vpn_base));
742	child_vmm->data_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->data_vpn_base));
743
744	child_vmm->entry_point = (intptr_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->entry_point));
745
746	hal_fence();
747
748	#if DEBUG_VMM_FORK_COPY
749	cycle = (uint32_t)hal_get_cycles();
750	if( DEBUG_VMM_FORK_COPY < cycle )
751	printk("\n[%s] thread[%x,%x] exit successfully / cycle %d\n",
752	__FUNCTION__ , this->process->pid, this->trdid , cycle );
753	#endif
754
755	return 0;
756
757	} // vmm_fork_copy()
758
759	///////////////////////////////////////
760	void vmm_destroy( process_t * process )
761	{
762	xptr_t vseg_xp;
763	vseg_t * vseg;
764
765	#if DEBUG_VMM_DESTROY
766	uint32_t cycle = (uint32_t)hal_get_cycles();
767	thread_t * this = CURRENT_THREAD;
768	if( DEBUG_VMM_DESTROY < cycle )
769	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
770	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
771	#endif
772
773	#if (DEBUG_VMM_DESTROY & 1 )
774	if( DEBUG_VMM_DESTROY < cycle )
775	vmm_display( process , true );
776	#endif
777
778	// get pointer on local VMM
779	vmm_t * vmm = &process->vmm;
780
781	// get extended pointer on VSL root and VSL lock
782	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
783
784	// scan the VSL to delete all registered vsegs
785	// (don't use a FOREACH for item deletion in xlist)
786
787	uint32_t count = 0;
788
789	while( !xlist_is_empty( root_xp ) && (count < 10 ) )
790	{
791	// get pointer on first vseg in VSL
792	vseg_xp = XLIST_FIRST( root_xp , vseg_t , xlist );
793	vseg = GET_PTR( vseg_xp );
794
795	// delete vseg and release physical pages
796	vmm_delete_vseg( process->pid , vseg->min );
797
798	#if( DEBUG_VMM_DESTROY & 1 )
799	if( DEBUG_VMM_DESTROY < cycle )
800	printk("\n[%s] %s vseg deleted / vpn_base %x / vpn_size %d\n",
801	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
802	#endif
803
804	count++;
805
806	}
807
808	// remove all vsegs from zombi_lists in MMAP allocator
809	uint32_t i;
810	for( i = 0 ; i<32 ; i++ )
811	{
812	while( !list_is_empty( &vmm->mmap_mgr.zombi_list[i] ) )
813	{
814	vseg = LIST_FIRST( &vmm->mmap_mgr.zombi_list[i] , vseg_t , zlist );
815
816	#if( DEBUG_VMM_DESTROY & 1 )
817	if( DEBUG_VMM_DESTROY < cycle )
818	printk("\n[%s] found zombi vseg / vpn_base %x / vpn_size %d\n",
819	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
820	#endif
821	// clean vseg descriptor
822	vseg->vmm = NULL;
823
824	// remove vseg from xlist
825	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
826
827	// release vseg descriptor
828	vseg_free( vseg );
829
830	#if( DEBUG_VMM_DESTROY & 1 )
831	if( DEBUG_VMM_DESTROY < cycle )
832	printk("\n[%s] zombi vseg released / vpn_base %x / vpn_size %d\n",
833	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
834	#endif
835	}
836	}
837
838	// release memory allocated to the GPT itself
839	hal_gpt_destroy( &vmm->gpt );
840
841	#if DEBUG_VMM_DESTROY
842	cycle = (uint32_t)hal_get_cycles();
843	if( DEBUG_VMM_DESTROY < cycle )
844	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
845	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy , cycle );
846	#endif
847
848	} // end vmm_destroy()
849
850	/////////////////////////////////////////////////
851	vseg_t * vmm_check_conflict( process_t * process,
852	vpn_t vpn_base,
853	vpn_t vpn_size )
854	{
855	vmm_t * vmm = &process->vmm;
856
857	// scan the VSL
858	vseg_t * vseg;
859	xptr_t iter_xp;
860	xptr_t vseg_xp;
861	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
862
863	XLIST_FOREACH( root_xp , iter_xp )
864	{
865	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
866	vseg = GET_PTR( vseg_xp );
867
868	if( ((vpn_base + vpn_size) > vseg->vpn_base) &&
869	(vpn_base < (vseg->vpn_base + vseg->vpn_size)) ) return vseg;
870	}
871	return NULL;
872
873	} // end vmm_check_conflict()
874
875	////////////////////////////////////////////////////////////////////////////////////////////
876	// This static function is called by the vmm_create_vseg() function, and implements
877	// the VMM stack_vseg specific allocator.
878	////////////////////////////////////////////////////////////////////////////////////////////
879	// @ vmm : pointer on VMM.
880	// @ vpn_base : (return value) first allocated page
881	// @ vpn_size : (return value) number of allocated pages
882	////////////////////////////////////////////////////////////////////////////////////////////
883	static error_t vmm_stack_alloc( vmm_t * vmm,
884	vpn_t * vpn_base,
885	vpn_t * vpn_size )
886	{
887	// get stack allocator pointer
888	stack_mgr_t * mgr = &vmm->stack_mgr;
889
890	// get lock on stack allocator
891	busylock_acquire( &mgr->lock );
892
893	// get first free slot index in bitmap
894	int32_t index = bitmap_ffc( &mgr->bitmap , 4 );
895	if( (index < 0) \|\| (index > 31) )
896	{
897	busylock_release( &mgr->lock );
898	return 0xFFFFFFFF;
899	}
900
901	// update bitmap
902	bitmap_set( &mgr->bitmap , index );
903
904	// release lock on stack allocator
905	busylock_release( &mgr->lock );
906
907	// returns vpn_base, vpn_size (one page non allocated)
908	vpn_base = mgr->vpn_base + index CONFIG_VMM_STACK_SIZE + 1;
909	*vpn_size = CONFIG_VMM_STACK_SIZE - 1;
910	return 0;
911
912	} // end vmm_stack_alloc()
913
914	////////////////////////////////////////////////////////////////////////////////////////////
915	// This static function is called by the vmm_create_vseg() function, and implements
916	// the VMM MMAP specific allocator.
917	////////////////////////////////////////////////////////////////////////////////////////////
918	// @ vmm : [in] pointer on VMM.
919	// @ npages : [in] requested number of pages.
920	// @ vpn_base : [out] first allocated page.
921	// @ vpn_size : [out] actual number of allocated pages.
922	////////////////////////////////////////////////////////////////////////////////////////////
923	static error_t vmm_mmap_alloc( vmm_t * vmm,
924	vpn_t npages,
925	vpn_t * vpn_base,
926	vpn_t * vpn_size )
927	{
928	uint32_t index;
929	vseg_t * vseg;
930	vpn_t base;
931	vpn_t size;
932	vpn_t free;
933
934	#if DEBUG_VMM_MMAP_ALLOC
935	thread_t * this = CURRENT_THREAD;
936	uint32_t cycle = (uint32_t)hal_get_cycles();
937	if( DEBUG_VMM_MMAP_ALLOC < cycle )
938	printk("\n[%s] thread[%x,%x] enter / cycle %d\n",
939	__FUNCTION__, this->process->pid, this->trdid, cycle );
940	#endif
941
942	// vseg size must be power of 2
943	// compute actual size and index in zombi_list array
944	size = POW2_ROUNDUP( npages );
945	index = bits_log2( size );
946
947	// get mmap allocator pointer
948	mmap_mgr_t * mgr = &vmm->mmap_mgr;
949
950	// get lock on mmap allocator
951	busylock_acquire( &mgr->lock );
952
953	// get vseg from zombi_list or from mmap zone
954	if( list_is_empty( &mgr->zombi_list[index] ) ) // from mmap zone
955	{
956	// check overflow
957	free = mgr->first_free_vpn;
958	if( (free + size) > mgr->vpn_size ) return -1;
959
960	// update MMAP allocator
961	mgr->first_free_vpn += size;
962
963	// compute base
964	base = free;
965	}
966	else // from zombi_list
967	{
968	// get pointer on zombi vseg from zombi_list
969	vseg = LIST_FIRST( &mgr->zombi_list[index] , vseg_t , zlist );
970
971	// remove vseg from free-list
972	list_unlink( &vseg->zlist );
973
974	// compute base
975	base = vseg->vpn_base;
976	}
977
978	// release lock on mmap allocator
979	busylock_release( &mgr->lock );
980
981	#if DEBUG_VMM_MMAP_ALLOC
982	cycle = (uint32_t)hal_get_cycles();
983	if( DEBUG_VMM_DESTROY < cycle )
984	printk("\n[%s] thread[%x,%x] exit / vpn_base %x / vpn_size %x / cycle %d\n",
985	__FUNCTION__, this->process->pid, this->trdid, base, size, cycle );
986	#endif
987
988	// returns vpn_base, vpn_size
989	*vpn_base = base;
990	*vpn_size = size;
991	return 0;
992
993	} // end vmm_mmap_alloc()
994
995	////////////////////////////////////////////////
996	vseg_t * vmm_create_vseg( process_t * process,
997	vseg_type_t type,
998	intptr_t base,
999	uint32_t size,
1000	uint32_t file_offset,
1001	uint32_t file_size,
1002	xptr_t mapper_xp,
1003	cxy_t cxy )
1004	{
1005	vseg_t * vseg; // created vseg pointer
1006	vpn_t vpn_base; // first page index
1007	vpn_t vpn_size; // number of pages covered by vseg
1008	error_t error;
1009
1010	#if DEBUG_VMM_CREATE_VSEG
1011	thread_t * this = CURRENT_THREAD;
1012	uint32_t cycle = (uint32_t)hal_get_cycles();
1013	if( DEBUG_VMM_CREATE_VSEG < cycle )
1014	printk("\n[%s] thread[%x,%x] enter for process %x / %s / cxy %x / cycle %d\n",
1015	__FUNCTION__, this->process->pid, this->trdid, process->pid, vseg_type_str(type), cxy, cycle );
1016	#endif
1017
1018	// get pointer on VMM
1019	vmm_t * vmm = &process->vmm;
1020
1021	// compute base, size, vpn_base, vpn_size, depending on vseg type
1022	// we use the VMM specific allocators for "stack", "file", "anon", & "remote" vsegs
1023
1024	if( type == VSEG_TYPE_STACK )
1025	{
1026	// get vpn_base and vpn_size from STACK allocator
1027	error = vmm_stack_alloc( vmm , &vpn_base , &vpn_size );
1028	if( error )
1029	{
1030	printk("\n[ERROR] in %s : no space for stack vseg / process %x in cluster %x\n",
1031	__FUNCTION__ , process->pid , local_cxy );
1032	return NULL;
1033	}
1034
1035	// compute vseg base and size from vpn_base and vpn_size
1036	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
1037	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
1038	}
1039	else if( type == VSEG_TYPE_FILE )
1040	{
1041	// compute page index (in mapper) for first byte
1042	vpn_t vpn_min = file_offset >> CONFIG_PPM_PAGE_SHIFT;
1043
1044	// compute page index (in mapper) for last byte
1045	vpn_t vpn_max = (file_offset + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
1046
1047	// compute offset in first page
1048	uint32_t offset = file_offset & CONFIG_PPM_PAGE_MASK;
1049
1050	// compute number of pages required in virtual space
1051	vpn_t npages = vpn_max - vpn_min + 1;
1052
1053	// get vpn_base and vpn_size from MMAP allocator
1054	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
1055	if( error )
1056	{
1057	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
1058	__FUNCTION__ , process->pid , local_cxy );
1059	return NULL;
1060	}
1061
1062	// set the vseg base (not always aligned for FILE)
1063	base = (vpn_base << CONFIG_PPM_PAGE_SHIFT) + offset;
1064	}
1065	else if( (type == VSEG_TYPE_ANON) \|\|
1066	(type == VSEG_TYPE_REMOTE) )
1067	{
1068	// compute number of required pages in virtual space
1069	vpn_t npages = size >> CONFIG_PPM_PAGE_SHIFT;
1070	if( size & CONFIG_PPM_PAGE_MASK) npages++;
1071
1072	// get vpn_base and vpn_size from MMAP allocator
1073	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
1074	if( error )
1075	{
1076	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
1077	__FUNCTION__ , process->pid , local_cxy );
1078	return NULL;
1079	}
1080
1081	// set vseg base (always aligned for ANON or REMOTE)
1082	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
1083	}
1084	else // VSEG_TYPE_DATA, VSEG_TYPE_CODE or KERNEL vseg
1085	{
1086	uint32_t vpn_min = base >> CONFIG_PPM_PAGE_SHIFT;
1087	uint32_t vpn_max = (base + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
1088
1089	vpn_base = vpn_min;
1090	vpn_size = vpn_max - vpn_min + 1;
1091	}
1092
1093	// check collisions
1094	vseg = vmm_check_conflict( process , vpn_base , vpn_size );
1095	if( vseg != NULL )
1096	{
1097	printk("\n[ERROR] in %s for process %x : new vseg [vpn_base %x / vpn_size %x]\n"
1098	" overlap existing vseg [vpn_base %x / vpn_size %x]\n",
1099	__FUNCTION__ , process->pid, vpn_base, vpn_size, vseg->vpn_base, vseg->vpn_size );
1100	return NULL;
1101	}
1102
1103	// allocate physical memory for vseg descriptor
1104	vseg = vseg_alloc();
1105	if( vseg == NULL )
1106	{
1107	printk("\n[ERROR] in %s for process %x : cannot allocate memory for vseg\n",
1108	__FUNCTION__ , process->pid );
1109	return NULL;
1110	}
1111
1112	#if DEBUG_VMM_CREATE_VSEG
1113	if( DEBUG_VMM_CREATE_VSEG < cycle )
1114	printk("\n[%s] thread[%x,%x] : base %x / size %x / vpn_base %x / vpn_size %x\n",
1115	__FUNCTION__, this->process->pid, this->trdid, base, size, vpn_base, vpn_size );
1116	#endif
1117
1118	// initialize vseg descriptor
1119	vseg_init( vseg,
1120	type,
1121	base,
1122	size,
1123	vpn_base,
1124	vpn_size,
1125	file_offset,
1126	file_size,
1127	mapper_xp,
1128	cxy );
1129
1130	// attach vseg to VSL
1131	vmm_attach_vseg_to_vsl( vmm , vseg );
1132
1133	#if DEBUG_VMM_CREATE_VSEG
1134	cycle = (uint32_t)hal_get_cycles();
1135	if( DEBUG_VMM_CREATE_VSEG < cycle )
1136	printk("\n[%s] thread[%x,%x] exit / %s / cxy %x / cycle %d\n",
1137	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), cxy, cycle );
1138	#endif
1139
1140	return vseg;
1141
1142	} // vmm_create_vseg()
1143
1144	///////////////////////////////////
1145	void vmm_delete_vseg( pid_t pid,
1146	intptr_t vaddr )
1147	{
1148	process_t * process; // local pointer on local process
1149	vmm_t * vmm; // local pointer on local process VMM
1150	vseg_t * vseg; // local pointer on local vseg containing vaddr
1151	gpt_t * gpt; // local pointer on local process GPT
1152	vpn_t vpn; // VPN of current PTE
1153	vpn_t vpn_min; // VPN of first PTE
1154	vpn_t vpn_max; // VPN of last PTE (excluded)
1155	ppn_t ppn; // current PTE ppn value
1156	uint32_t attr; // current PTE attributes
1157	kmem_req_t req; // request to release memory
1158	xptr_t page_xp; // extended pointer on page descriptor
1159	cxy_t page_cxy; // page descriptor cluster
1160	page_t * page_ptr; // page descriptor pointer
1161	xptr_t forks_xp; // extended pointer on pending forks counter
1162	xptr_t lock_xp; // extended pointer on lock protecting forks counter
1163	uint32_t forks; // actual number of pendinf forks
1164	uint32_t type; // vseg type
1165
1166	#if DEBUG_VMM_DELETE_VSEG
1167	uint32_t cycle = (uint32_t)hal_get_cycles();
1168	thread_t * this = CURRENT_THREAD;
1169	if( DEBUG_VMM_DELETE_VSEG < cycle )
1170	printk("\n[%s] thread[%x,%x] enter / process %x / vaddr %x / cycle %d\n",
1171	__FUNCTION__, this->process->pid, this->trdid, pid, vaddr, cycle );
1172	#endif
1173
1174	// get local pointer on local process descriptor
1175	process = cluster_get_local_process_from_pid( pid );
1176
1177	if( process == NULL )
1178	{
1179	printk("\n[ERRORR] in %s : cannot get local process descriptor\n",
1180	__FUNCTION__ );
1181	return;
1182	}
1183
1184	// get pointers on local process VMM an GPT
1185	vmm = &process->vmm;
1186	gpt = &process->vmm.gpt;
1187
1188	// get local pointer on vseg containing vaddr
1189	vseg = vmm_vseg_from_vaddr( vmm , vaddr );
1190
1191	if( vseg == NULL )
1192	{
1193	printk("\n[ERRORR] in %s : cannot get vseg descriptor\n",
1194	__FUNCTION__ );
1195	return;
1196	}
1197
1198	// get relevant vseg infos
1199	type = vseg->type;
1200	vpn_min = vseg->vpn_base;
1201	vpn_max = vpn_min + vseg->vpn_size;
1202
1203	// loop to invalidate all vseg PTEs in GPT
1204	for( vpn = vpn_min ; vpn < vpn_max ; vpn++ )
1205	{
1206	// get GPT entry
1207	hal_gpt_get_pte( XPTR( local_cxy , gpt ) , vpn , &attr , &ppn );
1208
1209	if( attr & GPT_MAPPED ) // entry is mapped
1210	{
1211
1212	#if( DEBUG_VMM_DELETE_VSEG & 1 )
1213	if( DEBUG_VMM_DELETE_VSEG < cycle )
1214	printk("- unmap vpn %x / ppn %x / vseg %s \n" , vpn , ppn, vseg_type_str(vseg->type) );
1215	#endif
1216	// unmap GPT entry in local GPT
1217	hal_gpt_reset_pte( gpt , vpn );
1218
1219	// the allocated page is not released to KMEM for kernel vseg
1220	if( (type != VSEG_TYPE_KCODE) &&
1221	(type != VSEG_TYPE_KDATA) &&
1222	(type != VSEG_TYPE_KDEV ) )
1223	{
1224
1225	// FIXME This code must be completely re-written, as the actual release must depend on
1226	// - the vseg type
1227	// - the reference cluster
1228	// - the page refcount and/or the forks counter
1229
1230	// get extended pointer on physical page descriptor
1231	page_xp = ppm_ppn2page( ppn );
1232	page_cxy = GET_CXY( page_xp );
1233	page_ptr = GET_PTR( page_xp );
1234
1235	// get extended pointers on forks and lock fields
1236	forks_xp = XPTR( page_cxy , &page_ptr->forks );
1237	lock_xp = XPTR( page_cxy , &page_ptr->lock );
1238
1239	// get the lock protecting the page
1240	remote_busylock_acquire( lock_xp );
1241
1242	// get pending forks counter
1243	forks = hal_remote_l32( forks_xp );
1244
1245	if( forks ) // decrement pending forks counter
1246	{
1247	hal_remote_atomic_add( forks_xp , -1 );
1248	}
1249	else // release physical page to relevant cluster
1250	{
1251	if( page_cxy == local_cxy ) // local cluster
1252	{
1253	req.type = KMEM_PAGE;
1254	req.ptr = page_ptr;
1255	kmem_free( &req );
1256	}
1257	else // remote cluster
1258	{
1259	rpc_pmem_release_pages_client( page_cxy , page_ptr );
1260	}
1261
1262	#if( DEBUG_VMM_DELETE_VSEG & 1 )
1263	if( DEBUG_VMM_DELETE_VSEG < cycle )
1264	printk("- release ppn %x\n", ppn );
1265	#endif
1266	}
1267
1268	// release the lock protecting the page
1269	remote_busylock_release( lock_xp );
1270	}
1271	}
1272	}
1273
1274	// remove vseg from VSL and release vseg descriptor (if not MMAP)
1275	vmm_detach_vseg_from_vsl( vmm , vseg );
1276
1277	#if DEBUG_VMM_DELETE_VSEG
1278	cycle = (uint32_t)hal_get_cycles();
1279	if( DEBUG_VMM_DELETE_VSEG < cycle )
1280	printk("\n[%s] thread[%x,%x] exit / process %x / vseg %s / base %x / cycle %d\n",
1281	__FUNCTION__, this->process->pid, this->trdid, pid, vseg_type_str(vseg->type), vaddr, cycle );
1282	#endif
1283
1284	} // end vmm_delete_vseg()
1285
1286	/////////////////////////////////////////////
1287	vseg_t * vmm_vseg_from_vaddr( vmm_t * vmm,
1288	intptr_t vaddr )
1289	{
1290	xptr_t iter_xp;
1291	xptr_t vseg_xp;
1292	vseg_t * vseg;
1293
1294	// get extended pointers on VSL lock and root
1295	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
1296	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
1297
1298	// get lock protecting the VSL
1299	remote_rwlock_rd_acquire( lock_xp );
1300
1301	// scan the list of vsegs in VSL
1302	XLIST_FOREACH( root_xp , iter_xp )
1303	{
1304	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
1305	vseg = GET_PTR( vseg_xp );
1306
1307	if( (vaddr >= vseg->min) && (vaddr < vseg->max) )
1308	{
1309	// return success
1310	remote_rwlock_rd_release( lock_xp );
1311	return vseg;
1312	}
1313	}
1314
1315	// return failure
1316	remote_rwlock_rd_release( lock_xp );
1317
1318	return NULL;
1319
1320	} // end vmm_vseg_from_vaddr()
1321
1322	/////////////////////////////////////////////
1323	error_t vmm_resize_vseg( process_t * process,
1324	intptr_t base,
1325	intptr_t size )
1326	{
1327	error_t error;
1328	vseg_t * new;
1329	vpn_t vpn_min;
1330	vpn_t vpn_max;
1331
1332	#if DEBUG_VMM_RESIZE_VSEG
1333	uint32_t cycle = (uint32_t)hal_get_cycles();
1334	thread_t * this = CURRENT_THREAD;
1335	if( DEBUG_VMM_RESIZE_VSEG < cycle )
1336	printk("\n[%s] thread[%x,%x] enter / process %x / base %x / size %d / cycle %d\n",
1337	__FUNCTION__, this->process->pid, this->trdid, process->pid, base, size, cycle );
1338	#endif
1339
1340	// get pointer on process VMM
1341	vmm_t * vmm = &process->vmm;
1342
1343	intptr_t addr_min = base;
1344	intptr_t addr_max = base + size;
1345
1346	// get pointer on vseg
1347	vseg_t * vseg = vmm_vseg_from_vaddr( vmm , base );
1348
1349	if( vseg == NULL)
1350	{
1351	printk("\n[ERROR] in %s : vseg(%x,%d) not found\n",
1352	__FUNCTION__, base , size );
1353	return -1;
1354	}
1355
1356	// resize depends on unmapped region base and size
1357	if( (vseg->min > addr_min) \|\| (vseg->max < addr_max) ) // not included in vseg
1358	{
1359	printk("\n[ERROR] in %s : unmapped region[%x->%x[ not included in vseg[%x->%x[\n",
1360	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
1361
1362	error = -1;
1363	}
1364	else if( (vseg->min == addr_min) && (vseg->max == addr_max) ) // vseg must be deleted
1365	{
1366
1367	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
1368	if( DEBUG_VMM_RESIZE_VSEG < cycle )
1369	printk("\n[%s] unmapped region[%x->%x[ equal vseg[%x->%x[\n",
1370	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
1371	#endif
1372	vmm_delete_vseg( process->pid , vseg->min );
1373
1374	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
1375	if( DEBUG_VMM_RESIZE_VSEG < cycle )
1376	printk("\n[%s] thread[%x,%x] deleted vseg\n",
1377	__FUNCTION__, this->process->pid, this->trdid );
1378	#endif
1379	error = 0;
1380	}
1381	else if( vseg->min == addr_min ) // vseg must be resized
1382	{
1383
1384	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
1385	if( DEBUG_VMM_RESIZE_VSEG < cycle )
1386	printk("\n[%s] unmapped region[%x->%x[ included in vseg[%x->%x[\n",
1387	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
1388	#endif
1389	// update vseg min address
1390	vseg->min = addr_max;
1391
1392	// update vpn_base and vpn_size
1393	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1394	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1395	vseg->vpn_base = vpn_min;
1396	vseg->vpn_size = vpn_max - vpn_min + 1;
1397
1398	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
1399	if( DEBUG_VMM_RESIZE_VSEG < cycle )
1400	printk("\n[%s] thread[%x,%x] changed vseg_min\n",
1401	__FUNCTION__, this->process->pid, this->trdid );
1402	#endif
1403	error = 0;
1404	}
1405	else if( vseg->max == addr_max ) // vseg must be resized
1406	{
1407
1408	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
1409	if( DEBUG_VMM_RESIZE_VSEG < cycle )
1410	printk("\n[%s] unmapped region[%x->%x[ included in vseg[%x->%x[\n",
1411	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
1412	#endif
1413	// update vseg max address
1414	vseg->max = addr_min;
1415
1416	// update vpn_base and vpn_size
1417	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1418	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1419	vseg->vpn_base = vpn_min;
1420	vseg->vpn_size = vpn_max - vpn_min + 1;
1421
1422	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
1423	if( DEBUG_VMM_RESIZE_VSEG < cycle )
1424	printk("\n[%s] thread[%x,%x] changed vseg_max\n",
1425	__FUNCTION__, this->process->pid, this->trdid );
1426	#endif
1427	error = 0;
1428
1429	}
1430	else // vseg cut in three regions
1431	{
1432
1433	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
1434	if( DEBUG_VMM_RESIZE_VSEG < cycle )
1435	printk("\n[%s] unmapped region[%x->%x[ included in vseg[%x->%x[\n",
1436	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
1437	#endif
1438	// resize existing vseg
1439	vseg->max = addr_min;
1440
1441	// update vpn_base and vpn_size
1442	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1443	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1444	vseg->vpn_base = vpn_min;
1445	vseg->vpn_size = vpn_max - vpn_min + 1;
1446
1447	// create new vseg
1448	new = vmm_create_vseg( process,
1449	vseg->type,
1450	addr_min,
1451	(vseg->max - addr_max),
1452	vseg->file_offset,
1453	vseg->file_size,
1454	vseg->mapper_xp,
1455	vseg->cxy );
1456
1457	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
1458	if( DEBUG_VMM_RESIZE_VSEG < cycle )
1459	printk("\n[%s] thread[%x,%x] replaced vseg by two smal vsegs\n",
1460	__FUNCTION__, this->process->pid, this->trdid );
1461	#endif
1462
1463	if( new == NULL ) error = -1;
1464	else error = 0;
1465	}
1466
1467	#if DEBUG_VMM_RESIZE_VSEG
1468	if( DEBUG_VMM_RESIZE_VSEG < cycle )
1469	printk("\n[%s] thread[%x,%x] exit / process %x / base %x / size %d / cycle %d\n",
1470	__FUNCTION__, this->process->pid, this->trdid, process->pid, base, size, cycle );
1471	#endif
1472
1473	return error;
1474
1475	} // vmm_resize_vseg()
1476
1477	///////////////////////////////////////////
1478	error_t vmm_get_vseg( process_t * process,
1479	intptr_t vaddr,
1480	vseg_t ** found_vseg )
1481	{
1482	xptr_t vseg_xp;
1483	vseg_t * vseg;
1484	vmm_t * vmm;
1485	error_t error;
1486
1487	// get pointer on local VMM
1488	vmm = &process->vmm;
1489
1490	// try to get vseg from local VMM
1491	vseg = vmm_vseg_from_vaddr( vmm , vaddr );
1492
1493	if( vseg == NULL ) // vseg not found in local cluster => try to get it from ref
1494	{
1495	// get extended pointer on reference process
1496	xptr_t ref_xp = process->ref_xp;
1497
1498	// get cluster and local pointer on reference process
1499	cxy_t ref_cxy = GET_CXY( ref_xp );
1500	process_t * ref_ptr = GET_PTR( ref_xp );
1501
1502	if( local_cxy == ref_cxy ) return -1; // local cluster is the reference
1503
1504	// get extended pointer on reference vseg
1505	rpc_vmm_get_vseg_client( ref_cxy , ref_ptr , vaddr , &vseg_xp , &error );
1506
1507	if( error ) return -1; // vseg not found => illegal user vaddr
1508
1509	// allocate a vseg in local cluster
1510	vseg = vseg_alloc();
1511
1512	if( vseg == NULL ) return -1; // cannot allocate a local vseg
1513
1514	// initialise local vseg from reference
1515	vseg_init_from_ref( vseg , vseg_xp );
1516
1517	// register local vseg in local VSL
1518	vmm_attach_vseg_to_vsl( vmm , vseg );
1519	}
1520
1521	// success
1522	*found_vseg = vseg;
1523	return 0;
1524
1525	} // end vmm_get_vseg()
1526
1527	//////////////////////////////////////////////////////////////////////////////////////
1528	// This static function compute the target cluster to allocate a physical page
1529	// for a given <vpn> in a given <vseg>, allocates the page (with an RPC if required)
1530	// and returns an extended pointer on the allocated page descriptor.
1531	// It can be called by a thread running in any cluster.
1532	// The vseg cannot have the FILE type.
1533	//////////////////////////////////////////////////////////////////////////////////////
1534	static xptr_t vmm_page_allocate( vseg_t * vseg,
1535	vpn_t vpn )
1536	{
1537
1538	#if DEBUG_VMM_ALLOCATE_PAGE
1539	uint32_t cycle = (uint32_t)hal_get_cycles();
1540	thread_t * this = CURRENT_THREAD;
1541	xptr_t this_xp = XPTR( local_cxy , this );
1542	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
1543	printk("\n[%s] thread[%x,%x] enter for vpn %x / cycle %d\n",
1544	__FUNCTION__ , this->process->pid, this->trdid, vpn, cycle );
1545	#endif
1546
1547	page_t * page_ptr;
1548	cxy_t page_cxy;
1549	kmem_req_t req;
1550	uint32_t index;
1551
1552	uint32_t type = vseg->type;
1553	uint32_t flags = vseg->flags;
1554	uint32_t x_size = LOCAL_CLUSTER->x_size;
1555	uint32_t y_size = LOCAL_CLUSTER->y_size;
1556
1557	// check vseg type
1558	assert( ( type != VSEG_TYPE_FILE ) , "illegal vseg type\n" );
1559
1560	if( flags & VSEG_DISTRIB ) // distributed => cxy depends on vpn LSB
1561	{
1562	index = vpn & ((x_size * y_size) - 1);
1563	page_cxy = HAL_CXY_FROM_XY( (index / y_size) , (index % y_size) );
1564
1565	// If the cluster selected from VPN's LSBs is empty, we select one randomly
1566	if ( cluster_is_active( page_cxy ) == false )
1567	{
1568	page_cxy = cluster_random_select();
1569	}
1570	}
1571	else // other cases => cxy specified in vseg
1572	{
1573	page_cxy = vseg->cxy;
1574	}
1575
1576	// allocate a physical page from target cluster
1577	if( page_cxy == local_cxy ) // target cluster is the local cluster
1578	{
1579	req.type = KMEM_PAGE;
1580	req.size = 0;
1581	req.flags = AF_NONE;
1582	page_ptr = (page_t *)kmem_alloc( &req );
1583	}
1584	else // target cluster is not the local cluster
1585	{
1586	rpc_pmem_get_pages_client( page_cxy , 0 , &page_ptr );
1587	}
1588
1589	#if DEBUG_VMM_ALLOCATE_PAGE
1590	cycle = (uint32_t)hal_get_cycles();
1591	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
1592	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle %d\n",
1593	__FUNCTION__ , this->process->pid, this->trdid, vpn,
1594	ppm_page2ppn( XPTR( page_cxy , page_ptr ) , cycle );
1595	#endif
1596
1597	if( page_ptr == NULL ) return XPTR_NULL;
1598	else return XPTR( page_cxy , page_ptr );
1599
1600	} // end vmm_page_allocate()
1601
1602	////////////////////////////////////////
1603	error_t vmm_get_one_ppn( vseg_t * vseg,
1604	vpn_t vpn,
1605	ppn_t * ppn )
1606	{
1607	error_t error;
1608	xptr_t page_xp; // extended pointer on physical page descriptor
1609	uint32_t page_id; // missing page index in vseg mapper
1610	uint32_t type; // vseg type;
1611
1612	type = vseg->type;
1613	page_id = vpn - vseg->vpn_base;
1614
1615	#if DEBUG_VMM_GET_ONE_PPN
1616	uint32_t cycle = (uint32_t)hal_get_cycles();
1617	thread_t * this = CURRENT_THREAD;
1618	if( DEBUG_VMM_GET_ONE_PPN < cycle )
1619	printk("\n[%s] thread[%x,%x] enter for vpn %x / type %s / page_id %d / cycle %d\n",
1620	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(type), page_id, cycle );
1621	#endif
1622
1623	// FILE type : get the physical page from the file mapper
1624	if( type == VSEG_TYPE_FILE )
1625	{
1626	// get extended pointer on mapper
1627	xptr_t mapper_xp = vseg->mapper_xp;
1628
1629	assert( (mapper_xp != XPTR_NULL),
1630	"mapper not defined for a FILE vseg\n" );
1631
1632	// get extended pointer on page descriptor
1633	page_xp = mapper_remote_get_page( mapper_xp , page_id );
1634
1635	if ( page_xp == XPTR_NULL ) return EINVAL;
1636	}
1637
1638	// Other types : allocate a physical page from target cluster,
1639	// as defined by vseg type and vpn value
1640	else
1641	{
1642	// allocate one physical page
1643	page_xp = vmm_page_allocate( vseg , vpn );
1644
1645	if( page_xp == XPTR_NULL ) return ENOMEM;
1646
1647	// initialise missing page from .elf file mapper for DATA and CODE types
1648	// the vseg->mapper_xp field is an extended pointer on the .elf file mapper
1649	if( (type == VSEG_TYPE_CODE) \|\| (type == VSEG_TYPE_DATA) )
1650	{
1651	// get extended pointer on mapper
1652	xptr_t mapper_xp = vseg->mapper_xp;
1653
1654	assert( (mapper_xp != XPTR_NULL),
1655	"mapper not defined for a CODE or DATA vseg\n" );
1656
1657	// compute missing page offset in vseg
1658	uint32_t offset = page_id << CONFIG_PPM_PAGE_SHIFT;
1659
1660	// compute missing page offset in .elf file
1661	uint32_t elf_offset = vseg->file_offset + offset;
1662
1663	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1664	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1665	printk("\n[%s] thread[%x,%x] for vpn = %x / elf_offset = %x\n",
1666	__FUNCTION__, this->process->pid, this->trdid, vpn, elf_offset );
1667	#endif
1668	// compute extended pointer on page base
1669	xptr_t base_xp = ppm_page2base( page_xp );
1670
1671	// file_size (in .elf mapper) can be smaller than vseg_size (BSS)
1672	uint32_t file_size = vseg->file_size;
1673
1674	if( file_size < offset ) // missing page fully in BSS
1675	{
1676
1677	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1678	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1679	printk("\n[%s] thread[%x,%x] for vpn %x / fully in BSS\n",
1680	__FUNCTION__, this->process->pid, this->trdid, vpn );
1681	#endif
1682	if( GET_CXY( page_xp ) == local_cxy )
1683	{
1684	memset( GET_PTR( base_xp ) , 0 , CONFIG_PPM_PAGE_SIZE );
1685	}
1686	else
1687	{
1688	hal_remote_memset( base_xp , 0 , CONFIG_PPM_PAGE_SIZE );
1689	}
1690	}
1691	else if( file_size >= (offset + CONFIG_PPM_PAGE_SIZE) ) // fully in mapper
1692	{
1693
1694	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1695	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1696	printk("\n[%s] thread[%x,%x] for vpn %x / fully in mapper\n",
1697	__FUNCTION__, this->process->pid, this->trdid, vpn );
1698	#endif
1699	error = mapper_move_kernel( mapper_xp,
1700	true, // to_buffer
1701	elf_offset,
1702	base_xp,
1703	CONFIG_PPM_PAGE_SIZE );
1704	if( error ) return EINVAL;
1705	}
1706	else // both in mapper and in BSS :
1707	// - (file_size - offset) bytes from mapper
1708	// - (page_size + offset - file_size) bytes from BSS
1709	{
1710
1711	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1712	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1713	printk("\n[%s] thread[%x,%x] for vpn %x / both mapper & BSS\n"
1714	" %d bytes from mapper / %d bytes from BSS\n",
1715	__FUNCTION__, this->process->pid, this->trdid, vpn,
1716	file_size - offset , offset + CONFIG_PPM_PAGE_SIZE - file_size );
1717	#endif
1718	// initialize mapper part
1719	error = mapper_move_kernel( mapper_xp,
1720	true, // to buffer
1721	elf_offset,
1722	base_xp,
1723	file_size - offset );
1724	if( error ) return EINVAL;
1725
1726	// initialize BSS part
1727	if( GET_CXY( page_xp ) == local_cxy )
1728	{
1729	memset( GET_PTR( base_xp ) + file_size - offset , 0 ,
1730	offset + CONFIG_PPM_PAGE_SIZE - file_size );
1731	}
1732	else
1733	{
1734	hal_remote_memset( base_xp + file_size - offset , 0 ,
1735	offset + CONFIG_PPM_PAGE_SIZE - file_size );
1736	}
1737	}
1738	} // end initialisation for CODE or DATA types
1739	}
1740
1741	// return ppn
1742	*ppn = ppm_page2ppn( page_xp );
1743
1744	#if DEBUG_VMM_GET_ONE_PPN
1745	cycle = (uint32_t)hal_get_cycles();
1746	if( DEBUG_VMM_GET_ONE_PPN < cycle )
1747	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle\n",
1748	__FUNCTION__ , this->process->pid, this->trdid , vpn , *ppn, cycle );
1749	#endif
1750
1751	return 0;
1752
1753	} // end vmm_get_one_ppn()
1754
1755	///////////////////////////////////////////////////
1756	error_t vmm_handle_page_fault( process_t * process,
1757	vpn_t vpn )
1758	{
1759	vseg_t * vseg; // vseg containing vpn
1760	uint32_t new_attr; // new PTE_ATTR value
1761	ppn_t new_ppn; // new PTE_PPN value
1762	uint32_t ref_attr; // PTE_ATTR value in reference GPT
1763	ppn_t ref_ppn; // PTE_PPN value in reference GPT
1764	cxy_t ref_cxy; // reference cluster for missing vpn
1765	process_t * ref_ptr; // reference process for missing vpn
1766	xptr_t local_gpt_xp; // extended pointer on local GPT
1767	xptr_t local_lock_xp; // extended pointer on local GPT lock
1768	xptr_t ref_gpt_xp; // extended pointer on reference GPT
1769	xptr_t ref_lock_xp; // extended pointer on reference GPT lock
1770	error_t error; // value returned by called functions
1771
1772	// get local vseg (access to reference VSL can be required)
1773	error = vmm_get_vseg( process,
1774	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
1775	&vseg );
1776	if( error )
1777	{
1778	printk("\n[ERROR] in %s : vpn %x in process %x not in a registered vseg\n",
1779	__FUNCTION__ , vpn , process->pid );
1780
1781	return EXCP_USER_ERROR;
1782	}
1783
1784	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1785	uint32_t cycle = (uint32_t)hal_get_cycles();
1786	thread_t * this = CURRENT_THREAD;
1787	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1788	printk("\n[%s] threadr[%x,%x] enter for vpn %x / %s / cycle %d\n",
1789	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(vseg->type), cycle );
1790	#endif
1791
1792	//////////////// private vseg => access only the local GPT
1793	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
1794	{
1795	// build extended pointer on local GPT and local GPT lock
1796	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1797	local_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
1798
1799	// take local GPT lock in write mode
1800	remote_rwlock_wr_acquire( local_lock_xp );
1801
1802	// check VPN still unmapped in local GPT
1803
1804	// do nothing if VPN has been mapped by a a concurrent page_fault
1805	hal_gpt_get_pte( local_gpt_xp,
1806	vpn,
1807	&new_attr,
1808	&new_ppn );
1809
1810	if( (new_attr & GPT_MAPPED) == 0 ) // VPN still unmapped
1811	{
1812	// allocate and initialise a physical page depending on the vseg type
1813	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
1814
1815	if( error )
1816	{
1817	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
1818	__FUNCTION__ , process->pid , vpn );
1819
1820	// release local GPT lock in write mode
1821	remote_rwlock_wr_release( local_lock_xp );
1822
1823	return EXCP_KERNEL_PANIC;
1824	}
1825
1826	// define new_attr from vseg flags
1827	new_attr = GPT_MAPPED \| GPT_SMALL;
1828	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
1829	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
1830	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
1831	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
1832
1833	// set PTE (PPN & attribute) to local GPT
1834	error = hal_gpt_set_pte( local_gpt_xp,
1835	vpn,
1836	new_attr,
1837	new_ppn );
1838	if ( error )
1839	{
1840	printk("\n[ERROR] in %s : cannot update local GPT / process %x / vpn = %x\n",
1841	__FUNCTION__ , process->pid , vpn );
1842
1843	// release local GPT lock in write mode
1844	remote_rwlock_wr_release( local_lock_xp );
1845
1846	return EXCP_KERNEL_PANIC;
1847	}
1848	}
1849
1850	// release local GPT lock in write mode
1851	remote_rwlock_wr_release( local_lock_xp );
1852
1853	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1854	cycle = (uint32_t)hal_get_cycles();
1855	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1856	printk("\n[%s] private page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
1857	__FUNCTION__, vpn, new_ppn, new_attr, cycle );
1858	#endif
1859	return EXCP_NON_FATAL;
1860
1861	} // end local GPT access
1862
1863	//////////// public vseg => access reference GPT
1864	else
1865	{
1866	// get reference process cluster and local pointer
1867	ref_cxy = GET_CXY( process->ref_xp );
1868	ref_ptr = GET_PTR( process->ref_xp );
1869
1870	// build extended pointer on reference GPT and reference GPT lock
1871	ref_gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
1872	ref_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt_lock );
1873
1874	// build extended pointer on local GPT and local GPT lock
1875	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1876	local_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
1877
1878	// take reference GPT lock in read mode
1879	remote_rwlock_rd_acquire( ref_lock_xp );
1880
1881	// get directly PPN & attributes from reference GPT
1882	// this can avoids a costly RPC for a false page fault
1883	hal_gpt_get_pte( ref_gpt_xp,
1884	vpn,
1885	&ref_attr,
1886	&ref_ppn );
1887
1888	// release reference GPT lock in read mode
1889	remote_rwlock_rd_release( ref_lock_xp );
1890
1891	if( ref_attr & GPT_MAPPED ) // false page fault => update local GPT
1892	{
1893	// take local GPT lock in write mode
1894	remote_rwlock_wr_acquire( local_lock_xp );
1895
1896	// check VPN still unmapped in local GPT
1897	hal_gpt_get_pte( local_gpt_xp,
1898	vpn,
1899	&new_attr,
1900	&new_ppn );
1901
1902	if( (new_attr & GPT_MAPPED) == 0 ) // VPN still unmapped
1903	{
1904	// update local GPT from reference GPT
1905	error = hal_gpt_set_pte( local_gpt_xp,
1906	vpn,
1907	ref_attr,
1908	ref_ppn );
1909	if( error )
1910	{
1911	printk("\n[ERROR] in %s : cannot update local GPT / process %x / vpn %x\n",
1912	__FUNCTION__ , process->pid , vpn );
1913
1914	// release local GPT lock in write mode
1915	remote_rwlock_wr_release( local_lock_xp );
1916
1917	return EXCP_KERNEL_PANIC;
1918	}
1919	}
1920	else // VPN has been mapped by a a concurrent page_fault
1921	{
1922	// keep PTE from local GPT
1923	ref_attr = new_attr;
1924	ref_ppn = new_ppn;
1925	}
1926
1927	// release local GPT lock in write mode
1928	remote_rwlock_wr_release( local_lock_xp );
1929
1930	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1931	cycle = (uint32_t)hal_get_cycles();
1932	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1933	printk("\n[%s] false page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
1934	__FUNCTION__, vpn, ref_ppn, ref_attr, cycle );
1935	#endif
1936	return EXCP_NON_FATAL;
1937	}
1938	else // true page fault => update reference GPT
1939	{
1940	// take reference GPT lock in write mode
1941	remote_rwlock_wr_acquire( ref_lock_xp );
1942
1943	// check VPN still unmapped in reference GPT
1944	// do nothing if VPN has been mapped by a a concurrent page_fault
1945	hal_gpt_get_pte( ref_gpt_xp,
1946	vpn,
1947	&ref_attr,
1948	&ref_ppn );
1949
1950	if( (ref_attr & GPT_MAPPED) == 0 ) // VPN actually unmapped
1951	{
1952	// allocate and initialise a physical page depending on the vseg type
1953	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
1954
1955	if( error )
1956	{
1957	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
1958	__FUNCTION__ , process->pid , vpn );
1959
1960	// release reference GPT lock in write mode
1961	remote_rwlock_wr_release( ref_lock_xp );
1962
1963	return EXCP_KERNEL_PANIC;
1964	}
1965
1966	// define new_attr from vseg flags
1967	new_attr = GPT_MAPPED \| GPT_SMALL;
1968	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
1969	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
1970	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
1971	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
1972
1973	// update reference GPT
1974	error = hal_gpt_set_pte( ref_gpt_xp,
1975	vpn,
1976	new_attr,
1977	new_ppn );
1978
1979	// update local GPT (protected by reference GPT lock)
1980	error \|= hal_gpt_set_pte( local_gpt_xp,
1981	vpn,
1982	new_attr,
1983	new_ppn );
1984
1985	if( error )
1986	{
1987	printk("\n[ERROR] in %s : cannot update GPT / process %x / vpn = %x\n",
1988	__FUNCTION__ , process->pid , vpn );
1989
1990	// release reference GPT lock in write mode
1991	remote_rwlock_wr_release( ref_lock_xp );
1992
1993	return EXCP_KERNEL_PANIC;
1994	}
1995	}
1996
1997	// release reference GPT lock in write mode
1998	remote_rwlock_wr_release( ref_lock_xp );
1999
2000	#if DEBUG_VMM_HANDLE_PAGE_FAULT
2001	cycle = (uint32_t)hal_get_cycles();
2002	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
2003	printk("\n[%s] true page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
2004	__FUNCTION__, vpn, new_ppn, new_attr, cycle );
2005	#endif
2006	return EXCP_NON_FATAL;
2007	}
2008	}
2009	} // end vmm_handle_page_fault()
2010
2011	////////////////////////////////////////////
2012	error_t vmm_handle_cow( process_t * process,
2013	vpn_t vpn )
2014	{
2015	vseg_t * vseg; // vseg containing vpn
2016	cxy_t ref_cxy; // reference cluster for missing vpn
2017	process_t * ref_ptr; // reference process for missing vpn
2018	xptr_t gpt_xp; // extended pointer on GPT
2019	xptr_t gpt_lock_xp; // extended pointer on GPT lock
2020	uint32_t old_attr; // current PTE_ATTR value
2021	ppn_t old_ppn; // current PTE_PPN value
2022	uint32_t new_attr; // new PTE_ATTR value
2023	ppn_t new_ppn; // new PTE_PPN value
2024	error_t error;
2025
2026	#if DEBUG_VMM_HANDLE_COW
2027	uint32_t cycle = (uint32_t)hal_get_cycles();
2028	thread_t * this = CURRENT_THREAD;
2029	xptr_t this_xp = XPTR( local_cxy , this );
2030	if( DEBUG_VMM_HANDLE_COW < cycle )
2031	printk("\n[%s] thread[%x,%x] enter for vpn %x / core[%x,%d] / cycle %d\n",
2032	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
2033	#endif
2034
2035	// access local GPT to get GPT_COW flag
2036	bool_t cow = hal_gpt_pte_is_cow( &(process->vmm.gpt), vpn );
2037
2038	if( cow == false ) return EXCP_USER_ERROR;
2039
2040	// get local vseg
2041	error = vmm_get_vseg( process,
2042	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
2043	&vseg );
2044	if( error )
2045	{
2046	printk("\n[PANIC] in %s : vpn %x in process %x not in a registered vseg\n",
2047	__FUNCTION__, vpn, process->pid );
2048
2049	return EXCP_KERNEL_PANIC;
2050	}
2051
2052	#if( DEBUG_VMM_HANDLE_COW & 1)
2053	if( DEBUG_VMM_HANDLE_COW < cycle )
2054	printk("\n[%s] thread[%x,%x] get vseg for vpn %x\n",
2055	__FUNCTION__, this->process->pid, this->trdid, vpn );
2056	#endif
2057
2058	// get reference GPT cluster and local pointer
2059	ref_cxy = GET_CXY( process->ref_xp );
2060	ref_ptr = GET_PTR( process->ref_xp );
2061
2062	// build relevant extended pointers on relevant GPT and GPT lock
2063	// - access local GPT for a private vseg
2064	// - access reference GPT for a public vseg
2065	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
2066	{
2067	gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
2068	gpt_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
2069	}
2070	else
2071	{
2072	gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
2073	gpt_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt_lock );
2074	}
2075
2076	// take GPT lock in write mode
2077	remote_rwlock_wr_acquire( gpt_lock_xp );
2078
2079	// get current PTE from reference GPT
2080	hal_gpt_get_pte( gpt_xp,
2081	vpn,
2082	&old_attr,
2083	&old_ppn );
2084
2085	#if( DEBUG_VMM_HANDLE_COW & 1)
2086	if( DEBUG_VMM_HANDLE_COW < cycle )
2087	printk("\n[%s] thread[%x,%x] get pte for vpn %x : ppn %x / attr %x\n",
2088	__FUNCTION__, this->process->pid, this->trdid, vpn, old_ppn, old_attr );
2089	#endif
2090
2091	// the PTE must be mapped for a COW
2092	if( (old_attr & GPT_MAPPED) == 0 )
2093	{
2094	printk("\n[PANIC] in %s : VPN %x in process %x unmapped\n",
2095	__FUNCTION__, vpn, process->pid );
2096
2097	// release GPT lock in write mode
2098	remote_rwlock_wr_release( gpt_lock_xp );
2099
2100	return EXCP_KERNEL_PANIC;
2101	}
2102
2103	// get pointers on physical page descriptor
2104	xptr_t page_xp = ppm_ppn2page( old_ppn );
2105	cxy_t page_cxy = GET_CXY( page_xp );
2106	page_t * page_ptr = GET_PTR( page_xp );
2107
2108	// get extended pointers on forks and lock field in page descriptor
2109	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
2110	xptr_t forks_lock_xp = XPTR( page_cxy , &page_ptr->lock );
2111
2112	// take lock protecting "forks" counter
2113	remote_busylock_acquire( forks_lock_xp );
2114
2115	// get number of pending forks from page descriptor
2116	uint32_t forks = hal_remote_l32( forks_xp );
2117
2118	#if( DEBUG_VMM_HANDLE_COW & 1)
2119	if( DEBUG_VMM_HANDLE_COW < cycle )
2120	printk("\n[%s] thread[%x,%x] get forks = %d for vpn %x\n",
2121	__FUNCTION__, this->process->pid, this->trdid, forks, vpn );
2122	#endif
2123
2124	if( forks ) // pending fork => allocate a new page, and copy old to new
2125	{
2126	// decrement pending forks counter in page descriptor
2127	hal_remote_atomic_add( forks_xp , -1 );
2128
2129	// release lock protecting "forks" counter
2130	remote_busylock_release( forks_lock_xp );
2131
2132	// allocate a new page
2133	page_xp = vmm_page_allocate( vseg , vpn );
2134
2135	if( page_xp == XPTR_NULL )
2136	{
2137	printk("\n[PANIC] in %s : no memory for vpn %x in process %x\n",
2138	__FUNCTION__ , vpn, process->pid );
2139
2140	// release GPT lock in write mode
2141	remote_rwlock_wr_acquire( gpt_lock_xp );
2142
2143	return EXCP_KERNEL_PANIC;
2144	}
2145
2146	// compute allocated page PPN
2147	new_ppn = ppm_page2ppn( page_xp );
2148
2149	#if( DEBUG_VMM_HANDLE_COW & 1)
2150	if( DEBUG_VMM_HANDLE_COW < cycle )
2151	printk("\n[%s] thread[%x,%x] get new ppn %x for vpn %x\n",
2152	__FUNCTION__, this->process->pid, this->trdid, new_ppn, vpn );
2153	#endif
2154
2155	// copy old page content to new page
2156	hal_remote_memcpy( ppm_ppn2base( new_ppn ),
2157	ppm_ppn2base( old_ppn ),
2158	CONFIG_PPM_PAGE_SIZE );
2159
2160	#if(DEBUG_VMM_HANDLE_COW & 1)
2161	if( DEBUG_VMM_HANDLE_COW < cycle )
2162	printk("\n[%s] thread[%x,%x] copied old page to new page\n",
2163	__FUNCTION__, this->process->pid, this->trdid );
2164	#endif
2165
2166	}
2167	else // no pending fork => keep the existing page
2168	{
2169	// release lock protecting "forks" counter
2170	remote_busylock_release( forks_lock_xp );
2171
2172	#if(DEBUG_VMM_HANDLE_COW & 1)
2173	if( DEBUG_VMM_HANDLE_COW < cycle )
2174	printk("\n[%s] thread[%x,%x] no pending forks / keep existing PPN %x\n",
2175	__FUNCTION__, this->process->pid, this->trdid, old_ppn );
2176	#endif
2177	new_ppn = old_ppn;
2178	}
2179
2180	// build new_attr : reset COW and set WRITABLE,
2181	new_attr = (old_attr \| GPT_WRITABLE) & (~GPT_COW);
2182
2183	// update the relevant GPT
2184	// - private vseg => update local GPT
2185	// - public vseg => update all GPT copies
2186	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
2187	{
2188	hal_gpt_set_pte( gpt_xp,
2189	vpn,
2190	new_attr,
2191	new_ppn );
2192	}
2193	else
2194	{
2195	if( ref_cxy == local_cxy ) // reference cluster is local
2196	{
2197	vmm_global_update_pte( process,
2198	vpn,
2199	new_attr,
2200	new_ppn );
2201	}
2202	else // reference cluster is remote
2203	{
2204	rpc_vmm_global_update_pte_client( ref_cxy,
2205	ref_ptr,
2206	vpn,
2207	new_attr,
2208	new_ppn );
2209	}
2210	}
2211
2212	// release GPT lock in write mode
2213	remote_rwlock_wr_release( gpt_lock_xp );
2214
2215	#if DEBUG_VMM_HANDLE_COW
2216	cycle = (uint32_t)hal_get_cycles();
2217	if( DEBUG_VMM_HANDLE_COW < cycle )
2218	printk("\n[%s] thread[%x,%x] exit for vpn %x / core[%x,%d] / cycle %d\n",
2219	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
2220	#endif
2221
2222	return EXCP_NON_FATAL;
2223
2224	} // end vmm_handle_cow()
2225

Note: See TracBrowser for help on using the repository browser.

Download in other formats: