source: trunk/kernel/mm/ppm.c @ 625

Last change on this file since 625 was 625, checked in by alain, 2 years ago

Fix a bug in the vmm_remove_vseg() function: the physical pages
associated to an user DATA vseg were released to the kernel when
the target process descriptor was in the reference cluster.
This physical pages release should be done only when the page
forks counter value is zero.
All other modifications are cosmetic.

File size: 17.4 KB
Line 
1/*
2 * ppm.c - Per-cluster Physical Pages Manager implementation
3 *
4 * Authors  Ghassan Almaless (2008,2009,2010,2011,2012)
5 *          Alain Greiner    (2016,2017,2018)
6 *
7 * Copyright (c) UPMC Sorbonne Universites
8 *
9 * This file is part of ALMOS-MKH.
10 *
11 * ALMOS-MKH.is free software; you can redistribute it and/or modify it
12 * under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; version 2.0 of the License.
14 *
15 * ALMOS-MKH.is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18 * General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with ALMOS-MKH.; if not, write to the Free Software Foundation,
22 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
23 */
24
25#include <kernel_config.h>
26#include <hal_kernel_types.h>
27#include <hal_special.h>
28#include <printk.h>
29#include <list.h>
30#include <bits.h>
31#include <page.h>
32#include <dqdt.h>
33#include <busylock.h>
34#include <queuelock.h>
35#include <thread.h>
36#include <cluster.h>
37#include <kmem.h>
38#include <process.h>
39#include <mapper.h>
40#include <ppm.h>
41#include <vfs.h>
42
43////////////////////////////////////////////////////////////////////////////////////////
44//     functions to  translate [ page <-> base <-> ppn ]
45////////////////////////////////////////////////////////////////////////////////////////
46
47////////////////////////////////////////////////
48inline bool_t ppm_page_is_valid( page_t * page )
49{
50        ppm_t    * ppm  = &LOCAL_CLUSTER->ppm;
51        uint32_t   pgnr = (uint32_t)( page - ppm->pages_tbl );
52        return (pgnr <= ppm->pages_nr);
53}
54
55/////////////////////////////////////////////
56inline xptr_t ppm_page2base( xptr_t page_xp )
57{
58        ppm_t  * ppm      = &LOCAL_CLUSTER->ppm;
59
60    cxy_t    page_cxy = GET_CXY( page_xp );
61    page_t * page_ptr = GET_PTR( page_xp );
62
63   void   * base_ptr = ppm->vaddr_base + 
64                       ((page_ptr - ppm->pages_tbl)<<CONFIG_PPM_PAGE_SHIFT);
65
66        return XPTR( page_cxy , base_ptr );
67
68} // end ppm_page2base()
69
70/////////////////////////////////////////////
71inline xptr_t ppm_base2page( xptr_t base_xp )
72{
73        ppm_t  * ppm = &LOCAL_CLUSTER->ppm;
74
75    cxy_t    base_cxy = GET_CXY( base_xp );
76    void   * base_ptr = GET_PTR( base_xp );
77
78        page_t * page_ptr = ppm->pages_tbl + 
79                        ((base_ptr - ppm->vaddr_base)>>CONFIG_PPM_PAGE_SHIFT);
80
81        return XPTR( base_cxy , page_ptr );
82
83}  // end ppm_base2page()
84
85
86
87///////////////////////////////////////////
88inline ppn_t ppm_page2ppn( xptr_t page_xp )
89{
90        ppm_t  * ppm      = &LOCAL_CLUSTER->ppm;
91
92    cxy_t    page_cxy = GET_CXY( page_xp );
93    page_t * page_ptr = GET_PTR( page_xp );
94
95    paddr_t  paddr    = PADDR( page_cxy , (page_ptr - ppm->pages_tbl)<<CONFIG_PPM_PAGE_SHIFT );
96
97    return (ppn_t)(paddr >> CONFIG_PPM_PAGE_SHIFT);
98
99}  // end hal_page2ppn()
100
101///////////////////////////////////////
102inline xptr_t ppm_ppn2page( ppn_t ppn )
103{
104        ppm_t   * ppm  = &LOCAL_CLUSTER->ppm;
105
106    paddr_t  paddr = ((paddr_t)ppn) << CONFIG_PPM_PAGE_SHIFT;
107
108    cxy_t    cxy   = CXY_FROM_PADDR( paddr );
109    lpa_t    lpa   = LPA_FROM_PADDR( paddr );
110
111    return XPTR( cxy , &ppm->pages_tbl[lpa>>CONFIG_PPM_PAGE_SHIFT] );
112
113}  // end hal_ppn2page
114
115
116
117///////////////////////////////////////
118inline xptr_t ppm_ppn2base( ppn_t ppn )
119{
120        ppm_t  * ppm   = &LOCAL_CLUSTER->ppm;
121   
122    paddr_t  paddr = ((paddr_t)ppn) << CONFIG_PPM_PAGE_SHIFT;
123
124    cxy_t    cxy   = CXY_FROM_PADDR( paddr );
125    lpa_t    lpa   = LPA_FROM_PADDR( paddr );
126
127        return XPTR( cxy , (void *)ppm->vaddr_base + lpa );
128
129}  // end ppm_ppn2base()
130
131///////////////////////////////////////////
132inline ppn_t ppm_base2ppn( xptr_t base_xp )
133{
134        ppm_t  * ppm      = &LOCAL_CLUSTER->ppm;
135
136    cxy_t    base_cxy = GET_CXY( base_xp );
137    void   * base_ptr = GET_PTR( base_xp );
138
139    paddr_t  paddr    = PADDR( base_cxy , (base_ptr - ppm->vaddr_base) );
140
141    return (ppn_t)(paddr >> CONFIG_PPM_PAGE_SHIFT);
142
143}  // end ppm_base2ppn()
144
145
146////////////////////////////////////////////////////////////////////////////////////////
147//     functions to  allocate / release  physical pages
148////////////////////////////////////////////////////////////////////////////////////////
149
150///////////////////////////////////////////
151void ppm_free_pages_nolock( page_t * page )
152{
153        page_t   * buddy;            // searched buddy page descriptor
154        uint32_t   buddy_index;      // buddy page index
155        page_t   * current;          // current (merged) page descriptor
156        uint32_t   current_index;    // current (merged) page index
157        uint32_t   current_order;    // current (merged) page order
158
159        ppm_t    * ppm         = &LOCAL_CLUSTER->ppm;
160        page_t   * pages_tbl   = ppm->pages_tbl;
161
162        assert( !page_is_flag( page , PG_FREE ) ,
163    "page already released : ppn = %x\n" , ppm_page2ppn(XPTR(local_cxy,page)) );
164
165        assert( !page_is_flag( page , PG_RESERVED ) ,
166    "reserved page : ppn = %x\n" , ppm_page2ppn(XPTR(local_cxy,page)) );
167
168        // update released page descriptor flags
169        page_set_flag( page , PG_FREE );
170
171        // search the buddy page descriptor
172        // - merge with current page descriptor if found
173        // - exit to release the current page descriptor if not found
174        current       = page ,
175        current_index = (uint32_t)(page - ppm->pages_tbl);
176        for( current_order = page->order ;
177             current_order < CONFIG_PPM_MAX_ORDER ;
178             current_order++ )
179        {
180                buddy_index = current_index ^ (1 << current_order);
181                buddy       = pages_tbl + buddy_index;
182
183                if( !page_is_flag( buddy , PG_FREE ) || (buddy->order != current_order) ) break;
184
185                // remove buddy from free list
186                list_unlink( &buddy->list );
187                ppm->free_pages_nr[current_order] --;
188
189                // merge buddy with current
190                buddy->order = 0;
191                current_index &= buddy_index;
192        }
193
194        // update merged page descriptor order
195        current        = pages_tbl + current_index;
196        current->order = current_order;
197
198        // insert current in free list
199        list_add_first( &ppm->free_pages_root[current_order] , &current->list );
200        ppm->free_pages_nr[current_order] ++;
201
202}  // end ppm_free_pages_nolock()
203
204////////////////////////////////////////////
205page_t * ppm_alloc_pages( uint32_t   order )
206{
207        uint32_t   current_order;
208        page_t   * remaining_block;
209        uint32_t   current_size;
210
211#if DEBUG_PPM_ALLOC_PAGES
212thread_t * this = CURRENT_THREAD;
213uint32_t cycle = (uint32_t)hal_get_cycles();
214if( DEBUG_PPM_ALLOC_PAGES < cycle )
215printk("\n[%s] thread[%x,%x] enter for %d page(s) / cycle %d\n",
216__FUNCTION__, this->process->pid, this->trdid, 1<<order, cycle );
217#endif
218
219#if(DEBUG_PPM_ALLOC_PAGES & 0x1)
220if( DEBUG_PPM_ALLOC_PAGES < cycle )
221ppm_print("enter ppm_alloc_pages");
222#endif
223
224        ppm_t    * ppm = &LOCAL_CLUSTER->ppm;
225
226// check order
227assert( (order < CONFIG_PPM_MAX_ORDER) , "illegal order argument = %d\n" , order );
228
229        page_t * block = NULL; 
230
231        // take lock protecting free lists
232        busylock_acquire( &ppm->free_lock );
233
234        // find a free block equal or larger to requested size
235        for( current_order = order ; current_order < CONFIG_PPM_MAX_ORDER ; current_order ++ )
236        {
237                if( !list_is_empty( &ppm->free_pages_root[current_order] ) )
238                {
239                        block = LIST_FIRST( &ppm->free_pages_root[current_order] , page_t , list );
240                        list_unlink( &block->list );
241                        break;
242                }
243        }
244
245        if( block == NULL ) // return failure
246        {
247                // release lock protecting free lists
248                busylock_release( &ppm->free_lock );
249
250#if DEBUG_PPM_ALLOC_PAGES
251cycle = (uint32_t)hal_get_cycles();
252if( DEBUG_PPM_ALLOC_PAGES < cycle )
253printk("\n[%s] thread[%x,%x] cannot allocate %d page(s) / cycle %d\n",
254__FUNCTION__, this->process->pid, this->trdid, 1<<order, cycle );
255#endif
256
257                return NULL;
258        }
259
260        // update free-lists after removing a block
261        ppm->free_pages_nr[current_order] --;
262        current_size = (1 << current_order);
263
264        // split the removed block in smaller sub-blocks if required
265        // and update the free-lists accordingly
266        while( current_order > order )
267        {
268                current_order --;
269                current_size >>= 1;
270
271                remaining_block = block + current_size;
272                remaining_block->order = current_order;
273
274                list_add_first( &ppm->free_pages_root[current_order] , &remaining_block->list );
275                ppm->free_pages_nr[current_order] ++;
276        }
277
278        // update page descriptor
279        page_clear_flag( block , PG_FREE );
280        page_refcount_up( block );
281        block->order = order;
282
283        // release lock protecting free lists
284        busylock_release( &ppm->free_lock );
285
286    // update DQDT
287    dqdt_increment_pages( order );
288
289#if DEBUG_PPM_ALLOC_PAGES
290cycle = (uint32_t)hal_get_cycles();
291if( DEBUG_PPM_ALLOC_PAGES < cycle )
292printk("\n[%s] thread[%x,%x] exit for %d page(s) / ppn = %x / cycle %d\n",
293__FUNCTION__, this->process->pid, this->trdid, 
2941<<order, ppm_page2ppn(XPTR( local_cxy , block )), cycle );
295#endif
296
297#if(DEBUG_PPM_ALLOC_PAGES & 0x1)
298if( DEBUG_PPM_ALLOC_PAGES < cycle )
299ppm_print("exit ppm_alloc_pages");
300#endif
301
302        return block;
303
304}  // end ppm_alloc_pages()
305
306
307////////////////////////////////////
308void ppm_free_pages( page_t * page )
309{
310        ppm_t * ppm = &LOCAL_CLUSTER->ppm;
311
312#if DEBUG_PPM_FREE_PAGES
313uint32_t cycle = (uint32_t)hal_get_cycles();
314if( DEBUG_PPM_FREE_PAGES < cycle )
315printk("\n[%s] thread[%x,%x] enter for %d page(s) / ppn %x / cycle %d\n",
316__FUNCTION__, this->process->pid, this->trdid, 
3171<<page->order, ppm_page2ppn(XPTR(local_cxy , page)), cycle );
318#endif
319
320#if(DEBUG_PPM_FREE_PAGES & 0x1)
321if( DEBUG_PPM_FREE_PAGES < cycle )
322ppm_print("enter ppm_free_pages");
323#endif
324
325        // get lock protecting free_pages[] array
326        busylock_acquire( &ppm->free_lock );
327
328        ppm_free_pages_nolock( page );
329
330        // release lock protecting free_pages[] array
331        busylock_release( &ppm->free_lock );
332
333    // update DQDT
334    dqdt_decrement_pages( page->order );
335
336#if DEBUG_PPM_FREE_PAGES
337cycle = (uint32_t)hal_get_cycles();
338if( DEBUG_PPM_FREE_PAGES < cycle )
339printk("\n[%s] thread[%x,%x] exit for %d page(s) / ppn %x / cycle %d\n",
340__FUNCTION__, this->process->pid, this->trdid, 
3411<<page->order, ppm_page2ppn(XPTR(local_cxy , page)), cycle );
342#endif
343
344#if(DEBUG_PPM_FREE_PAGES & 0x1)
345if( DEBUG_PPM_FREE_PAGES < cycle )
346ppm_print("exit ppm_free_pages");
347#endif
348
349}  // end ppm_free_pages()
350
351////////////////////////
352void ppm_display( void )
353{
354        uint32_t       order;
355        list_entry_t * iter;
356        page_t       * page;
357
358    ppm_t * ppm = &LOCAL_CLUSTER->ppm;
359
360        // get lock protecting free lists
361        busylock_acquire( &ppm->free_lock );
362
363        printk("\n***** PPM in cluster %x / %d pages\n", local_cxy , ppm->pages_nr );
364
365        for( order = 0 ; order < CONFIG_PPM_MAX_ORDER ; order++ )
366        {
367                printk("- order = %d / free_pages = %d\t: ",
368                       order , ppm->free_pages_nr[order] );
369
370                LIST_FOREACH( &ppm->free_pages_root[order] , iter )
371                {
372                        page = LIST_ELEMENT( iter , page_t , list );
373                        printk("%x," , page - ppm->pages_tbl );
374                }
375
376                printk("\n");
377        }
378
379        // release lock protecting free lists
380        busylock_release( &ppm->free_lock );
381}
382
383///////////////////////////////////////
384error_t ppm_assert_order( ppm_t * ppm )
385{
386        uint32_t       order;
387        list_entry_t * iter;
388        page_t       * page;
389
390        for( order=0 ; order < CONFIG_PPM_MAX_ORDER ; order++ )
391        {
392                if( list_is_empty( &ppm->free_pages_root[order] ) ) continue;
393
394                LIST_FOREACH( &ppm->free_pages_root[order] , iter )
395                {
396                        page = LIST_ELEMENT( iter , page_t , list );
397                        if( page->order != order )  return -1;
398                }
399        }
400
401        return 0;
402}
403
404
405//////////////////////////////////////////////////////////////////////////////////////
406//     functions to handle  dirty physical pages
407//////////////////////////////////////////////////////////////////////////////////////
408
409//////////////////////////////////////////
410bool_t ppm_page_do_dirty( xptr_t page_xp )
411{
412        bool_t done = false;
413
414    // get page cluster and local pointer
415    page_t * page_ptr = GET_PTR( page_xp );
416    cxy_t    page_cxy = GET_CXY( page_xp );
417
418    // get local pointer on PPM (same in all clusters)
419        ppm_t * ppm = &LOCAL_CLUSTER->ppm;
420
421    // build extended pointers on page lock, page flags, and PPM dirty list lock
422    xptr_t page_lock_xp  = XPTR( page_cxy , &page_ptr->lock  );   
423    xptr_t page_flags_xp = XPTR( page_cxy , &page_ptr->flags );
424    xptr_t dirty_lock_xp = XPTR( page_cxy , &ppm->dirty_lock );
425           
426        // lock the remote PPM dirty_list
427        remote_queuelock_acquire( dirty_lock_xp );
428
429    // lock the remote page
430    remote_busylock_acquire( page_lock_xp );
431
432    // get remote page flags
433    uint32_t flags = hal_remote_l32( page_flags_xp );
434
435        if( (flags & PG_DIRTY) == 0 )
436        {
437                // set dirty flag in page descriptor
438        hal_remote_s32( page_flags_xp , flags | PG_DIRTY );
439
440                // The PPM dirty list is a LOCAL list !!!
441        // We must update 4 pointers to insert a new page in this list.
442        // We can use the standard LIST API when the page is local,
443        // but we cannot use the standard API if the page is remote...
444
445        if( page_cxy == local_cxy )         // locally update the PPM dirty list
446        {
447            list_add_first( &ppm->dirty_root , &page_ptr->list );
448        } 
449        else                                // remotely update the PPM dirty list
450        {
451            // get local and remote pointers on "root" list entry
452            list_entry_t * root    = &ppm->dirty_root;
453            xptr_t         root_xp = XPTR( page_cxy , root );
454
455            // get local and remote pointers on "page" list entry
456            list_entry_t * list    = &page_ptr->list;
457            xptr_t         list_xp = XPTR( page_cxy , list );
458
459            // get local and remote pointers on first dirty page
460            list_entry_t * dirt    = hal_remote_lpt( XPTR( page_cxy, &root->next ) );
461            xptr_t         dirt_xp = XPTR( page_cxy , dirt );
462
463            // set root.next, list.next, list pred, curr.pred in remote cluster
464            hal_remote_spt( root_xp                    , list );
465            hal_remote_spt( list_xp                    , dirt );
466            hal_remote_spt( list_xp + sizeof(intptr_t) , root );
467            hal_remote_spt( dirt_xp + sizeof(intptr_t) , list );
468        }
469
470                done = true;
471        }
472
473    // unlock the remote page
474    remote_busylock_release( page_lock_xp );
475
476        // unlock the remote PPM dirty_list
477        remote_queuelock_release( dirty_lock_xp );
478
479        return done;
480
481} // end ppm_page_do_dirty()
482
483////////////////////////////////////////////
484bool_t ppm_page_undo_dirty( xptr_t page_xp )
485{
486        bool_t done = false;
487
488    // get page cluster and local pointer
489    page_t * page_ptr = GET_PTR( page_xp );
490    cxy_t    page_cxy = GET_CXY( page_xp );
491
492    // get local pointer on PPM (same in all clusters)
493        ppm_t * ppm = &LOCAL_CLUSTER->ppm;
494
495    // build extended pointers on page lock, page flags, and PPM dirty list lock
496    xptr_t page_lock_xp  = XPTR( page_cxy , &page_ptr->lock  );
497    xptr_t page_flags_xp = XPTR( page_cxy , &page_ptr->flags );
498    xptr_t dirty_lock_xp = XPTR( page_cxy , &ppm->dirty_lock );
499           
500        // lock the remote PPM dirty_list
501        remote_queuelock_acquire( XPTR( page_cxy , &ppm->dirty_lock ) );
502
503    // lock the remote page
504    remote_busylock_acquire( page_lock_xp );
505
506    // get remote page flags
507    uint32_t flags = hal_remote_l32( page_flags_xp );
508
509        if( (flags & PG_DIRTY) )  // page is dirty
510        {
511                // reset dirty flag in page descriptor
512        hal_remote_s32( page_flags_xp , flags & (~PG_DIRTY) );
513
514                // The PPM dirty list is a LOCAL list !!!
515        // We must update 4 pointers to remove a page from this list.
516        // we can use the standard LIST API when the page is local,
517        // but we cannot use the standard API if the page is remote...
518
519        if( page_cxy == local_cxy )         // locally update the PPM dirty list
520        {
521            list_unlink( &page_ptr->list );
522        } 
523        else                                // remotely update the PPM dirty list
524        {
525            // get local and remote pointers on "page" list entry
526            list_entry_t * list    = &page_ptr->list;
527            xptr_t         list_xp = XPTR( page_cxy , list );
528
529            // get local and remote pointers on "next" page list entry
530            list_entry_t * next    = hal_remote_lpt( list_xp );
531            xptr_t         next_xp = XPTR( page_cxy , next );
532
533            // get local and remote pointers on "pred" page list entry
534            list_entry_t * pred    = hal_remote_lpt( list_xp + sizeof(intptr_t) );
535            xptr_t         pred_xp = XPTR( page_cxy , pred );
536
537            // set root.next, list.next, list pred, curr.pred in remote cluster
538            hal_remote_spt( pred_xp                    , next );
539            hal_remote_spt( list_xp                    , NULL );
540            hal_remote_spt( list_xp + sizeof(intptr_t) , NULL );
541            hal_remote_spt( next_xp + sizeof(intptr_t) , pred );
542        }
543
544                done = true;
545        }
546
547    // unlock the remote page
548    remote_busylock_release( page_lock_xp );
549
550        // unlock the remote PPM dirty_list
551        remote_queuelock_release( dirty_lock_xp );
552
553        return done;
554
555}  // end ppm_page_undo_dirty()
556
557/////////////////////////////////
558void ppm_sync_dirty_pages( void )
559{
560        ppm_t * ppm = &LOCAL_CLUSTER->ppm;
561
562    // get local pointer on PPM dirty_root
563    list_entry_t * dirty_root = &ppm->dirty_root;
564
565    // build extended pointer on PPM dirty_lock
566    xptr_t dirty_lock_xp = XPTR( local_cxy , &ppm->dirty_lock );
567
568        // get the PPM dirty_list lock
569        remote_queuelock_acquire( dirty_lock_xp );
570
571        while( !list_is_empty( &ppm->dirty_root ) )
572        {
573                page_t * page = LIST_FIRST( dirty_root ,  page_t , list );
574        xptr_t   page_xp = XPTR( local_cxy , page );
575
576        // build extended pointer on page lock
577        xptr_t page_lock_xp = XPTR( local_cxy , &page->lock );
578
579                // get the page lock
580                remote_busylock_acquire( page_lock_xp );
581
582                // sync the page
583                vfs_fs_move_page( page_xp , false );  // from mapper to device
584
585                // release the page lock
586                remote_busylock_release( page_lock_xp );
587        }
588
589        // release the PPM dirty_list lock
590        remote_queuelock_release( dirty_lock_xp );
591
592}  // end ppm_sync_dirty_pages()
593
Note: See TracBrowser for help on using the repository browser.