source: trunk/kernel/mm/ppm.c @ 619

Last change on this file since 619 was 611, checked in by alain, 5 years ago

Introduce sigificant modifs in VFS to support the <ls> command,
and the . and .. directories entries.

File size: 17.5 KB
RevLine 
[1]1/*
2 * ppm.c - Per-cluster Physical Pages Manager implementation
3 *
4 * Authors  Ghassan Almaless (2008,2009,2010,2011,2012)
[567]5 *          Alain Greiner    (2016,2017,2018)
[1]6 *
7 * Copyright (c) UPMC Sorbonne Universites
8 *
9 * This file is part of ALMOS-MKH.
10 *
11 * ALMOS-MKH.is free software; you can redistribute it and/or modify it
12 * under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; version 2.0 of the License.
14 *
15 * ALMOS-MKH.is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18 * General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with ALMOS-MKH.; if not, write to the Free Software Foundation,
22 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
23 */
24
[14]25#include <kernel_config.h>
[457]26#include <hal_kernel_types.h>
[1]27#include <hal_special.h>
28#include <printk.h>
29#include <list.h>
30#include <bits.h>
31#include <page.h>
[585]32#include <dqdt.h>
[567]33#include <busylock.h>
34#include <queuelock.h>
[1]35#include <thread.h>
36#include <cluster.h>
37#include <kmem.h>
38#include <process.h>
[567]39#include <mapper.h>
[1]40#include <ppm.h>
[606]41#include <vfs.h>
[1]42
[567]43////////////////////////////////////////////////////////////////////////////////////////
44//     functions to  translate [ page <-> base <-> ppn ]
45////////////////////////////////////////////////////////////////////////////////////////
46
[1]47////////////////////////////////////////////////
48inline bool_t ppm_page_is_valid( page_t * page )
49{
[160]50        ppm_t    * ppm  = &LOCAL_CLUSTER->ppm;
[1]51        uint32_t   pgnr = (uint32_t)( page - ppm->pages_tbl );
52        return (pgnr <= ppm->pages_nr);
53}
54
[50]55/////////////////////////////////////////////
[315]56inline xptr_t ppm_page2base( xptr_t page_xp )
[1]57{
[315]58        ppm_t  * ppm      = &LOCAL_CLUSTER->ppm;
[1]59
[315]60    cxy_t    page_cxy = GET_CXY( page_xp );
[437]61    page_t * page_ptr = GET_PTR( page_xp );
[315]62
[406]63   void   * base_ptr = ppm->vaddr_base + 
64                       ((page_ptr - ppm->pages_tbl)<<CONFIG_PPM_PAGE_SHIFT);
65
[315]66        return XPTR( page_cxy , base_ptr );
67
68} // end ppm_page2base()
69
70/////////////////////////////////////////////
71inline xptr_t ppm_base2page( xptr_t base_xp )
[1]72{
[315]73        ppm_t  * ppm = &LOCAL_CLUSTER->ppm;
[1]74
[315]75    cxy_t    base_cxy = GET_CXY( base_xp );
[437]76    void   * base_ptr = GET_PTR( base_xp );
[315]77
78        page_t * page_ptr = ppm->pages_tbl + 
79                        ((base_ptr - ppm->vaddr_base)>>CONFIG_PPM_PAGE_SHIFT);
80
81        return XPTR( base_cxy , page_ptr );
82
83}  // end ppm_base2page()
84
85
86
[50]87///////////////////////////////////////////
[315]88inline ppn_t ppm_page2ppn( xptr_t page_xp )
89{
90        ppm_t  * ppm      = &LOCAL_CLUSTER->ppm;
91
92    cxy_t    page_cxy = GET_CXY( page_xp );
[437]93    page_t * page_ptr = GET_PTR( page_xp );
[315]94
95    paddr_t  paddr    = PADDR( page_cxy , (page_ptr - ppm->pages_tbl)<<CONFIG_PPM_PAGE_SHIFT );
96
[437]97    return (ppn_t)(paddr >> CONFIG_PPM_PAGE_SHIFT);
[315]98
99}  // end hal_page2ppn()
100
101///////////////////////////////////////
102inline xptr_t ppm_ppn2page( ppn_t ppn )
103{
[437]104        ppm_t   * ppm  = &LOCAL_CLUSTER->ppm;
[315]105
[437]106    paddr_t  paddr = ((paddr_t)ppn) << CONFIG_PPM_PAGE_SHIFT;
[315]107
[437]108    cxy_t    cxy   = CXY_FROM_PADDR( paddr );
109    lpa_t    lpa   = LPA_FROM_PADDR( paddr );
[315]110
[437]111    return XPTR( cxy , &ppm->pages_tbl[lpa>>CONFIG_PPM_PAGE_SHIFT] );
[315]112
113}  // end hal_ppn2page
114
115
116
117///////////////////////////////////////
118inline xptr_t ppm_ppn2base( ppn_t ppn )
119{
[437]120        ppm_t  * ppm   = &LOCAL_CLUSTER->ppm;
[315]121   
[437]122    paddr_t  paddr = ((paddr_t)ppn) << CONFIG_PPM_PAGE_SHIFT;
[315]123
[437]124    cxy_t    cxy   = CXY_FROM_PADDR( paddr );
125    lpa_t    lpa   = LPA_FROM_PADDR( paddr );
[315]126
[437]127        return XPTR( cxy , (void *)ppm->vaddr_base + lpa );
[315]128
129}  // end ppm_ppn2base()
130
131///////////////////////////////////////////
132inline ppn_t ppm_base2ppn( xptr_t base_xp )
133{
134        ppm_t  * ppm      = &LOCAL_CLUSTER->ppm;
135
136    cxy_t    base_cxy = GET_CXY( base_xp );
[437]137    void   * base_ptr = GET_PTR( base_xp );
[315]138
139    paddr_t  paddr    = PADDR( base_cxy , (base_ptr - ppm->vaddr_base) );
140
[437]141    return (ppn_t)(paddr >> CONFIG_PPM_PAGE_SHIFT);
[315]142
143}  // end ppm_base2ppn()
144
145
[567]146////////////////////////////////////////////////////////////////////////////////////////
147//     functions to  allocate / release  physical pages
148////////////////////////////////////////////////////////////////////////////////////////
[315]149
150///////////////////////////////////////////
[50]151void ppm_free_pages_nolock( page_t * page )
[1]152{
[7]153        page_t   * buddy;            // searched buddy page descriptor
154        uint32_t   buddy_index;      // buddy page index
155        page_t   * current;          // current (merged) page descriptor
156        uint32_t   current_index;    // current (merged) page index
[50]157        uint32_t   current_order;    // current (merged) page order
[7]158
[160]159        ppm_t    * ppm         = &LOCAL_CLUSTER->ppm;
160        page_t   * pages_tbl   = ppm->pages_tbl;
[1]161
[492]162        assert( !page_is_flag( page , PG_FREE ) ,
[407]163    "page already released : ppn = %x\n" , ppm_page2ppn(XPTR(local_cxy,page)) );
[177]164
[492]165        assert( !page_is_flag( page , PG_RESERVED ) ,
[407]166    "reserved page : ppn = %x\n" , ppm_page2ppn(XPTR(local_cxy,page)) );
167
[160]168        // update released page descriptor flags
[1]169        page_set_flag( page , PG_FREE );
170
[160]171        // search the buddy page descriptor
172        // - merge with current page descriptor if found
173        // - exit to release the current page descriptor if not found
174        current       = page ,
175        current_index = (uint32_t)(page - ppm->pages_tbl);
[18]176        for( current_order = page->order ;
[160]177             current_order < CONFIG_PPM_MAX_ORDER ;
178             current_order++ )
179        {
[7]180                buddy_index = current_index ^ (1 << current_order);
181                buddy       = pages_tbl + buddy_index;
[18]182
[7]183                if( !page_is_flag( buddy , PG_FREE ) || (buddy->order != current_order) ) break;
[1]184
[160]185                // remove buddy from free list
[7]186                list_unlink( &buddy->list );
[1]187                ppm->free_pages_nr[current_order] --;
[18]188
[160]189                // merge buddy with current
[7]190                buddy->order = 0;
191                current_index &= buddy_index;
[1]192        }
[18]193
[160]194        // update merged page descriptor order
[7]195        current        = pages_tbl + current_index;
196        current->order = current_order;
[1]197
[160]198        // insert current in free list
[7]199        list_add_first( &ppm->free_pages_root[current_order] , &current->list );
[1]200        ppm->free_pages_nr[current_order] ++;
201
[433]202}  // end ppm_free_pages_nolock()
203
[1]204////////////////////////////////////////////
205page_t * ppm_alloc_pages( uint32_t   order )
206{
[160]207        uint32_t   current_order;
[1]208        page_t   * remaining_block;
209        uint32_t   current_size;
[551]210
[438]211#if DEBUG_PPM_ALLOC_PAGES
[611]212thread_t * this = CURRENT_THREAD;
[433]213uint32_t cycle = (uint32_t)hal_get_cycles();
[438]214if( DEBUG_PPM_ALLOC_PAGES < cycle )
[611]215printk("\n[%s] thread[%x,%x] enter for %d page(s) / cycle %d\n",
216__FUNCTION__, this->process->pid, this->trdid, 1<<order, cycle );
[433]217#endif
[1]218
[438]219#if(DEBUG_PPM_ALLOC_PAGES & 0x1)
220if( DEBUG_PPM_ALLOC_PAGES < cycle )
[611]221ppm_print("enter ppm_alloc_pages");
[433]222#endif
223
[160]224        ppm_t    * ppm = &LOCAL_CLUSTER->ppm;
[1]225
[611]226// check order
227assert( (order < CONFIG_PPM_MAX_ORDER) , "illegal order argument = %d\n" , order );
[1]228
[406]229        page_t * block = NULL; 
[1]230
[160]231        // take lock protecting free lists
[567]232        busylock_acquire( &ppm->free_lock );
[1]233
[160]234        // find a free block equal or larger to requested size
[1]235        for( current_order = order ; current_order < CONFIG_PPM_MAX_ORDER ; current_order ++ )
236        {
237                if( !list_is_empty( &ppm->free_pages_root[current_order] ) )
238                {
239                        block = LIST_FIRST( &ppm->free_pages_root[current_order] , page_t , list );
240                        list_unlink( &block->list );
241                        break;
242                }
243        }
244
245        if( block == NULL ) // return failure
246        {
[160]247                // release lock protecting free lists
[567]248                busylock_release( &ppm->free_lock );
[1]249
[438]250#if DEBUG_PPM_ALLOC_PAGES
[433]251cycle = (uint32_t)hal_get_cycles();
[438]252if( DEBUG_PPM_ALLOC_PAGES < cycle )
[611]253printk("\n[%s] thread[%x,%x] cannot allocate %d page(s) / cycle %d\n",
254__FUNCTION__, this->process->pid, this->trdid, 1<<order, cycle );
[433]255#endif
256
[160]257                return NULL;
258        }
[18]259
[160]260        // update free-lists after removing a block
[18]261        ppm->free_pages_nr[current_order] --;
[1]262        current_size = (1 << current_order);
263
[160]264        // split the removed block in smaller sub-blocks if required
265        // and update the free-lists accordingly
[1]266        while( current_order > order )
267        {
268                current_order --;
269                current_size >>= 1;
[18]270
[1]271                remaining_block = block + current_size;
272                remaining_block->order = current_order;
273
274                list_add_first( &ppm->free_pages_root[current_order] , &remaining_block->list );
275                ppm->free_pages_nr[current_order] ++;
276        }
[18]277
[160]278        // update page descriptor
279        page_clear_flag( block , PG_FREE );
[1]280        page_refcount_up( block );
281        block->order = order;
282
[160]283        // release lock protecting free lists
[567]284        busylock_release( &ppm->free_lock );
[18]285
[585]286    // update DQDT
287    dqdt_increment_pages( order );
288
[438]289#if DEBUG_PPM_ALLOC_PAGES
[433]290cycle = (uint32_t)hal_get_cycles();
[438]291if( DEBUG_PPM_ALLOC_PAGES < cycle )
[611]292printk("\n[%s] thread[%x,%x] exit for %d page(s) / ppn = %x / cycle %d\n",
293__FUNCTION__, this->process->pid, this->trdid, 
[567]2941<<order, ppm_page2ppn(XPTR( local_cxy , block )), cycle );
[433]295#endif
[7]296
[611]297#if(DEBUG_PPM_ALLOC_PAGES & 0x1)
298if( DEBUG_PPM_ALLOC_PAGES < cycle )
299ppm_print("exit ppm_alloc_pages");
300#endif
301
[1]302        return block;
303
[433]304}  // end ppm_alloc_pages()
[1]305
[433]306
[1]307////////////////////////////////////
308void ppm_free_pages( page_t * page )
309{
310        ppm_t * ppm = &LOCAL_CLUSTER->ppm;
[18]311
[438]312#if DEBUG_PPM_FREE_PAGES
[433]313uint32_t cycle = (uint32_t)hal_get_cycles();
[438]314if( DEBUG_PPM_FREE_PAGES < cycle )
[611]315printk("\n[%s] thread[%x,%x] enter for %d page(s) / ppn %x / cycle %d\n",
316__FUNCTION__, this->process->pid, this->trdid, 
[567]3171<<page->order, ppm_page2ppn(XPTR(local_cxy , page)), cycle );
[433]318#endif
319
[438]320#if(DEBUG_PPM_FREE_PAGES & 0x1)
321if( DEBUG_PPM_FREE_PAGES < cycle )
[611]322ppm_print("enter ppm_free_pages");
[433]323#endif
324
[160]325        // get lock protecting free_pages[] array
[567]326        busylock_acquire( &ppm->free_lock );
[1]327
[18]328        ppm_free_pages_nolock( page );
[1]329
[160]330        // release lock protecting free_pages[] array
[567]331        busylock_release( &ppm->free_lock );
[433]332
[585]333    // update DQDT
334    dqdt_decrement_pages( page->order );
335
[438]336#if DEBUG_PPM_FREE_PAGES
[433]337cycle = (uint32_t)hal_get_cycles();
[438]338if( DEBUG_PPM_FREE_PAGES < cycle )
[611]339printk("\n[%s] thread[%x,%x] exit for %d page(s) / ppn %x / cycle %d\n",
340__FUNCTION__, this->process->pid, this->trdid, 
[567]3411<<page->order, ppm_page2ppn(XPTR(local_cxy , page)), cycle );
[433]342#endif
343
[611]344#if(DEBUG_PPM_FREE_PAGES & 0x1)
345if( DEBUG_PPM_FREE_PAGES < cycle )
346ppm_print("exit ppm_free_pages");
347#endif
348
[567]349}  // end ppm_free_pages()
[1]350
[611]351///////////////////////////////
352void ppm_print( char * string )
[1]353{
354        uint32_t       order;
355        list_entry_t * iter;
356        page_t       * page;
357
[433]358    ppm_t * ppm = &LOCAL_CLUSTER->ppm;
359
[160]360        // get lock protecting free lists
[567]361        busylock_acquire( &ppm->free_lock );
[1]362
[611]363        printk("\n***  PPM in cluster %x / %s / %d pages ***\n",
364    local_cxy , string, ppm->pages_nr );
[18]365
[1]366        for( order = 0 ; order < CONFIG_PPM_MAX_ORDER ; order++ )
367        {
[433]368                printk("- order = %d / free_pages = %d\t: ",
[160]369                       order , ppm->free_pages_nr[order] );
[18]370
[1]371                LIST_FOREACH( &ppm->free_pages_root[order] , iter )
372                {
373                        page = LIST_ELEMENT( iter , page_t , list );
[433]374                        printk("%x," , page - ppm->pages_tbl );
[1]375                }
[18]376
[433]377                printk("\n");
[1]378        }
379
[160]380        // release lock protecting free lists
[567]381        busylock_release( &ppm->free_lock );
[160]382}
[1]383
[53]384///////////////////////////////////////
385error_t ppm_assert_order( ppm_t * ppm )
[1]386{
387        uint32_t       order;
388        list_entry_t * iter;
389        page_t       * page;
[18]390
[407]391        for( order=0 ; order < CONFIG_PPM_MAX_ORDER ; order++ )
[1]392        {
393                if( list_is_empty( &ppm->free_pages_root[order] ) ) continue;
[18]394
[1]395                LIST_FOREACH( &ppm->free_pages_root[order] , iter )
396                {
397                        page = LIST_ELEMENT( iter , page_t , list );
[160]398                        if( page->order != order )  return -1;
[1]399                }
400        }
401
[160]402        return 0;
403}
[53]404
[567]405
406//////////////////////////////////////////////////////////////////////////////////////
407//     functions to handle  dirty physical pages
408//////////////////////////////////////////////////////////////////////////////////////
409
[606]410//////////////////////////////////////////
411bool_t ppm_page_do_dirty( xptr_t page_xp )
[567]412{
413        bool_t done = false;
414
[606]415    // get page cluster and local pointer
416    page_t * page_ptr = GET_PTR( page_xp );
417    cxy_t    page_cxy = GET_CXY( page_xp );
418
419    // get local pointer on PPM (same in all clusters)
[567]420        ppm_t * ppm = &LOCAL_CLUSTER->ppm;
421
[606]422    // build extended pointers on page lock, page flags, and PPM dirty list lock
423    xptr_t page_lock_xp  = XPTR( page_cxy , &page_ptr->lock  );   
424    xptr_t page_flags_xp = XPTR( page_cxy , &page_ptr->flags );
425    xptr_t dirty_lock_xp = XPTR( page_cxy , &ppm->dirty_lock );
426           
427        // lock the remote PPM dirty_list
428        remote_queuelock_acquire( dirty_lock_xp );
[567]429
[606]430    // lock the remote page
431    remote_busylock_acquire( page_lock_xp );
432
433    // get remote page flags
434    uint32_t flags = hal_remote_l32( page_flags_xp );
435
436        if( (flags & PG_DIRTY) == 0 )
[567]437        {
438                // set dirty flag in page descriptor
[606]439        hal_remote_s32( page_flags_xp , flags | PG_DIRTY );
[567]440
[606]441                // The PPM dirty list is a LOCAL list !!!
442        // We must update 4 pointers to insert a new page in this list.
443        // We can use the standard LIST API when the page is local,
444        // but we cannot use the standard API if the page is remote...
445
446        if( page_cxy == local_cxy )         // locally update the PPM dirty list
447        {
448            list_add_first( &ppm->dirty_root , &page_ptr->list );
449        } 
450        else                                // remotely update the PPM dirty list
451        {
452            // get local and remote pointers on "root" list entry
453            list_entry_t * root    = &ppm->dirty_root;
454            xptr_t         root_xp = XPTR( page_cxy , root );
455
456            // get local and remote pointers on "page" list entry
457            list_entry_t * list    = &page_ptr->list;
458            xptr_t         list_xp = XPTR( page_cxy , list );
459
460            // get local and remote pointers on first dirty page
461            list_entry_t * dirt    = hal_remote_lpt( XPTR( page_cxy, &root->next ) );
462            xptr_t         dirt_xp = XPTR( page_cxy , dirt );
463
464            // set root.next, list.next, list pred, curr.pred in remote cluster
465            hal_remote_spt( root_xp                    , list );
466            hal_remote_spt( list_xp                    , dirt );
467            hal_remote_spt( list_xp + sizeof(intptr_t) , root );
468            hal_remote_spt( dirt_xp + sizeof(intptr_t) , list );
469        }
470
[567]471                done = true;
472        }
473
[606]474    // unlock the remote page
475    remote_busylock_release( page_lock_xp );
[567]476
[606]477        // unlock the remote PPM dirty_list
478        remote_queuelock_release( dirty_lock_xp );
479
[567]480        return done;
481
[606]482} // end ppm_page_do_dirty()
483
484////////////////////////////////////////////
485bool_t ppm_page_undo_dirty( xptr_t page_xp )
[567]486{
487        bool_t done = false;
488
[606]489    // get page cluster and local pointer
490    page_t * page_ptr = GET_PTR( page_xp );
491    cxy_t    page_cxy = GET_CXY( page_xp );
492
493    // get local pointer on PPM (same in all clusters)
[567]494        ppm_t * ppm = &LOCAL_CLUSTER->ppm;
495
[606]496    // build extended pointers on page lock, page flags, and PPM dirty list lock
497    xptr_t page_lock_xp  = XPTR( page_cxy , &page_ptr->lock  );
498    xptr_t page_flags_xp = XPTR( page_cxy , &page_ptr->flags );
499    xptr_t dirty_lock_xp = XPTR( page_cxy , &ppm->dirty_lock );
500           
501        // lock the remote PPM dirty_list
502        remote_queuelock_acquire( XPTR( page_cxy , &ppm->dirty_lock ) );
[567]503
[606]504    // lock the remote page
505    remote_busylock_acquire( page_lock_xp );
506
507    // get remote page flags
508    uint32_t flags = hal_remote_l32( page_flags_xp );
509
510        if( (flags & PG_DIRTY) )  // page is dirty
[567]511        {
[606]512                // reset dirty flag in page descriptor
513        hal_remote_s32( page_flags_xp , flags & (~PG_DIRTY) );
[567]514
[606]515                // The PPM dirty list is a LOCAL list !!!
516        // We must update 4 pointers to remove a page from this list.
517        // we can use the standard LIST API when the page is local,
518        // but we cannot use the standard API if the page is remote...
519
520        if( page_cxy == local_cxy )         // locally update the PPM dirty list
521        {
522            list_unlink( &page_ptr->list );
523        } 
524        else                                // remotely update the PPM dirty list
525        {
526            // get local and remote pointers on "page" list entry
527            list_entry_t * list    = &page_ptr->list;
528            xptr_t         list_xp = XPTR( page_cxy , list );
529
530            // get local and remote pointers on "next" page list entry
531            list_entry_t * next    = hal_remote_lpt( list_xp );
532            xptr_t         next_xp = XPTR( page_cxy , next );
533
534            // get local and remote pointers on "pred" page list entry
535            list_entry_t * pred    = hal_remote_lpt( list_xp + sizeof(intptr_t) );
536            xptr_t         pred_xp = XPTR( page_cxy , pred );
537
538            // set root.next, list.next, list pred, curr.pred in remote cluster
539            hal_remote_spt( pred_xp                    , next );
540            hal_remote_spt( list_xp                    , NULL );
541            hal_remote_spt( list_xp + sizeof(intptr_t) , NULL );
542            hal_remote_spt( next_xp + sizeof(intptr_t) , pred );
543        }
544
[567]545                done = true;
546        }
547
[606]548    // unlock the remote page
549    remote_busylock_release( page_lock_xp );
[567]550
[606]551        // unlock the remote PPM dirty_list
552        remote_queuelock_release( dirty_lock_xp );
553
[567]554        return done;
555
[606]556}  // end ppm_page_undo_dirty()
557
558/////////////////////////////////
559void ppm_sync_dirty_pages( void )
[567]560{
[606]561        ppm_t * ppm = &LOCAL_CLUSTER->ppm;
[567]562
[606]563    // get local pointer on PPM dirty_root
564    list_entry_t * dirty_root = &ppm->dirty_root;
565
566    // build extended pointer on PPM dirty_lock
567    xptr_t dirty_lock_xp = XPTR( local_cxy , &ppm->dirty_lock );
568
[567]569        // get the PPM dirty_list lock
[606]570        remote_queuelock_acquire( dirty_lock_xp );
[567]571
572        while( !list_is_empty( &ppm->dirty_root ) )
573        {
[606]574                page_t * page = LIST_FIRST( dirty_root ,  page_t , list );
575        xptr_t   page_xp = XPTR( local_cxy , page );
[567]576
[606]577        // build extended pointer on page lock
578        xptr_t page_lock_xp = XPTR( local_cxy , &page->lock );
579
[567]580                // get the page lock
[606]581                remote_busylock_acquire( page_lock_xp );
[567]582
583                // sync the page
[606]584                vfs_fs_move_page( page_xp , false );  // from mapper to device
[567]585
586                // release the page lock
[606]587                remote_busylock_release( page_lock_xp );
[567]588        }
589
590        // release the PPM dirty_list lock
[606]591        remote_queuelock_release( dirty_lock_xp );
[567]592
[606]593}  // end ppm_sync_dirty_pages()
594
Note: See TracBrowser for help on using the repository browser.