source: trunk/kernel/mm/ppm.c @ 605

Last change on this file since 605 was 585, checked in by alain, 6 years ago

Introduce two separate vmm_handle_page_fault() & vmm_handle_cow() functions in vmm.c

File size: 12.7 KB
RevLine 
[1]1/*
2 * ppm.c - Per-cluster Physical Pages Manager implementation
3 *
4 * Authors  Ghassan Almaless (2008,2009,2010,2011,2012)
[567]5 *          Alain Greiner    (2016,2017,2018)
[1]6 *
7 * Copyright (c) UPMC Sorbonne Universites
8 *
9 * This file is part of ALMOS-MKH.
10 *
11 * ALMOS-MKH.is free software; you can redistribute it and/or modify it
12 * under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; version 2.0 of the License.
14 *
15 * ALMOS-MKH.is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18 * General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with ALMOS-MKH.; if not, write to the Free Software Foundation,
22 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
23 */
24
[14]25#include <kernel_config.h>
[457]26#include <hal_kernel_types.h>
[1]27#include <hal_special.h>
28#include <printk.h>
29#include <list.h>
30#include <bits.h>
31#include <page.h>
[585]32#include <dqdt.h>
[567]33#include <busylock.h>
34#include <queuelock.h>
[1]35#include <thread.h>
36#include <cluster.h>
37#include <kmem.h>
38#include <process.h>
[567]39#include <mapper.h>
[1]40#include <ppm.h>
41
[567]42////////////////////////////////////////////////////////////////////////////////////////
43//     functions to  translate [ page <-> base <-> ppn ]
44////////////////////////////////////////////////////////////////////////////////////////
45
[1]46////////////////////////////////////////////////
47inline bool_t ppm_page_is_valid( page_t * page )
48{
[160]49        ppm_t    * ppm  = &LOCAL_CLUSTER->ppm;
[1]50        uint32_t   pgnr = (uint32_t)( page - ppm->pages_tbl );
51        return (pgnr <= ppm->pages_nr);
52}
53
[50]54/////////////////////////////////////////////
[315]55inline xptr_t ppm_page2base( xptr_t page_xp )
[1]56{
[315]57        ppm_t  * ppm      = &LOCAL_CLUSTER->ppm;
[1]58
[315]59    cxy_t    page_cxy = GET_CXY( page_xp );
[437]60    page_t * page_ptr = GET_PTR( page_xp );
[315]61
[406]62   void   * base_ptr = ppm->vaddr_base + 
63                       ((page_ptr - ppm->pages_tbl)<<CONFIG_PPM_PAGE_SHIFT);
64
[315]65        return XPTR( page_cxy , base_ptr );
66
67} // end ppm_page2base()
68
69/////////////////////////////////////////////
70inline xptr_t ppm_base2page( xptr_t base_xp )
[1]71{
[315]72        ppm_t  * ppm = &LOCAL_CLUSTER->ppm;
[1]73
[315]74    cxy_t    base_cxy = GET_CXY( base_xp );
[437]75    void   * base_ptr = GET_PTR( base_xp );
[315]76
77        page_t * page_ptr = ppm->pages_tbl + 
78                        ((base_ptr - ppm->vaddr_base)>>CONFIG_PPM_PAGE_SHIFT);
79
80        return XPTR( base_cxy , page_ptr );
81
82}  // end ppm_base2page()
83
84
85
[50]86///////////////////////////////////////////
[315]87inline ppn_t ppm_page2ppn( xptr_t page_xp )
88{
89        ppm_t  * ppm      = &LOCAL_CLUSTER->ppm;
90
91    cxy_t    page_cxy = GET_CXY( page_xp );
[437]92    page_t * page_ptr = GET_PTR( page_xp );
[315]93
94    paddr_t  paddr    = PADDR( page_cxy , (page_ptr - ppm->pages_tbl)<<CONFIG_PPM_PAGE_SHIFT );
95
[437]96    return (ppn_t)(paddr >> CONFIG_PPM_PAGE_SHIFT);
[315]97
98}  // end hal_page2ppn()
99
100///////////////////////////////////////
101inline xptr_t ppm_ppn2page( ppn_t ppn )
102{
[437]103        ppm_t   * ppm  = &LOCAL_CLUSTER->ppm;
[315]104
[437]105    paddr_t  paddr = ((paddr_t)ppn) << CONFIG_PPM_PAGE_SHIFT;
[315]106
[437]107    cxy_t    cxy   = CXY_FROM_PADDR( paddr );
108    lpa_t    lpa   = LPA_FROM_PADDR( paddr );
[315]109
[437]110    return XPTR( cxy , &ppm->pages_tbl[lpa>>CONFIG_PPM_PAGE_SHIFT] );
[315]111
112}  // end hal_ppn2page
113
114
115
116///////////////////////////////////////
117inline xptr_t ppm_ppn2base( ppn_t ppn )
118{
[437]119        ppm_t  * ppm   = &LOCAL_CLUSTER->ppm;
[315]120   
[437]121    paddr_t  paddr = ((paddr_t)ppn) << CONFIG_PPM_PAGE_SHIFT;
[315]122
[437]123    cxy_t    cxy   = CXY_FROM_PADDR( paddr );
124    lpa_t    lpa   = LPA_FROM_PADDR( paddr );
[315]125
[437]126        return XPTR( cxy , (void *)ppm->vaddr_base + lpa );
[315]127
128}  // end ppm_ppn2base()
129
130///////////////////////////////////////////
131inline ppn_t ppm_base2ppn( xptr_t base_xp )
132{
133        ppm_t  * ppm      = &LOCAL_CLUSTER->ppm;
134
135    cxy_t    base_cxy = GET_CXY( base_xp );
[437]136    void   * base_ptr = GET_PTR( base_xp );
[315]137
138    paddr_t  paddr    = PADDR( base_cxy , (base_ptr - ppm->vaddr_base) );
139
[437]140    return (ppn_t)(paddr >> CONFIG_PPM_PAGE_SHIFT);
[315]141
142}  // end ppm_base2ppn()
143
144
[567]145////////////////////////////////////////////////////////////////////////////////////////
146//     functions to  allocate / release  physical pages
147////////////////////////////////////////////////////////////////////////////////////////
[315]148
149///////////////////////////////////////////
[50]150void ppm_free_pages_nolock( page_t * page )
[1]151{
[7]152        page_t   * buddy;            // searched buddy page descriptor
153        uint32_t   buddy_index;      // buddy page index
154        page_t   * current;          // current (merged) page descriptor
155        uint32_t   current_index;    // current (merged) page index
[50]156        uint32_t   current_order;    // current (merged) page order
[7]157
[160]158        ppm_t    * ppm         = &LOCAL_CLUSTER->ppm;
159        page_t   * pages_tbl   = ppm->pages_tbl;
[1]160
[492]161        assert( !page_is_flag( page , PG_FREE ) ,
[407]162    "page already released : ppn = %x\n" , ppm_page2ppn(XPTR(local_cxy,page)) );
[177]163
[492]164        assert( !page_is_flag( page , PG_RESERVED ) ,
[407]165    "reserved page : ppn = %x\n" , ppm_page2ppn(XPTR(local_cxy,page)) );
166
[160]167        // update released page descriptor flags
[1]168        page_set_flag( page , PG_FREE );
169
[160]170        // search the buddy page descriptor
171        // - merge with current page descriptor if found
172        // - exit to release the current page descriptor if not found
173        current       = page ,
174        current_index = (uint32_t)(page - ppm->pages_tbl);
[18]175        for( current_order = page->order ;
[160]176             current_order < CONFIG_PPM_MAX_ORDER ;
177             current_order++ )
178        {
[7]179                buddy_index = current_index ^ (1 << current_order);
180                buddy       = pages_tbl + buddy_index;
[18]181
[7]182                if( !page_is_flag( buddy , PG_FREE ) || (buddy->order != current_order) ) break;
[1]183
[160]184                // remove buddy from free list
[7]185                list_unlink( &buddy->list );
[1]186                ppm->free_pages_nr[current_order] --;
[18]187
[160]188                // merge buddy with current
[7]189                buddy->order = 0;
190                current_index &= buddy_index;
[1]191        }
[18]192
[160]193        // update merged page descriptor order
[7]194        current        = pages_tbl + current_index;
195        current->order = current_order;
[1]196
[160]197        // insert current in free list
[7]198        list_add_first( &ppm->free_pages_root[current_order] , &current->list );
[1]199        ppm->free_pages_nr[current_order] ++;
200
[433]201}  // end ppm_free_pages_nolock()
202
[1]203////////////////////////////////////////////
204page_t * ppm_alloc_pages( uint32_t   order )
205{
[160]206        uint32_t   current_order;
[1]207        page_t   * remaining_block;
208        uint32_t   current_size;
[551]209
[438]210#if DEBUG_PPM_ALLOC_PAGES
[433]211uint32_t cycle = (uint32_t)hal_get_cycles();
[438]212if( DEBUG_PPM_ALLOC_PAGES < cycle )
[567]213printk("\n[DBG] in %s : thread %x in process %x enter for %d page(s) / cycle %d\n",
214__FUNCTION__, CURRENT_THREAD->trdid, CURRENT_THREAD->process->pid, 1<<order, cycle );
[433]215#endif
[1]216
[438]217#if(DEBUG_PPM_ALLOC_PAGES & 0x1)
218if( DEBUG_PPM_ALLOC_PAGES < cycle )
[433]219ppm_print();
220#endif
221
[160]222        ppm_t    * ppm = &LOCAL_CLUSTER->ppm;
[1]223
[492]224        assert( (order < CONFIG_PPM_MAX_ORDER) ,
[407]225    "illegal order argument = %x\n" , order );
[1]226
[406]227        page_t * block = NULL; 
[1]228
[160]229        // take lock protecting free lists
[567]230        busylock_acquire( &ppm->free_lock );
[1]231
[160]232        // find a free block equal or larger to requested size
[1]233        for( current_order = order ; current_order < CONFIG_PPM_MAX_ORDER ; current_order ++ )
234        {
235                if( !list_is_empty( &ppm->free_pages_root[current_order] ) )
236                {
237                        block = LIST_FIRST( &ppm->free_pages_root[current_order] , page_t , list );
238                        list_unlink( &block->list );
239                        break;
240                }
241        }
242
243        if( block == NULL ) // return failure
244        {
[160]245                // release lock protecting free lists
[567]246                busylock_release( &ppm->free_lock );
[1]247
[438]248#if DEBUG_PPM_ALLOC_PAGES
[433]249cycle = (uint32_t)hal_get_cycles();
[438]250if( DEBUG_PPM_ALLOC_PAGES < cycle )
[567]251printk("\n[DBG] in %s : thread %x in process %x cannot allocate %d page(s) / cycle %d\n",
252__FUNCTION__, CURRENT_THREAD->trdid, CURRENT_THREAD->process->pid, 1<<order, cycle );
[433]253#endif
254
[160]255                return NULL;
256        }
[18]257
[160]258        // update free-lists after removing a block
[18]259        ppm->free_pages_nr[current_order] --;
[1]260        current_size = (1 << current_order);
261
[160]262        // split the removed block in smaller sub-blocks if required
263        // and update the free-lists accordingly
[1]264        while( current_order > order )
265        {
266                current_order --;
267                current_size >>= 1;
[18]268
[1]269                remaining_block = block + current_size;
270                remaining_block->order = current_order;
271
272                list_add_first( &ppm->free_pages_root[current_order] , &remaining_block->list );
273                ppm->free_pages_nr[current_order] ++;
274        }
[18]275
[160]276        // update page descriptor
277        page_clear_flag( block , PG_FREE );
[1]278        page_refcount_up( block );
279        block->order = order;
280
[160]281        // release lock protecting free lists
[567]282        busylock_release( &ppm->free_lock );
[18]283
[585]284    // update DQDT
285    dqdt_increment_pages( order );
286
[438]287#if DEBUG_PPM_ALLOC_PAGES
[433]288cycle = (uint32_t)hal_get_cycles();
[438]289if( DEBUG_PPM_ALLOC_PAGES < cycle )
[567]290printk("\n[DBG] in %s : thread %x in process %x exit for %d page(s) / ppn = %x / cycle %d\n",
291__FUNCTION__, CURRENT_THREAD->trdid, CURRENT_THREAD->process->pid, 
2921<<order, ppm_page2ppn(XPTR( local_cxy , block )), cycle );
[433]293#endif
[7]294
[1]295        return block;
296
[433]297}  // end ppm_alloc_pages()
[1]298
[433]299
[1]300////////////////////////////////////
301void ppm_free_pages( page_t * page )
302{
303        ppm_t * ppm = &LOCAL_CLUSTER->ppm;
[18]304
[438]305#if DEBUG_PPM_FREE_PAGES
[433]306uint32_t cycle = (uint32_t)hal_get_cycles();
[438]307if( DEBUG_PPM_FREE_PAGES < cycle )
[567]308printk("\n[DBG] in %s : thread %x in process %x enter for %d page(s) / ppn %x / cycle %d\n",
309__FUNCTION__, CURRENT_THREAD->trdid, CURRENT_THREAD->process->pid, 
3101<<page->order, ppm_page2ppn(XPTR(local_cxy , page)), cycle );
[433]311#endif
312
[438]313#if(DEBUG_PPM_FREE_PAGES & 0x1)
314if( DEBUG_PPM_FREE_PAGES < cycle )
[433]315ppm_print();
316#endif
317
[160]318        // get lock protecting free_pages[] array
[567]319        busylock_acquire( &ppm->free_lock );
[1]320
[18]321        ppm_free_pages_nolock( page );
[1]322
[160]323        // release lock protecting free_pages[] array
[567]324        busylock_release( &ppm->free_lock );
[433]325
[585]326    // update DQDT
327    dqdt_decrement_pages( page->order );
328
[438]329#if DEBUG_PPM_FREE_PAGES
[433]330cycle = (uint32_t)hal_get_cycles();
[438]331if( DEBUG_PPM_FREE_PAGES < cycle )
[567]332printk("\n[DBG] in %s : thread %x in process %x exit for %d page(s) / ppn %x / cycle %d\n",
333__FUNCTION__, CURRENT_THREAD->trdid, CURRENT_THREAD->process->pid, 
3341<<page->order, ppm_page2ppn(XPTR(local_cxy , page)), cycle );
[433]335#endif
336
[567]337}  // end ppm_free_pages()
[1]338
[567]339//////////////////////
[486]340void ppm_print( void )
[1]341{
342        uint32_t       order;
343        list_entry_t * iter;
344        page_t       * page;
345
[433]346    ppm_t * ppm = &LOCAL_CLUSTER->ppm;
347
[160]348        // get lock protecting free lists
[567]349        busylock_acquire( &ppm->free_lock );
[1]350
[433]351        printk("\n***  PPM in cluster %x : %d pages ***\n", local_cxy , ppm->pages_nr );
[18]352
[1]353        for( order = 0 ; order < CONFIG_PPM_MAX_ORDER ; order++ )
354        {
[433]355                printk("- order = %d / free_pages = %d\t: ",
[160]356                       order , ppm->free_pages_nr[order] );
[18]357
[1]358                LIST_FOREACH( &ppm->free_pages_root[order] , iter )
359                {
360                        page = LIST_ELEMENT( iter , page_t , list );
[433]361                        printk("%x," , page - ppm->pages_tbl );
[1]362                }
[18]363
[433]364                printk("\n");
[1]365        }
366
[160]367        // release lock protecting free lists
[567]368        busylock_release( &ppm->free_lock );
[160]369}
[1]370
[53]371///////////////////////////////////////
372error_t ppm_assert_order( ppm_t * ppm )
[1]373{
374        uint32_t       order;
375        list_entry_t * iter;
376        page_t       * page;
[18]377
[407]378        for( order=0 ; order < CONFIG_PPM_MAX_ORDER ; order++ )
[1]379        {
380                if( list_is_empty( &ppm->free_pages_root[order] ) ) continue;
[18]381
[1]382                LIST_FOREACH( &ppm->free_pages_root[order] , iter )
383                {
384                        page = LIST_ELEMENT( iter , page_t , list );
[160]385                        if( page->order != order )  return -1;
[1]386                }
387        }
388
[160]389        return 0;
390}
[53]391
[567]392
393//////////////////////////////////////////////////////////////////////////////////////
394//     functions to handle  dirty physical pages
395//////////////////////////////////////////////////////////////////////////////////////
396
397/////////////////////////////////////////
398bool_t ppm_page_do_dirty( page_t * page )
399{
400        bool_t done = false;
401
402        ppm_t * ppm = &LOCAL_CLUSTER->ppm;
403
404        // lock the PPM dirty_list
405        queuelock_acquire( &ppm->dirty_lock );
406
407        if( !page_is_flag( page , PG_DIRTY ) )
408        {
409                // set dirty flag in page descriptor
410                page_set_flag( page , PG_DIRTY );
411
412                // register page in PPM dirty list
413                list_add_first( &ppm->dirty_root , &page->list );
414                done = true;
415        }
416
417        // unlock the PPM dirty_list
418        queuelock_release( &ppm->dirty_lock );
419
420        return done;
421}
422
423///////////////////////////////////////////
424bool_t ppm_page_undo_dirty( page_t * page )
425{
426        bool_t done = false;
427
428        ppm_t * ppm = &LOCAL_CLUSTER->ppm;
429
430        // lock the dirty_list
431        queuelock_acquire( &ppm->dirty_lock );
432
433        if( page_is_flag( page , PG_DIRTY) )
434        {
435                // clear dirty flag in page descriptor
436                page_clear_flag( page , PG_DIRTY );
437
438                // remove page from PPM dirty list
439                list_unlink( &page->list );
440                done = true;
441        }
442
443        // unlock the dirty_list
444        queuelock_release( &ppm->dirty_lock );
445
446        return done;
447}
448
449///////////////////////////////
450void ppm_sync_all_pages( void )
451{
452        page_t   * page;
453        ppm_t    * ppm = &LOCAL_CLUSTER->ppm;
454
455        // get the PPM dirty_list lock
456        queuelock_acquire( &ppm->dirty_lock );
457
458        while( !list_is_empty( &ppm->dirty_root ) )
459        {
460                page = LIST_FIRST( &ppm->dirty_root ,  page_t , list );
461
462                // get the page lock
463                remote_busylock_acquire( XPTR( local_cxy, &page->lock ) );
464
465                // sync the page
466                vfs_mapper_move_page( page , false );  // from mapper
467
468                // release the page lock
469                remote_busylock_release( XPTR( local_cxy , &page->lock ) );
470        }
471
472        // release the PPM dirty_list lock
473        queuelock_release( &ppm->dirty_lock );
474}
475
Note: See TracBrowser for help on using the repository browser.