/* * mapper.c - Kernel cache for FS files or directories implementation. * * Authors Mohamed Lamine Karaoui (2015) * Alain Greiner (2016,2017,2018) * * Copyright (c) UPMC Sorbonne Universites * * This file is part of ALMOS-MKH. * * ALMOS-MKH is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by * the Free Software Foundation; version 2.0 of the License. * * ALMOS-MKH is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with ALMOS-MKH; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include ////////////////////////////////////////////// mapper_t * mapper_create( vfs_fs_type_t type ) { mapper_t * mapper; kmem_req_t req; error_t error; // allocate memory for mapper req.type = KMEM_MAPPER; req.size = sizeof(mapper_t); req.flags = AF_KERNEL | AF_ZERO; mapper = (mapper_t *)kmem_alloc( &req ); if( mapper == NULL ) { printk("\n[ERROR] in %s : no memory for mapper descriptor\n", __FUNCTION__ ); return NULL; } // initialize refcount & inode mapper->refcount = 0; mapper->inode = NULL; // initialize radix tree error = grdxt_init( &mapper->rt, CONFIG_MAPPER_GRDXT_W1, CONFIG_MAPPER_GRDXT_W2, CONFIG_MAPPER_GRDXT_W3 ); if( error ) { printk("\n[ERROR] in %s : cannot initialize radix tree\n", __FUNCTION__ ); req.type = KMEM_MAPPER; req.ptr = mapper; kmem_free( &req ); return NULL; } // initialize mapper type mapper->type = type; // initialize mapper lock remote_rwlock_init( XPTR( local_cxy , &mapper->lock ) , LOCK_MAPPER_STATE ); // initialize waiting threads xlist (empty) xlist_root_init( XPTR( local_cxy , &mapper->wait_root ) ); // initialize vsegs xlist (empty) xlist_root_init( XPTR( local_cxy , &mapper->vsegs_root ) ); return mapper; } // end mapper_create() //////////////////////////////////////// void mapper_destroy( mapper_t * mapper ) { page_t * page; uint32_t found_index = 0; uint32_t start_index = 0; kmem_req_t req; // scan radix tree do { // get page from radix tree page = (page_t *)grdxt_get_first( &mapper->rt , start_index , &found_index ); // release registered pages to PPM if( page != NULL ) { // remove page from mapper and release to PPM mapper_release_page( mapper , page ); // update start_key value for next page start_index = found_index; } } while( page != NULL ); // release the memory allocated to radix tree itself grdxt_destroy( &mapper->rt ); // release memory for mapper descriptor req.type = KMEM_MAPPER; req.ptr = mapper; kmem_free( &req ); } // end mapper_destroy() //////////////////////////////////////////////////// xptr_t mapper_remote_get_page( xptr_t mapper_xp, uint32_t page_id ) { error_t error; mapper_t * mapper_ptr; cxy_t mapper_cxy; xptr_t lock_xp; // extended pointer on mapper lock xptr_t page_xp; // extended pointer on searched page descriptor xptr_t rt_xp; // extended pointer on radix tree in mapper thread_t * this = CURRENT_THREAD; // get mapper cluster and local pointer mapper_ptr = GET_PTR( mapper_xp ); mapper_cxy = GET_CXY( mapper_xp ); #if DEBUG_MAPPER_GET_PAGE uint32_t cycle = (uint32_t)hal_get_cycles(); char name[CONFIG_VFS_MAX_NAME_LENGTH]; vfs_inode_t * inode = hal_remote_lpt( XPTR( mapper_cxy , &mapper_ptr->inode ) ); vfs_inode_get_name( XPTR( mapper_cxy , inode ) , name ); if( DEBUG_MAPPER_GET_PAGE < cycle ) printk("\n[%s] thread [%x,%x] enter for page %d of <%s> / cycle %d\n", __FUNCTION__, this->process->pid, this->trdid, page_id, name, cycle ); #endif // check thread can yield thread_assert_can_yield( this , __FUNCTION__ ); // build extended pointer on mapper lock and mapper rt lock_xp = XPTR( mapper_cxy , &mapper_ptr->lock ); rt_xp = XPTR( mapper_cxy , &mapper_ptr->rt ); // take mapper lock in READ_MODE remote_rwlock_rd_acquire( lock_xp ); // search page in radix tree page_xp = grdxt_remote_lookup( rt_xp , page_id ); // test mapper miss if( page_xp == XPTR_NULL ) // miss => try to handle it { // release the lock in READ_MODE and take it in WRITE_MODE remote_rwlock_rd_release( lock_xp ); remote_rwlock_wr_acquire( lock_xp ); // second test on missing page because the page status can be modified // by another thread, when passing from READ_MODE to WRITE_MODE. // from this point there is no concurrent accesses to mapper. page_xp = grdxt_remote_lookup( rt_xp , page_id ); if ( page_xp == XPTR_NULL ) // miss confirmed => handle it { #if (DEBUG_MAPPER_GET_PAGE & 1) if( DEBUG_MAPPER_GET_PAGE < cycle ) printk("\n[%s] missing page => load it from IOC device\n", __FUNCTION__ ); #endif if( mapper_cxy == local_cxy ) // mapper is local { error = mapper_handle_miss( mapper_ptr, page_id, &page_xp ); } else { rpc_mapper_handle_miss_client( mapper_cxy, mapper_ptr, page_id, &page_xp, &error ); } if ( error ) { printk("\n[ERROR] in %s : thread[%x,%x] cannot handle mapper miss\n", __FUNCTION__ , this->process->pid, this->trdid ); remote_rwlock_wr_release( lock_xp ); return XPTR_NULL; } } // release mapper lock from WRITE_MODE remote_rwlock_wr_release( lock_xp ); } else // hit { // release mapper lock from READ_MODE remote_rwlock_rd_release( lock_xp ); } #if DEBUG_MAPPER_GET_PAGE cycle = (uint32_t)hal_get_cycles(); if( DEBUG_MAPPER_GET_PAGE < cycle ) printk("\n[%s] thread[%x,%x] exit for page %d of <%s> / ppn %x / cycle %d\n", __FUNCTION__, this->process->pid, this->trdid, page_id, name, ppm_page2ppn( page_xp ), cycle ); #endif return page_xp; } // end mapper_remote_get_page() ////////////////////////////////////////////// error_t mapper_handle_miss( mapper_t * mapper, uint32_t page_id, xptr_t * page_xp ) { kmem_req_t req; page_t * page; error_t error; thread_t * this = CURRENT_THREAD; #if DEBUG_MAPPER_HANDLE_MISS uint32_t cycle = (uint32_t)hal_get_cycles(); char name[CONFIG_VFS_MAX_NAME_LENGTH]; vfs_inode_t * inode = mapper->inode; vfs_inode_get_name( XPTR( local_cxy , inode ) , name ); if( DEBUG_MAPPER_HANDLE_MISS < cycle ) printk("\n[%s] enter for page %d in <%s> / cycle %d\n", __FUNCTION__, page_id, name, cycle ); if( DEBUG_MAPPER_HANDLE_MISS & 1 ) grdxt_display( &mapper->rt , name ); #endif // allocate one page from the mapper cluster req.type = KMEM_PAGE; req.size = 0; req.flags = AF_NONE; page = kmem_alloc( &req ); if( page == NULL ) { printk("\n[ERROR] in %s : thread [%x,%x] cannot allocate page in cluster %x\n", __FUNCTION__ , this->process->pid, this->trdid , local_cxy ); return -1; } // initialize the page descriptor page_init( page ); page_set_flag( page , PG_INIT ); page_refcount_up( page ); page->mapper = mapper; page->index = page_id; // insert page in mapper radix tree error = grdxt_insert( &mapper->rt , page_id , page ); if( error ) { printk("\n[ERROR] in %s : thread[%x,%x] cannot insert page in mapper\n", __FUNCTION__ , this->process->pid, this->trdid ); mapper_release_page( mapper , page ); req.ptr = page; req.type = KMEM_PAGE; kmem_free(&req); return -1; } // launch I/O operation to load page from device to mapper error = vfs_fs_move_page( XPTR( local_cxy , page ) , true ); if( error ) { printk("\n[ERROR] in %s : thread[%x,%x] cannot load page from device\n", __FUNCTION__ , this->process->pid, this->trdid ); mapper_release_page( mapper , page ); req.ptr = page; req.type = KMEM_PAGE; kmem_free( &req ); return -1; } // set extended pointer on allocated page *page_xp = XPTR( local_cxy , page ); #if DEBUG_MAPPER_HANDLE_MISS cycle = (uint32_t)hal_get_cycles(); if( DEBUG_MAPPER_HANDLE_MISS < cycle ) printk("\n[%s] exit for page %d in <%s> / ppn %x / cycle %d\n", __FUNCTION__, page_id, name, ppm_page2ppn( *page_xp ), cycle ); if( DEBUG_MAPPER_HANDLE_MISS & 1 ) grdxt_display( &mapper->rt , name ); #endif return 0; } // end mapper_handle_miss() //////////////////////////////////////////// void mapper_release_page( mapper_t * mapper, page_t * page ) { // build extended pointer on mapper lock xptr_t mapper_lock_xp = XPTR( local_cxy , &mapper->lock ); // take mapper lock in WRITE_MODE remote_rwlock_wr_acquire( mapper_lock_xp ); // remove physical page from radix tree grdxt_remove( &mapper->rt , page->index ); // release mapper lock from WRITE_MODE remote_rwlock_wr_release( mapper_lock_xp ); // release page to PPM kmem_req_t req; req.type = KMEM_PAGE; req.ptr = page; kmem_free( &req ); } // end mapper_release_page() //////////////////////////////////////////// error_t mapper_move_user( mapper_t * mapper, bool_t to_buffer, uint32_t file_offset, void * buffer, uint32_t size ) { xptr_t mapper_xp; // extended pointer on local mapper uint32_t page_offset; // first byte to move to/from a mapper page uint32_t page_count; // number of bytes to move to/from a mapper page uint32_t page_id; // current mapper page index uint32_t done; // number of moved bytes xptr_t page_xp; // extended pointer on current mapper page descriptor #if DEBUG_MAPPER_MOVE_USER uint32_t cycle = (uint32_t)hal_get_cycles(); thread_t * this = CURRENT_THREAD; if( DEBUG_MAPPER_MOVE_USER < cycle ) printk("\n[%s] thread[%x,%x] : to_buf %d / buffer %x / size %d / offset %d / cycle %d\n", __FUNCTION__, this->process->pid, this->trdid, to_buffer, buffer, size, file_offset, cycle ); #endif // build extended pointer on mapper mapper_xp = XPTR( local_cxy , mapper ); // compute offsets of first and last bytes in file uint32_t min_byte = file_offset; uint32_t max_byte = file_offset + size - 1; // compute indexes of pages for first and last byte in mapper uint32_t first = min_byte >> CONFIG_PPM_PAGE_SHIFT; uint32_t last = max_byte >> CONFIG_PPM_PAGE_SHIFT; #if (DEBUG_MAPPER_MOVE_USER & 1) if( DEBUG_MAPPER_MOVE_USER < cycle ) printk("\n[%s] first_page %d / last_page %d\n", __FUNCTION__, first, last ); #endif done = 0; // loop on pages in mapper for( page_id = first ; page_id <= last ; page_id++ ) { // compute page_offset if( page_id == first ) page_offset = min_byte & CONFIG_PPM_PAGE_MASK; else page_offset = 0; // compute number of bytes in page if ( first == last ) page_count = size; else if ( page_id == first ) page_count = CONFIG_PPM_PAGE_SIZE - page_offset; else if ( page_id == last ) page_count = (max_byte & CONFIG_PPM_PAGE_MASK) + 1; else page_count = CONFIG_PPM_PAGE_SIZE; #if (DEBUG_MAPPER_MOVE_USER & 1) if( DEBUG_MAPPER_MOVE_USER < cycle ) printk("\n[%s] page_id = %d / page_offset = %d / page_count = %d\n", __FUNCTION__ , page_id , page_offset , page_count ); #endif // get extended pointer on page descriptor page_xp = mapper_remote_get_page( mapper_xp , page_id ); if ( page_xp == XPTR_NULL ) return -1; // compute pointer in mapper xptr_t base_xp = ppm_page2base( page_xp ); uint8_t * map_ptr = (uint8_t *)GET_PTR( base_xp ) + page_offset; // compute pointer in buffer uint8_t * buf_ptr = (uint8_t *)buffer + done; // move fragment if( to_buffer ) { hal_copy_to_uspace( buf_ptr , map_ptr , page_count ); } else { ppm_page_do_dirty( page_xp ); hal_copy_from_uspace( map_ptr , buf_ptr , page_count ); } done += page_count; } #if DEBUG_MAPPER_MOVE_USER cycle = (uint32_t)hal_get_cycles(); if( DEBUG_MAPPER_MOVE_USER < cycle ) printk("\n[%s] thread[%x,%x] exit / cycle %d\n", __FUNCTION__, this->process->pid, this->trdid, cycle ); #endif return 0; } // end mapper_move_user() //////////////////////////////////////////////// error_t mapper_move_kernel( xptr_t mapper_xp, bool_t to_buffer, uint32_t file_offset, xptr_t buffer_xp, uint32_t size ) { uint32_t page_offset; // first byte to move to/from a mapper page uint32_t page_count; // number of bytes to move to/from a mapper page uint32_t page_id; // current mapper page index uint32_t done; // number of moved bytes xptr_t page_xp; // extended pointer on current mapper page descriptor uint8_t * src_ptr; // source buffer local pointer cxy_t src_cxy; // source cluster uint8_t * dst_ptr; // destination buffer local pointer cxy_t dst_cxy; // destination cluster // get buffer cluster and local pointer cxy_t buffer_cxy = GET_CXY( buffer_xp ); uint8_t * buffer_ptr = GET_PTR( buffer_xp ); // get mapper cluster cxy_t mapper_cxy = GET_CXY( mapper_xp ); #if DEBUG_MAPPER_MOVE_KERNEL uint32_t cycle = (uint32_t)hal_get_cycles(); thread_t * this = CURRENT_THREAD; if( DEBUG_MAPPER_MOVE_KERNEL < cycle ) printk("\n[%s] thread[%x,%x] enter / to_buf %d / buf_cxy %x / buf_ptr %x / cycle %d\n", __FUNCTION__, this->process->pid, this->trdid, to_buffer, buffer_cxy, buffer_ptr, cycle ); #endif // compute offsets of first and last bytes in file uint32_t min_byte = file_offset; uint32_t max_byte = file_offset + size -1; // compute indexes for first and last pages in mapper uint32_t first = min_byte >> CONFIG_PPM_PAGE_SHIFT; uint32_t last = max_byte >> CONFIG_PPM_PAGE_SHIFT; #if (DEBUG_MAPPER_MOVE_KERNEL & 1) if( DEBUG_MAPPER_MOVE_KERNEL < cycle ) printk("\n[%s] first_page %d / last_page %d\n", __FUNCTION__, first, last ); #endif // compute source and destination clusters if( to_buffer ) { dst_cxy = buffer_cxy; src_cxy = mapper_cxy; } else { src_cxy = buffer_cxy; dst_cxy = mapper_cxy; } done = 0; // loop on pages in mapper for( page_id = first ; page_id <= last ; page_id++ ) { // compute page_offset if( page_id == first ) page_offset = min_byte & CONFIG_PPM_PAGE_MASK; else page_offset = 0; // compute number of bytes to move in page if ( first == last ) page_count = size; else if ( page_id == first ) page_count = CONFIG_PPM_PAGE_SIZE - page_offset; else if ( page_id == last ) page_count = (max_byte & CONFIG_PPM_PAGE_MASK) + 1; else page_count = CONFIG_PPM_PAGE_SIZE; #if (DEBUG_MAPPER_MOVE_KERNEL & 1) if( DEBUG_MAPPER_MOVE_KERNEL < cycle ) printk("\n[%s] page_id = %d / offset = %d / bytes = %d\n", __FUNCTION__ , page_id , page_offset , page_count ); #endif // get extended pointer on page descriptor page_xp = mapper_remote_get_page( mapper_xp , page_id ); if ( page_xp == XPTR_NULL ) return -1; // get page base address xptr_t base_xp = ppm_page2base( page_xp ); uint8_t * base_ptr = (uint8_t *)GET_PTR( base_xp ); // compute source and destination pointers if( to_buffer ) { dst_ptr = buffer_ptr + done; src_ptr = base_ptr + page_offset; } else { src_ptr = buffer_ptr + done; dst_ptr = base_ptr + page_offset; ppm_page_do_dirty( page_xp ); } // move fragment hal_remote_memcpy( XPTR( dst_cxy , dst_ptr ), XPTR( src_cxy , src_ptr ), page_count ); done += page_count; } #if DEBUG_MAPPER_MOVE_KERNEL cycle = (uint32_t)hal_get_cycles(); if( DEBUG_MAPPER_MOVE_KERNEL < cycle ) printk("\n[%s] thread[%x,%x] exit / to_buf %d / buf_cxy %x / buf_ptr %x / cycle %d\n", __FUNCTION__, this->process->pid, this->trdid, to_buffer, buffer_cxy, buffer_ptr, cycle ); #endif return 0; } // end mapper_move_kernel() /////////////////////////////////////////////////// error_t mapper_remote_get_32( xptr_t mapper_xp, uint32_t word_id, uint32_t * p_value ) { uint32_t page_id; // page index in file uint32_t local_id; // word index in page xptr_t page_xp; // extended pointer on searched page descriptor xptr_t base_xp; // extended pointer on searched page base // get page index and local word index page_id = word_id >> 10; local_id = word_id & 0x3FF; // get page containing the searched word page_xp = mapper_remote_get_page( mapper_xp , page_id ); if( page_xp == XPTR_NULL ) return -1; // get page base base_xp = ppm_page2base( page_xp ); // get the value from mapper *p_value = hal_remote_l32( base_xp + (local_id<<2) ); return 0; } // end mapper_remote_get_32() /////////////////////////////////////////////////// error_t mapper_remote_set_32( xptr_t mapper_xp, uint32_t word_id, uint32_t value ) { uint32_t page_id; // page index in file uint32_t local_id; // word index in page xptr_t page_xp; // extended pointer on searched page descriptor xptr_t base_xp; // extended pointer on searched page base // get page index and local vord index page_id = word_id >> 10; local_id = word_id & 0x3FF; // get page containing the searched word page_xp = mapper_remote_get_page( mapper_xp , page_id ); if( page_xp == XPTR_NULL ) return -1; // get page base base_xp = ppm_page2base( page_xp ); // set value to mapper hal_remote_s32( (base_xp + (local_id << 2)) , value ); // set the dirty flag ppm_page_do_dirty( page_xp ); return 0; } // end mapper_remote_set_32()