/* * vmm.h - virtual memory management related operations * * Authors Ghassan Almaless (2008,2009,2010,2011, 2012) * Alain Greiner (2016,2017,2018,2019,2020)) * * Copyright (c) UPMC Sorbonne Universites * * This file is part of ALMOS-MKH. * * ALMOS-MKH is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by * the Free Software Foundation; version 2.0 of the License. * * ALMOS-MKH is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with ALMOS-MKH; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #ifndef _VMM_H_ #define _VMM_H_ #include #include #include #include #include #include #include #include /**** Forward declarations ****/ struct process_s; struct vseg_s; /********************************************************************************************* * This structure defines the STACK allocator used by the VMM to dynamically handle * vseg allocation or release requests for an user thread. * This allocator handles a fixed size array of fixed size slots in STACK zone of user space. * The stack size and the number of slots are defined by the CONFIG_VMM_STACK_SIZE, and * CONFIG_VMM_STACK_BASE parameters. * Each slot can contain one user stack vseg. The first 4 Kbytes page in the slot is not * mapped to detect stack overflow. * In this implementation, the slot index is defined by the user thead LTID. * All allocated stacks are registered in a bitmap defining the STACK zone state: * - The allocator checks that the requested slot has not been already allocated, and set the * corresponding bit in the bitmap. * - The de-allocator reset the corresponding bit in the bitmap. * * An architecture dependant hal_vmm_display() function is defined in file. ********************************************************************************************/ typedef struct stack_mgr_s { busylock_t lock; /*! lock protecting STACK allocator */ vpn_t vpn_base; /*! first page of STACK zone */ bitmap_t bitmap; /*! bit vector of allocated stacks */ } stack_mgr_t; /********************************************************************************************* * This structure defines the MMAP allocator used by the VMM to dynamically handle MMAP vsegs * requested or released by an user process. It must be called in the reference cluster. * - allocation policy : * This allocator implements the buddy algorithm. All allocated vsegs occupy an integer * number of pages, that is power of 2, and are aligned (vpn_base is multiple of vpn_size). * The requested number of pages is rounded if required. The global allocator state is * completely defined by the free_pages_root[] array indexed by the vseg order. * These free lists are local, but are implemented as xlist because we use the existing * vseg.xlist to register a free vseg in its free list. * - release policy : * A released vseg is recursively merged with the "buddy" vseg when it is free, in * order to build the largest possible aligned free vsegs. The resulting vseg.vpn_size * field is updated. * Implementation note: * The only significant (and documented) fiels in the vsegs registered in the MMAP allocator * free lists are "xlist", "vpn_base", and "vpn_size". ********************************************************************************************/ typedef struct mmap_mgr_s { busylock_t lock; /*! lock protecting MMAP allocator */ vpn_t vpn_base; /*! first page of MMAP zone */ vpn_t vpn_size; /*! number of pages in MMAP zone */ xlist_entry_t free_list_root[CONFIG_VMM_HEAP_MAX_ORDER + 1]; /* roots of free lists */ } mmap_mgr_t; /********************************************************************************************* * This structure defines the Virtual Memory Manager for a given process in a given cluster. * This VMM implements four main services: * 1) It contains the local copy of the vseg list (VSL), only complete in reference cluster. * 2) It contains the local copy of the generic page table (GPT), only complete in reference. * 3) The stack manager dynamically allocates virtual memory space for the STACK vsegs. * 4) The mmap manager dynamically allocates virtual memory for the (FILE/ANON/REMOTE) vsegs. ******************************************************a************************************** * Implementation notes: * 1. In most clusters, the VSL and GPT are only partial copies of the reference VSL and GPT * structures, stored in the reference cluster. * 2. The VSL contains only local vsegs, but it is implemented as an xlist, and protected by * a remote_rwlock, because it can be accessed by a thread running in a remote cluster. * An example is the vmm_fork_copy() function. * 3. The GPT in the reference cluster can be directly accessed by remote threads to handle * false page-fault (page is mapped in the reference GPT, but the PTE copy is missing * in the local GPT). As each PTE can be protected by a specific GPT_LOCKED attribute * for exclusive access, it is NOT protected by a global lock. ********************************************************************************************/ typedef struct vmm_s { remote_queuelock_t vsl_lock; /*! lock protecting the local VSL */ xlist_entry_t vsegs_root; /*! Virtual Segment List root */ uint32_t vsegs_nr; /*! total number of local vsegs */ gpt_t gpt; /*! Generic Page Table descriptor */ stack_mgr_t stack_mgr; /*! embedded STACK vsegs allocator */ mmap_mgr_t mmap_mgr; /*! embedded MMAP vsegs allocator */ uint32_t false_pgfault_nr; /*! false page fault counter (for all threads) */ uint32_t local_pgfault_nr; /*! false page fault counter (for all threads) */ uint32_t global_pgfault_nr; /*! false page fault counter (for all threads) */ uint32_t false_pgfault_cost; /*! cumulated cost (for all threads) */ uint32_t local_pgfault_cost; /*! cumulated cost (for all threads) */ uint32_t global_pgfault_cost; /*! cumulated cost (for all threads) */ vpn_t args_vpn_base; /*! args vseg first page */ vpn_t envs_vpn_base; /*! envs vseg first page */ vpn_t code_vpn_base; /*! code vseg first page */ vpn_t data_vpn_base; /*! data vseg first page */ vpn_t heap_vpn_base; /*! heap zone first page */ intptr_t entry_point; /*! main thread entry point */ } vmm_t; /********************************************************************************************* * This function makes a partial initialisation of the VMM attached to an user process. * It intializes the STACK and MMAP allocators, the VSL lock, and the instrumentation * counters, but it does not register any vseg in the VSL: * - The "kernel" vsegs are registered, by the hal_vmm_kernel_update() function. * - The "args" & "envs" vsegs are registered by the process_make_exec() function. * - The "code" and "data" vsegs are registered by the elf_load_process() function. * - The "stack" vsegs are registered by process_make_exec() / thread_user_create(). * - The "file", "anon", "remote" vsegs are dynamically registered by the mmap() syscall. ********************************************************************************************* * @ process : pointer on process descriptor * @ return 0 if success / return -1 if failure. ********************************************************************************************/ error_t vmm_user_init( struct process_s * process ); /********************************************************************************************* * This function is called by the process_make_exec() function to re-initialises the VMM * attached to an user process: * - It removes from the VMM of the process identified by the argument all user * vsegs, by calling the vmm_remove_vseg() function: the vsegs are removed from the VSL, * the corresponding GPT entries are removed from the GPT, and the physical pages are * released to the relevant kmem when they are not shared. * - The VSL and the GPT are not modified for the kernel vsegs. * - Finally, it calls the vmm_user_init() function to re-initialize the STAK and MMAP * allocators, and the lock protecting GPT. ********************************************************************************************* * @ process : pointer on process descriptor. ********************************************************************************************/ void vmm_user_reset( struct process_s * process ); /********************************************************************************************* * This function is called by the process_make_fork() function. It partially copies * the content of a remote parent process VMM to the local child process VMM: * - The KERNEL vsegs required by the architecture must have been previously * created in the child VMM, using the hal_vmm_kernel_update() function. * - The DATA, ANON, REMOTE vsegs registered in the parent VSL are registered in the * child VSL. All valid PTEs in parent GPT are copied to the child GPT. * The WRITABLE and COW flags are not modified, as it will be done later for those * shared pages by the vmm_set_cow() function. * - The CODE vsegs registered in the parent VSL are registered in the child VSL, but the * GPT entries are not copied in the child GPT, and will be dynamically updated from * the .elf file when a page fault is reported. * - The FILE vsegs registered in the parent VSL are registered in the child VSL, and all * valid GPT entries in parent GPT are copied to the child GPT. The COW flag is not set. * - No STACK vseg is copied from parent VMM to child VMM: the child stack vseg is copied * later from the cluster containing the user thread requesting the fork(). ********************************************************************************************* * @ child_process : local pointer on local child process descriptor. * @ parent_process_xp : extended pointer on remote parent process descriptor. * @ return 0 if success / return -1 if failure. ********************************************************************************************/ error_t vmm_fork_copy( struct process_s * child_process, xptr_t parent_process_xp ); /********************************************************************************************* * This function is called by the process_make_fork() function to update the COW attribute * in the parent parent process vsegs. It set the COW flag, and reset the WRITABLE flag of * all GPT entries of the DATA, MMAP, and REMOTE vsegs of the argument. * It must be called by a thread running in the reference cluster, that contains the complete * VSL and GPT (use the rpc_vmm_set_cow_client() when the calling thread client is remote). * It updates all copies of the process in all clusters, to maintain coherence in GPT copies, * using the list of copies stored in the owner process, and using remote_write accesses to * update the remote GPTs. It atomically increment the pending_fork counter, in all involved * physical page descriptors. It cannot fail, as only mapped entries in GPTs are updated. ********************************************************************************************* * @ process : local pointer on local reference process descriptor. ********************************************************************************************/ void vmm_set_cow( struct process_s * process ); /********************************************************************************************* * This function modifies the vseg identified by and arguments in all * clusters containing a VSL copy, as defined by and arguments. * The new vseg, defined by the and arguments must be included * in the existing vseg. The target VSL size and base fields are modified in the VSL. * This is done in all clusters containing a VMM copy to maintain VMM coherence. * It is called by the sys_munmap() and dev_fbf_resize_window() functions. * It can be called by a thread running in any cluster, as it uses the vmm_resize_vseg() in * the local cluster, and parallel RPC_VMM_RESIZE_VSEG for remote clusters. * It cannot fail, as only vseg registered in VSL copies are updated. ********************************************************************************************* * @ process : local pointer on process descriptor. * @ base : current vseg base address in user space. * @ new_base : new vseg base. * @ new_size : new vseg size. ********************************************************************************************/ void vmm_global_resize_vseg( struct process_s * process, intptr_t base, intptr_t new_base, intptr_t new_size ); /********************************************************************************************* * This function removes the vseg identified by the and arguments from * the VSL and remove all associated PTE entries from the GPT. * This is done in all clusters containing a VMM copy to maintain VMM coherence. * It is called by the sys_munmap() and dev_fbf_resize_window() functions. * It can be called by a thread running in any cluster, as it uses the vmm_remove_vseg() in * the local cluster, and parallel RPC_VMM_REMOVE_VSEG for remote clusters. * It cannot fail, as only vseg registered in VSL copies are deleted. ********************************************************************************************* * @ pid : local pointer on process identifier. * @ base : vseg base address in user space. ********************************************************************************************/ void vmm_global_delete_vseg( struct process_s * process, intptr_t base ); /********************************************************************************************* * This function modifies one GPT entry identified by the and arguments * in all clusters containing a process copy. It maintains coherence in GPT copies, * using remote_write accesses. * It cannot fail, as only mapped PTE2 in GPT copies are updated. ********************************************************************************************* * @ process : local pointer on local process descriptor. * @ vpn : PTE index. * @ attr : PTE / attributes. * @ ppn : PTE / physical page index. ********************************************************************************************/ void vmm_global_update_pte( struct process_s * process, vpn_t vpn, uint32_t attr, ppn_t ppn ); /********************************************************************************************* * This function deletes, in the local cluster, all vsegs registered in the VSL * of the process identified by the argument. For each vseg: * - it unmaps all vseg PTEs from the GPT (release the physical pages when required). * - it removes the vseg from the local VSL. * - it releases the memory allocated to the local vseg descriptors. * - it releases the memory allocated to the GPT itself. ********************************************************************************************* * @ process : pointer on process descriptor. ********************************************************************************************/ void vmm_destroy( struct process_s * process ); /********************************************************************************************* * This function scans the list of vsegs registered in the VMM of a given process descriptor * to check if a given virtual region (defined by a base and size) overlap an existing vseg. ********************************************************************************************* * @ process : pointer on process descriptor. * @ base : region virtual base address. * @ size : region size (bytes). * @ returns NULL if no conflict / return conflicting vseg pointer if conflict. ********************************************************************************************/ vseg_t * vmm_check_conflict( struct process_s * process, vpn_t base, vpn_t size ); /********************************************************************************************* * This function allocates memory for a vseg descriptor, initialises it, and register it * in the VSL of the local process descriptor. * - For the FILE, ANON, & REMOTE types, it does not use the argument, but uses * the specific VMM MMAP allocator. * - For the STACK type, it does not use the and arguments, but uses the * the specific VMM STACK allocator. * It checks collision with pre-existing vsegs. * To comply with the "on-demand" paging policy, this function does NOT modify the GPT, * and does not allocate physical memory for vseg data. * It should be called by a local thread (could be a RPC thread if the client thread is not * running in the reference cluster). ********************************************************************************************* * @ process : pointer on local processor descriptor. * @ type : vseg type. * @ base : vseg base address (or user thread ltid for an user stack vseg). * @ size : vseg size (bytes). * @ file_offset : offset in file for CODE, DATA, FILE types. * @ file_size : can be smaller than "size" for DATA type. * @ mapper_xp : extended pointer on mapper for CODE, DATA, FILE types. * @ cxy : physical mapping cluster (for non distributed vsegs). * @ returns pointer on vseg if success / returns NULL if no memory, or conflict. ********************************************************************************************/ vseg_t * vmm_create_vseg( struct process_s * process, vseg_type_t type, intptr_t base, uint32_t size, uint32_t file_offset, uint32_t file_size, xptr_t mapper_xp, cxy_t cxy ); /********************************************************************************************* * This function removes from the VMM of a local process descriptor, identified by * the argument, the vseg identified by the argument. * It is called by the vmm_user_reset(), vmm_global_delete_vseg(), vmm_destroy() functions. * It must be called by a local thread, running in the cluster containing the modified VMM. * Use the RPC_VMM_REMOVE_VSEG if required. * It makes a kernel panic if the process is not registered in the local cluster. * For all vseg types, the vseg is detached from local VSL, and all associated PTEs are * unmapped from local GPT. Other actions depend on the vseg type: * Regarding the vseg descriptor release: * . for ANON and REMOTE, the vseg is not released, but registered in local zombi_list. * . for STACK the vseg is released to the local stack allocator. * . for all other types, the vseg descriptor is released to the local kmem. * Regarding the physical pages release: * . for KERNEL and FILE, the pages are not released to kmem. * . for CODE and STACK, the pages are released to local kmem. * . for DATA, ANON and REMOTE, the pages are released to relevant kmem only when * the local cluster is the reference cluster. * The VSL lock protecting the VSL must be taken by the caller. ********************************************************************************************* * @ process : local pointer on process descriptor. * @ vseg : local pointer on target vseg. ********************************************************************************************/ void vmm_remove_vseg( struct process_s * process, struct vseg_s * vseg ); /********************************************************************************************* * This function resize a local vseg identified by the and arguments. * Both the "size" and "base" fields are modified in the process VSL. When the new vseg * contains less pages than the target vseg, the relevant pages are removed from the GPT. * It is called by the vmm_global_resize() and dev_fbf_resize_window() functions. * It must be called by a local thread, running in the cluster containing the modified VSL. * Use the RPC_VMM_RESIZE_VSEG if required. * The VSL lock protecting the VSL must be taken by the caller. ********************************************************************************************* * @ process : local pointer on process descriptor * @ vseg : local pointer on target vseg * @ new_base : vseg base address * @ new_size : vseg size (bytes) ********************************************************************************************/ void vmm_resize_vseg( struct process_s * process, struct vseg_s * vseg, intptr_t new_base, intptr_t new_size ); /********************************************************************************************* * This function checks that a given virtual address in a given is * contained in a registered vseg. It can be called by any thread running in any cluster. * - if the vseg is registered in the local process VSL, it returns the local vseg pointer. * - if the vseg is missing in local VSL, it access directly the reference VSL. * - if the vseg is found in reference VSL, it updates the local VSL and returns this pointer. * It returns an error when the vseg is missing in the reference VMM, or when there is * not enough memory for a new vseg descriptor in the calling thread cluster. * For both the local and the reference VSL, it takes the VSL lock before scanning the VSL. ********************************************************************************************* * @ process : [in] pointer on process descriptor. * @ vaddr : [in] virtual address. * @ vseg : [out] local pointer on local vseg. * @ returns 0 if success / returns -1 if user error ********************************************************************************************/ error_t vmm_get_vseg( struct process_s * process, intptr_t vaddr, vseg_t ** vseg ); /********************************************************************************************* * This function is called by the generic exception handler in case of page-fault event, * detected for a given . The argument is used to access the relevant VMM. * It checks the missing VPN and returns an user error if it is not in a registered vseg. * For a legal VPN, there is actually 3 cases: * 1) if the missing VPN belongs to a private vseg (STACK or CODE segment types, non * replicated in all clusters), it allocates a new physical page, computes the attributes, * depending on vseg type, and updates directly the local GPT. * 2) if the missing VPN belongs to a public vseg, it can be a false page-fault, when the VPN * is mapped in the reference GPT, but not in the local GPT. For this false page-fault, * the local GPT is simply updated from the reference GPT. * 3) if the missing VPN is public, and unmapped in the ref GPT, it is a true page fault. * The calling thread allocates a new physical page, computes the attributes, depending * on vseg type, and updates directly (without RPC) the local GPT and the reference GPT. * Other GPT copies will updated on demand. * Concurrent accesses to the GPT(s) are handled, by locking the target PTE before accessing * the local and/or reference GPT(s). ********************************************************************************************* * @ process : local pointer on local process. * @ vpn : VPN of the missing PTE. * @ returns EXCP_NON_FATAL / EXCP_USER_ERROR / EXCP_KERNEL_PANIC after analysis ********************************************************************************************/ error_t vmm_handle_page_fault( struct process_s * process, vpn_t vpn ); /********************************************************************************************* * This function is called by the generic exception handler in case of WRITE violation event, * detected for a given . The argument is used to access the relevant VMM. * It returns a kernel panic if the faulty VPN is not in a registered vseg, or is not mapped. * For a legal mapped vseg there is two cases: * 1) If the missing VPN belongs to a private vseg (STACK), it access only the local GPT. * It access the forks counter in the current physical page descriptor. * If there is a pending fork, it allocates a new physical page from the cluster defined * by the vseg type, copies the old physical page content to the new physical page, * and decrements the pending_fork counter in old physical page descriptor. * Finally, it reset the COW flag and set the WRITE flag in local GPT. * 2) If the missing VPN is public, it access only the reference GPT. * It access the forks counter in the current physical page descriptor. * If there is a pending fork, it allocates a new physical page from the cluster defined * by the vseg type, copies the old physical page content to the new physical page, * and decrements the pending_fork counter in old physical page descriptor. * Finally it calls the vmm_global_update_pte() function to reset the COW flag and set * the WRITE flag in all the GPT copies, using a RPC if the reference cluster is remote. * In both cases, concurrent accesses to the GPT are handled by locking the target PTE * before accessing the GPT. ********************************************************************************************* * @ process : pointer on local process descriptor copy. * @ vpn : VPN of the faulting PTE. * @ returns EXCP_NON_FATAL / EXCP_USER_ERROR / EXCP_KERNEL_PANIC after analysis ********************************************************************************************/ error_t vmm_handle_cow( struct process_s * process, vpn_t vpn ); /********************************************************************************************* * This function is called by the vmm_get_pte() function when a page is unmapped. * Depending on the vseg type, defined by the argument, it returns the PPN * (Physical Page Number) associated to a missing page defined by the argument. * - For the FILE type, it returns directly the physical page from the file mapper. * - For the CODE and DATA types, it allocates a new physical page from the cluster defined * by the cxy> field, or by the MSB bits for a distributed vseg, * and initialize this page from the .elf file mapper. * - For all other types, it allocates a new physical page from the cluster defined * by the cxy> field, or by the MSB bits for a distributed vseg, * but the new page is not initialized. ********************************************************************************************* * @ vseg : local pointer on vseg containing the mising page. * @ vpn : Virtual Page Number identifying the missing page. * @ ppn : [out] returned Physical Page Number. * return 0 if success / return EINVAL or ENOMEM if error. ********************************************************************************************/ error_t vmm_get_one_ppn( vseg_t * vseg, vpn_t vpn, ppn_t * ppn ); #endif /* _VMM_H_ */