source: trunk/kernel/kern/process.c @ 428

/*
 * process.c - process related management
 *
 * Authors  Ghassan Almaless (2008,2009,2010,2011,2012)
 *          Mohamed Lamine Karaoui (2015)
 *          Alain Greiner (2016,2017)
 *
 * Copyright (c) UPMC Sorbonne Universites
 *
 * This file is part of ALMOS-MKH.
 *
 * ALMOS-MKH is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by
 * the Free Software Foundation; version 2.0 of the License.
 *
 * ALMOS-MKH is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with ALMOS-MKH; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include <kernel_config.h>
#include <hal_types.h>
#include <hal_remote.h>
#include <hal_uspace.h>
#include <hal_irqmask.h>
#include <errno.h>
#include <printk.h>
#include <memcpy.h>
#include <bits.h>
#include <kmem.h>
#include <page.h>
#include <vmm.h>
#include <vfs.h>
#include <core.h>
#include <thread.h>
#include <chdev.h>
#include <list.h>
#include <string.h>
#include <scheduler.h>
#include <remote_spinlock.h>
#include <dqdt.h>
#include <cluster.h>
#include <ppm.h>
#include <boot_info.h>
#include <process.h>
#include <elf.h>
#include <syscalls.h>
#include <signal.h>

//////////////////////////////////////////////////////////////////////////////////////////
// Extern global variables
//////////////////////////////////////////////////////////////////////////////////////////

extern process_t           process_zero;     // allocated in kernel_init.c
extern chdev_directory_t   chdev_dir;        // allocated in kernel_init.c

//////////////////////////////////////////////////////////////////////////////////////////
// Process initialisation related functions
//////////////////////////////////////////////////////////////////////////////////////////

///////////////////////////
process_t * process_alloc()
{
        kmem_req_t   req;

    req.type  = KMEM_PROCESS;
        req.size  = sizeof(process_t);
        req.flags = AF_KERNEL;

    return (process_t *)kmem_alloc( &req );
}

////////////////////////////////////////
void process_free( process_t * process )
{
    kmem_req_t  req;

        req.type = KMEM_PROCESS;
        req.ptr  = process;
        kmem_free( &req );
}

/////////////////////////////////////////////////
void process_reference_init( process_t * process,
                             pid_t       pid,
                             xptr_t      parent_xp,
                             xptr_t      model_xp )
{
    cxy_t       parent_cxy;
    process_t * parent_ptr;
    cxy_t       model_cxy;
    process_t * model_ptr;
    xptr_t      stdin_xp;
    xptr_t      stdout_xp;
    xptr_t      stderr_xp;
    uint32_t    stdin_id;
    uint32_t    stdout_id;
    uint32_t    stderr_id;
    error_t     error;
    uint32_t    txt_id;
    char        rx_path[40];
    char        tx_path[40];
    xptr_t      chdev_xp;
    chdev_t *   chdev_ptr;
    cxy_t       chdev_cxy;
    pid_t       model_pid;
    pid_t       parent_pid;

    // get model process cluster and local pointer
    model_cxy = GET_CXY( model_xp );
    model_ptr = (process_t *)GET_PTR( model_xp );

    // get parent process cluster and local pointer
    parent_cxy = GET_CXY( parent_xp );
    parent_ptr = (process_t *)GET_PTR( parent_xp );

    // get model_pid and parent_pid
    parent_pid = hal_remote_lw( XPTR( parent_cxy , &parent_ptr->pid ) );
    model_pid  = hal_remote_lw( XPTR( model_cxy  , &model_ptr->pid ) );

process_dmsg("\n[DBG] %s : core[%x,%d] enters / pid = %x / ppid = %x / model_pid = %x\n",
__FUNCTION__ , local_cxy , CURRENT_THREAD->core->lid , pid , parent_pid , model_pid );

    // initialize PID, REF_XP, PARENT_XP, and STATE
        process->pid       = pid;
    process->ref_xp    = XPTR( local_cxy , process );
    process->parent_xp = parent_xp;
    process->state     = PROCESS_STATE_RUNNING;

    // initialize vmm as empty 
    error = vmm_init( process );
    assert( (error == 0) , __FUNCTION__ , "cannot initialize VMM\n" );
 
process_dmsg("\n[DBG] %s : core[%x,%d] / vmm inialised as empty for process %x\n", 
__FUNCTION__ , local_cxy , CURRENT_THREAD->core->lid , pid );

    // initialize fd_array as empty
    process_fd_init( process );

    // define the stdin/stdout/stderr pseudo files <=> select a TXT terminal.
    // - if INIT (pid == 1)         => link to kernel TXT[0] 
    // - if KSH[i] (model_pid == 1) => allocate a free TXT[i] 
    // - if USER process            => same terminal as model

    if( (pid == 1) || (model_pid == 1)) // INIT or KSH process
    {
        if (pid == 1 )  txt_id = 0;                    // INIT
        else            txt_id = process_txt_alloc();  // KSH[i]

        // attach process to TXT[txt_id]
        process_txt_attach( process , txt_id ); 

        // build path to TXT_RX[i] and TXT_TX[i] chdevs
        snprintf( rx_path , 40 , "/dev/external/txt%d_rx", txt_id );
        snprintf( tx_path , 40 , "/dev/external/txt%d_tx", txt_id );

        // create stdin pseudo file          
        error = vfs_open( process,
                           rx_path,
                           O_RDONLY, 
                           0,                // FIXME chmod
                           &stdin_xp, 
                           &stdin_id );

        assert( (error == 0) , __FUNCTION__ , "cannot open stdin pseudo file" );
        assert( (stdin_id == 0) , __FUNCTION__ , "stdin index must be 0" );

        // create stdout pseudo file          
        error = vfs_open( process,
                           tx_path,
                           O_WRONLY, 
                           0,                // FIXME chmod
                           &stdout_xp, 
                           &stdout_id );

        assert( (error == 0) , __FUNCTION__ , "cannot open stdout pseudo file" );
        assert( (stdout_id == 1) , __FUNCTION__ , "stdout index must be 1" );

        // create stderr pseudo file          
        error = vfs_open( process,
                           tx_path,
                           O_WRONLY, 
                           0,                // FIXME chmod
                           &stderr_xp, 
                           &stderr_id );

        assert( (error == 0) , __FUNCTION__ , "cannot open stderr pseudo file" );
        assert( (stderr_id == 2) , __FUNCTION__ , "stderr index must be 2" );

    }
    else                                            // normal user process 
    {
        // get extended pointer on model process TXT chdev
        chdev_xp = chdev_from_file( model_ptr->fd_array.array[0] );
 
        // get cluster and local pointer on chdev
        chdev_cxy = GET_CXY( chdev_xp );
        chdev_ptr = (chdev_t *)GET_PTR( chdev_xp );
 
        // get TXT terminal index
        txt_id = hal_remote_lw( XPTR( chdev_cxy , &chdev_ptr->channel ) );

        // attach process to TXT[txt_id]
        process_txt_attach( process , txt_id ); 

        // copy all open files from model process fd_array to this process
        process_fd_remote_copy( XPTR( local_cxy , &process->fd_array ),
                                XPTR( model_cxy , &model_ptr->fd_array ) );
    }

    // initialize specific inodes root and cwd
    process->vfs_root_xp = (xptr_t)hal_remote_lwd( XPTR( model_cxy,
                                                         &model_ptr->vfs_root_xp ) );
    process->vfs_cwd_xp  = (xptr_t)hal_remote_lwd( XPTR( model_cxy,
                                                         &model_ptr->vfs_cwd_xp ) );
    vfs_inode_remote_up( process->vfs_root_xp );
    vfs_inode_remote_up( process->vfs_cwd_xp );

    remote_rwlock_init( XPTR( local_cxy , &process->cwd_lock ) );

process_dmsg("\n[DBG] %s : core[%x,%d] / fd array initialised for process %x\n", 
__FUNCTION__ , local_cxy , CURRENT_THREAD->core->lid , pid );

    // reset children list root 
    xlist_root_init( XPTR( local_cxy , &process->children_root ) );
    process->children_nr     = 0;
    remote_spinlock_init( XPTR( local_cxy , &process->children_lock ) );

    // reset semaphore / mutex / barrier / condvar list roots 
    xlist_root_init( XPTR( local_cxy , &process->sem_root ) );
    xlist_root_init( XPTR( local_cxy , &process->mutex_root ) );
    xlist_root_init( XPTR( local_cxy , &process->barrier_root ) );
    xlist_root_init( XPTR( local_cxy , &process->condvar_root ) );
    remote_spinlock_init( XPTR( local_cxy , &process->sync_lock ) );

    // register new process in the local cluster manager pref_tbl[]
    lpid_t lpid = LPID_FROM_PID( pid );
    LOCAL_CLUSTER->pmgr.pref_tbl[lpid] = XPTR( local_cxy , process );

    // register new process descriptor in local cluster manager local_list
    cluster_process_local_link( process );

    // register new process descriptor in local cluster manager copies_list 
    cluster_process_copies_link( process );

    // reset th_tbl[] array as empty in process descriptor
    uint32_t i;
    for( i = 0 ; i < CONFIG_THREAD_MAX_PER_CLUSTER ; i++ )
        {
        process->th_tbl[i] = NULL;
    }
    process->th_nr  = 0;
    spinlock_init( &process->th_lock );

        hal_fence();

process_dmsg("\n[DBG] %s : core[%x,%d] exit for process %x\n",
__FUNCTION__ , local_cxy , CURRENT_THREAD->core->lid , pid );

}  // process_reference_init()

/////////////////////////////////////////////////////
error_t process_copy_init( process_t * local_process,
                           xptr_t      reference_process_xp )
{
    error_t error;

    // get reference process cluster and local pointer
    cxy_t       ref_cxy = GET_CXY( reference_process_xp );
    process_t * ref_ptr = (process_t *)GET_PTR( reference_process_xp );

    // initialize PID, REF_XP, PARENT_XP, and STATE
    local_process->pid       = hal_remote_lw(  XPTR( ref_cxy , &ref_ptr->pid ) );
    local_process->parent_xp = hal_remote_lwd( XPTR( ref_cxy , &ref_ptr->parent_xp ) );
    local_process->ref_xp    = reference_process_xp;
    local_process->state     = PROCESS_STATE_RUNNING;

process_dmsg("\n[DBG] %s : core[%x,%d] enters for process %x\n",
__FUNCTION__ , local_cxy , CURRENT_THREAD->core->lid , local_process->pid );

    // reset local process vmm
    error = vmm_init( local_process );
    assert( (error == 0) , __FUNCTION__ , "cannot initialize VMM\n");

    // reset process file descriptors array
        process_fd_init( local_process );

    // reset vfs_root_xp / vfs_bin_xp / vfs_cwd_xp fields
    local_process->vfs_root_xp = hal_remote_lwd( XPTR( ref_cxy , &ref_ptr->vfs_root_xp ) );
    local_process->vfs_bin_xp  = hal_remote_lwd( XPTR( ref_cxy , &ref_ptr->vfs_bin_xp ) );
    local_process->vfs_cwd_xp  = XPTR_NULL;

    // reset children list root (not used in a process descriptor copy)
    xlist_root_init( XPTR( local_cxy , &local_process->children_root ) );
    local_process->children_nr   = 0;
    remote_spinlock_init( XPTR( local_cxy , &local_process->children_lock ) );

    // reset children_list (not used in a process descriptor copy)
    xlist_entry_init( XPTR( local_cxy , &local_process->children_list ) );

    // reset semaphores list root (not used in a process descriptor copy)
    xlist_root_init( XPTR( local_cxy , &local_process->sem_root ) );
    xlist_root_init( XPTR( local_cxy , &local_process->mutex_root ) );
    xlist_root_init( XPTR( local_cxy , &local_process->barrier_root ) );
    xlist_root_init( XPTR( local_cxy , &local_process->condvar_root ) );

    // reset th_tbl[] array as empty
    uint32_t i;
    for( i = 0 ; i < CONFIG_THREAD_MAX_PER_CLUSTER ; i++ )
        {
        local_process->th_tbl[i] = NULL;
    }
    local_process->th_nr  = 0;
    spinlock_init( &local_process->th_lock );

    // register new process descriptor in local cluster manager local_list
    cluster_process_local_link( local_process );

    // register new process descriptor in owner cluster manager copies_list
    cluster_process_copies_link( local_process );

        hal_fence();

process_dmsg("\n[DBG] %s : core[%x,%d] exit for process %x\n",
__FUNCTION__ , local_cxy , CURRENT_THREAD->core->lid , local_process->pid );

    return 0;

} // end process_copy_init()

///////////////////////////////////////////
void process_destroy( process_t * process )
{
    xptr_t      parent_xp;
    process_t * parent_ptr;
    cxy_t       parent_cxy;
    xptr_t      parent_thread_xp;
    xptr_t      children_lock_xp;
    xptr_t      copies_lock_xp;

        assert( (process->th_nr == 0) , __FUNCTION__ ,
    "process %x in cluster %x has still active threads", process->pid , local_cxy );

process_dmsg("\n[DBG] %s : core[%x,%d] enter for process %x\n",
__FUNCTION__ , local_cxy , CURRENT_THREAD->core->lid , process->pid );

    // get local process manager pointer
    pmgr_t * pmgr = &LOCAL_CLUSTER->pmgr;

    // remove process from local_list in cluster manager
    remote_spinlock_lock( XPTR( local_cxy , &pmgr->local_lock ) );
    xlist_unlink( XPTR( local_cxy , &process->local_list ) );
    remote_spinlock_unlock( XPTR( local_cxy , &pmgr->local_lock ) );

    // get extended pointer on copies_lock in owner cluster manager
    cxy_t  owner_cxy = CXY_FROM_PID( process->pid );
        lpid_t lpid      = LPID_FROM_PID( process->pid );
    copies_lock_xp   = XPTR( owner_cxy , &pmgr->copies_lock[lpid] );

    // remove local process from copies_list
    remote_spinlock_lock( copies_lock_xp );
    xlist_unlink( XPTR( local_cxy , &process->copies_list ) );
    remote_spinlock_unlock( copies_lock_xp );

    // for reference process only
    if( XPTR( local_cxy , process ) == process->ref_xp )
    {
        // remove reference process from txt_list
        process_txt_detach( process );

        // get pointers on parent process
        parent_xp  = process->parent_xp;
        parent_cxy = GET_CXY( parent_xp );
        parent_ptr = GET_PTR( parent_xp );

        // get extended pointer on children_lock in parent process
        children_lock_xp = XPTR( parent_cxy , &parent_ptr->children_lock );

        // remove process from children_list
        remote_spinlock_lock( children_lock_xp );
        xlist_unlink( XPTR( local_cxy , &process->children_list ) );
        remote_spinlock_unlock( children_lock_xp );

        // get extende pointer on parent main thread
        parent_thread_xp = XPTR( parent_cxy ,
                           hal_remote_lpt( XPTR( parent_cxy , &parent_ptr->th_tbl[1] )));
        
        // unblock parent process main thread
        thread_unblock( parent_thread_xp , THREAD_BLOCKED_WAIT );
    }

    // release the process PID to cluster manager
    cluster_pid_release( process->pid );

    // FIXME close all open files and update dirty [AG]

    // decrease refcount for bin file, root file and cwd file
        if( process->vfs_bin_xp  != XPTR_NULL ) vfs_file_count_down( process->vfs_bin_xp );
        if( process->vfs_root_xp != XPTR_NULL ) vfs_file_count_down( process->vfs_root_xp );
        if( process->vfs_cwd_xp  != XPTR_NULL ) vfs_file_count_down( process->vfs_cwd_xp );

    // Destroy VMM
    vmm_destroy( process );

    // release memory allocated to process descriptor
    process_free( process );

process_dmsg("\n[DBG] %s : core[%x,%d] exit for process %x\n",
__FUNCTION__ , local_cxy , CURRENT_THREAD->core->lid , process->pid );

}  // end process_destroy()

/////////////////////////////////////////////////
char * process_action_str( uint32_t action_type )
{
    if     ( action_type == BLOCK_ALL_THREADS   ) return "BLOCK";
    else if( action_type == UNBLOCK_ALL_THREADS ) return "UNBLOCK";
    else if( action_type == DELETE_ALL_THREADS  ) return "DELETE";
    else                                          return "undefined";
}

////////////////////////////////////////////
void process_sigaction( process_t * process,
                        uint32_t    action_type )
{
    cxy_t              owner_cxy;         // owner cluster identifier
    lpid_t             lpid;              // process index in owner cluster
    cluster_t        * cluster;           // pointer on cluster manager
    xptr_t             root_xp;           // extended pointer on root of copies
    xptr_t             lock_xp;           // extended pointer on lock protecting copies
    xptr_t             iter_xp;           // iterator on copies list
    xptr_t             process_xp;        // extended pointer on process copy
    cxy_t              process_cxy;       // process copy cluster identifier
    process_t        * process_ptr;       // local pointer on process copy
    uint32_t           responses;         // number of remote process copies
    uint32_t           rsp_count;         // used to assert number of copies

    rpc_desc_t         rpc;               // rpc descriptor allocated in stack

process_dmsg("\n[DBG] %s : enter to %s process %x in cluster %x\n",
__FUNCTION__ , process_action_str( action_type ) , process->pid , local_cxy );

    thread_t         * client = CURRENT_THREAD;
    xptr_t             client_xp = XPTR( local_cxy , client );

    // get local pointer on local cluster manager
    cluster = LOCAL_CLUSTER;

    // get owner cluster identifier and process lpid
    owner_cxy = CXY_FROM_PID( process->pid );
    lpid      = LPID_FROM_PID( process->pid );

    // check owner cluster
    assert( (owner_cxy == local_cxy) , __FUNCTION__ , "must be executed in owner cluster\n" ); 
   
    // get number of remote copies
    responses = cluster->pmgr.copies_nr[lpid] - 1;
    rsp_count = 0;

    // check action type
    assert( ((action_type == DELETE_ALL_THREADS ) ||
             (action_type == BLOCK_ALL_THREADS )  ||
             (action_type == UNBLOCK_ALL_THREADS )), __FUNCTION__ , "illegal action type" );
             
    // initialise rpc descriptor
    rpc.index    = RPC_PROCESS_SIGACTION;
    rpc.response = responses;
    rpc.blocking = false;
    rpc.thread   = client;

    // get extended pointers on copies root and lock
    root_xp   = XPTR( local_cxy , &cluster->pmgr.copies_root[lpid] );
    lock_xp   = XPTR( local_cxy , &cluster->pmgr.copies_lock[lpid] );

    // take the lock protecting the copies
    remote_spinlock_lock( lock_xp );

    // send RPCs to remote clusters
    XLIST_FOREACH( root_xp , iter_xp )
    {
        process_xp  = XLIST_ELEMENT( iter_xp , process_t , copies_list );
        process_cxy = GET_CXY( process_xp );
        process_ptr = (process_t *)GET_PTR( process_xp );

        // send RPC to remote clusters
        if( process_cxy != local_cxy )  
        {

process_dmsg("\n[DBG] %s : send RPC to remote cluster %x\n",
__FUNCTION__ , process_cxy );

            rpc.args[0] = (uint64_t)action_type;
            rpc.args[1] = (uint64_t)(intptr_t)process_ptr;
            rpc_process_sigaction_client( process_cxy , &rpc );
            rsp_count++;
        }
    }
   
    // release the lock protecting process copies
    remote_spinlock_unlock( lock_xp );

    // check number of copies...
    assert( (rsp_count == responses) , __FUNCTION__ ,
    "unconsistent number of process copies : rsp_count = %d / responses = %d",
    rsp_count , responses );

    // block and deschedule to wait RPC responses if required
    if( responses )
    {   
        thread_block( CURRENT_THREAD , THREAD_BLOCKED_RPC );
        sched_yield("BLOCKED on RPC_PROCESS_SIGACTION");
    }

process_dmsg("\n[DBG] %s : make action in owner cluster %x\n",
__FUNCTION__ , local_cxy );


    // call directly the relevant function in local owner cluster
    if      (action_type == DELETE_ALL_THREADS  ) process_delete_threads ( process , client_xp ); 
    else if (action_type == BLOCK_ALL_THREADS   ) process_block_threads  ( process , client_xp ); 
    else if (action_type == UNBLOCK_ALL_THREADS ) process_unblock_threads( process             );

process_dmsg("\n[DBG] %s : exit after %s process %x in cluster %x\n",
__FUNCTION__ , process_action_str( action_type ) , process->pid , local_cxy );

}  // end process_sigaction()

////////////////////////////////////////////////
void process_block_threads( process_t * process,
                            xptr_t      client_xp )
{
    thread_t          * target;         // pointer on target thread 
    uint32_t            ltid;           // index in process th_tbl
    thread_t          * requester;      // requesting thread pointer
    uint32_t            count;          // requests counter
    volatile uint32_t   rsp_count;      // responses counter

    // get calling thread pointer
    requester = CURRENT_THREAD;

sigaction_dmsg("\n[DBG] %s : enter for process %x in cluster %x\n",
__FUNCTION__ , process->pid , local_cxy );

    // get lock protecting process th_tbl[]
    spinlock_lock( &process->th_lock );

    // initialize local responses counter 
    rsp_count = process->th_nr;

    // loop on process threads to block and deschedule all threads in cluster
    // we use both "ltid" and "count" because it can exist "holes" in th_tbl
    for( ltid = 0 , count = 0 ; count < process->th_nr ; ltid++ )
    {
        target = process->th_tbl[ltid];

        if( target != NULL )             // thread found
        {
            count++;

            // - if the target thread is the client thread, we do nothing,
            //   and we simply decrement the responses counter.
            // - if the calling thread and the target thread are on the same core,
            //   we block the target thread, we don't need confirmation from scheduler,
            //   and we simply decrement the responses counter.
            // - if the calling thread and the target thread are not running on the same
            //   core, we ask the target scheduler to acknowlege the blocking
            //   to be sure that the target thread is not running.
            
            if( XPTR( local_cxy , target ) == client_xp )
            {
                // decrement responses counter
                hal_atomic_add( (void *)&rsp_count , -1 );
            }
            else if( requester->core->lid == target->core->lid )
            {
                // set the global blocked bit in target thread descriptor.
                thread_block( target , THREAD_BLOCKED_GLOBAL );

                // decrement responses counter
                hal_atomic_add( (void *)&rsp_count , -1 );
            }
            else
            {
                // set the global blocked bit in target thread descriptor.
                thread_block( target , THREAD_BLOCKED_GLOBAL );

                // set FLAG_REQ_ACK and &ack_rsp_count in target descriptor
                thread_set_req_ack( target , (void *)&rsp_count );

                // force scheduling on target thread
                dev_pic_send_ipi( local_cxy , target->core->lid );
            }
        }
    }

    // release lock protecting process th_tbl[]
    spinlock_unlock( &process->th_lock );

    // wait all responses from schedulers 
    while( 1 )
    {
        // exit loop when all local responses received
        if ( rsp_count == 0 ) break;
    
        // wait 1000 cycles before retry
        hal_fixed_delay( 1000 );
    }

sigaction_dmsg("\n[DBG] %s : exit for process %x in cluster %x / %d threads blocked\n",
__FUNCTION__ , process->pid , local_cxy , count );

}  // end process_block_threads()

///////////////////////////////////////////////////
void process_unblock_threads( process_t * process )
{
    thread_t          * target;        // pointer on target thead 
    uint32_t            ltid;          // index in process th_tbl
    uint32_t            count;         // requests counter

sigaction_dmsg("\n[DBG] %s : enter for process %x in cluster %x\n",
__FUNCTION__ , process->pid , local_cxy );

    // get lock protecting process th_tbl[]
    spinlock_lock( &process->th_lock );

    // loop on process threads to unblock all threads in cluster
    // we use both "ltid" and "count" because it can exist "holes" in th_tbl
    for( ltid = 0 , count = 0 ; count < process->th_nr ; ltid++ )
    {
        target = process->th_tbl[ltid];

        if( target != NULL )             // thread found
        {
            count++;

            // reset the global blocked bit in target thread descriptor.
            thread_unblock( XPTR( local_cxy , target ) , THREAD_BLOCKED_GLOBAL );
        }
    }

    // release lock protecting process th_tbl[]
    spinlock_unlock( &process->th_lock );

sigaction_dmsg("\n[DBG] %s : exit for process %x in cluster %x / %d threads blocked\n",
__FUNCTION__ , process->pid , local_cxy , count );

}  // end process_unblock_threads()

/////////////////////////////////////////////////
void process_delete_threads( process_t * process,
                             xptr_t      client_xp )
{
    thread_t          * target;        // pointer on target thread 
    uint32_t            ltid;          // index in process th_tbl
    uint32_t            count;         // request counter

sigaction_dmsg("\n[DBG] %s : enter for process %x in cluster %x at cycle %d\n",
__FUNCTION__ , process->pid , local_cxy , (uint32_t)hal_get_cycles() );

    // get lock protecting process th_tbl[]
    spinlock_lock( &process->th_lock );

    // loop on threads to set the REQ_DELETE flag
    // we use both "ltid" and "count" because it can exist "holes" in th_tbl
    for( ltid = 0 , count = 0  ; count < process->th_nr ; ltid++ )
    {
        target = process->th_tbl[ltid];

        if( target != NULL )             // thread found
        {
            count++;

            // delete only if the target is not the client
            if( XPTR( local_cxy , target ) != client_xp ) 
            { 
                hal_atomic_or( &target->flags , THREAD_FLAG_REQ_DELETE );
            }
        }
    }

    // release lock protecting process th_tbl[]
    spinlock_unlock( &process->th_lock );

sigaction_dmsg("\n[DBG] %s : exit for process %x in cluster %x at cycle %d\n",
__FUNCTION__ , process->pid , local_cxy , (uint32_t)hal_get_cycles() );

}  // end process_delete_threads()

///////////////////////////////////////////////
process_t * process_get_local_copy( pid_t pid )
{
    error_t        error;
    process_t    * process_ptr;   // local pointer on process
    xptr_t         process_xp;    // extended pointer on process

    cluster_t * cluster = LOCAL_CLUSTER;

    // get lock protecting local list of processes
    remote_spinlock_lock( XPTR( local_cxy , &cluster->pmgr.local_lock ) );

    // scan the local list of process descriptors to find the process
    xptr_t  iter;
    bool_t  found = false;
    XLIST_FOREACH( XPTR( local_cxy , &cluster->pmgr.local_root ) , iter )
    {
        process_xp  = XLIST_ELEMENT( iter , process_t , local_list );
        process_ptr = (process_t *)GET_PTR( process_xp );
        if( process_ptr->pid == pid )
        {
            found = true;
            break;
        }
    }

    // release lock protecting local list of processes
    remote_spinlock_unlock( XPTR( local_cxy , &cluster->pmgr.local_lock ) );

    // allocate memory for a new local process descriptor
    // and initialise it from reference cluster if required
    if( !found )
    {
        // get extended pointer on reference process descriptor
        xptr_t ref_xp = cluster_get_reference_process_from_pid( pid );

        assert( (ref_xp != XPTR_NULL) , __FUNCTION__ , "illegal pid\n" );

        // allocate memory for local process descriptor
        process_ptr = process_alloc();
        if( process_ptr == NULL )  return NULL;

        // initialize local process descriptor copy
        error = process_copy_init( process_ptr , ref_xp );
        if( error ) return NULL;
    }

    return process_ptr;

}  // end process_get_local_copy()

//////////////////////////////////////////////////////////////////////////////////////////
// File descriptor array related functions
//////////////////////////////////////////////////////////////////////////////////////////

///////////////////////////////////////////
void process_fd_init( process_t * process )
{
    uint32_t fd;

    remote_spinlock_init( XPTR( local_cxy , &process->fd_array.lock ) );

    process->fd_array.current = 0;

    // initialize array
    for ( fd = 0 ; fd < CONFIG_PROCESS_FILE_MAX_NR ; fd++ )
    {
        process->fd_array.array[fd] = XPTR_NULL;
    }
}

//////////////////////////////
bool_t process_fd_array_full()
{
    // get extended pointer on reference process
    xptr_t ref_xp = CURRENT_THREAD->process->ref_xp;

    // get reference process cluster and local pointer
    process_t * ref_ptr = (process_t *)GET_PTR( ref_xp );
    cxy_t       ref_cxy = GET_CXY( ref_xp );

    // get number of open file descriptors from reference fd_array
    uint32_t current = hal_remote_lw( XPTR( ref_cxy , &ref_ptr->fd_array.current ) );

        return ( current >= CONFIG_PROCESS_FILE_MAX_NR );
}

/////////////////////////////////////////////////
error_t process_fd_register( process_t * process,
                             xptr_t      file_xp,
                             uint32_t  * fdid )
{
    bool_t    found;
    uint32_t  id;
    xptr_t    xp;

    // get reference process cluster and local pointer
    xptr_t ref_xp = process->ref_xp;
    process_t * ref_ptr = (process_t *)GET_PTR( ref_xp );
    cxy_t       ref_cxy = GET_CXY( ref_xp );

    // take lock protecting reference fd_array
        remote_spinlock_lock( XPTR( ref_cxy , &ref_ptr->fd_array.lock ) );

    found   = false;

    for ( id = 0; id < CONFIG_PROCESS_FILE_MAX_NR ; id++ )
    {
        xp = hal_remote_lwd( XPTR( ref_cxy , &ref_ptr->fd_array.array[id] ) );
        if ( xp == XPTR_NULL )
        {
            found = true;
            hal_remote_swd( XPTR( ref_cxy , &ref_ptr->fd_array.array[id] ) , file_xp );
                hal_remote_atomic_add( XPTR( ref_cxy , &ref_ptr->fd_array.current ) , 1 );
                        *fdid = id;
            break;
        }
    }

    // release lock protecting reference fd_array
        remote_spinlock_unlock( XPTR( ref_cxy , &ref_ptr->fd_array.lock ) );

    if ( !found ) return -1;
    else          return 0;
}

////////////////////////////////////////////////
xptr_t process_fd_get_xptr( process_t * process,
                            uint32_t    fdid )
{
    xptr_t  file_xp;

    // access local copy of process descriptor
    file_xp = process->fd_array.array[fdid];

    if( file_xp == XPTR_NULL )
    {
        // get reference process cluster and local pointer
        xptr_t      ref_xp  = process->ref_xp;
        cxy_t       ref_cxy = GET_CXY( ref_xp );
        process_t * ref_ptr = (process_t *)GET_PTR( ref_xp );

        // access reference process descriptor
        file_xp = hal_remote_lwd( XPTR( ref_cxy , &ref_ptr->fd_array.array[fdid] ) );

        // update local fd_array if found
        if( file_xp != XPTR_NULL )
        {
            process->fd_array.array[fdid] = file_xp;
        }
    }

    return file_xp;

}  // end process_fd_get_xptr()

///////////////////////////////////////////
void process_fd_remote_copy( xptr_t dst_xp,
                             xptr_t src_xp )
{
    uint32_t fd;
    xptr_t   entry;

    // get cluster and local pointer for src fd_array
    cxy_t        src_cxy = GET_CXY( src_xp );
    fd_array_t * src_ptr = (fd_array_t *)GET_PTR( src_xp );

    // get cluster and local pointer for dst fd_array
    cxy_t        dst_cxy = GET_CXY( dst_xp );
    fd_array_t * dst_ptr = (fd_array_t *)GET_PTR( dst_xp );

    // get the remote lock protecting the src fd_array
        remote_spinlock_lock( XPTR( src_cxy , &src_ptr->lock ) );

    // loop on all fd_array entries
    for( fd = 0 ; fd < CONFIG_PROCESS_FILE_MAX_NR ; fd++ )
        {
                entry = (xptr_t)hal_remote_lwd( XPTR( src_cxy , &src_ptr->array[fd] ) );

                if( entry != XPTR_NULL )
                {
            // increment file descriptor ref count
            vfs_file_count_up( entry );

                        // copy entry in destination process fd_array
                        hal_remote_swd( XPTR( dst_cxy , &dst_ptr->array[fd] ) , entry );
                }
        }

    // release lock on source process fd_array
        remote_spinlock_unlock( XPTR( src_cxy , &src_ptr->lock ) );

}  // end process_fd_remote_copy()

////////////////////////////////////////////////////////////////////////////////////
//  Thread related functions
////////////////////////////////////////////////////////////////////////////////////

/////////////////////////////////////////////////////
error_t process_register_thread( process_t * process,
                                 thread_t  * thread,
                                 trdid_t   * trdid )
{
    ltid_t   ltid;
    bool_t   found = false;

    assert( (process != NULL) , __FUNCTION__ , "process argument is NULL" );

    assert( (thread != NULL) , __FUNCTION__ , "thread argument is NULL" );

    // take lock protecting th_tbl
    spinlock_lock( &process->th_lock );

    // search a free slot in th_tbl[] 
    for( ltid = 0 ; ltid < CONFIG_THREAD_MAX_PER_CLUSTER ; ltid++ )
    {
        if( process->th_tbl[ltid] == NULL )
        {
            found = true;
            break;
        }
    }

    if( found )
    {
        // register thread in th_tbl[]
        process->th_tbl[ltid] = thread;
        process->th_nr++;

        // returns trdid
        *trdid = TRDID( local_cxy , ltid );
    }


    // release lock protecting th_tbl
    hal_fence();
    spinlock_unlock( &process->th_lock );

    return (found) ? 0 : ENOMEM;

}  // end process_register_thread()

///////////////////////////////////////////////
void process_remove_thread( thread_t * thread )
{
    assert( (thread != NULL) , __FUNCTION__ , "thread argument is NULL" );

    process_t * process = thread->process;

    // get thread local index
    ltid_t  ltid = LTID_FROM_TRDID( thread->trdid );

    // take lock protecting th_tbl
    spinlock_lock( &process->th_lock );

    assert( (process->th_nr) , __FUNCTION__ , "process th_nr cannot be 0\n" );

    // remove thread from th_tbl[]
    process->th_tbl[ltid] = NULL;
    process->th_nr--;

    hal_fence();

    // release lock protecting th_tbl
    spinlock_unlock( &process->th_lock );

}  // process_remove_thread()

/////////////////////////////////////////////////////////
error_t process_make_fork( xptr_t      parent_process_xp,
                           xptr_t      parent_thread_xp,
                           pid_t     * child_pid,
                           thread_t ** child_thread )
{
    process_t * process;         // local pointer on child process descriptor
    thread_t  * thread;          // local pointer on child thread descriptor
    pid_t       new_pid;         // process identifier for child process
    pid_t       parent_pid;      // process identifier for parent process
    xptr_t      ref_xp;          // extended pointer on reference process
    xptr_t      vfs_bin_xp;      // extended pointer on .elf file
    error_t     error;

    // get cluster and local pointer for parent process
    cxy_t       parent_process_cxy = GET_CXY( parent_process_xp );
    process_t * parent_process_ptr = (process_t *)GET_PTR( parent_process_xp );

    // get parent process PID and extended pointer on .elf file
    parent_pid = hal_remote_lw (XPTR( parent_process_cxy , &parent_process_ptr->pid));
    vfs_bin_xp = hal_remote_lwd(XPTR( parent_process_cxy , &parent_process_ptr->vfs_bin_xp));

    // check parent process is the reference
    ref_xp = hal_remote_lwd( XPTR( parent_process_cxy , &parent_process_ptr->ref_xp ) );
    assert( (parent_process_xp == ref_xp ) , __FUNCTION__ ,
    "parent process must be the reference process\n" );

fork_dmsg("\n[DBG] %s : core[%x,%d] enter at cycle %d\n",
__FUNCTION__, local_cxy, CURRENT_THREAD->core->lid , (uint32_t)hal_get_cycles() );

    // allocate a process descriptor
    process = process_alloc();
    if( process == NULL )
    {
        printk("\n[ERROR] in %s : cannot get process in cluster %x\n", 
        __FUNCTION__, local_cxy ); 
        return -1;
    }

fork_dmsg("\n[DBG] %s : core[%x,%d] created child process %x at cycle %d\n",
 __FUNCTION__, local_cxy, CURRENT_THREAD->core->lid, process, (uint32_t)hal_get_cycles() );

    // allocate a child PID from local cluster
    error = cluster_pid_alloc( process , &new_pid );
    if( error ) 
    {
        printk("\n[ERROR] in %s : cannot get PID in cluster %x\n", 
        __FUNCTION__, local_cxy ); 
        process_free( process );
        return -1;
    }

fork_dmsg("\n[DBG] %s : core[%x, %d] child process PID = %x at cycle %d\n",
 __FUNCTION__ , local_cxy, CURRENT_THREAD->core->lid, new_pid , (uint32_t)hal_get_cycles() );

    // initializes child process descriptor from parent process descriptor 
    process_reference_init( process,
                            new_pid,
                            parent_process_xp,
                            parent_process_xp );

fork_dmsg("\n[DBG] %s : core[%x, %d] child process initialised at cycle %d\n",
__FUNCTION__ , local_cxy, CURRENT_THREAD->core->lid, hal_get_cycles() );

    // copy VMM from parent descriptor to child descriptor
    error = vmm_fork_copy( process,
                           parent_process_xp );
    if( error )
    {
        printk("\n[ERROR] in %s : cannot copy VMM in cluster %x\n", 
        __FUNCTION__, local_cxy ); 
        process_free( process );
        cluster_pid_release( new_pid );
        return -1;
    }

fork_dmsg("\n[DBG] %s : core[%x, %d] child process VMM copied at cycle %d\n",
__FUNCTION__ , local_cxy, CURRENT_THREAD->core->lid, (uint32_t)hal_get_cycles() );

    // update extended pointer on .elf file
    process->vfs_bin_xp = vfs_bin_xp;

    // create child thread descriptor from parent thread descriptor
    error = thread_user_fork( parent_thread_xp,
                              process,
                              &thread );
    if( error )
    {
        printk("\n[ERROR] in %s : cannot create thread in cluster %x\n",
        __FUNCTION__, local_cxy ); 
        process_free( process );
        cluster_pid_release( new_pid );
        return -1;
    }

    // check main thread index
    assert( (thread->trdid == 0) , __FUNCTION__ , "main thread must have index 0\n" );

fork_dmsg("\n[DBG] %s : core[%x,%d] child thread created at cycle %d\n", 
__FUNCTION__ , local_cxy, CURRENT_THREAD->core->lid, (uint32_t)hal_get_cycles() );

    // update parent process GPT to set Copy_On_Write for shared data vsegs 
    // this includes all replicated GPT copies
    if( parent_process_cxy == local_cxy )   // reference is local
    {
        vmm_set_cow( parent_process_ptr );
    }
    else                                    // reference is remote
    {
        rpc_vmm_set_cow_client( parent_process_cxy,
                                parent_process_ptr );
    }

fork_dmsg("\n[DBG] %s : core[%x,%d] COW set in parent_process at cycle %d\n",
__FUNCTION__ , local_cxy, CURRENT_THREAD->core->lid, (uint32_t)hal_get_cycles() );

    // get extended pointers on parent children_root, children_lock and children_nr
    xptr_t children_root_xp = XPTR( parent_process_cxy , &parent_process_ptr->children_root );
    xptr_t children_lock_xp = XPTR( parent_process_cxy , &parent_process_ptr->children_lock );
    xptr_t children_nr_xp   = XPTR( parent_process_cxy , &parent_process_ptr->children_nr   );

    // register process in parent children list
    remote_spinlock_lock( children_lock_xp );
        xlist_add_last( children_root_xp , XPTR( local_cxy , &process->children_list ) );
        hal_remote_atomic_add( children_nr_xp , 1 );
    remote_spinlock_unlock( children_lock_xp );

    // return success
    *child_thread = thread;
    *child_pid    = new_pid;


fork_dmsg("\n[DBG] %s : core[%x,%d] exit at cycle %d\n",
__FUNCTION__, local_cxy, CURRENT_THREAD->core->lid, (uint32_t)hal_get_cycles() );

    return 0;

}   // end process_make_fork()


/////////////////////////////////////////////////////
error_t process_make_exec( exec_info_t  * exec_info )
{
    char           * path;                    // pathname to .elf file
    pid_t            pid;                     // old_process PID given to new_process
    pid_t            temp_pid;                // temporary PID given to old_process 
    process_t      * old_process;             // local pointer on old process
    process_t      * new_process;             // local pointer on new process
    thread_t       * new_thread;              // local pointer on main thread
    pthread_attr_t   attr;                    // main thread attributes
    lid_t            lid;                     // selected core local index
        error_t          error;

        // get .elf pathname and PID from exec_info
        path     = exec_info->path;
    pid      = exec_info->pid;

    // this function must be executed by a thread running in owner cluster
    assert( (CXY_FROM_PID( pid ) == local_cxy), __FUNCTION__,
    "local cluster %x is not owner for process %x\n", local_cxy, pid );

exec_dmsg("\n[DBG] %s : core[%x,%d] enters for process %x / %s / cycle %d\n",
__FUNCTION__, local_cxy, CURRENT_THREAD->core->lid, pid, path, (uint32_t)hal_get_cycles() );

    // get old_process local pointer
    old_process = (process_t *)cluster_get_local_process_from_pid( pid );
    
    if( old_process == NULL )
    {
        printk("\n[ERROR] in %s : cannot get old process descriptor\n", __FUNCTION__ );
        return -1;
    }

     // allocate memory for new_process descriptor
    new_process = process_alloc();

    if( new_process == NULL )
    {
        printk("\n[ERROR] in %s : cannot allocate process descriptor in cluster %x\n",
        __FUNCTION__ , local_cxy );
        return -1;
    }

    // get a new PID for old_process
    error = cluster_pid_alloc( old_process , &temp_pid );
    if( error ) 
    {
        printk("\n[ERROR] in %s : cannot get PID in cluster %x\n", 
        __FUNCTION__ , local_cxy ); 
        process_free( new_process );
        return -1;
    }

    // request blocking for all threads in old_process (but the calling thread)
    process_sigaction( old_process , BLOCK_ALL_THREADS );

    // request destruction for all threads in old_process (but the calling thread)
    process_sigaction( old_process , DELETE_ALL_THREADS );

exec_dmsg("\n[DBG] %s : core[%x,%d] marked old threads for destruction / cycle %d\n",
__FUNCTION__ , local_cxy, CURRENT_THREAD->core->lid , (uint32_t)hal_get_cycles() );

    // set new PID to old_process 
    old_process->pid = temp_pid;

    // initialize new process descriptor
    process_reference_init( new_process,
                            pid,
                            old_process->parent_xp,             // parent_process_xp
                            XPTR(local_cxy , old_process) );    // model_process_xp

    // give TXT ownership to new_process
    process_txt_set_ownership( XPTR( local_cxy , new_process ) );

exec_dmsg("\n[DBG] %s : core[%x,%d] initialised new process %x / cycle %d \n",
__FUNCTION__, local_cxy, CURRENT_THREAD->core->lid, new_process, (uint32_t)hal_get_cycles() );

    // register code & data vsegs as well as entry-point in new process VMM,
    // and register extended pointer on .elf file in process descriptor
        if( elf_load_process( path , new_process ) )
        {
                printk("\n[ERROR] in %s : failed to access .elf file for path %s\n",
                __FUNCTION__ , path );
        process_destroy( new_process );
        return -1;
        }

exec_dmsg("\n[DBG] %s : core[%x,%d] vsegs registered in new process %x / cycle %d\n",
__FUNCTION__, local_cxy, CURRENT_THREAD->core->lid, new_process, (uint32_t)hal_get_cycles() );

    // select a core in local cluster to execute the main thread
    lid  = cluster_select_local_core();

    // initialize pthread attributes for main thread
    attr.attributes = PT_ATTR_DETACH | PT_ATTR_CLUSTER_DEFINED | PT_ATTR_CORE_DEFINED;
    attr.cxy        = local_cxy;
    attr.lid        = lid;

    // create and initialize main thread in local cluster
        error = thread_user_create( pid,
                                (void *)new_process->vmm.entry_point,
                                exec_info->args_pointers,
                                &attr,
                                &new_thread );
        if( error )
        {
                printk("\n[ERROR] in %s : cannot create thread for process %x\n",
            __FUNCTION__ , new_process );
        process_destroy( new_process );
        return -1;
        }

    // check main thread index
    assert( (new_thread->trdid == 0) , __FUNCTION__ , "main thread must have index 0\n" );

exec_dmsg("\n[DBG] %s : core[%x,%d] created new_process main thread / cycle %d\n",
__FUNCTION__ , local_cxy, CURRENT_THREAD->core->lid, (uint32_t)hal_get_cycles() );

    // get pointers on parent process
    xptr_t      parent_xp  = new_process->parent_xp;
    process_t * parent_ptr = GET_PTR( parent_xp );
    cxy_t       parent_cxy = GET_CXY( parent_xp );

    // get extended pointers on parent children_root, children_lock and children_nr
    xptr_t root_xp = XPTR( parent_cxy , &parent_ptr->children_root );
    xptr_t lock_xp = XPTR( parent_cxy , &parent_ptr->children_lock );
    xptr_t nr_xp   = XPTR( parent_cxy , &parent_ptr->children_nr   );

    // register new_process in parent children list
    remote_spinlock_lock( lock_xp );
        xlist_add_last( root_xp , XPTR( local_cxy , &new_process->children_list ) );
        hal_remote_atomic_add( nr_xp , 1 );
    remote_spinlock_unlock( lock_xp );

exec_dmsg("\n[DBG] %s : core[%x,%d] updated parent process children list / cycle %d\n",
__FUNCTION__ , local_cxy, CURRENT_THREAD->core->lid, (uint32_t)hal_get_cycles() );
   
    // block and mark calling thread for deletion
    // only when it is an user thread
    thread_t * this = CURRENT_THREAD;
    if( this->type == THREAD_USER )
    {
        thread_block( this , THREAD_BLOCKED_GLOBAL );
        hal_atomic_or( &this->flags , THREAD_FLAG_REQ_DELETE );
    }

    // activate new thread
        thread_unblock( XPTR( local_cxy , new_thread ) , THREAD_BLOCKED_GLOBAL );

    hal_fence();

exec_dmsg("\n[DBG] %s : core[%x,%d] exit for path = %s / cycle %d\n",
__FUNCTION__, local_cxy, CURRENT_THREAD->core->lid, path , (uint32_t)hal_get_cycles() );

        return 0;

}  // end process_make_exec()

///////////////////////////////////////
void process_make_kill( pid_t      pid,
                        uint32_t   sig_id )
{
    // this function must be executed by a thread running in owner cluster
    assert( (CXY_FROM_PID( pid ) == local_cxy) , __FUNCTION__ ,
    "must execute in owner cluster" );

    thread_t * this = CURRENT_THREAD;

kill_dmsg("\n[DBG] %s : core[%x,%d] enter / process %x / sig %d\n",
__FUNCTION__, local_cxy, this->core->lid, pid , sig_id );

    // get pointer on local target process descriptor
    process_t * process = process_get_local_copy( pid );

    // does nothing if process does not exist
    if( process == NULL )
    {
        printk("\n[WARNING] %s : process %x does not exist => do nothing\n",
        __FUNCTION__ , pid );
        return;
    }

    // analyse signal type
    switch( sig_id )
    {
        case SIGSTOP:     
        {
            // block all threads in all clusters
            process_sigaction( process , BLOCK_ALL_THREADS );

            // remove TXT ownership to target process 
            process_txt_reset_ownership( XPTR( local_cxy , process ) );
        }
        break;
        case SIGCONT:     // unblock all threads in all clusters
        {
            process_sigaction( process , UNBLOCK_ALL_THREADS );
        }
        break;
        case SIGKILL:  // block all threads, then delete all threads
        {
            // block all threads in all clusters
            process_sigaction( process , BLOCK_ALL_THREADS );

            // remove TXT ownership to target process 
            process_txt_reset_ownership( XPTR( local_cxy , process ) );

            // delete all threads (but the calling thread)
            process_sigaction( process , DELETE_ALL_THREADS );

            // delete the calling thread if required
            if( CURRENT_THREAD->process == process )
            {
                // set REQ_DELETE flag
                hal_atomic_or( &this->flags , THREAD_FLAG_REQ_DELETE );

                // deschedule
                sched_yield( "suicide after kill" ); 
            }
        }
        break;
    }

kill_dmsg("\n[DBG] %s : core[%x,%d] exit / process %x / sig %d \n",
__FUNCTION__, local_cxy, CURRENT_THREAD->core->lid, pid , sig_id );

}  // end process_make_kill()

/////////////////////////////////////////
void process_make_exit( pid_t       pid,
                        uint32_t    status )
{
    // this function must be executed by a thread running in owner cluster
    assert( (CXY_FROM_PID( pid ) == local_cxy) , __FUNCTION__ ,
    "must execute in owner cluster" );

    // get pointer on local process descriptor
    process_t * process = process_get_local_copy( pid );

    // does nothing if process does not exist
    if( process == NULL )
    {
        printk("\n[WARNING] %s : process %x does not exist => do nothing\n",
        __FUNCTION__ , pid );
        return;
    }

    // block all threads in all clusters (but the calling thread)
    process_sigaction( process , BLOCK_ALL_THREADS );

    // delete all threads in all clusters (but the calling thread)
    process_sigaction( process , DELETE_ALL_THREADS );

    // delete the calling thread 
    hal_atomic_or( &CURRENT_THREAD->flags , THREAD_FLAG_REQ_DELETE );

    // deschedule
    sched_yield( "suicide after exit" ); 

}  // end process_make_exit()

///////////////////////////////////////////////
void process_zero_create( process_t * process )
{

process_dmsg("\n[DBG] %s : core[%x,%d] enter at cycle %d\n",
__FUNCTION__, local_cxy, CURRENT_THREAD->core->lid, (uint32_t)hal_get_cycles() );

    // initialize PID, REF_XP, PARENT_XP, and STATE
    process->pid       = 0;
    process->ref_xp    = XPTR( local_cxy , process );
    process->parent_xp = XPTR_NULL;
    process->state     = PROCESS_STATE_RUNNING;

    // reset th_tbl[] array as empty
    uint32_t i;
    for( i = 0 ; i < CONFIG_THREAD_MAX_PER_CLUSTER ; i++ )
        {
        process->th_tbl[i] = NULL;
    }
    process->th_nr  = 0;
    spinlock_init( &process->th_lock );

    // reset children list as empty
    xlist_root_init( XPTR( local_cxy , &process->children_root ) );
    remote_spinlock_init( XPTR( local_cxy , &process->children_lock ) );
    process->children_nr = 0;

        hal_fence();

process_dmsg("\n[DBG] %s : core[%x,%d] exit at cycle %d\n",
__FUNCTION__ , local_cxy , CURRENT_THREAD->core->lid , (uint32_t)hal_get_cycles() );

}  // end process_zero_init()

//////////////////////////
void process_init_create()
{
    process_t      * process;       // local pointer on process descriptor
    pid_t            pid;           // process_init identifier
    thread_t       * thread;        // local pointer on main thread
    pthread_attr_t   attr;          // main thread attributes
    lid_t            lid;           // selected core local index for main thread
    error_t          error;

process_dmsg("\n[DBG] %s :  core[%x,%d] enters at cycle %d\n", 
__FUNCTION__ , local_cxy, CURRENT_THREAD->core->lid );

    // allocates memory for process descriptor from local cluster
        process = process_alloc(); 
        if( process == NULL )
    {
                printk("\n[PANIC] in %s : no memory for process descriptor in cluster %x\n",
                __FUNCTION__, local_cxy  );
    }

    // get PID from local cluster
    error = cluster_pid_alloc( process , &pid );
    if( error )
    {
                printk("\n[PANIC] in %s : cannot allocate PID in cluster %x\n",
                __FUNCTION__, local_cxy );
        process_free( process );
    }

    // check allocated PID
    assert( (pid == 1) , __FUNCTION__ , "process INIT must be first process in cluster 0\n" );

    // initialize process descriptor / parent is local process_zero
    process_reference_init( process,
                            pid,
                            XPTR( local_cxy , &process_zero ),     // parent
                            XPTR( local_cxy , &process_zero ) );   // model

process_dmsg("\n[DBG] %s : core[%x,%d] / initialisation done\n", 
__FUNCTION__ , local_cxy, CURRENT_THREAD->core->lid );

    // register "code" and "data" vsegs as well as entry-point
    // in process VMM, using information contained in the elf file.
        if( elf_load_process( CONFIG_PROCESS_INIT_PATH , process ) )
        {
                printk("\n[PANIC] in %s : cannot access .elf file / path = %s\n",
                __FUNCTION__, CONFIG_PROCESS_INIT_PATH );
        process_destroy( process );
        }

process_dmsg("\n[DBG] %s : core[%x,%d] vsegs registered / path = %s\n",
__FUNCTION__, local_cxy, CURRENT_THREAD->core->lid, CONFIG_PROCESS_INIT_PATH );

    // get extended pointers on process_zero children_root, children_lock
    xptr_t children_root_xp = XPTR( local_cxy , &process_zero.children_root );
    xptr_t children_lock_xp = XPTR( local_cxy , &process_zero.children_lock );

    // register process INIT in parent local process_zero 
    remote_spinlock_lock( children_lock_xp );
        xlist_add_last( children_root_xp , XPTR( local_cxy , &process->children_list ) );
        hal_atomic_add( &process_zero.children_nr , 1 );
    remote_spinlock_unlock( children_lock_xp );

    // select a core in local cluster to execute the main thread
    lid  = cluster_select_local_core();

    // initialize pthread attributes for main thread
    attr.attributes = PT_ATTR_DETACH | PT_ATTR_CLUSTER_DEFINED | PT_ATTR_CORE_DEFINED;
    attr.cxy        = local_cxy;
    attr.lid        = lid;

    // create and initialize thread descriptor
        error = thread_user_create( pid,
                                (void *)process->vmm.entry_point,
                                NULL,
                                &attr,
                                &thread );
        if( error )
        {
                printk("\n[PANIC] in %s : cannot create main thread / path = %s\n",
                __FUNCTION__, CONFIG_PROCESS_INIT_PATH );
        process_destroy( process );
        }

    // check main thread index
    assert( (thread->trdid == 0) , __FUNCTION__ , "main thread must have index 0\n" );

    // activate thread
        thread_unblock( XPTR( local_cxy , thread ) , THREAD_BLOCKED_GLOBAL );

    hal_fence();

process_dmsg("\n[DBG] %s : core[%x,%d] exit / main thread = %x\n",
__FUNCTION__, local_cxy, CURRENT_THREAD->core->lid, thread );

}  // end process_init_create()

//////////////////////////////////////////
char * process_state_str( uint32_t state )
{
    if     ( state == PROCESS_STATE_RUNNING ) return "RUNNING";
    else if( state == PROCESS_STATE_KILLED  ) return "KILLED";
    else if( state == PROCESS_STATE_EXITED  ) return "EXITED";
    else                                      return "undefined";
}

/////////////////////////////////////////
void process_display( xptr_t process_xp )
{
    process_t   * process_ptr;
    cxy_t         process_cxy;
    xptr_t        parent_xp;       // extended pointer on parent process
    process_t   * parent_ptr;
    cxy_t         parent_cxy;

    pid_t         pid;
    pid_t         ppid;
    uint32_t      state;
    xptr_t        ref_xp; 
    uint32_t      th_nr;

    xptr_t        txt_file_xp;     // extended pointer on TXT_RX pseudo file
    xptr_t        chdev_xp;        // extended pointer on TXT_RX chdev
    chdev_t     * chdev_ptr;
    cxy_t         chdev_cxy;
    xptr_t        owner_xp;        // extended pointer on TXT owner process 

    xptr_t        elf_file_xp;     // extended pointer on .elf file 
    cxy_t         elf_file_cxy;
    vfs_file_t  * elf_file_ptr;
    vfs_inode_t * elf_inode_ptr;   // local pointer on .elf inode

    char          txt_name[CONFIG_VFS_MAX_NAME_LENGTH];
    char          elf_name[CONFIG_VFS_MAX_NAME_LENGTH];

    // get cluster and local pointer on process
    process_ptr = GET_PTR( process_xp );
    process_cxy = GET_CXY( process_xp );

    // check reference process
    ref_xp = hal_remote_lwd( XPTR( process_cxy , &process_ptr->ref_xp ) );
    assert( (process_xp == ref_xp) , __FUNCTION__ , "process is not the reference\n");

    // get PID and state
    pid   = hal_remote_lw( XPTR( process_cxy , &process_ptr->pid ) );
    state = hal_remote_lw( XPTR( process_cxy , &process_ptr->state ) );

    // get PPID
    parent_xp  = hal_remote_lwd( XPTR( process_cxy , &process_ptr->parent_xp ) );
    parent_cxy = GET_CXY( parent_xp );
    parent_ptr = GET_PTR( parent_xp );
    ppid       = hal_remote_lw( XPTR( parent_cxy , &parent_ptr->pid ) );

    // get number of threads
    th_nr      = hal_remote_lw( XPTR( process_cxy , &process_ptr->th_nr ) );

    // get TXT name and process owner
    txt_file_xp = hal_remote_lwd( XPTR( process_cxy , &process_ptr->fd_array.array[0] ) );

    assert( (txt_file_xp != XPTR_NULL) , __FUNCTION__ , 
    "process must be attached to one TXT terminal\n" ); 

    chdev_xp  = chdev_from_file( txt_file_xp );
    chdev_cxy = GET_CXY( chdev_xp );
    chdev_ptr = (chdev_t *)GET_PTR( chdev_xp );
    hal_remote_strcpy( XPTR( local_cxy , txt_name ) ,
                           XPTR( chdev_cxy , chdev_ptr->name ) );
    owner_xp = (xptr_t)hal_remote_lwd( XPTR( chdev_cxy , &chdev_ptr->ext.txt.owner_xp ) );
    
    // get process .elf name
    elf_file_xp   = hal_remote_lwd( XPTR( process_cxy , &process_ptr->vfs_bin_xp ) );

    elf_file_cxy  = GET_CXY( elf_file_xp );
    elf_file_ptr  = (vfs_file_t *)GET_PTR( elf_file_xp );
    elf_inode_ptr = (vfs_inode_t *)hal_remote_lpt( XPTR( elf_file_cxy , &elf_file_ptr->inode ) );
    vfs_inode_get_name( XPTR( elf_file_cxy , elf_inode_ptr ) , elf_name );

    // display process info
    if( owner_xp == process_xp )
    {
        printk("PID %X | PPID %X | %s\t| %s (FG) | %X | %d | %s\n", 
        pid, ppid, process_state_str(state), txt_name, process_ptr, th_nr, elf_name );
    }
    else
    {
        printk("PID %X | PPID %X | %s\t| %s (BG) | %X | %d | %s\n", 
        pid, ppid, process_state_str(state), txt_name, process_ptr, th_nr, elf_name );
    }
}  // end process_display() 


////////////////////////////////////////////////////////////////////////////////////////
//     Terminals related functions
////////////////////////////////////////////////////////////////////////////////////////

////////////////////////////
uint32_t process_txt_alloc()
{
    uint32_t  index;       // TXT terminal index
    xptr_t    chdev_xp;    // extended pointer on TXT_RX chdev
    chdev_t * chdev_ptr;   // local pointer on TXT_RX chdev
    cxy_t     chdev_cxy;   // TXT_RX chdev cluster
    xptr_t    root_xp;     // extended pointer on owner field in chdev

    // scan the user TXT_RX chdevs (TXT0 is reserved for kernel)
    for( index = 1 ; index < LOCAL_CLUSTER->nb_txt_channels ; index ++ )
    {
        // get pointers on TXT_RX[index]
        chdev_xp  = chdev_dir.txt_rx[index];
        chdev_cxy = GET_CXY( chdev_xp );
        chdev_ptr = GET_PTR( chdev_xp );

        // get extended pointer on root of attached process
        root_xp = XPTR( chdev_cxy , &chdev_ptr->ext.txt.root );

        // return free TXT index if found
        if( xlist_is_empty( root_xp ) ) return index;  
    }

    assert( false , __FUNCTION__ , "no free TXT terminal found" );

    return -1;

} // end process_txt_alloc()

/////////////////////////////////////////////
void process_txt_attach( process_t * process,
                         uint32_t    txt_id )
{
    xptr_t      chdev_xp;     // extended pointer on TXT_RX chdev
    cxy_t       chdev_cxy;    // TXT_RX chdev cluster
    chdev_t *   chdev_ptr;    // local pointer on TXT_RX chdev
    xptr_t      root_xp;      // extended pointer on list root in chdev
    xptr_t      lock_xp;      // extended pointer on list lock in chdev

process_dmsg("\n[DBG] %s : core[%x,%d] enter for process %x at cycle\n",
__FUNCTION__, local_cxy, CURRENT_THREAD->core->lid, process->pid, (uint32_t)hal_get_cycles() );

    // check process is reference
    assert( (process->ref_xp == XPTR( local_cxy , process )) , __FUNCTION__ ,
    "process is not the reference descriptor" );

    // check terminal index
    assert( (txt_id < LOCAL_CLUSTER->nb_txt_channels) ,
    __FUNCTION__ , "illegal TXT terminal index" );

    // get pointers on TXT_RX[txt_id] chdev
    chdev_xp  = chdev_dir.txt_rx[txt_id];
    chdev_cxy = GET_CXY( chdev_xp );
    chdev_ptr = GET_PTR( chdev_xp );

    // get extended pointer on root & lock of attached process list
    root_xp = XPTR( chdev_cxy , &chdev_ptr->ext.txt.root );
    lock_xp = XPTR( chdev_cxy , &chdev_ptr->ext.txt.lock );

    // insert process in attached process list
    remote_spinlock_lock( lock_xp );
    xlist_add_last( root_xp , XPTR( local_cxy , &process->txt_list ) );
    remote_spinlock_unlock( lock_xp );

process_dmsg("\n[DBG] %s : core[%x,%d] exit for process %x at cycle\n",
__FUNCTION__, local_cxy, CURRENT_THREAD->core->lid, process->pid, (uint32_t)hal_get_cycles() );

} // end process_txt_attach()

//////////////////////////////////////////////
void process_txt_detach( process_t * process )
{
    xptr_t      chdev_xp;     // extended pointer on TXT_RX chdev
    cxy_t       chdev_cxy;    // TXT_RX chdev cluster
    chdev_t *   chdev_ptr;    // local pointer on TXT_RX chdev
    xptr_t      lock_xp;      // extended pointer on list lock in chdev

process_dmsg("\n[DBG] %s : core[%x,%d] enter for process %x at cycle\n",
__FUNCTION__, local_cxy, CURRENT_THREAD->core->lid, process->pid, (uint32_t)hal_get_cycles() );

    // check process is reference
    assert( (process->ref_xp == XPTR( local_cxy , process )) , __FUNCTION__ ,
    "process is not the reference descriptor" );

    // get extended pointer on TXT_RX chdev
    chdev_xp  = chdev_from_file( process->fd_array.array[0] );
    chdev_cxy = GET_CXY( chdev_xp );
    chdev_ptr = (chdev_t *)GET_PTR( chdev_xp );

    // get extended pointer on lock of attached process list
    lock_xp = XPTR( chdev_cxy , &chdev_ptr->ext.txt.lock );

    // unlink process from attached process list
    remote_spinlock_lock( lock_xp );
    xlist_unlink( XPTR( local_cxy , &process->txt_list ) );
    remote_spinlock_unlock( lock_xp );
   
process_dmsg("\n[DBG] %s : core[%x,%d] exit for process %x at cycle %d\n",
__FUNCTION__, local_cxy, CURRENT_THREAD->core->lid, process->pid, (uint32_t)hal_get_cycles() );

} // end process_txt_detach()

///////////////////////////////////////////////////
void process_txt_set_ownership( xptr_t process_xp )
{
    process_t * process_ptr;
    cxy_t       process_cxy;
    xptr_t      file_xp;
    xptr_t      txt_xp;     
    chdev_t   * txt_ptr;
    cxy_t       txt_cxy;

    // get cluster and local pointer on process
    process_cxy = GET_CXY( process_xp );
    process_ptr = (process_t *)GET_PTR( process_xp );

    // get extended pointer on stdin pseudo file
    file_xp = hal_remote_lwd( XPTR( process_cxy , &process_ptr->fd_array.array[0] ) );

    // get pointers on TXT chdev
    txt_xp  = chdev_from_file( file_xp );
    txt_cxy = GET_CXY( txt_xp );
    txt_ptr = (chdev_t *)GET_PTR( txt_xp );

    // set owner field in TXT chdev
    hal_remote_swd( XPTR( txt_cxy , &txt_ptr->ext.txt.owner_xp ) , process_xp );

}  // end process_txt_set ownership()

/////////////////////////////////////////////////////
void process_txt_reset_ownership( xptr_t process_xp )
{
    process_t * process_ptr;
    cxy_t       process_cxy;
    xptr_t      parent_xp;       // extended pointer on parent process
    process_t * parent_ptr;
    cxy_t       parent_cxy;
    xptr_t      file_xp;         // extended pointer on TXT_RX pseudo file
    xptr_t      txt_xp;          // extended pointer on TXT_RX chdev
    chdev_t   * txt_ptr;
    cxy_t       txt_cxy;
    xptr_t      owner_xp;        // extended pointer on current TXT_RX owner
    xptr_t      root_xp;         // extended pointer on root of attached process list
    xptr_t      iter_xp;         // iterator for xlist
    xptr_t      current_xp;      // extended pointer on current process
    process_t * current_ptr;
    cxy_t       current_cxy;
    pid_t       ppid;

    // get cluster and local pointer on process
    process_cxy = GET_CXY( process_xp );
    process_ptr = (process_t *)GET_PTR( process_xp );

    // get extended pointer on stdin pseudo file
    file_xp = hal_remote_lwd( XPTR( process_cxy , &process_ptr->fd_array.array[0] ) );

    // get pointers on TXT chdev
    txt_xp  = chdev_from_file( file_xp );
    txt_cxy = GET_CXY( txt_xp );
    txt_ptr = (chdev_t *)GET_PTR( txt_xp );

    // get extended pointer on TXT_RX owner
    owner_xp = hal_remote_lwd( XPTR( txt_cxy , &txt_ptr->ext.txt.owner_xp ) );

    // transfer ownership to KSH if required
    if( owner_xp == process_xp )   
    {
        // get extended pointer on root of list of attached processes
        root_xp = hal_remote_lwd( XPTR( txt_cxy , &txt_ptr->ext.txt.root ) );

        // scan attached process list to find KSH process
        XLIST_FOREACH( root_xp , iter_xp )
        {
            current_xp  = XLIST_ELEMENT( iter_xp , process_t , txt_list );
            current_cxy = GET_CXY( current_xp );
            current_ptr = GET_PTR( current_xp );
            parent_xp   = hal_remote_lwd( XPTR( current_cxy , &current_ptr->parent_xp ) );

            parent_cxy  = GET_CXY( parent_xp );
            parent_ptr  = GET_PTR( parent_xp );
            ppid        = hal_remote_lw( XPTR( parent_cxy , &parent_ptr->pid ) );

            if( ppid == 1 )  // current is KSH
            {
                // set owner field in TXT chdev 
                hal_remote_swd( XPTR( txt_cxy , &txt_ptr->ext.txt.owner_xp ) , current_xp );
                return;
            }
        }
    }

    assert( false , __FUNCTION__ , "KSH process not found" ); 

}  // end process_txt_reset_ownership()