Index: trunk/libs/libalmosmkh/almosmkh.c
===================================================================
--- trunk/libs/libalmosmkh/almosmkh.c	(revision 626)
+++ trunk/libs/libalmosmkh/almosmkh.c	(revision 637)
@@ -2,5 +2,5 @@
  * almosmkh.c - User level ALMOS-MKH specific library implementation.
  * 
- * Author     Alain Greiner (2016,2017,2018)
+ * Author     Alain Greiner (2016,2017,2018,2019)
  *
  * Copyright (c) UPMC Sorbonne Universites
@@ -24,4 +24,5 @@
 #include <almosmkh.h>
 #include <hal_user.h>
+#include <hal_macros.h>
 #include <hal_shared_types.h>
 #include <syscalls_numbers.h>
@@ -32,7 +33,10 @@
 #include <mman.h>
 
-#define  MALLOC_DEBUG    0
+#define  DEBUG_REMOTE_MALLOC     0
+#define  DEBUG_PTHREAD_PARALLEL  1
  
-/////////////     Non standard system calls    /////////////////////////////////
+//////////////////////////////////////////////////////////////////////////////////////
+/////////////     Non standard system calls    ///////////////////////////////////////
+//////////////////////////////////////////////////////////////////////////////////////
 
 //////////////////////////
@@ -63,11 +67,33 @@
 }
 
-/////////////////////////////////
-int get_core( unsigned int * cxy,
-              unsigned int * lid )
-{
-    return hal_user_syscall( SYS_GET_CORE,
+////////////////////////////////////
+int get_core_id( unsigned int * cxy,
+                 unsigned int * lid )
+{
+    return hal_user_syscall( SYS_GET_CORE_ID,
                              (reg_t)cxy,
                              (reg_t)lid, 0, 0 );
+}
+
+/////////////////////////////////////
+int get_nb_cores( unsigned int   cxy,
+                  unsigned int * ncores )
+{
+    return hal_user_syscall( SYS_GET_NB_CORES,
+                             (reg_t)cxy,
+                             (reg_t)ncores, 0, 0 );
+}
+
+///////////////////////////////////////////
+int get_best_core( unsigned int   base_cxy,
+                   unsigned int   level,
+                   unsigned int * cxy,
+                   unsigned int * lid )
+{
+    return hal_user_syscall( SYS_GET_BEST_CORE,
+                             (reg_t)base_cxy,
+                             (reg_t)level,
+                             (reg_t)cxy,
+                             (reg_t)lid );
 }
 
@@ -250,6 +276,7 @@
 }  // end get_string()
 
-
-///////////////    non standard debug functions    //////////////////////////
+//////////////////////////////////////////////////////////////////////////////////////
+///////////////    non standard debug functions    ///////////////////////////////////
+//////////////////////////////////////////////////////////////////////////////////////
 
 ////////////////////////////////////
@@ -496,10 +523,12 @@
 
 
-///////////////    non standard malloc functions    //////////////////////////
+/////////////////////////////////////////////////////////////////////////////////////////
+///////////////    non standard remote_malloc    ////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////////////
 
 /////////////////////////////////////////////////////////////////////////////////////////
 // Global variable defining the allocator array (one per cluster)
 // This array (about 16 Kbytes ) will be stored in the data segment
-// of any application linked with this malloc libray.
+// of any application linked with this libray.
 /////////////////////////////////////////////////////////////////////////////////////////
 
@@ -546,5 +575,5 @@
 ////////////////////////////////////////////////////////////////////////////////////////////
 
-#if MALLOC_DEBUG
+#if DEBUG_REMOTE_MALLOC
 static void display_free_array( unsigned int cxy )
 {
@@ -594,7 +623,10 @@
     unsigned int   iter;             // iterator
 
-#if MALLOC_DEBUG
-printf("\n[MALLOC] %s : enter for store[%x] / size = %x\n",
-__FUNCTION__, cxy, store_size );
+#if DEBUG_REMOTE_MALLOC
+unsigned int core_cxy;
+unsigned int core_lid;
+get_core_id( &core_cxy , &core_lid );
+printf("\n[%s] core[%x,%d] enter for store[%x] / size = %x\n",
+__FUNCTION__, core_cxy, core_lid, cxy, store_size );
 #endif
 
@@ -635,7 +667,7 @@
     }
 
-#if MALLOC_DEBUG
-printf("\n[MALLOC] %s : mmap done for store[%x] / base = %x\n",
-__FUNCTION__, cxy, store_base );
+#if DEBUG_REMOTE_MALLOC
+printf("\n[%s] core[%x,%d] created vseg %x for store[%x]\n",
+__FUNCTION__, core_cxy, core_lid, store_base, cxy );
 #endif
 
@@ -656,9 +688,4 @@
     }
 
-    // DEPRECATED: we don't reset the alloc_base array 
-    // because we don't want to allocate the physical memory
-    // when the heap is created  [AG]
-    // memset( (void *)alloc_base , 0 , alloc_size ); 
- 
     // split the store into various sizes blocks,
     // initializes the free[] array and NEXT pointers
@@ -690,8 +717,10 @@
 
 
-#if MALLOC_DEBUG
-printf("\n[MALLOC] %s : completes store[%x] initialisation\n",
-__FUNCTION__, cxy );
-
+#if DEBUG_REMOTE_MALLOC
+printf("\n[%s] core[%x,%d] completed store[%x] initialisation\n",
+__FUNCTION__, core_cxy, core_lid, cxy );
+#endif
+
+#if (DEBUG_REMOTE_MALLOC & 1)
 display_free_array( cxy );
 #endif
@@ -762,7 +791,10 @@
     int error;
 
-#if MALLOC_DEBUG
-printf("\n[MALLOC] %s : enter for size = %x / cxy = %x\n",
-__FUNCTION__ , size , cxy );
+#if DEBUG_REMOTE_MALLOC
+unsigned int core_cxy;
+unsigned int core_lid;
+get_core_id( &core_cxy , &core_lid );
+printf("\n[%s] core[%x,%d] enter for size = %x / target_cxy = %x\n",
+__FUNCTION__ , core_cxy, core_lid, size , cxy );
 #endif
 
@@ -828,15 +860,4 @@
     unsigned char * ptr    = (unsigned char*)(store[cxy].alloc_base + offset);
 
-    // DEPRECATED : we cannot check the alloc[] array,
-    // because it has not been initialised by store_init,
-    // to avoid physical memory allocation at heap creation [AG]
-    // if ( *ptr != 0 )
-    // {
-    //    pthread_mutex_unlock( &store[cxy].mutex );
-    //    printf("\n[PANIC] in %s : allocate an already allocated block...\n",
-    //    __FUNCTION__ );
-    //    return NULL;
-    // }
-
     // update alloc_array
     *ptr = requested_index;
@@ -845,7 +866,7 @@
     pthread_mutex_unlock( &store[cxy].mutex );
  
-#if MALLOC_DEBUG
-printf("\n[MALLOC] %s : exit / base = %x / size = %x / from store[%x]\n",
-__FUNCTION__, base , size , cxy );
+#if DEBUG_REMOTE_MALLOC
+printf("\n[%s] core[%x,%d] exit / base = %x / size = %x / from store[%x]\n",
+__FUNCTION__, core_cxy, core_lid, base , size , cxy );
 #endif
 
@@ -853,6 +874,4 @@
 
 } // end remote_malloc()
-
-
 
 //////////////////////////////////////////
@@ -920,5 +939,7 @@
 
     return new_ptr;
-}
+
+}  // end remote_realloc()
+
 
 //////////////////////////////////////////////////////
@@ -991,5 +1012,5 @@
 {
 
-#if MALLOC_DEBUG
+#if DEBUG_REMOTE_MALLOC
 printf("\n[MALLOC] %s : enter for block = %x / cxy = %x\n",
 __FUNCTION__, ptr, cxy );
@@ -1052,5 +1073,5 @@
     pthread_mutex_unlock( &store[cxy].mutex );
 
-#if MALLOC_DEBUG
+#if DEBUG_REMOTE_MALLOC
 printf("\n[MALLOC] %s : conmpletes for block = %x / cxy = %x\n",
 __FUNCTION__, ptr, cxy );
@@ -1058,4 +1079,365 @@
 
 } // end remote_free()
+
+/////////////////////////////////////////////////////////////////////////////////////////
+///////////////    non standard pthread_parallel_create    //////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////////////
+
+#define X_MAX                   16              // max number of clusters in a row
+#define Y_MAX                   16              // max number of clusters in a column
+#define CLUSTERS_MAX            X_MAX * Y_MAX
+#define LEVEL_MAX               5
+#define CORES_MAX               4               // max number of cores per cluster
+
+typedef struct build_args_s           
+{
+    unsigned char       cxy;                    // this thread cluster identifier
+    unsigned char       level;                  // this thread level in quad-tree
+    unsigned char       parent_cxy;             // parent thread cluster identifier
+    unsigned char       root_level;             // quad-tree root level 
+    void              * work_func;              // pointer on work function pointer
+    void              * work_args_array;        // pointer on 2D array of pointers
+    pthread_barrier_t * parent_barriers_array;  // pointer on 1D array of barriers
+    unsigned int        error;                  // return value : 0 if success 
+}
+build_args_t;
+
+/////////////////////////////////////////////////////////////////////////////////////////
+//      Global variables used for inter-thread communications
+/////////////////////////////////////////////////////////////////////////////////////////
+
+pthread_attr_t    build_attr   [CLUSTERS_MAX][LEVEL_MAX];   // POSIX thread attributes
+
+build_args_t      build_args   [CLUSTERS_MAX][LEVEL_MAX];   // build function arguments
+
+pthread_barrier_t build_barrier[CLUSTERS_MAX][LEVEL_MAX];   // parent/child synchro
+
+pthread_attr_t    work_attr    [CLUSTERS_MAX][CORES_MAX];    // POSIX thread attributes
+
+//////////////////////////////////////////////////////////
+static void pthread_recursive_build( build_args_t * args )
+{
+    unsigned int   trdid;         // unused (required by pthread_create()
+
+    // get arguments
+    unsigned int        cxy                   = args->cxy;
+    unsigned int        level                 = args->level;
+    unsigned int        parent_cxy            = args->parent_cxy;
+    unsigned int        root_level            = args->root_level;
+    void              * work_func             = args->work_func;
+    void              * work_args_array       = args->work_args_array;
+    pthread_barrier_t * parent_barriers_array = args->parent_barriers_array;
+
+    // set error default value 
+    build_args[cxy][level].error = 0;
+
+    ///////////////////////////////////////////////////////////
+    if( level == 0 )             // children are "work" threads
+    {
+        unsigned int   lid;           // core local index
+        unsigned int   ncores;        // number of cores in a cluster
+
+        // get number of cores per cluster
+        get_nb_cores( cxy , &ncores );
+
+        // kill process if no active core in cluster
+        // TODO this "if" should be replaced by an "assert" [AG]
+        if( ncores == 0 )
+        {
+            printf("\n[PANIC] in %s : no active core in cluster %x\n",
+            __FUNCTION__ , cxy );
+
+            // report error to parent
+            build_args[parent_cxy][level+1].error = 1;
+
+            // kill process
+            exit( EXIT_FAILURE );
+        }
+
+        // initialize the parent_barrier 
+        if( pthread_barrier_init( &parent_barriers_array[cxy] , NULL , ncores + 1 ) )
+        {
+            printf("\n[ERROR] in %s : cannot initialise barrier for build thread[%x][%d]\n",
+            __FUNCTION__ , cxy , level );
+
+            // report error to parent
+            build_args[parent_cxy][level+1].error = 1;
+        }
+
+#if DEBUG_PTHREAD_PARALLEL
+printf("\n[%s] <build> thread[%x][%d] created barrier / %d children\n",
+__FUNCTION__, cxy, level, ncores + 1 );
+#endif
+        // create (ncores) "work" threads
+        for ( lid = 0 ; lid < ncores ; lid++ )
+        {
+            // set attributes for thread[cxy][lid]
+            work_attr[cxy][lid].attributes = PT_ATTR_DETACH |
+                                             PT_ATTR_CLUSTER_DEFINED |
+                                             PT_ATTR_CORE_DEFINED;
+            work_attr[cxy][lid].cxy        = cxy;
+            work_attr[cxy][lid].lid        = lid;
+
+            // compute pointer on thread[cxy][lid] arguments
+            void * work_args = *((void **)work_args_array + (cxy * CORES_MAX) + lid);
+
+            // create thread
+            if ( pthread_create( &trdid,                  // unused
+                                 &work_attr[cxy][lid],
+                                 work_func,
+                                 work_args ) ) 
+            {
+                printf("\n[ERROR] in %s : cannot create work thread[%x,%x]\n",
+                __FUNCTION__ , cxy , lid );
+
+                // report error to parent
+                build_args[parent_cxy][level+1].error = 1;
+            }
+
+#if DEBUG_PTHREAD_PARALLEL
+printf("\n[%s] <build> thread[%x][%d] created <work> thread[%x][%d]\n",
+__FUNCTION__, cxy, level, cxy, lid );
+#endif
+        }
+
+        // wait on barrier until "work" children threads completed
+        if( pthread_barrier_wait( &parent_barriers_array[cxy] ) )
+        {
+            printf("\n[ERROR] in %s / first barrier for <build> thread[%x][%d]\n",
+            __FUNCTION__ , cxy , level );
+
+            // report error to parent
+            build_args[parent_cxy][level+1].error = 1;
+        }
+
+#if DEBUG_PTHREAD_PARALLEL
+printf("\n[%s] <build> thread[%x][%d] resume after children completion\n",
+__FUNCTION__, cxy, level );
+#endif
+
+    }  // end level == 0
+
+    ////////////////////////////////////////////////////////////
+    else                        // children are "build" threads 
+    {
+        // the 4 children threads can be created in any core of each quarters
+        // of the parent macro-cluster
+
+        unsigned int parent_x;          // X coordinate of parent macro-cluster 
+        unsigned int parent_y;          // Y coordinate of parent macro-cluster
+        unsigned int child_x;           // X coordinate of child macro-cluster
+        unsigned int child_y;           // Y coordinate of child macro-cluster 
+        unsigned int child_cxy[2][2];   // selected cluster for child thread
+        unsigned int child_lid[2][2];   // selected core index for child thread
+        int          child_sts[2][2];   // -1 if error / 0 if success / +1 if not found
+        unsigned int x;                 // X loop index for children
+        unsigned int y;                 // Y loop index for children
+        
+        unsigned int nb_children = 0;
+
+        // get parent macro-cluster mask and half-size from level
+        unsigned int mask = (1 << level) - 1;
+        unsigned int half = (level > 0) ? (1 << (level - 1)) : 0;
+
+        // get parent macro-cluster coordinates 
+        parent_x = HAL_X_FROM_CXY( cxy ) & ~mask;
+        parent_y = HAL_Y_FROM_CXY( cxy ) & ~mask;
+
+        // get child_cxy and child_lid for up to 4 children threads : 00 / 01 / 10 / 11
+        for (x = 0 ; x < 2 ; x++)
+        {
+            // compute child macro-cluster X coordinate 
+            child_x = (x == 0) ? parent_x : (parent_x + half);
+
+            for (y = 0 ; y < 2 ; y++)
+            {
+                // compute child macro-cluster Y coordinate 
+                child_y = (y == 0) ? parent_y : (parent_y + half);
+
+                // select the best core in macro-cluster 
+                child_sts[x][y] = get_best_core( HAL_CXY_FROM_XY( child_x , child_y ),
+                                                 level-1,
+                                                 &child_cxy[x][y],
+                                                 &child_lid[x][y] );
+
+                if( child_sts[x][y] < 0 )  // failure => report error
+                {
+                    printf("\n[ERROR] in %s : illegal arguments for <build> thread[%x,%x]\n",
+                    __FUNCTION__ , cxy , level );
+
+                    // report error to parent
+                    build_args[parent_cxy][level+1].error = 1;
+                }
+                else if (child_sts[x][y] > 0 )  // macro-cluster undefined => does nothing
+                {
+                }
+                else                            // core found
+                {
+                    nb_children++;
+                }
+            }  // end for y
+        }  // end for x
+
+        // kill process if no active core in cluster
+        // TODO this "if" should be replaced by an "assert" [AG]
+        if( nb_children == 0 )
+        {
+            printf("\n[PANIC] in %s : no active core in macro cluster [%x,%d]\n",
+            __FUNCTION__ , cxy , level );
+
+            // report error to parent
+            build_args[parent_cxy][level+1].error = 1;
+
+            // kill process
+            exit( EXIT_FAILURE );
+        }
+
+        // initialize the barrier for (nb_children + 1)
+        if( pthread_barrier_init( &build_barrier[cxy][level], NULL , nb_children + 1 ) )
+        {
+            printf("\n[error] in %s : cannot initialise barrier for build thread[%x][%d]\n",
+            __FUNCTION__ , cxy , level );
+
+            // report error to parent
+            build_args[parent_cxy][level+1].error = 1;
+        }
+
+#if DEBUG_PTHREAD_PARALLEL
+printf("\n[%s] <build> thread[%x][%d] created barrier / %d children\n",
+__FUNCTION__, cxy, level, nb_children + 1 );
+#endif
+        // create 1 to 4 children threads
+        for (x = 0 ; x < 2 ; x++)
+        {
+            for (y = 0 ; y < 2 ; y++)
+            {
+                // thread is created only if macro-cluster is active
+                if( child_sts[x][y] == 0 )
+                {
+                    unsigned int tgt_cxy = child_cxy[x][y];
+                    unsigned int tgt_lid = child_lid[x][y];
+
+                    // set child thread attributes
+                    build_attr[tgt_cxy][level-1].attributes = PT_ATTR_DETACH |
+                                                              PT_ATTR_CLUSTER_DEFINED |
+                                                              PT_ATTR_CORE_DEFINED;
+                    build_attr[tgt_cxy][level-1].cxy        = tgt_cxy;
+                    build_attr[tgt_cxy][level-1].lid        = tgt_lid;
+
+                    // propagate build function arguments
+                    build_args[tgt_cxy][level-1].cxy                   = child_cxy[x][y];
+                    build_args[tgt_cxy][level-1].level                 = level-1;
+                    build_args[tgt_cxy][level-1].parent_cxy            = cxy;
+                    build_args[tgt_cxy][level-1].root_level            = root_level;
+                    build_args[tgt_cxy][level-1].work_func             = work_func;
+                    build_args[tgt_cxy][level-1].work_args_array       = work_args_array;
+                    build_args[tgt_cxy][level-1].parent_barriers_array = parent_barriers_array;
+                    
+                    // create thread
+                    if( pthread_create( &trdid,                          
+                                        &build_attr[tgt_cxy][level-1],    
+                                        &pthread_recursive_build,                          
+                                        &build_args[tgt_cxy][level-1] ) )
+                    {
+                        printf("\n[ERROR] in %s : cannot create build thread[%x][%d]\n",
+                        __FUNCTION__ , child_cxy , level -1 );
+
+                        // report error to parent
+                        build_args[parent_cxy][level+1].error = 1;
+                    }
+
+#if DEBUG_PTHREAD_PARALLEL
+printf("\n[%s] <build> thread[%x][%d] created <build> thread[%x][%d] on core[%x,%d]\n",
+__FUNCTION__, cxy, level, tgt_cxy, level - 1, tgt_cxy, tgt_lid );
+#endif
+                }  //end if sts[x][y]
+            }  // end for y
+        }  // end for x
+        
+        // wait on barrier until "build" children threads completed
+        if( pthread_barrier_wait( &build_barrier[cxy][level] ) )
+        {
+            printf("\n[ERROR] in %s / first barrier for <build> thread[%x][%d]\n",
+            __FUNCTION__ , cxy , level );
+
+            // report error to parent
+            build_args[parent_cxy][level+1].error = 1;
+        }
+
+#if DEBUG_PTHREAD_PARALLEL
+printf("\n[%s] <build> thread[%x][%d] resume after children completion\n",
+__FUNCTION__, cxy, level );
+#endif
+
+    }  // end level > 0
+
+    // report error to parent when required
+    if( build_args[cxy][level].error )
+    {
+        build_args[parent_cxy][level+1].error = 1;
+    }
+
+    // all <build> threads - but the root - 
+    // signal completion to parent thread and exit 
+    if( level < root_level )
+    {
+        if( pthread_barrier_wait( &build_barrier[parent_cxy][level+1] ) )
+        {
+            printf("\n[ERROR] in %s / second barrier for <build> thread[%x][%d]\n",
+            __FUNCTION__ , cxy , level );
+
+            // report error to parent
+            build_args[parent_cxy][level+1].error = 1;
+        }
+   
+#if DEBUG_PTHREAD_PARALLEL
+printf("\n[%s] <build> thread[%x][%d] exit\n",
+__FUNCTION__, cxy , level );
+#endif
+        // "build" thread exit
+        pthread_exit( NULL );
+    }
+}  // end pthread_recursive_build()
+
+///////////////////////////////////////////////////////
+int pthread_parallel_create( unsigned int   root_level,
+                             void         * work_func,
+                             void         * work_args_array,
+                             void         * parent_barriers_array )
+{
+    unsigned int   root_cxy;
+    unsigned int   root_lid;    // unused, but required by get_core_id()
+    
+#if DEBUG_PTHREAD_PARALLEL
+printf("\n[%s] enter / root_level %d / func %x / args %x / barriers %x\n",
+__FUNCTION__, root_level, work_func, work_args_array, parent_barriers_array );
+#endif
+
+    // get calling thread cluster
+    get_core_id( &root_cxy , &root_lid );
+
+    // set the build function arguments for the root <build> thread
+    build_args[root_cxy][root_level].cxy                   = root_cxy; 
+    build_args[root_cxy][root_level].level                 = root_level;
+    build_args[root_cxy][root_level].root_level            = root_level;
+    build_args[root_cxy][root_level].work_func             = work_func;
+    build_args[root_cxy][root_level].work_args_array       = work_args_array;
+    build_args[root_cxy][root_level].parent_barriers_array = parent_barriers_array;
+    
+    // call the recursive build function
+    pthread_recursive_build( &build_args[root_cxy][root_level] );
+
+    // check error
+    if( build_args[root_cxy][root_level].error )
+    {
+        printf("\n[error] in  %s\n", __FUNCTION__ );
+        return -1;
+    }
+
+    return 0;
+
+}  // end pthread_parallel_create()
+
+
 
 // Local Variables:
