Index: trunk/kernel/libk/remote_condvar.c
===================================================================
--- trunk/kernel/libk/remote_condvar.c	(revision 457)
+++ trunk/kernel/libk/remote_condvar.c	(revision 563)
@@ -1,6 +1,6 @@
 /*
- * remote_condvar.c - distributed kernel condvar implementaion 
- * 
- * Author   Alain Greiner (2016)
+ * remote_condvar.c - remote kernel condition variable implementation.
+ *
+ * Authors     Alain Greiner (2016,2017,2018)
  *
  * Copyright (c) UPMC Sorbonne Universites
@@ -22,15 +22,13 @@
  */
 
+#include <kernel_config.h>
 #include <hal_kernel_types.h>
-#include <hal_remote.h>
-#include <hal_irqmask.h>
 #include <thread.h>
-#include <kmem.h>
-#include <printk.h>
-#include <process.h>
-#include <vmm.h>
+#include <scheduler.h>
 #include <xlist.h>
 #include <remote_mutex.h>
+#include <remote_busylock.h>
 #include <remote_condvar.h>
+
 
 ///////////////////////////////////////////////////
@@ -45,14 +43,18 @@
     // get cluster and local pointer on reference process 
     cxy_t          ref_cxy = GET_CXY( ref_xp );
-    process_t    * ref_ptr = (process_t *)GET_PTR( ref_xp );
-
-    // get extended pointer on root of condvars list 
+    process_t    * ref_ptr = GET_PTR( ref_xp );
+
+    // get extended pointer on condvars list 
     xptr_t root_xp = XPTR( ref_cxy , &ref_ptr->condvar_root );
-   
-    // scan reference process condvars list
+    xptr_t lock_xp = XPTR( ref_cxy , &ref_ptr->sync_lock );
+
+    // get lock protecting synchro lists
+    remote_queuelock_acquire( lock_xp );
+ 
+    // scan reference process condvar list
     xptr_t             iter_xp;
     xptr_t             condvar_xp;
     cxy_t              condvar_cxy;
-    struct remote_condvar_s * condvar_ptr;
+    remote_condvar_t * condvar_ptr;
     intptr_t           current;
     bool_t             found = false;
@@ -62,7 +64,8 @@
         condvar_xp  = XLIST_ELEMENT( iter_xp , remote_condvar_t , list );
         condvar_cxy = GET_CXY( condvar_xp );
-        condvar_ptr = (remote_condvar_t *)GET_PTR( condvar_xp );
-        current     = (intptr_t)hal_remote_lpt( XPTR( condvar_cxy , &condvar_ptr->ident ) );   
-        if( ident == current )
+        condvar_ptr = GET_PTR( condvar_xp );
+        current = (intptr_t)hal_remote_lpt( XPTR( condvar_cxy , &condvar_ptr->ident ) );   
+
+        if( current == ident )
         {
             found = true;
@@ -71,4 +74,7 @@
     }
 
+    // relese lock protecting synchros lists
+    remote_queuelock_release( lock_xp );
+ 
     if( found == false )  return XPTR_NULL;
     else                  return condvar_xp;
@@ -76,9 +82,9 @@
 }  // end remote_condvar_from_ident()
 
-///////////////////////////////////////////////
-error_t remote_condvar_create( intptr_t ident )
-{
+/////////////////////////////////////////////////
+error_t remote_condvar_create( intptr_t   ident )
+{
+    remote_condvar_t * condvar_ptr;
     xptr_t             condvar_xp;
-    remote_condvar_t * condvar_ptr;
 
     // get pointer on local process descriptor
@@ -92,33 +98,34 @@
     process_t * ref_ptr = (process_t *)GET_PTR( ref_xp );
 
-    // allocate memory for condvar descriptor
-    if( ref_cxy == local_cxy )                  // local cluster is the reference 
+    // allocate memory for new condvar in reference cluster
+    if( ref_cxy == local_cxy )                              // local cluster is the reference 
     {
         kmem_req_t req;   
-        req.type      = KMEM_CONDVAR;
-        req.flags     = AF_ZERO;
-        condvar_ptr   = kmem_alloc( &req );
-        condvar_xp    = XPTR( local_cxy , condvar_ptr );
-    }
-    else                                       // reference is remote
+        req.type    = KMEM_CONDVAR;
+        req.flags   = AF_ZERO;
+        condvar_ptr = kmem_alloc( &req );
+        condvar_xp  = XPTR( local_cxy , condvar_ptr );
+    }
+    else                                                   // reference cluster is remote
     {
         rpc_kcm_alloc_client( ref_cxy , KMEM_CONDVAR , &condvar_xp );
-        condvar_ptr = (remote_condvar_t *)GET_PTR( condvar_xp );
-    }
-
-    if( condvar_ptr == NULL ) return ENOMEM;
+        condvar_ptr = GET_PTR( condvar_xp );
+    }
+
+    if( condvar_xp == XPTR_NULL ) return 0xFFFFFFFF;
 
     // initialise condvar 
-    hal_remote_spt( XPTR( ref_cxy , &condvar_ptr->ident      ) , (void*)ident );
-    xlist_entry_init( XPTR( ref_cxy , &condvar_ptr->list ) );
-    xlist_root_init( XPTR( ref_cxy , &condvar_ptr->root ) );
-
-    // register  condvar in reference process xlist
+	hal_remote_spt( XPTR( ref_cxy , &condvar_ptr->ident ) , (void *)ident );
+	xlist_root_init( XPTR( ref_cxy , &condvar_ptr->root ) );
+	xlist_entry_init( XPTR( ref_cxy , &condvar_ptr->list ) );
+    remote_busylock_init( XPTR( ref_cxy , &condvar_ptr->lock ), LOCK_CONDVAR_STATE );
+
+    // register condvar in reference process xlist
     xptr_t root_xp = XPTR( ref_cxy , &ref_ptr->condvar_root );
-    xptr_t xp_list = XPTR( ref_cxy , &condvar_ptr->list );
-
-    remote_spinlock_lock( XPTR( ref_cxy , &ref_ptr->sync_lock ) );
-    xlist_add_first( root_xp , xp_list );
-    remote_spinlock_unlock( XPTR( ref_cxy , &ref_ptr->sync_lock ) );
+    xptr_t list_xp = XPTR( ref_cxy , &condvar_ptr->list );
+
+    remote_queuelock_acquire( XPTR( ref_cxy , &ref_ptr->sync_lock ) );
+    xlist_add_first( root_xp , list_xp );
+    remote_queuelock_release( XPTR( ref_cxy , &ref_ptr->sync_lock ) );
 
     return 0;
@@ -137,20 +144,30 @@
     // get reference process cluster and local pointer
     cxy_t       ref_cxy = GET_CXY( ref_xp );
-    process_t * ref_ptr = (process_t *)GET_PTR( ref_xp );
-
-    // get condvar cluster and local pointer
-    cxy_t              condvar_cxy = GET_CXY( condvar_xp );
-    remote_condvar_t * condvar_ptr = (remote_condvar_t *)GET_PTR( condvar_xp );
+    process_t * ref_ptr = GET_PTR( ref_xp );
+
+    // get condvar cluster and local pointer
+    cxy_t              condvar_cxy = GET_CXY( condvar_xp );
+    remote_condvar_t * condvar_ptr = GET_PTR( condvar_xp );
+
+    // get remote pointer on waiting queue root
+    xptr_t root_xp = XPTR( condvar_cxy , &condvar_ptr->root );
+  
+    if( !xlist_is_empty( root_xp ) )   // user error
+    {
+        printk("WARNING in %s for thread %x in process %x : "
+               "destroy condvar, but  waiting threads queue not empty\n", 
+               __FUNCTION__ , CURRENT_THREAD->trdid , CURRENT_THREAD->process->pid );
+    }
 
     // remove condvar from reference process xlist
-    remote_spinlock_lock( XPTR( ref_cxy , &ref_ptr->sync_lock ) );
+    remote_queuelock_acquire( XPTR( ref_cxy , &ref_ptr->sync_lock ) );
     xlist_unlink( XPTR( condvar_cxy , &condvar_ptr->list ) );
-    remote_spinlock_unlock( XPTR( ref_cxy , &ref_ptr->sync_lock ) );
-
-    // release memory allocated for condvaraphore descriptor
+    remote_queuelock_release( XPTR( ref_cxy , &ref_ptr->sync_lock ) );
+
+    // release memory allocated for condvar descriptor
     if( condvar_cxy == local_cxy )                            // reference is local
     {
         kmem_req_t  req;
-        req.type = KMEM_BARRIER;
+        req.type = KMEM_SEM;
         req.ptr  = condvar_ptr;
         kmem_free( &req );
@@ -158,8 +175,8 @@
     else                                                  // reference is remote
     {
-        rpc_kcm_free_client( condvar_cxy , condvar_ptr , KMEM_BARRIER );
-    }
-
-}  // end remote_condvar_destroy()
+        rpc_kcm_free_client( condvar_cxy , condvar_ptr , KMEM_CONDVAR );
+    }
+
+}  // end remote_convar_destroy()
 
 ////////////////////////////////////////////
@@ -167,30 +184,29 @@
                           xptr_t mutex_xp )
 {
-    // unlock the mutex
+    thread_t * this = CURRENT_THREAD;
+
+// check calling thread can yield
+assert( (this->busylocks == 0),
+"cannot yield : busylocks = %d\n", this->busylocks );
+
+    // get condvar cluster and local pointer
+    remote_condvar_t * condvar_ptr = GET_PTR( condvar_xp );
+    cxy_t              condvar_cxy = GET_CXY( condvar_xp );
+
+    // register the calling thread in condvar waiting queue
+    xlist_add_last( XPTR( condvar_cxy , &condvar_ptr->root ),
+                    XPTR( local_cxy   , &this->wait_xlist ) );
+
+    // block the calling thread
+    thread_block( XPTR( local_cxy , this ) , THREAD_BLOCKED_USERSYNC );
+
+    // release the mutex
     remote_mutex_unlock( mutex_xp );
 
-    thread_t * this = CURRENT_THREAD; 
-
-    // get condvar cluster an local pointer
-    remote_condvar_t * condvar_ptr = (remote_condvar_t *)GET_PTR( condvar_xp );
-    cxy_t              condvar_cxy = GET_CXY( condvar_xp );
-
-    // get extended pointer on condvar waiting queue 
-    xptr_t root_xp  = XPTR( condvar_cxy , &condvar_ptr->root );
-
-    // get extended pointer on calling thread xlist_entry
-    xptr_t entry_xp = XPTR( local_cxy , &this->wait_list ); 
-
-    // register the calling thread in the condvar waiting queue
-    remote_spinlock_lock( XPTR( condvar_cxy , &condvar_ptr->lock ) ); 
-    xlist_add_last( root_xp , entry_xp );
-    remote_spinlock_unlock( XPTR( condvar_cxy , &condvar_ptr->lock ) ); 
-
-    // block the calling thread 
-    thread_block( XPTR( local_cxy , CURRENT_THREAD ) , THREAD_BLOCKED_USERSYNC );
+    // deschedule
     sched_yield("blocked on condvar");
 
-    // lock the mutex before return
-    remote_mutex_unlock( mutex_xp );
+    // re-acquire the mutex
+    remote_mutex_lock( mutex_xp );
 
 }  // end remote_condvar_wait()
@@ -199,31 +215,23 @@
 void remote_condvar_signal( xptr_t condvar_xp )
 {
-    reg_t     irq_state;
-
-    // get condvar cluster an local pointer
-    remote_condvar_t * condvar_ptr = (remote_condvar_t *)GET_PTR( condvar_xp );
-    cxy_t              condvar_cxy = GET_CXY( condvar_xp );
-
-    // get extended pointer on condvar waiting queue 
-    xptr_t root_xp  = XPTR( condvar_cxy , &condvar_ptr->root );
-
-    if( xlist_is_empty( root_xp ) ) return;
-
-    // disable interrupts
-	hal_disable_irq( &irq_state );
-  
-    // get extended pointer on the first waiting thread
-    xptr_t thread_xp = XLIST_FIRST_ELEMENT( root_xp , thread_t , wait_list );
-
-    // remove the first waiting thread from queue
-    remote_spinlock_lock( XPTR( condvar_cxy , &condvar_ptr->lock ) );
-    xlist_unlink( XPTR( condvar_cxy , &condvar_ptr->list ) );
-    remote_spinlock_unlock( XPTR( condvar_cxy , &condvar_ptr->lock ) );
-
-    // unblock first waiting thread
-    thread_unblock( thread_xp , THREAD_BLOCKED_USERSYNC ); 
-
-    // restore interrupts
-	hal_restore_irq( irq_state );
+    // get condvar cluster and local pointer
+    remote_condvar_t * condvar_ptr = GET_PTR( condvar_xp );
+    cxy_t              condvar_cxy = GET_CXY( condvar_xp );
+
+    // does nothing if waiting queue empty
+    if( xlist_is_empty( XPTR( condvar_cxy, &condvar_ptr->root ) ) == false )
+    {
+         // get first waiting thread
+         xptr_t thread_xp = XLIST_FIRST( XPTR( condvar_cxy , &condvar_ptr->root ),
+                                         thread_t , wait_xlist );
+
+         // remove this waiting thread from queue
+         thread_t * thread_ptr = GET_PTR( thread_xp );
+         cxy_t      thread_cxy = GET_CXY( thread_xp );
+         xlist_unlink( XPTR( thread_cxy , &thread_ptr->wait_xlist ) );
+
+         // unblock this waiting thread
+         thread_unblock( thread_xp , THREAD_BLOCKED_USERSYNC );
+    }
 
 }  // end remote_condvar_signal()
@@ -232,37 +240,23 @@
 void remote_condvar_broadcast( xptr_t condvar_xp )
 {
-    reg_t    irq_state;
-
-    // get condvar cluster an local pointer
-    remote_condvar_t * condvar_ptr = (remote_condvar_t *)GET_PTR( condvar_xp );
-    cxy_t              condvar_cxy = GET_CXY( condvar_xp );
-
-    // get extended pointer on condvar waiting queue 
-    xptr_t root_xp  = XPTR( condvar_cxy , &condvar_ptr->root );
-
-    if( xlist_is_empty( root_xp ) ) return;
-
-    // disable interrupts
-    hal_disable_irq( &irq_state );
-  
-    // loop on waiting threads
-    xptr_t  iter_xp;
-    xptr_t  thread_xp;
-    XLIST_FOREACH( root_xp , iter_xp )
-    {
-        // get extended pointer on waiting thread
-        thread_xp = XLIST_ELEMENT( iter_xp , thread_t , wait_list );
-        
-        // remove waiting thread from queue
-        remote_spinlock_lock( XPTR( condvar_cxy , &condvar_ptr->lock ) );
-        xlist_unlink( XPTR( condvar_cxy , &condvar_ptr->list ) );
-        remote_spinlock_unlock( XPTR( condvar_cxy , &condvar_ptr->lock ) );
-
-        // unblock waiting thread
-        thread_unblock( thread_xp , THREAD_BLOCKED_USERSYNC ); 
-    }
-
-    // restore interrupts
-    hal_restore_irq( irq_state );
-
+    // get condvar cluster and local pointer
+    remote_condvar_t * condvar_ptr = GET_PTR( condvar_xp );
+    cxy_t              condvar_cxy = GET_CXY( condvar_xp );
+
+    // does nothing if waiting queue empty
+    while( xlist_is_empty( XPTR( condvar_cxy , &condvar_ptr->root ) ) == false )
+    {
+         // get first waiting thread
+         xptr_t thread_xp = XLIST_FIRST( XPTR( condvar_cxy , &condvar_ptr->root ),
+                                         thread_t , wait_xlist );
+
+         // remove this waiting thread from queue
+         thread_t * thread_ptr = GET_PTR( thread_xp );
+         cxy_t      thread_cxy = GET_CXY( thread_xp );
+         xlist_unlink( XPTR( thread_cxy , &thread_ptr->wait_xlist ) );
+
+         // unblock this waiting thread
+         thread_unblock( thread_xp , THREAD_BLOCKED_USERSYNC );
+    }
 }  // end remote_condvar_broadcast()
+
