Index: trunk/hal/tsar_mips32/core/hal_gpt.c
===================================================================
--- trunk/hal/tsar_mips32/core/hal_gpt.c	(revision 637)
+++ trunk/hal/tsar_mips32/core/hal_gpt.c	(revision 640)
@@ -25,4 +25,5 @@
 #include <hal_gpt.h>
 #include <hal_special.h>
+#include <hal_irqmask.h>
 #include <printk.h>
 #include <bits.h>
@@ -133,5 +134,5 @@
 ///////////////////////////////////////////////////////////////////////////////////////
 
-#define GPT_LOCK_WATCHDOG   1000000
+#define GPT_LOCK_WATCHDOG   100000
 
 /////////////////////////////////////
@@ -166,6 +167,10 @@
     }
 
-    gpt->ptr = base;
-	gpt->ppn = ppm_base2ppn( XPTR( local_cxy , base ) );
+    // initialze the GPT descriptor
+    gpt->ptr              = base;
+    gpt->pte1_wait_events = 0;
+    gpt->pte1_wait_iters  = 0;
+    gpt->pte2_wait_events = 0;
+    gpt->pte2_wait_iters  = 0;
 
 #if DEBUG_HAL_GPT_CREATE
@@ -173,5 +178,6 @@
 if( DEBUG_HAL_GPT_CREATE < cycle )
 printk("\n[%s] thread[%x,%x] exit / pt1_base %x / pt1_ppn %x / cycle %d\n", 
-__FUNCTION__, this->process->pid, this->trdid, gpt->ptr, gpt->ppn, cycle );
+__FUNCTION__, this->process->pid, this->trdid, 
+base, ppm_base2ppn( XPTR( local_cxy , base ) ), cycle );
 #endif
 
@@ -192,7 +198,8 @@
 	kmem_req_t   req;
 
+    thread_t * this = CURRENT_THREAD;
+
 #if DEBUG_HAL_GPT_DESTROY
 uint32_t   cycle = (uint32_t)hal_get_cycles();
-thread_t * this  = CURRENT_THREAD;
 if( DEBUG_HAL_GPT_DESTROY < cycle )
 printk("\n[%s] thread[%x,%x] enter / cycle %d\n", 
@@ -212,6 +219,6 @@
             if( (pte1 & TSAR_PTE_SMALL) == 0 )   // BIG page
             {
-                printk("\n[WARNING] in %s : mapped big page / ix1 %x\n", 
-                __FUNCTION__ , ix1 );
+                printk("\n[WARNING] %s : valid PTE1 / thread[%x,%x] / ix1 %x\n", 
+                __FUNCTION__, this->process->pid, this->trdid, ix1 );
             }
             else                             // PT2 exist
@@ -228,6 +235,6 @@
                     if( (attr & TSAR_PTE_MAPPED) != 0 )  // PTE2 mapped
                     {
-                        printk("\n[WARNING] in %s : mapped small page / ix1 %x / ix2 %x\n", 
-                        __FUNCTION__ , ix1, ix2 );
+                        printk("\n[WARNING] %s : valid PTE2 / thread[%x,%x] / ix1 %x / ix2 %x\n", 
+                        __FUNCTION__, this->process->pid, this->trdid, ix1, ix2 );
                     }
                 }
@@ -272,9 +279,7 @@
     uint32_t            pte2_attr;       // PT2[ix2].attr current value   
     uint32_t            pte2_ppn;        // PT2[ix2].ppn current value   
-	bool_t              atomic;
-
-#if GPT_LOCK_WATCHDOG
-    uint32_t count = 0;
-#endif
+	bool_t              success;         // used for both PTE1 and PTE2 mapping 
+    uint32_t            count;           // watchdog
+    uint32_t            sr_save;         // for critical section
 
     // get cluster and local pointer on GPT
@@ -285,5 +290,6 @@
 thread_t * this  = CURRENT_THREAD;
 uint32_t   cycle = (uint32_t)hal_get_cycles();
-if( DEBUG_HAL_GPT_LOCK_PTE < cycle )
+// if( DEBUG_HAL_GPT_LOCK_PTE < cycle )
+if( (vpn == 0x3600) && (gpt_cxy == 0x11) )
 printk("\n[%s] thread[%x,%x] enters / vpn %x in cluster %x / cycle %d\n", 
 __FUNCTION__, this->process->pid, this->trdid, vpn, gpt_cxy, cycle );
@@ -303,19 +309,28 @@
     pte1 = hal_remote_l32( pte1_xp );
 
-    // If PTE1 is unmapped and unlocked, try to atomically lock this PT1 entry.
-    // This PTE1 locking prevent multiple concurrent PT2 allocations
-    // - only the thread that successfully locked the PTE1 allocates a new PT2
-    //   and updates the PTE1
-    // - all other threads simply wait until the missing PTE1 is mapped.
-
-    if( pte1 == 0 )  
+    // If PTE1 is unmapped, the calling thread try to map this PTE1.
+    // To prevent multiple concurrent PT2 allocations, only the thread that 
+    // successfully locked the PTE1 allocates a new PT2 and updates the PTE1.
+    // All other threads simply wait until the missing PTE1 is mapped.
+
+    if( (pte1 & TSAR_PTE_MAPPED) == 0 )  
 	{
-        // try to atomically lock the PTE1 to prevent concurrent PT2 allocations
-        atomic = hal_remote_atomic_cas( pte1_xp,
-                                        pte1,
-                                        pte1 | TSAR_PTE_LOCKED );
-        if( atomic )  
-		{
-            // allocate one 4 Kbytes physical page for PT2
+        if( (pte1 & TSAR_PTE_LOCKED)  == 0 )
+        { 
+            // try to atomically lock the PTE1
+            success = hal_remote_atomic_cas( pte1_xp,
+                                             pte1,
+                                             TSAR_PTE_LOCKED );
+        }
+        else
+        {
+            success = false;
+        }
+    
+        if( success )       // winner thread allocates one 4 Kbytes page for PT2
+        {
+            // enter critical section
+            hal_disable_irq( &sr_save );
+
             req.type  = KMEM_PPM;
             req.order = 0;  
@@ -336,41 +351,80 @@
             pte1 = TSAR_PTE_MAPPED | TSAR_PTE_SMALL | pt2_ppn;
 
-            // set the PTE1 value in PT1
-            // this unlocks the PTE1
+            // set the PTE1 value in PT1 / this unlocks the PTE1
             hal_remote_s32( pte1_xp , pte1 );
             hal_fence();
 
-#if (DEBUG_HAL_GPT_LOCK_PTE & 1)
-if( DEBUG_HAL_GPT_LOCK_PTE < cycle )
-printk("\n[%s] thread[%x,%x] allocates a new PT2 for vpn %x in cluster %x\n", 
+            // exit critical section
+            hal_restore_irq( sr_save );
+
+#if DEBUG_HAL_GPT_LOCK_PTE
+// if( DEBUG_HAL_GPT_LOCK_PTE < cycle )
+if( (vpn == 0x3600) && (gpt_cxy == 0x11) )
+printk("\n[%s] PTE1 unmapped : winner thread[%x,%x] allocates a PT2 for vpn %x in cluster %x\n", 
 __FUNCTION__, this->process->pid, this->trdid, vpn, gpt_cxy );
 #endif
 
-        }  // end if atomic
-    }  // end if (pte1 == 0)
-
-    // wait until PTE1 is mapped by another thread
-    while( (pte1 & TSAR_PTE_MAPPED) == 0 )
-    {
-        pte1 = hal_remote_l32( pte1_xp );
-
-#if GPT_LOCK_WATCHDOG
-if( count > GPT_LOCK_WATCHDOG )  
-{
-    thread_t * thread = CURRENT_THREAD;
-    printk("\n[PANIC] in %s : thread[%x,%x] waiting PTE1 / vpn %x / cxy %x / %d iterations\n",
-    __FUNCTION__, thread->process->pid, thread->trdid, vpn, gpt_cxy, count );
-    hal_core_sleep();
-}
-count++;
-#endif
-
-    }
-
-// check pte1 because only small page can be locked
-assert( (pte1 & TSAR_PTE_SMALL), "cannot lock a big page\n");
-
-#if (DEBUG_HAL_GPT_LOCK_PTE & 1)
-if( DEBUG_HAL_GPT_LOCK_PTE < cycle )
+        }
+        else                 // other threads wait until PTE1 mapped by the winner
+        {
+
+#if DEBUG_HAL_GPT_LOCK_PTE
+// if( DEBUG_HAL_GPT_LOCK_PTE < cycle )
+if( (vpn == 0x3600) && (gpt_cxy == 0x11) )
+printk("\n[%s] PTE1 unmapped : loser thread[%x,%x] wait PTE1 for vpn %x in cluster %x\n", 
+__FUNCTION__, this->process->pid, this->trdid, vpn, gpt_cxy );
+#endif
+
+            count = 0;
+            do
+            {
+                // get current pte1 value
+                pte1 = hal_remote_l32( pte1_xp );
+
+                // check iterations number
+                if( count > GPT_LOCK_WATCHDOG )
+                {
+                    thread_t * this  = CURRENT_THREAD;
+                    uint32_t   cycle = (uint32_t)hal_get_cycles();
+                    printk("\n[PANIC] in %s for PTE1 after %d iterations\n"
+                    "  thread[%x,%x] / vpn %x / cluster %x / pte1 %x / cycle %d\n",
+                    __FUNCTION__, count, this->process->pid, this->trdid,
+                    vpn, gpt_cxy, pte1, cycle );
+
+                    xptr_t process_xp = cluster_get_process_from_pid_in_cxy( gpt_cxy,
+                                                                             this->process->pid );
+                    hal_vmm_display( process_xp , true );
+ 
+                    hal_core_sleep();
+                }
+
+                // increment watchdog
+                count++;
+            }
+            while( (pte1 & TSAR_PTE_MAPPED) == 0 ); 
+
+#if CONFIG_INSTRUMENTATION_GPT
+hal_remote_atomic_add( XPTR( gpt_cxy , &gpt_ptr->pte1_wait_events ) , 1 );
+hal_remote_atomic_add( XPTR( gpt_cxy , &gpt_ptr->pte1_wait_iters  ) , count );
+#endif
+
+
+#if DEBUG_HAL_GPT_LOCK_PTE
+// if( DEBUG_HAL_GPT_LOCK_PTE < cycle )
+if( (vpn == 0x3600) && (gpt_cxy == 0x11) )
+printk("\n[%s] PTE1 unmapped : loser thread[%x,%x] get PTE1 for vpn %x in cluster %x\n", 
+__FUNCTION__, this->process->pid, this->trdid, vpn, gpt_cxy );
+#endif
+        }
+    }  // end if pte1 unmapped
+
+    // This code is executed by all calling threads
+ 
+// check PTE1 : only small and mapped pages can be locked
+assert( (pte1 & (TSAR_PTE_SMALL | TSAR_PTE_MAPPED)) , "cannot lock a big or unmapped page\n");
+
+#if DEBUG_HAL_GPT_LOCK_PTE 
+// if( DEBUG_HAL_GPT_LOCK_PTE < cycle )
+if( (vpn == 0x3600) && (gpt_cxy == 0x11) )
 printk("\n[%s] thread[%x,%x] get pte1 %x for vpn %x in cluster %x\n", 
 __FUNCTION__, this->process->pid, this->trdid, pte1, vpn, gpt_cxy );
@@ -384,37 +438,53 @@
     pte2_xp = XPTR( gpt_cxy , &pt2[2 * ix2] );
 
-    // wait until PTE2 atomically set using a remote CAS
+    // initialize external loop watchdog
+    count = 0;
+
+    // in this busy waiting loop, each thread try to atomically
+    // lock the PTE2, after checking that the PTE2 is not locked 
+    
     do
     {
-
-#if GPT_LOCK_WATCHDOG
-count = 0;
-#endif
-
-        // wait until PTE lock released by the current owner
-        do
+        // get current value of pte2_attr
+        pte2_attr = hal_remote_l32( pte2_xp );
+
+        // check loop watchdog
+        if( count > GPT_LOCK_WATCHDOG )
         {
-            pte2_attr = hal_remote_l32( pte2_xp );
-
-#if GPT_LOCK_WATCHDOG
-if( count > GPT_LOCK_WATCHDOG )  
-{
-    thread_t * thread = CURRENT_THREAD;
-    printk("\n[PANIC] in %s : thread[%x,%x] waiting PTE2 / vpn %x / cxy %x / %d iterations\n",
-    __FUNCTION__, thread->process->pid, thread->trdid, vpn, gpt_cxy, count );
-    hal_core_sleep();
-}
-count++;
-#endif
-      
+            thread_t * this  = CURRENT_THREAD;
+            uint32_t   cycle = (uint32_t)hal_get_cycles();
+            printk("\n[PANIC] in %s for PTE2 after %d iterations\n"
+            "  thread[%x,%x] / vpn %x / cluster %x / pte2_attr %x / cycle %d\n",
+            __FUNCTION__, count, this->process->pid, this->trdid,
+            vpn, gpt_cxy, pte2_attr, cycle );
+
+                    xptr_t process_xp = cluster_get_process_from_pid_in_cxy( gpt_cxy,
+                                                                             this->process->pid );
+                    hal_vmm_display( process_xp , true );
+ 
+            hal_core_sleep();
         }
-        while( (pte2_attr & TSAR_PTE_LOCKED) != 0 );
-
-        // try to atomically set the TSAR_PTE_LOCKED attribute    
-		atomic = hal_remote_atomic_cas( pte2_xp,
-                                        pte2_attr,
-                                        (pte2_attr | TSAR_PTE_LOCKED) );
+
+        // increment loop watchdog
+        count++;
+
+        if( (pte2_attr & TSAR_PTE_LOCKED) == 0 )
+        {
+            // try to atomically set the TSAR_PTE_LOCKED attribute    
+		    success = hal_remote_atomic_cas( pte2_xp,
+                                             pte2_attr,
+                                             (pte2_attr | TSAR_PTE_LOCKED) );
+        }
+        else
+        {
+            success = false;
+        }
     }
-    while( atomic == 0 );
+    while( success == false );
+
+#if CONFIG_INSTRUMENTATION_GPT
+hal_remote_atomic_add( XPTR( gpt_cxy , &gpt_ptr->pte2_wait_events ) , 1 );
+hal_remote_atomic_add( XPTR( gpt_cxy , &gpt_ptr->pte2_wait_iters  ) , count );
+#endif
 
     // get PTE2.ppn
@@ -423,6 +493,7 @@
 #if DEBUG_HAL_GPT_LOCK_PTE
 cycle = (uint32_t)hal_get_cycles();
-if( DEBUG_HAL_GPT_LOCK_PTE < cycle )
-printk("\n[%s] thread[%x,%x] exit / vpn %x in cluster %x / attr %x / ppn %x / cycle %d\n", 
+// if( DEBUG_HAL_GPT_LOCK_PTE < cycle )
+if( (vpn == 0x3600) && (gpt_cxy == 0x11) )
+printk("\n[%s] thread[%x,%x] success / vpn %x in cluster %x / attr %x / ppn %x / cycle %d\n", 
 __FUNCTION__, this->process->pid, this->trdid, vpn, gpt_cxy, pte2_attr, pte2_ppn, cycle );
 #endif
@@ -452,49 +523,44 @@
     gpt_t * gpt_ptr = GET_PTR( gpt_xp );
 
+    // compute indexes in P1 and PT2
+    uint32_t  ix1 = TSAR_MMU_IX1_FROM_VPN( vpn ); 
+    uint32_t  ix2 = TSAR_MMU_IX2_FROM_VPN( vpn );
+
+    // get local pointer on PT1
+    pt1 = hal_remote_lpt( XPTR( gpt_cxy , &gpt_ptr->ptr ) );
+
+    // build extended pointer on PTE1 == PT1[ix1]
+	pte1_xp = XPTR( gpt_cxy , &pt1[ix1] );
+
+    // get current pte1 value
+    pte1 = hal_remote_l32( pte1_xp );
+
+assert( ((pte1 & TSAR_PTE_MAPPED) != 0),
+"PTE1 for vpn %x in cluster %x is unmapped / pte1 = %x\n", vpn, gpt_cxy, pte1 );
+
+assert( ((pte1 & TSAR_PTE_SMALL ) != 0),
+"PTE1 for vpn %x in cluster %x is not small / pte1 = %x\n", vpn, gpt_cxy, pte1 );
+
+    // get pointer on PT2 base  
+	pt2_ppn = TSAR_MMU_PPN2_FROM_PTE1( pte1 );
+	pt2     = GET_PTR( ppm_ppn2base( pt2_ppn ) );
+
+    // build extended pointers on PT2[ix2].attr  
+    pte2_xp = XPTR( gpt_cxy , &pt2[2 * ix2] );
+
+    // get PT2[ix2].attr
+    pte2_attr = hal_remote_l32( pte2_xp );
+
+assert( ((pte2_attr & TSAR_PTE_LOCKED) != 0),
+"PTE2 for vpn %x in cluster %x is unlocked / pte2_attr = %x\n", vpn, gpt_cxy, pte2_attr );
+
+    // reset TSAR_PTE_LOCKED attribute 
+    hal_remote_s32( pte2_xp , pte2_attr & ~TSAR_PTE_LOCKED );
+
 #if DEBUG_HAL_GPT_LOCK_PTE
 thread_t * this  = CURRENT_THREAD;
 uint32_t   cycle = (uint32_t)hal_get_cycles();
-if( DEBUG_HAL_GPT_LOCK_PTE < cycle )
-printk("\n[%s] thread[%x,%x] enters for vpn %x in cluster %x / cycle %d\n", 
-__FUNCTION__, this->process->pid, this->trdid, vpn, gpt_cxy, cycle );
-#endif
-
-    // compute indexes in P1 and PT2
-    uint32_t  ix1 = TSAR_MMU_IX1_FROM_VPN( vpn ); 
-    uint32_t  ix2 = TSAR_MMU_IX2_FROM_VPN( vpn );
-
-    // get local pointer on PT1
-    pt1 = hal_remote_lpt( XPTR( gpt_cxy , &gpt_ptr->ptr ) );
-
-    // build extended pointer on PTE1 == PT1[ix1]
-	pte1_xp = XPTR( gpt_cxy , &pt1[ix1] );
-
-    // get current pte1 value
-    pte1 = hal_remote_l32( pte1_xp );
-
-// check PTE1 attributes
-assert( ((pte1 & TSAR_PTE_MAPPED) != 0), "unmapped PTE1\n");
-assert( ((pte1 & TSAR_PTE_SMALL ) != 0), "big page PTE1\n");
-
-    // get pointer on PT2 base  
-	pt2_ppn = TSAR_MMU_PPN2_FROM_PTE1( pte1 );
-	pt2     = GET_PTR( ppm_ppn2base( pt2_ppn ) );
-
-    // build extended pointers on PT2[ix2].attr  
-    pte2_xp = XPTR( gpt_cxy , &pt2[2 * ix2] );
-
-    // get PT2[ix2].attr
-    pte2_attr = hal_remote_l32( pte2_xp );
-
-// check PTE2 attributes
-assert( ((pte2_attr & TSAR_PTE_MAPPED) != 0), "unmapped PTE2\n");
-assert( ((pte2_attr & TSAR_PTE_LOCKED) != 0), "unlocked PTE2\n");
-
-    // reset TSAR_PTE_LOCKED attribute 
-    hal_remote_s32( pte2_xp , pte2_attr & ~TSAR_PTE_LOCKED );
-
-#if DEBUG_HAL_GPT_LOCK_PTE
-cycle = (uint32_t)hal_get_cycles();
-if( DEBUG_HAL_GPT_LOCK_PTE < cycle )
+// if( DEBUG_HAL_GPT_LOCK_PTE < cycle )
+if( (vpn == 0xc5fff) && (gpt_cxy == 0x1) )
 printk("\n[%s] thread[%x,%x] unlocks vpn %x in cluster %x / cycle %d\n", 
 __FUNCTION__, this->process->pid, this->trdid, vpn, gpt_cxy, cycle );
@@ -587,5 +653,6 @@
 
 // PTE1 must be mapped because PTE2 must be locked
-assert( (pte1 & TSAR_PTE_MAPPED), "PTE1 must be mapped\n" );
+assert( (pte1 & TSAR_PTE_MAPPED),
+"PTE1 for vpn %x in cluster %x must be mapped / pte1 = %x\n", vpn, gpt_cxy, pte1 );
 
         // get PT2 base 
@@ -601,5 +668,6 @@
 
 // PTE2 must be locked
-assert( (pte2_attr & TSAR_PTE_LOCKED), "PTE2 must be locked\n" );
+assert( (pte2_attr & TSAR_PTE_LOCKED),
+"PTE2 for vpn %x in cluster %x must be locked / pte2_attr = %x\n", vpn, gpt_cxy, pte2_attr );
  
         // set PTE2 in PT2 (in this order)
