Index: trunk/hal/tsar_mips32/core/hal_uspace.c
===================================================================
--- trunk/hal/tsar_mips32/core/hal_uspace.c	(revision 625)
+++ trunk/hal/tsar_mips32/core/hal_uspace.c	(revision 626)
@@ -2,6 +2,5 @@
  * hal_uspace.c - implementation of Generic User Space Access API for MIPS32
  * 
- * Author  Mohamed Karaoui (2015)
- *         Alain Greiner   (2016)
+ * Author        Alain Greiner   (2016,2017,2018,2019)
  *
  * Copyright (c) UPMC Sorbonne Universites
@@ -23,8 +22,7 @@
  */
 
-#include <errno.h>
-#include <vmm.h>
 #include <hal_kernel_types.h>
 #include <hal_uspace.h>
+#include <hal_special.h>
 #include <hal_irqmask.h>
 
@@ -32,123 +30,206 @@
 #include <thread.h>
 
-///////////////////////////////////////////
-void hal_copy_from_uspace( void     * k_dst,
+///////////////////////////////////////////////////////////////////////////////////////
+// This function moves <size> bytes from a source buffer in user virtual space,
+// defined by the <u_src> argument, to a destination kernel buffer, defined by the
+// <k_cxy> and <k_dst> arguments.
+// It works in a critical section, as it modifies briefly two CP2 registers:
+// It activates briefly the DATA_MMU by writing into the CP2_MODE register to access the
+// user buffer, and modifies the CP2_DATA_EXT register to access the kernel buffer.
+// If the two buffers are aligned on a word boundary, it moves the data word per word
+// in a first loop, and moves byte per byte the remaining bytes in a second loop.
+// If the buffers are not aligned, it moves all data byte per byte.
+///////////////////////////////////////////////////////////////////////////////////////
+// @ k_cxy    : cluster of destination kernel buffer
+// @ k_dst    : pointer on destination kernel buffer
+// @ u_src    : pointer on source user buffer
+// @ size     : number of bytes to move
+///////////////////////////////////////////////////////////////////////////////////////
+void hal_copy_from_uspace( cxy_t      k_cxy,
+                           void     * k_dst,
                            void     * u_src,
                            uint32_t   size )  
 {
     uint32_t save_sr;
-	uint32_t i;
-	uint32_t wsize;                        // number of words 
+	uint32_t words;                        // number of words (if buffers aligned)
     uint32_t src = (uint32_t)u_src;
     uint32_t dst = (uint32_t)k_dst;
-
+ 
 #if DEBUG_HAL_USPACE
-thread_t * this = CURRENT_THREAD;
-printk("\n[%s] thread[%x,%x] enter in cluster %x / u_src %x / k_dst %x / size %d\n", 
-__FUNCTION__, this->process->pid, this->trdid, local_cxy, u_src, k_dst, size );
+thread_t * this  = CURRENT_THREAD;
+uint32_t   cycle = (uint32_t)hal_get_cycles();
+if( cycle > DEBUG_HAL_USPACE )
+printk("\n[%s] thread[%x,%x] enter / %d bytes / u_buf(%x,%x) -> k_buf(%x,%x) / cycle %d\n", 
+__FUNCTION__, this->process->pid, this->trdid, size, local_cxy, u_src, k_cxy, k_dst, cycle );
 #endif
 
-	if( (dst & 0x3) || (src & 0x3) ) wsize = 0;          // do it all in bytes
-    else                             wsize = size >> 2;
-
+	if( (dst & 0x3) || (src & 0x3) ) words = 0;          // do it all in bytes
+    else                             words = size >> 2;
+
+    // enter critical section
     hal_disable_irq( &save_sr );
 
-
-	for( i = 0 ; i < wsize ; i++ )          // transfer one word per iteration
-	{
-        asm volatile(
-        "mfc2   $15,   $1           \n"   /* save   MMU_MODE                */
-        "ori    $14,   $0,  0x7	    \n"  
-        "mtc2   $14,   $1			\n"   /* MMU_MODE <= DTLB ON            */
-        "lw	    $13,   0(%0)        \n"   /* read data from user space      */
-        "mtc2   $15,   $1			\n"   /* restore MMU_MODE               */
-	    "sw	    $13,   0(%1)        \n"   /* store data to kernel space     */
-        : : "r"( src ) , "r"( dst ) : "$13","$14","$15", "memory" );
-
-        src += 4;
-        dst += 4;
-    }
-
-	for( i = wsize << 2 ; i < size ; i++ )  // transfer one byte per iteration
-	{
-        asm volatile(
-        "mfc2   $15,   $1           \n"   /* save   MMU_MODE                */
-        "ori    $14,   $0,  0x7	    \n"  
-        "mtc2   $14,   $1			\n"   /* MMU_MODE <= DTLB ON            */
-        "lb	    $13,   0(%0)        \n"   /* read data from user space      */
-        "mtc2   $15,   $1			\n"   /* restore MMU_MODE               */
-	    "sb	    $13,   0(%1)        \n"   /* store data to kernel space     */
-        : : "r"( src ) , "r"( dst ) : "$13","$14","$15", "memory" );
-
-        src += 1;
-        dst += 1;
-    }
-
+    asm volatile( ".set noreorder             \n"
+ 
+                  /* initialise registers                                         */
+                  "move   $8,    %0           \n"   /* $8 <= src                  */
+                  "move   $9,    %1           \n"   /* $9 <= dst                  */
+                  "move   $10,   %2           \n"   /* $10 <= words               */
+                  "move   $11,   %3           \n"   /* $11 <= size                */
+                  "mfc2   $12,   $1           \n"   /* $12 <= old MMU_MODE        */
+                  "ori    $13,   $12,   0x4	  \n"   /* $13 <= MMU_MODE with DTLB  */
+
+                  /* save old MMU_DATA_EXT and set k_cxy in it                    */
+                  "mfc2   $16,   $24          \n"   /* $16 <= old MMU_DATA_EXT    */
+                  "mtc2   %4,    $24          \n"   /* MMU_DATA_EXT <= k_cxy      */
+
+                  /* transfer one word per iteration in first loop if aligned     */
+                  "move   $15,   $10          \n"   /* $15 <= words ($15 == i)    */
+                  "1:                         \n"
+                  "beq    $15,   $0,    2f    \n"   /* exit loop if (i==0)        */
+                  "nop                        \n"
+                  "mtc2   $13,   $1			  \n"   /* MMU_MODE <= DTLB ON        */
+                  "lw	  $14,   0($8)        \n"   /* word from user space       */
+                  "mtc2   $12,   $1			  \n"   /* restore old MMU_MODE       */
+	              "sw     $14,   0($9)        \n"   /* word to kernel space       */
+                  "addi   $15,   $15,   -1    \n"   /* i--                        */
+                  "addi   $8,    $8,    4     \n"   /* src += 4 bytes             */
+                  "j             1b           \n"   
+                  "addi   $9,    $9,    4     \n"   /* dst += 4 bytes             */
+
+                  /* transfer one byte per iteration in this second loop          */
+                  "2:                         \n"
+                  "sll    $15,   $10,   2     \n"   /* $15 <= words*4 ($15 == i)  */
+                  "3:                         \n"
+                  "beq    $15,   $11,   4f    \n"   /* exit loop if (i == size)   */
+                  "nop                        \n"
+                  "mtc2   $13,   $1			  \n"   /* MMU_MODE <= DTLB ON        */
+                  "lb	  $14,   0($8)        \n"   /* byte from user space       */
+                  "mtc2   $12,   $1			  \n"   /* restore omd MMU_MODE       */
+	              "sb     $14,   0($9)        \n"   /* byte to kernel space       */
+                  "addi   $15,   $15,   1     \n"   /* i++                        */
+                  "addi   $8,    $8,    1     \n"   /* src += 1 byte              */
+                  "j             3b           \n"   
+                  "addi   $9,    $9,    1     \n"   /* dst += 1 byte              */
+
+                  /* restore old MMU_DATA_EXT register                            */
+                  "4:                         \n"
+                  "mtc2   $16,   $24          \n"   /* MMU__DATA_EXT <= $16       */
+                  ".set reorder               \n"
+                  : 
+                  : "r"(src) , "r"(dst) , "r"(words) , "r"(size) , "r"(k_cxy)
+                  : "$8","$9","$10","$11","$12","$13","$14","$15","$16","memory" );
+
+    // exit critical section
     hal_restore_irq( save_sr );
 
 #if DEBUG_HAL_USPACE
-printk("\n[%s] thread[%x,%x] exit\n", 
-__FUNCTION__, this->process->pid, this->trdid );
+cycle = (uint32_t)hal_get_cycles();
+if( cycle > DEBUG_HAL_USPACE )
+printk("\n[%s] thread[%x,%x] moved %d bytes / u_buf(%x,%x) -> k_buf(%x,%x) / cycle %d\n", 
+__FUNCTION__, this->process->pid, this->trdid, size, local_cxy, u_src, k_cxy, k_dst, cycle );
 #endif
 
 }  // end hal_copy_from_uspace()
 
-///////////////////////////////////////////
-void hal_copy_to_uspace( void     * u_dst,
+///////////////////////////////////////////////////////////////////////////////////////
+// This function moves <size> bytes from a source kernel buffer, defined by the
+// <k_cxy> and <k_src> arguments, to a destination buffer in user virtual space,
+// defined by the <u_dst> argument.
+// It works in a critical section, as it modifies briefly two CP2 registers:
+// It activates briefly the DATA_MMU by writing into the CP2_MODE register to access the
+// user buffer, and modifies the CP2_DATA_EXT register to access the kernel buffer.
+// If the two buffers are aligned on a word boundary, it moves the data word per word
+// in a first loop, and moves byte per byte the remaining bytes in a second loop.
+// If the buffers are not aligned, it moves all data byte per byte.
+///////////////////////////////////////////////////////////////////////////////////////
+// @ k_cxy    : cluster of destination kernel buffer
+// @ k_dst    : pointer on destination kernel buffer
+// @ u_src    : pointer on source user buffer
+// @ size     : number of bytes to move
+///////////////////////////////////////////////////////////////////////////////////////
+void hal_copy_to_uspace( cxy_t      k_cxy,
                          void     * k_src,
+                         void     * u_dst,
                          uint32_t   size )
 {
     uint32_t save_sr;
-	uint32_t i;
-	uint32_t wsize;                   // number of words if aligned
+	uint32_t words;                   // number of words (if buffers aligned)
     uint32_t src = (uint32_t)k_src;
     uint32_t dst = (uint32_t)u_dst;
 
 #if DEBUG_HAL_USPACE
-thread_t * this = CURRENT_THREAD;
-printk("\n[%s] thread[%x,%x] enter in cluster %x / k_src %x / u_dst %x / size %d\n", 
-__FUNCTION__, this->process->pid, this->trdid, local_cxy, k_src, u_dst, size );
+thread_t * this  = CURRENT_THREAD;
+uint32_t   cycle = (uint32_t)hal_get_cycles();
+if( cycle > DEBUG_HAL_USPACE )
+printk("\n[%s] thread[%x,%x] enter / %d bytes / k_buf(%x,%x) -> u_buf(%x,%x) / cycle %d\n", 
+__FUNCTION__, this->process->pid, this->trdid, size, k_cxy, k_src, local_cxy, u_dst, cycle );
 #endif
 
-	if( (dst & 0x3) || (src & 0x3) ) wsize = 0;          // not aligned
-    else                             wsize = size >> 2;
-
+	if( (dst & 0x3) || (src & 0x3) ) words = 0;          // not aligned
+    else                             words = size >> 2;
+
+    // enter critical section
     hal_disable_irq( &save_sr );
 
-	for( i = 0 ; i < wsize ; i++ )          // transfer one word per iteration
-	{
-        asm volatile(
-        "mfc2   $15,   $1           \n"   /* save   MMU_MODE                */
-        "lw	    $13,   0(%0)        \n"   /* read data from kernel space    */
-        "ori    $14,   $0,  0x7	    \n"  
-        "mtc2   $14,   $1			\n"   /* MMU_MODE <= DTLB ON            */
-	    "sw	    $13,   0(%1)        \n"   /* store data to user space       */
-        "mtc2   $15,   $1			\n"   /* restore MMU_MODE               */
-        : : "r"( src ) , "r"( dst ) : "$13","$14","$15", "memory" );
-
-        src += 4;
-        dst += 4;
-    }
-
-	for( i = wsize << 2 ; i < size ; i++ )  // transfer one byte per iteration
-	{
-        asm volatile(
-        "mfc2   $15,   $1           \n"   /* save   MMU_MODE                */
-        "lb	    $13,   0(%0)        \n"   /* read data from kernel space    */
-        "ori    $14,   $0,  0x7	    \n"  
-        "mtc2   $14,   $1			\n"   /* MMU_MODE <= DTLB ON            */
-	    "sb	    $13,   0(%1)        \n"   /* store data to user space       */
-        "mtc2   $15,   $1			\n"   /* restore MMU_MODE               */
-        : : "r"( src ) , "r"( dst ) : "$13","$14","$15", "memory" );
-
-        src += 1;
-        dst += 1;
-    }
-
+    asm volatile( ".set noreorder             \n"
+ 
+                  /* initialise registers                                         */
+                  "move   $8,    %0           \n"   /* $8 <= k_src                */
+                  "move   $9,    %1           \n"   /* $9 <= u_dst                */
+                  "move   $10,   %2           \n"   /* $10 <= words               */
+                  "move   $11,   %3           \n"   /* $11 <= size                */
+                  "mfc2   $12,   $1           \n"   /* $12 <= old MMU_MODE        */
+                  "ori    $13,   $12,   0x4	  \n"   /* $13 <= MMU_MODE with DTLB  */
+
+                  /* save old MMU_DATA_EXT and set k_cxy in it                    */
+                  "mfc2   $16,   $24          \n"   /* $16 <= old MMU_DATA_EXT    */
+                  "mtc2   %4,    $24          \n"   /* MMU_DATA_EXT <= k_cxy      */
+
+                  /* transfer one word per iteration in first loop if aligned     */
+                  "move   $15,   $10          \n"   /* $15 <= words ($15 == i)    */
+                  "1:                         \n"
+                  "beq    $15,   $0,    2f    \n"   /* exit loop if (i==0)        */
+                  "nop                        \n"
+                  "lw	  $14,   0($8)        \n"   /* load from kernel space     */
+                  "mtc2   $13,   $1			  \n"   /* MMU_MODE <= DTLB ON        */
+	              "sw     $14,   0($9)        \n"   /* store to user space        */
+                  "mtc2   $12,   $1			  \n"   /* restore old MMU_MODE       */
+                  "addi   $15,   $15,   -1    \n"   /* i--                        */
+                  "addi   $8,    $8,    4     \n"   /* src += 4 bytes             */
+                  "j             1b           \n"   
+                  "addi   $9,    $9,    4     \n"   /* dst += 4 bytes             */
+
+                  /* transfer one byte per iteration in this second loop          */
+                  "2:                         \n"
+                  "sll    $15,   $10,   2     \n"   /* $15 <= words*4 ($15 == i)  */
+                  "3:                         \n"
+                  "beq    $15,   $11,   4f    \n"   /* exit loop if (i == size)   */
+                  "nop                        \n"
+                  "lb	  $14,   0($8)        \n"   /* byte from kernel space     */
+                  "mtc2   $13,   $1			  \n"   /* MMU_MODE <= DTLB ON        */
+	              "sb     $14,   0($9)        \n"   /* byte to user space         */
+                  "mtc2   $12,   $1			  \n"   /* restore omd MMU_MODE       */
+                  "addi   $15,   $15,   1     \n"   /* i++                        */
+                  "addi   $8,    $8,    1     \n"   /* src += 1 byte              */
+                  "j             3b           \n"   
+                  "addi   $9,    $9,    1     \n"   /* dst += 1 byte              */
+
+                  /* restore old MMU_DATA_EXT register                            */
+                  "4:                         \n"
+                  "mtc2   $16,   $24          \n"   /* MMU__DATA_EXT <= $16       */
+                  ".set reorder               \n"
+                  : 
+                  : "r"(src) , "r"(dst) , "r"(words) , "r"(size) , "r"(k_cxy)
+                  : "$8","$9","$10","$11","$12","$13","$14","$15","$16","memory" );
+
+    // exit critical section
     hal_restore_irq( save_sr );
 
 #if DEBUG_HAL_USPACE
-printk("\n[%s] thread[%x,%x] exit\n", 
-__FUNCTION__, this->process->pid, this->trdid );
+cycle = (uint32_t)hal_get_cycles();
+if( cycle > DEBUG_HAL_USPACE )
+printk("\n[%s] thread[%x,%x] moved %d bytes / k_buf(%x,%x) -> u_buf(%x,%x) / cycle %d\n", 
+__FUNCTION__, this->process->pid, this->trdid, size, k_cxy, k_src, local_cxy, u_dst, cycle );
 #endif
 
