////////////////////////////////////////////////////////////////////////////////////////// // File : transpose.c // Date : september 2015 // author : Alain Greiner ////////////////////////////////////////////////////////////////////////////////////////// // This multi-threaded aplication read a raw image (one byte per pixel) // stored on disk, transpose it, display the result on the frame buffer, // and store the transposed image on disk. // The input image can be interactively selected if the INTERACTIVE flag is set. // It can run on a multi-processors, multi-clusters architecture, with one thread // per processor, and uses the POSIX threads API. // It uses the giet_fat_mmap() to directly access the input and output files // in the kernel files cache. It does not use the CMA to display the result image. // // The main() function can be launched on any processor P[x,y,l]. // It makes the initialisations, launch (N-1) threads to run the execute() function // on the (N-1) other processors than P[x,y,l], call himself the execute() function, // and finally call the instrument() function to display instrumentation results // when the parallel execution is completed. // // The buf_in[x,y] and buf_out[put buffers containing the direct ans transposed images // are distributed in clusters: // In each cluster[x,y], the thread running on processor P[x,y,0] uses the giet_fat_mmap() // function to map the buf_in[x,y] and buf_out[x,y] buffers containing a set of lines. // Then, all threads in cluster[x,y] read pixels from the local buf_in[x,y] buffer, and // write the pixels to the remote buf_out[x,y] buffers. Finally, each thread display // a part of the transposed image to the frame buffer. // // - The image size must fit the frame buffer width and height, that must be power of 2. // - The number of clusters must be a power of 2 no larger than 256. // - The number of processors per cluster must be a power of 2 no larger than 4. // - The number of clusters cannot be larger than (image_size * image_size) / 4096, // because the size of buf_in[x,y] and buf_out[x,y] must be multiple of 4096. // // The transpose_rw.c file contains a variant that use the giet_fat_read() // and giet_fat_write() system calls, to access the files. ////////////////////////////////////////////////////////////////////////////////////////// #include "stdio.h" #include "stdlib.h" #include "string.h" #include "user_barrier.h" #include "malloc.h" #define BLOCK_SIZE 512 // block size on disk #define X_MAX 16 // max number of clusters in row #define Y_MAX 16 // max number of clusters in column #define PROCS_MAX 4 // max number of procs per cluster #define CLUSTER_MAX (X_MAX * Y_MAX) // max number of clusters #define IMAGE_SIZE 128 // default image size #define INPUT_FILE_PATH "/misc/images_128.raw" // default input file pathname #define OUTPUT_FILE_PATH "/home/trsp_128.raw" // default output file pathname #define INTERACTIVE 0 // interactive capture of filenames #define VERBOSE 0 // print comments on TTY // macro to use a shared TTY #define printf(...); { lock_acquire( &tty_lock ); \ giet_tty_printf(__VA_ARGS__); \ lock_release( &tty_lock ); } /////////////////////////////////////////////////////// // global variables stored in seg_data in cluster(0,0) /////////////////////////////////////////////////////// // instrumentation counters for each processor in each cluster unsigned int MMAP_START[X_MAX][Y_MAX][PROCS_MAX] = {{{ 0 }}}; unsigned int MMAP_END [X_MAX][Y_MAX][PROCS_MAX] = {{{ 0 }}}; unsigned int TRSP_START[X_MAX][Y_MAX][PROCS_MAX] = {{{ 0 }}}; unsigned int TRSP_END [X_MAX][Y_MAX][PROCS_MAX] = {{{ 0 }}}; unsigned int DISP_START[X_MAX][Y_MAX][PROCS_MAX] = {{{ 0 }}}; unsigned int DISP_END [X_MAX][Y_MAX][PROCS_MAX] = {{{ 0 }}}; // arrays of pointers on distributed buffers // one input buffer & one output buffer per cluster unsigned char* buf_in [CLUSTER_MAX]; unsigned char* buf_out[CLUSTER_MAX]; // lock protecting shared TTY user_lock_t tty_lock; // synchronisation barrier (all threads) giet_sqt_barrier_t barrier; // input & output files pathname and size char input_file_name[256]; char output_file_name[256]; unsigned int image_size; // input & output file descriptors int fd_in; int fd_out; //////////////////////////////////////////////////////////////// __attribute__ ((constructor)) void execute( unsigned int* tid ) //////////////////////////////////////////////////////////////// { unsigned int l; // line index for loops unsigned int p; // pixel index for loops // get processor identifiers unsigned int x_id; // x cluster coordinate unsigned int y_id; // y cluster coordinate unsigned int p_id; // local processor index giet_proc_xyp( &x_id, &y_id, &p_id); // get & check plat-form parameters unsigned int x_size; // number of clusters in a row unsigned int y_size; // number of clusters in a column unsigned int nprocs; // number of processors per cluster giet_procs_number( &x_size , &y_size , &nprocs ); unsigned int nclusters = x_size * y_size; // number of clusters unsigned int nthreads = x_size * y_size * nprocs; // number of threads unsigned int npixels = image_size * image_size; // pixels per image unsigned int cluster_id = (x_id * y_size) + y_id; // "continuous" index unsigned int thread_id = (cluster_id * nprocs) + p_id; // "continuous" index if ( VERBOSE ) { printf("\n@@@ thread %d / %d running on P[%d,%d,%d]\n", *tid , nthreads , x_id , y_id , p_id ); } // parallel load of image: // thread running on processor[x,y,0] // map input & output files in buf_in & buf_out buffers. MMAP_START[x_id][y_id][p_id] = giet_proctime(); if ( p_id == 0 ) { // map buf_in unsigned int length = npixels / nclusters; unsigned int offset = length * cluster_id; buf_in[cluster_id] = giet_fat_mmap( NULL, length, MAP_PROT_READ, MAP_SHARED, fd_in, offset ); if ( buf_in[cluster_id] == NULL ) { printf("\n[TRANSPOSE ERROR] Thread[%d,%d,%d] cannot map input file\n", x_id , y_id , p_id ); giet_pthread_exit( NULL ); } if ( VERBOSE ) { printf("\n@@@ thread %d running on P[%d,%d,%d] call mmap for input file\n" " length = %x / offset = %x / buf_in = %x\n", *tid , x_id , y_id , p_id , length , offset , buf_in[cluster_id] ); } // map buf_out buf_out[cluster_id] = giet_fat_mmap( NULL, length, MAP_PROT_WRITE, MAP_SHARED, fd_out, offset ); if ( buf_out[cluster_id] == NULL ) { printf("\n[TRANSPOSE ERROR] Thread[%d,%d,%d] cannot map output file\n", x_id , y_id , p_id ); giet_pthread_exit( NULL ); } if ( VERBOSE ) { printf("\n@@@ thread %d running on P[%d,%d,%d] call mmap for output file\n" " length = %x / offset = %x / buf_out = %x\n", *tid , x_id , y_id , p_id , length , offset , buf_out[cluster_id] ); } } MMAP_END[x_id][y_id][p_id] = giet_proctime(); ///////////////////////////// sqt_barrier_wait( &barrier ); ///////////////////////////// // parallel transpose from buf_in to buf_out // each thread makes the transposition for nlt lines (nlt = image_size/nthreads) // from line [thread_id*nlt] to line [(thread_id + 1)*nlt - 1] // (p,l) are the absolute pixel coordinates in the source image TRSP_START[x_id][y_id][p_id] = giet_proctime(); unsigned int nlt = image_size / nthreads; // number of lines per thread unsigned int nlc = image_size / nclusters; // number of lines per cluster unsigned int src_cluster; unsigned int src_index; unsigned int dst_cluster; unsigned int dst_index; unsigned char byte; unsigned int first = thread_id * nlt; // first line index for a given thread unsigned int last = first + nlt; // last line index for a given thread for ( l = first ; l < last ; l++ ) { // in each iteration we transfer one byte for ( p = 0 ; p < image_size ; p++ ) { // read one byte from local buf_in src_cluster = l / nlc; src_index = (l % nlc)*image_size + p; byte = buf_in[src_cluster][src_index]; // write one byte to remote buf_out dst_cluster = p / nlc; dst_index = (p % nlc)*image_size + l; buf_out[dst_cluster][dst_index] = byte; } } if ( (p_id == 0) && (x_id==0) && (y_id==0) ) { printf("\n[TRANSPOSE] Thread[%d,%d,%d] completes transpose at cycle %d\n", x_id, y_id, p_id, giet_proctime() ); } TRSP_END[x_id][y_id][p_id] = giet_proctime(); ///////////////////////////// sqt_barrier_wait( &barrier ); ///////////////////////////// // parallel display from local buf_out to frame buffer // all threads contribute to display using memcpy... DISP_START[x_id][y_id][p_id] = giet_proctime(); unsigned int npt = npixels / nthreads; // number of pixels per thread giet_fbf_sync_write( npt * thread_id, &buf_out[cluster_id][p_id*npt], npt ); if ( (x_id==0) && (y_id==0) && (p_id==0) ) { printf("\n[TRANSPOSE] Thread[%d,%d,%d] completes display at cycle %d\n", x_id, y_id, p_id, giet_proctime() ); } DISP_END[x_id][y_id][p_id] = giet_proctime(); ///////////////////////////// sqt_barrier_wait( &barrier ); ///////////////////////////// // all threads, but thread[0,0,0], suicide if ( (x_id != 0) || (y_id != 0) || (p_id != 0) ) giet_pthread_exit( "completed" ); } // end execute() ////////////////////////////////////// void instrument( unsigned int x_size, unsigned int y_size, unsigned int nprocs ) ////////////////////////////////////// { unsigned int x, y, l; unsigned int min_load_start = 0xFFFFFFFF; unsigned int max_load_start = 0; unsigned int min_load_ended = 0xFFFFFFFF; unsigned int max_load_ended = 0; unsigned int min_trsp_start = 0xFFFFFFFF; unsigned int max_trsp_start = 0; unsigned int min_trsp_ended = 0xFFFFFFFF; unsigned int max_trsp_ended = 0; unsigned int min_disp_start = 0xFFFFFFFF; unsigned int max_disp_start = 0; unsigned int min_disp_ended = 0xFFFFFFFF; unsigned int max_disp_ended = 0; char string[64]; snprintf( string , 64 , "/home/transpose_%d_%d_%d" , x_size , y_size , nprocs ); // open instrumentation file unsigned int fd = giet_fat_open( string , O_CREAT ); if ( fd < 0 ) { printf("\n[TRANSPOSE ERROR] cannot open instrumentation file %s\n", string ); giet_pthread_exit( NULL ); } for (x = 0; x < x_size; x++) { for (y = 0; y < y_size; y++) { for ( l = 0 ; l < nprocs ; l++ ) { if (MMAP_START[x][y][l] < min_load_start) min_load_start = MMAP_START[x][y][l]; if (MMAP_START[x][y][l] > max_load_start) max_load_start = MMAP_START[x][y][l]; if (MMAP_END[x][y][l] < min_load_ended) min_load_ended = MMAP_END[x][y][l]; if (MMAP_END[x][y][l] > max_load_ended) max_load_ended = MMAP_END[x][y][l]; if (TRSP_START[x][y][l] < min_trsp_start) min_trsp_start = TRSP_START[x][y][l]; if (TRSP_START[x][y][l] > max_trsp_start) max_trsp_start = TRSP_START[x][y][l]; if (TRSP_END[x][y][l] < min_trsp_ended) min_trsp_ended = TRSP_END[x][y][l]; if (TRSP_END[x][y][l] > max_trsp_ended) max_trsp_ended = TRSP_END[x][y][l]; if (DISP_START[x][y][l] < min_disp_start) min_disp_start = DISP_START[x][y][l]; if (DISP_START[x][y][l] > max_disp_start) max_disp_start = DISP_START[x][y][l]; if (DISP_END[x][y][l] < min_disp_ended) min_disp_ended = DISP_END[x][y][l]; if (DISP_END[x][y][l] > max_disp_ended) max_disp_ended = DISP_END[x][y][l]; } } } giet_tty_printf( "\n ------ %s ------\n" , string ); giet_fat_fprintf( fd , "\n ------ %s ------\n" , string ); giet_tty_printf( " - MMAP_START : min = %d / max = %d / med = %d / delta = %d\n", min_load_start, max_load_start, (min_load_start+max_load_start)/2, max_load_start-min_load_start ); giet_fat_fprintf( fd , " - MMAP_START : min = %d / max = %d / med = %d / delta = %d\n", min_load_start, max_load_start, (min_load_start+max_load_start)/2, max_load_start-min_load_start ); giet_tty_printf( " - MMAP_END : min = %d / max = %d / med = %d / delta = %d\n", min_load_ended, max_load_ended, (min_load_ended+max_load_ended)/2, max_load_ended-min_load_ended ); giet_fat_fprintf( fd , " - MMAP_END : min = %d / max = %d / med = %d / delta = %d\n", min_load_ended, max_load_ended, (min_load_ended+max_load_ended)/2, max_load_ended-min_load_ended ); giet_tty_printf( " - TRSP_START : min = %d / max = %d / med = %d / delta = %d\n", min_trsp_start, max_trsp_start, (min_trsp_start+max_trsp_start)/2, max_trsp_start-min_trsp_start ); giet_fat_fprintf( fd , " - TRSP_START : min = %d / max = %d / med = %d / delta = %d\n", min_trsp_start, max_trsp_start, (min_trsp_start+max_trsp_start)/2, max_trsp_start-min_trsp_start ); giet_tty_printf( " - TRSP_END : min = %d / max = %d / med = %d / delta = %d\n", min_trsp_ended, max_trsp_ended, (min_trsp_ended+max_trsp_ended)/2, max_trsp_ended-min_trsp_ended ); giet_fat_fprintf( fd , " - TRSP_END : min = %d / max = %d / med = %d / delta = %d\n", min_trsp_ended, max_trsp_ended, (min_trsp_ended+max_trsp_ended)/2, max_trsp_ended-min_trsp_ended ); giet_tty_printf( " - DISP_START : min = %d / max = %d / med = %d / delta = %d\n", min_disp_start, max_disp_start, (min_disp_start+max_disp_start)/2, max_disp_start-min_disp_start ); giet_fat_fprintf( fd , " - DISP_START : min = %d / max = %d / med = %d / delta = %d\n", min_disp_start, max_disp_start, (min_disp_start+max_disp_start)/2, max_disp_start-min_disp_start ); giet_tty_printf( " - DISP_END : min = %d / max = %d / med = %d / delta = %d\n", min_disp_ended, max_disp_ended, (min_disp_ended+max_disp_ended)/2, max_disp_ended-min_disp_ended ); giet_fat_fprintf( fd , " - DISP_END : min = %d / max = %d / med = %d / delta = %d\n", min_disp_ended, max_disp_ended, (min_disp_ended+max_disp_ended)/2, max_disp_ended-min_disp_ended ); giet_fat_close( fd ); } // end instrument() ////////////////////////////////////////// __attribute__ ((constructor)) void main() ////////////////////////////////////////// { // indexes for loops unsigned int x , y , n; // get identifiers for proc executing main unsigned int x_id; // x cluster coordinate unsigned int y_id; // y cluster coordinate unsigned int p_id; // local processor index giet_proc_xyp( &x_id , &y_id , &p_id ); // get & check plat-form parameters unsigned int x_size; // number of clusters in a row unsigned int y_size; // number of clusters in a column unsigned int nprocs; // number of processors per cluster giet_procs_number( &x_size , &y_size , &nprocs ); giet_pthread_assert( ((nprocs == 1) || (nprocs == 2) || (nprocs == 4)), "[TRANSPOSE ERROR] number of procs per cluster must be 1, 2 or 4"); giet_pthread_assert( ((x_size == 1) || (x_size == 2) || (x_size == 4) || (x_size == 8) || (x_size == 16)), "[TRANSPOSE ERROR] x_size must be 1,2,4,8,16"); giet_pthread_assert( ((y_size == 1) || (y_size == 2) || (y_size == 4) || (y_size == 8) || (y_size == 16)), "[TRANSPOSE ERROR] y_size must be 1,2,4,8,16"); // compute number of threads unsigned int nthreads = x_size * y_size * nprocs; // shared TTY allocation giet_tty_alloc( 1 ); lock_init( &tty_lock); // get FBF ownership and FBF size unsigned int width; unsigned int height; giet_fbf_alloc(); giet_fbf_size( &width , &height ); printf("\n[TRANSPOSE] start at cycle %d on %d cores / FBF = %d * %d pixels\n", giet_proctime(), nthreads , width , height ); if ( INTERACTIVE ) // input_file_name, output_file_name, and size acquisition { printf("\n[TRANSPOSE] enter path for input file / default is : %s\n> ", INPUT_FILE_PATH ); giet_tty_gets( input_file_name , 256 ); printf("\n"); if ( strcmp( input_file_name , "" ) == 0 ) strcpy( input_file_name , INPUT_FILE_PATH ); printf("\n[TRANSPOSE] enter path for output file / default is : %s\n> ", OUTPUT_FILE_PATH ); giet_tty_gets( output_file_name , 256 ); printf("\n"); if ( strcmp( output_file_name , "" ) == 0 ) strcpy( output_file_name , OUTPUT_FILE_PATH ); printf("\n[TRANSPOSE] enter image size / default is : %d\n> ", IMAGE_SIZE ); giet_tty_getw( &image_size ); printf("\n"); if ( image_size == 0 ) image_size = IMAGE_SIZE; } else { strcpy( input_file_name , INPUT_FILE_PATH ); strcpy( output_file_name , OUTPUT_FILE_PATH ); image_size = IMAGE_SIZE; } // check image size / number of clusters giet_pthread_assert( ((((image_size * image_size) / (x_size * y_size)) & 0xFFF) == 0) , "[TRANSPOSE ERROR] pixels per cluster must be multiple of 4096"); printf("\n[TRANSPOSE] input = %s / output = %s / size = %d\n", input_file_name, output_file_name, image_size ); // distributed heap initialisation for ( x = 0 ; x < x_size ; x++ ) { for ( y = 0 ; y < y_size ; y++ ) { heap_init( x , y ); } } // open input and output files fd_in = giet_fat_open( input_file_name , O_RDONLY ); // read_only if ( fd_in < 0 ) { printf("\n[TRANSPOSE ERROR] main cannot open file %s\n", input_file_name ); giet_pthread_exit( NULL ); } else { printf("\n[TRANSPOSE] main open file %s / fd = %d\n", input_file_name , fd_in ); } fd_out = giet_fat_open( output_file_name , O_CREAT ); // create if required if ( fd_out < 0 ) { printf("\n[TRANSPOSE ERROR] main cannot open file %s\n", output_file_name ); giet_pthread_exit(" open() failure"); } else { printf("\n[TRANSPOSE] main open file %s / fd = %d\n", output_file_name , fd_out ); } // allocate thread[] array : thread identidiers defined by the kernel pthread_t* thread = malloc( nthreads * sizeof(pthread_t) ); // allocate tid[] array : continuous thred index defined by the user unsigned int* tid = malloc( nthreads * sizeof(unsigned int) ); // barrier initialisation sqt_barrier_init( &barrier, x_size , y_size , nprocs ); // Initialisation completed printf("\n[TRANSPOSE] main completes initialisation\n"); // launch other threads to run execute() function for ( n = 1 ; n < nthreads ; n++ ) { tid[n] = n; if ( giet_pthread_create( &thread[n], NULL, // no attribute &execute, &tid[n] ) ) // pointer on thread index { printf("\n[TRANSPOSE ERROR] creating thread %d / id = %x\n", tid[n] , thread[n] ); giet_pthread_exit( NULL ); } } // run the execute() function execute( &tid[0] ); // wait other threads completion for ( n = 1 ; n < nthreads ; n++ ) { if ( giet_pthread_join( thread[n], NULL ) ) { printf("\n[TRANSPOSE ERROR] joining thread %d / id = %x\n", tid[n] , thread[n] ); giet_pthread_exit( NULL ); } else { printf("\n[TRANSPOSE] thread %x joined at cycle %d\n", thread[n] , giet_proctime() ); } } // call the instrument() function instrument( x_size , y_size , nprocs ); // close input and output files giet_fat_close( fd_in ); giet_fat_close( fd_out ); // suicide giet_pthread_exit( "completed" ); } // end main()