source: branches/RWT/soft/validation/scripts/run_simus.py @ 843

#!/usr/bin/python

import subprocess
import os
import sys
import shutil
import random

# User parameters
use_omp = True
nb_procs = 4
protocol = 'dhccp'

nb_max_incr = 50
nb_max_trans = 100
nb_ml = 7 # max number of memory lines
nb_cl = 2 # max number of cache lines
cache_line_size = 16
nb_cache_lines = 256

data_dir = 'data'
test_gen_tool_dir = 'TestGenerator'
test_gen_binary = 'generate_test'

log_init_name = 'log_init_'
log_term_name = 'log_term_'
res_natif = 'res_natif.txt'

all_protocols = [ 'dhccp', 'rwt', 'hmesi', 'wtidl' ]

# Path
top_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")
config_name = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.py")

scripts_path       = os.path.join(top_path, 'scripts')
almos_path         = os.path.join(top_path, 'almos')
soclib_conf_name   = os.path.join(top_path, "soclib.conf")
topcell_name       = os.path.join(top_path, "top.cpp")
arch_info_name     = os.path.join(almos_path, "arch-info-gen.info")
arch_info_bib_name = os.path.join(almos_path, 'arch-info.bib')
hdd_img_file_name  = os.path.join(almos_path, "hdd-img.bin")
shrc_file_name     = os.path.join(almos_path, "shrc")
hard_config_name   = os.path.join(almos_path, "hard_config.h")
app_path           = os.path.join(scripts_path, 'soft')

app_name = 'gen_test'
app_source = os.path.join(scripts_path, 'gen_test.c')

# Checks
if protocol not in all_protocols:
    help_str = '''
*** Error: variable protocol has an unsupported value
'''
    print help_str
    sys.exit()

if not os.path.isfile(config_name):
    help_str = '''
You should create a file named config.py in this directory with the following definitions:
 - almos_src_dir: path to almos source directory (for kernel and bootloader binaries)
 - hdd_img_name:  path to the hdd image to use (will be copied but not modified)
 - tsar_dir:      path to tsar repository
Optional definitions (necessary if you want to use alternative protocols):
 - rwt_dir:       path to the RWT repository
 - hmesi_dir:     path to HMESI directory
 - wtidl_dir:     path to the ideal write-through protocol directory
*** Stopping execution
'''
    print help_str
    sys.exit()

# Loading config
exec(file(config_name))

# Check that variables and paths exist
for var in [ 'almos_src_dir', 'hdd_img_name', 'tsar_dir' ]:
    if eval(var) == "":
        print "*** Error: variable %s not defined in config file" % (var)
        sys.exit()
    if not os.path.exists(eval(var)):
        print "*** Error: variable %s does not define a valid path" % (var)
        sys.exit()

if protocol == "rwt":
    if rwt_dir == "":
        print "*** Error: variable rwt_dir not defined in config file"
        sys.exit()
    if not os.path.exists(rwt_dir):
        print "*** Error: variable rwt_dir does not define a valid path"
        sys.exit()

if protocol == "hmesi":
    if hmesi_dir == "":
        print "*** Error: variable hmesi_dir not defined in config file"
        sys.exit()
    if not os.path.exists(hmesi_dir):
        print "*** Error: variable hmesi_dir does not define a valid path"
        sys.exit()

if protocol == "wtidl":
    if wtidl_dir == "":
        print "*** Error: variable wtidl_dir not defined in config file"
        sys.exit()
    if not os.path.exists(wtidl_dir):
        print "*** Error: variable wtidl_dir does not define a valid path"
        sys.exit()


random.seed()


def get_x_y(nb_procs):
    x = 1
    y = 1
    to_x = True
    while (x * y * 4 < nb_procs):
        if to_x:
            x = x * 2
        else:
            y = y * 2
        to_x = not to_x
    return x, y


def gen_soclib_conf():

    if os.path.isfile(soclib_conf_name):
        print "Updating file %s" % (soclib_conf_name)
        # First, remove lines containing "addDescPath"
        f = open(soclib_conf_name, "r")
        lines = f.readlines()
        f.close()

        f = open(soclib_conf_name, "w")

        for line in lines:
            if not ("addDescPath" in line):
                f.write(line)
        f.close()
    else:
        print "Creating file %s" % (soclib_conf_name)
        f = open(soclib_conf_name, "w")
        f.close()

    # Defining common and specific modules 
    common_modules = [
            'lib/generic_llsc_global_table',
            'modules/dspin_router_tsar', 
            'modules/sdmmc',
            'modules/vci_block_device_tsar',
            'modules/vci_ethernet_tsar',
            'modules/vci_io_bridge',
            'modules/vci_iox_network',
            'modules/vci_spi',
            'platforms/tsar_generic_xbar/tsar_xbar_cluster'
    ]

    specific_modules = [
            'communication',
            'lib/generic_cache_tsar',
            'modules/vci_cc_vcache_wrapper',
            'modules/vci_mem_cache',
    ]

    f = open(soclib_conf_name, "a")
    # Adding common modules
    for common_module in common_modules:
        f.write("config.addDescPath(\"%s/%s\")\n" % (tsar_dir, common_module))
    #f.write("\n")

    if protocol == "dhccp":
        arch_dir = tsar_dir
    elif protocol == "rwt":
        arch_dir = rwt_dir
    elif protocol == "hmesi":
        arch_dir = hmesi_dir
    elif protocol == "wtidl":
        arch_dir = wtidl_dir
    else:
        assert(False)

    for specific_module in specific_modules:
        f.write("config.addDescPath(\"%s/%s\")\n" % (arch_dir, specific_module))

    #f.write("\n")
    f.close()


def gen_hard_config(x, y, hard_config):
   header = '''
/* Generated from run_simus.py */

#ifndef _HD_CONFIG_H
#define _HD_CONFIG_H

#define X_SIZE              %(x)d
#define Y_SIZE              %(y)d
#define NB_CLUSTERS         %(nb_clus)d
#define NB_PROCS_MAX        4
#define NB_TASKS_MAX        8

#define NB_TIM_CHANNELS     32
#define NB_DMA_CHANNELS     1

#define NB_TTY_CHANNELS     4
#define NB_IOC_CHANNELS     1
#define NB_NIC_CHANNELS     0
#define NB_CMA_CHANNELS     0

#define USE_XICU            1
#define IOMMU_ACTIVE        0

#define IRQ_PER_PROCESSOR   1
''' % dict(x = x, y = y, nb_clus = x * y)

   if protocol == 'wtidl':
       header += '#define WT_IDL\n'

   header += '#endif //_HD_CONFIG_H\n'

   file = open(hard_config, 'w')
   file.write(header)
   file.close()



def gen_arch_info(x, y, arch_info, arch_info_bib):
   old_path = os.getcwd()
   print "cd", scripts_path
   os.chdir(scripts_path)
   print "./gen_arch_info_large.sh", str(x), str(y), ">", arch_info
   output = subprocess.Popen([ './gen_arch_info_large.sh', str(x), str(y) ], stdout = subprocess.PIPE).communicate()[0]
   os.chdir(almos_path)
   file = open(arch_info, 'w')
   file.write(output)
   file.close()
   print "./info2bib -i", arch_info, "-o", arch_info_bib
   subprocess.call([ './info2bib', '-i', arch_info, '-o', arch_info_bib ])
   print "cd", old_path
   os.chdir(old_path)


def gen_sym_links():
   old_path = os.getcwd()
   print "cd", almos_path
   os.chdir(almos_path)

   target = os.path.join(almos_src_dir, 'tools/soclib-bootloader/bootloader-tsar-mipsel.bin')
   link_name = 'bootloader-tsar-mipsel.bin'
   if not os.path.isfile(link_name):
      print "ln -s", target, link_name
      os.symlink(target, link_name)

   target = os.path.join(almos_src_dir, 'kernel/obj.tsar/almix-tsar-mipsel.bin')
   link_name = 'kernel-soclib.bin'
   if not os.path.isfile(link_name):
      print "ln -s", target, link_name
      os.symlink(target, link_name)

   target = os.path.join(almos_src_dir, 'tools/bin/info2bib')
   link_name = 'info2bib'
   if not os.path.isfile(link_name):
      print "ln -s", target, link_name
      os.symlink(target, link_name)

   copied_hdd = 'hdd-img.bin'
   print "cp", hdd_img_name, copied_hdd
   shutil.copy(hdd_img_name, copied_hdd)




# Loop of simulation

print "make -C", test_gen_tool_dir
subprocess.call([ 'make', '-C', test_gen_tool_dir ])

print "cp", os.path.join(test_gen_tool_dir, test_gen_binary), os.path.join(scripts_path, test_gen_binary)
subprocess.call([ 'cp', os.path.join(test_gen_tool_dir, test_gen_binary), os.path.join(scripts_path, test_gen_binary)])

print "mkdir -p", os.path.join(scripts_path, data_dir)
subprocess.call([ 'mkdir', '-p', os.path.join(scripts_path, data_dir) ])

gen_sym_links()
gen_soclib_conf()

while True:
   x, y = get_x_y(nb_procs)
   nthreads = min(4, x * y)
   gen_hard_config(x, y, hard_config_name)
   gen_arch_info(x, y, arch_info_name, arch_info_bib_name)

   # Generating test
   print test_gen_binary, nb_procs, nb_max_incr, nb_max_trans, nb_ml, nb_cl, cache_line_size, nb_cache_lines, app_source
   subprocess.call([ os.path.join(scripts_path, test_gen_binary), str(nb_procs), str(nb_max_incr), str(nb_max_trans), str(nb_ml), str(nb_cl), str(cache_line_size), str(nb_cache_lines), app_source ])
   
   print "cd", scripts_path
   os.chdir(scripts_path)

   # Compiling and executing generated test in native
   print "make -f Makefile.nat"
   subprocess.call([ 'make', '-f', 'Makefile.nat' ])

   print "./test_natif >", os.path.join(data_dir, res_natif)
   output = subprocess.Popen([ './test_natif' ], stdout = subprocess.PIPE).communicate()[0]
   file = open(os.path.join(data_dir, res_natif), 'w')
   file.write(output)
   file.close()

   # Building simulated soft
   print "cp", app_source, app_path
   subprocess.call([ 'cp', app_source, app_path ])

   old_path = os.getcwd()
   print "cd", app_path
   os.chdir(app_path)

   # Compilation process is different in splash and other apps
   print "make clean"
   subprocess.call([ 'make', 'clean' ])

   print "make TARGET=tsar"
   subprocess.call([ 'make', 'TARGET=tsar' ])

   # Creation/Modification du shrc de almos
   shrc = "exec -p 0 /bin/gen_test\n"
   
   file = open(shrc_file_name, 'w')
   file.write(shrc)
   file.close()

   # Copie du binaire et du shrc dans l'image disque
   print "mcopy -o -i", hdd_img_file_name, shrc_file_name, "::/etc/"
   subprocess.call([ 'mcopy', '-o', '-i', hdd_img_file_name, shrc_file_name, '::/etc/' ])
   print "mcopy -o -i", hdd_img_file_name, app_name, "::/bin/"
   subprocess.call([ 'mcopy', '-o', '-i', hdd_img_file_name, app_name, '::/bin/' ])

   print "cd", old_path
   os.chdir(old_path)

   # Compiling topcell
   print "cd", top_path
   os.chdir(top_path)
   print "touch", topcell_name
   subprocess.call([ 'touch', topcell_name ])
   print "make"
   retval = subprocess.call([ 'make' ])
   if retval != 0:
       sys.exit()
   
   # Launching simulation
   if use_omp:
      print "./simul.x -THREADS", nthreads, ">", os.path.join(scripts_path, data_dir, log_init_name + str(nb_procs))
      output = subprocess.Popen([ './simul.x', '-THREADS', str(nthreads) ], stdout = subprocess.PIPE).communicate()[0]
   else:
      print "./simul.x >" , os.path.join(scripts_path, data_dir, log_init_name + str(nb_procs))
      output = subprocess.Popen([ './simul.x' ], stdout = subprocess.PIPE).communicate()[0]

   # Writing simulation results to data directory
   print "cd", scripts_path
   os.chdir(scripts_path)
   filename = os.path.join(data_dir, log_init_name + str(nb_procs))
   file = open(filename, 'w')
   file.write(output)
   file.close()

   term_filename = os.path.join(scripts_path, data_dir, log_term_name + str(nb_procs))
   print "tail -n +5", os.path.join(top_path, 'term1'), ">", term_filename
   output = subprocess.Popen([ 'tail', '-n', '+5', os.path.join(top_path, 'term1') ], stdout = subprocess.PIPE).communicate()[0]
   file = open(term_filename, 'w')
   file.write(output)
   file.close()
   
   # Quiting if results obtained by simulation are incorrect
   print "diff", term_filename, os.path.join(data_dir, res_natif)
   output = subprocess.Popen([ 'diff', term_filename, os.path.join(data_dir, res_natif) ], stdout = subprocess.PIPE).communicate()[0]
   if output != "":
      print "*** Difference found, stopping"
      break;
   else:
      print "No diff found, continuing"


## Enf of simulations