source: soft/giet_vm/applications/rosenfeld/scripts/create_simul_dirs.py @ 827

#!/usr/bin/python
# -*- coding: utf-8 -*-

import shutil
from common import *


granularities = [16]
threads = [16, 64]
use_rand_images = True
img_size = 2048
use_dsk = True
use_rand_img = True
nb_simus_wrt_density = 2 # e.g. if 2, each simus processes 50 or 51 images

rand_seed = 7

configs = [
        {'SLOW':'0', 'FAST':'1', 'FEATURES':'0', 'PARMERGE':'1', 'ARSP':'1'},
        #{'SLOW':'0', 'FAST':'1', 'FEATURES':'1', 'PARMERGE':'1', 'ARSP':'1'},
]


mode = 'simu'

if mode == 'test':
    data_dir = 'data_test'
else:
    data_dir = 'data'



# Directories definitions
scripts_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
giet_dir    = os.path.abspath(os.path.join(scripts_dir, "..", "..", ".."))
pyconf_file = os.path.join(scripts_dir, "config.py")
# Contains run_simus_soclib.py, 
template_scripts_dir = os.path.join(scripts_dir, "template_scripts")


check_pyconf_file(pyconf_file)

exec(file(pyconf_file))

try:
    dsk_dir
    if not os.path.exists(dsk_dir):
        print("mkdir %s", dsk_dir)
        os.mkdir(dsk_dir)
except NameError:
    print("*** Error: variable dsk_dir is not defined in %s file" % (short_path(pyconf_file)))
    sys.exit(1)
except OSError:
    print("*** Error: Impossible to create directory %s" % (dsk_dir))
    sys.exit(1)

try:
    tsar_dir
    tsar_iob_dir = os.path.join(tsar_dir, "platforms", "tsar_generic_iob")
except NameError:
    print("*** Error: variable tsar_dir is not defined in %s file" % (short_path(pyconf_file)))
    sys.exit(1)

try:
    rand_img_dir
    if not os.path.exists(rand_img_dir):
        print("mkdir %s", rand_img_dir)
        os.mkdir(rand_img_dir)
except NameError:
    print("*** Error: variable rand_img_dir, containing the path the random images (either pre-existing or to be generated), is not defined in %s file" % (short_path(pyconf_file)))
    sys.exit(1)
except OSError:
    print("*** Error: Impossible to create directory %s" % (rand_img_dir))
    sys.exit(1)

try:
    preloader_dir
except NameError:
    print("*** Error: variable preloader_dir is not defined in %s file" % (short_path(pyconf_file)))
    sys.exit(1)


# Topcell files
top_file  = os.path.join(tsar_iob_dir, "top.cpp")
desc_file = os.path.join(tsar_iob_dir, "top.desc")
make_file = os.path.join(tsar_iob_dir, "Makefile")
arch_file = os.path.join(tsar_iob_dir, "arch.py")



def create_config_file(filename, granularities, threads, start_density, end_density, config):
    print "# Creating file %s" % (filename)
    f = open(filename, "w")
    f.write("use_rand_img = %s\n" % str(use_rand_img))
    f.write("granularities = [%s]\n" % str(granularities))
    f.write("threads = [%s]\n" % str(threads))
    f.write("start_density = %s\n" % str(start_density))
    f.write("end_density = %s\n" % str(end_density))
    f.write("img_size = %s\n" % str(img_size))
    f.write("configs = [\n")
    f.write("    {")
    config_keys = list(config.keys()) # duplicates list
    config_keys = sorted(config_keys)
    first = True
    for key in config_keys:
        if not first:
            f.write(", ")
        f.write("'%s':'%s'" % (key, config[key]))
        first = False
    f.write("},\n")
    f.write("]\n")
    f.write('tsar_dir = "%s"\n' % tsar_dir)
    f.write('preloader_dir = "%s"\n' % preloader_dir)
    f.close()
    

def giet_ignore_list(dir, files):
    ignore_list = []
    if dir == giet_dir:
        ok_list = ["applications", "create_dmg", "hdd", "giet_boot", "giet_common", "giet_config.h", "giet_drivers", "giet_fat32", "giet_kernel", "giet_libs", "giet_python", "giet_xml", "images", "Makefile"]
        for file in files:
            if not file in ok_list:
                ignore_list.append(file)
    elif dir == os.path.join(giet_dir, "applications"):
        ok_list = ["rosenfeld"]
        for file in files:
            if not file in ok_list:
                ignore_list.append(file)
    elif dir == os.path.join(giet_dir, "applications", "rosenfeld"):
        ok_list = ["include", "Makefile", "nrc2", "rosenfeld.ld", "rosenfeld.py", "src", "src-par"]
        for file in files:
            if not file in ok_list:
                ignore_list.append(file)
    elif dir == os.path.join(giet_dir, "hdd"):
        for file in files:
            ignore_list.append(file)
    else:
        pass
    return ignore_list



def get_x_y_p(nb_procs):
    if nb_procs < 4:
        return 1, 1, nb_procs
    cpu_per_cluster = 4
    x = 1
    y = 1
    to_x = True
    while (x * y * cpu_per_cluster < nb_procs):
        if to_x:
            x = x * 2
        else:
            y = y * 2
        to_x = not to_x
    return x, y, cpu_per_cluster



def gen_params_mk(copied_giet_dir, simul_dir, nthreads):
    x_size, y_size, cpu_per_cluster = get_x_y_p(nthreads)
    tmpl = '''
ARCH      ?= %(arch_dir)s
X_SIZE    ?= %(x_size)d
Y_SIZE    ?= %(y_size)d
NB_PROCS  ?= %(nb_procs)d
NB_TTYS   ?= 3
FBF_WIDTH ?= 256
IOC_TYPE  ?= BDV
MWR_TYPE  ?= CPY
APPLIS    ?= rosenfeld
''' % dict(arch_dir = simul_dir,
           x_size = x_size,
           y_size = y_size,
           nb_procs = cpu_per_cluster)
    params_mk_file = os.path.join(copied_giet_dir, "params.mk")
    f = open(params_mk_file, 'w')
    f.write(tmpl)
    f.close()
 
# Computing start_density and end_density
nb_density_base_per_simu = 101 / nb_simus_wrt_density
nb_simus_left = 101 - nb_density_base_per_simu * nb_simus_wrt_density
curr_density = 0
start_densities = {}
end_densities = {}
for simu_wrt_density in range(0, nb_simus_wrt_density):
    start_densities[simu_wrt_density] = curr_density
    if nb_simus_left > 0:
        curr_nb_simus = nb_density_base_per_simu + 1
        nb_simus_left -= 1
    else:
        curr_nb_simus = nb_density_base_per_simu
    end_densities[simu_wrt_density] = curr_density + curr_nb_simus - 1
    curr_density += curr_nb_simus



for gran in granularities:
    for th in threads:
        for config in configs:
            for simu_wrt_density in range(0, nb_simus_wrt_density):
                start_density = start_densities[simu_wrt_density]
                end_density = end_densities[simu_wrt_density]

                features = config['FEATURES'] == '1'
                simul_dir = get_dirname(dsk_dir, th, config, features, gran, start_density, end_density, "rosen")
                if not os.path.exists(simul_dir):
                    my_mkdir(simul_dir)
                my_cp(top_file, simul_dir)
                my_cp(desc_file, simul_dir)
                my_cp(make_file, simul_dir)
                my_cp(arch_file, simul_dir)

                create_config_file(os.path.join(simul_dir, "config.py"), gran, th, start_density, end_density, config)


                tmpl_script_files = [f for f in os.listdir(template_scripts_dir) if os.path.isfile(os.path.join(template_scripts_dir, f))]
                for tmpl_script_file in tmpl_script_files:
                    my_cp(os.path.join(template_scripts_dir, tmpl_script_file), simul_dir)

                # Copy and compile the giet
                copied_giet_dir = os.path.join(simul_dir, "giet_vm")
                if not os.path.exists(copied_giet_dir):
                    shutil.copytree(giet_dir, copied_giet_dir, symlinks = False, ignore = giet_ignore_list)
                my_chdir(copied_giet_dir)
                # Copy the correct image files
                if use_rand_img:
                    for density in range(start_density, end_density + 1):
                        img_filename = os.path.join(rand_img_dir, get_random_img_file(density, img_size, img_size, gran, rand_seed))
                        if not os.path.isfile(img_filename):
                            print("# Generating random image %s with granularity = %d and density = %d" % (img_filename, gran, density))
                            gen_random_image(img_filename, img_size, img_size, gran, float(density) / 100, rand_seed)
                        # We are forced to use a short image name for the giet because the hdd truncates after 8 characters, so all the
                        # distinctive informations should be contained in 8 chars.
                        target_img_filename = get_short_random_img_file(density, gran)
                        my_cp(img_filename, os.path.join(copied_giet_dir, "images", target_img_filename))
                else:
                    print("*** Error: Only random images are currently supported\n");
                    sys.exit(1)
             
                # Generate the file params.mk
                gen_params_mk(copied_giet_dir, simul_dir, th)

                #cmd = ['make', 'clean-disk']
                #print_and_call(cmd)

                #cmd = ['make']
                #print_and_call(cmd)

                my_chdir(scripts_dir)
            # end density range
        # end config
    # end threads
# end granularity
            
print("# End of %s" % __file__)