diff --git a/data_parser.py b/data_parser.py index 8136473..d3c57a7 100644 --- a/data_parser.py +++ b/data_parser.py @@ -6,14 +6,18 @@ tasks = [] single_tasks = [] job_ID = 0 -def parse_from_files(): +def parse_from_files(fileSuffix=''): for state in ('good','bad'): - with open('data/'+state+'_tasksets','r') as file: + with open('data/'+state+'_tasksets'+fileSuffix,'r') as file: + print('data/'+state+'_tasksets'+fileSuffix) # each line in the file represents a tuple (tasksetsize, listOfTasksets) # listOfTasksets is a list of tuples (success:bool, Taskset) for line in file: - level, listOfTasksets = eval(line) - for success_taskset in listOfTasksets: + level = eval(line[1:2]) + splitlist = (line[5:-2]+',').split('(') + for text in splitlist[1:]: + dataTuple = '('+text[:-2] + success_taskset = eval(dataTuple) taskset_list.append(success_taskset) if level == 1: #print('there was a level 1') @@ -147,6 +151,10 @@ def create_tables(): if __name__ == "__main__": name = sys.argv[1] + try: + fileSuffix = sys.argv[2] + except IndexError: + fileSuffix = '' database = sqlite3.connect(name+'.db') db = database.cursor() @@ -155,10 +163,14 @@ def create_tables(): db.execute('DROP TABLE IF EXISTS TaskSet') create_tables() - - parse_from_files() + if fileSuffix == 'A' or fileSuffix == 'B': + parse_from_files(fileSuffix) + elif fileSuffix == '': + parse_from_files() + else: + raise ValueError("sys.argv[2] was '"+fileSuffix+"' but viable options are only 'A', 'B' or nothing" ) write_tasks_to_db() write_taskset_and_job_to_db() database.commit() - db.close() \ No newline at end of file + db.close() diff --git a/main.py b/main.py index e1a4808..66a33e0 100644 --- a/main.py +++ b/main.py @@ -1,9 +1,10 @@ import sys +import select sys.path.append('../') -from signal import signal, alarm, SIGALRM from distributor.distributorClass import Distributor from distributor.monitors.dataGenerationMonitor import DataGenerationMonitor from distributor.clean import clean_function, clean_panda + import value_init as VI from taskgen.task import Task from taskgen.taskset import TaskSet @@ -89,13 +90,19 @@ def load_tasksets(include_possibilities=True): global TASKSETS global BADTASKSETS global POSSIBLETASKSETS - + print('loading tasksets') for option in ('good', 'bad'): + print(option) with open('./data/{}_tasksets'.format(option), 'r') as taskset_file: for line in taskset_file:# line format: (int, [ (bool,[{}]) ] ) - level, tasksetList = eval(line)# level indicates size of tasksets + #print('got a line') + level = eval(line[1:2])# level indicates size of tasksets add_if_not_exists(level) - for successful, tasksetInfo in tasksetList: + #print(level) + splitlist = (line[5:-2]+',').split('(') + for text in splitlist[1:]: + dataTuple = '('+text[:-2] + successful, tasksetInfo = eval(dataTuple) taskset = TaskSet([]) for taskDict in tasksetInfo: taskset.append(Task(taskDict)) @@ -103,14 +110,16 @@ def load_tasksets(include_possibilities=True): TASKSETS[level].append((successful,taskset)) else: BADTASKSETS[level].append((successful,taskset)) + if include_possibilities: + print('possible') try: with open('./data/possible_tasksets','r') as taskset_file: for line in taskset_file:# format: [taskset_hash], taskset_hash is type string POSSIBLETASKSETS += eval(line) except FileNotFoundError as e: print('There were no possible tasksets.') - + print('done') """ write_tasksets_to_file() is for basic book-keeping and we will write the good,bad and possible tasksets into the appropriate files. @@ -199,7 +208,7 @@ def add_job(distributor, numberOfTasksets=1, tasksetSize=1): except IndexError: break if tasksetList: - distributor.add_job(tasksetList, monitor=monitor, is_list=True) + distributor.add_job(tasksetList, monitor=monitor) MONITORLISTS.append([len(tasksetList), 0, monitor.out]) @@ -286,14 +295,14 @@ def show_status(): for pkg in PC.taskTypes: print("Number of tasks in TASKS[", pkg, "]: ", len(TASKS[pkg])) - try: - print('you can increase the current level (i)') - print('or you can set the level to one of these values: {}'.format(list(TASKSETS.keys()))) - if CURRENTTASKSETSIZE == 1: - print('you can also add more tasks for a pkg, just type the name of one of these: {}'.format(PC.taskTypes)) - alarm(10) - option = input() - alarm(0) + print('you can increase the current level (i)') + print('or you can set the level to one of these values: {}'.format(list(TASKSETS.keys()))) + if CURRENTTASKSETSIZE == 1: + print('you can also add more tasks for a pkg, just type the name of one of these: {}'.format(PC.taskTypes)) + #wait for input + i, _, _ = select.select( [sys.stdin], [], [], 10 ) + if i: + option = sys.stdin.readline().strip() if option == 'i': print('RUNNINGTASKSETS will be finished and then the level will be raised.') SAVE_POSSIBLES[CURRENTTASKSETSIZE] = POSSIBLETASKSETS @@ -309,17 +318,18 @@ def show_status(): if intOption in TASKSETS: # print(intOption,type(intOption),[x for k,l in TASKS.items() for x in l],'\n', TASKS) if intOption == 1 and not [x for k,l in TASKS.items() for x in l]: - try: - print('There is no unexecuted Tasks, do you want to add more of everything? [y/n]') - alarm(5) - option = input() - alarm(0) + print('There is no unexecuted Tasks, do you want to add more of everything? [y/n]') + i, _, _ = select.select( [sys.stdin], [], [], 5 ) + if i: + option = sys.stdin.readline().strip() if option == 'y': for task in PC.taskTypes: PC.make_tasks(task) load_tasks(packages=[task], addToPossible=False) - except ZeroDivisionError: - pass + elif option =='n': + pass + else: + print('option was',option, 'not "y" or "n"') newLevel = intOption print('RUNNINGTASKSETS will be finished and then the level will be set to {}.'.format(intOption)) SAVE_POSSIBLES[CURRENTTASKSETSIZE] = POSSIBLETASKSETS @@ -329,8 +339,7 @@ def show_status(): print('{} was not a viable option.'.format(intOption)) except ValueError: print('{} was not a viable option.'.format(option)) - except ZeroDivisionError: - pass + return @@ -346,11 +355,10 @@ def halt_machines(distributor, hard=False): else: distributor.shut_down_all_machines() # ask if current running should be cleared (clear jobQueue) - try: - print('Do you also want to clear the current RUNNINGTASKSETS?[y/n]') - alarm(10) - option = input() - alarm(0) + print('Do you also want to clear the current RUNNINGTASKSETS?[y/n]') + i, _, _ = select.select( [sys.stdin], [], [], 10 ) + if i: + option = sys.stdin.readline().strip() if option == 'y': print('will clear distributor jobQueue, RUNNINGTASKSETS and MONITORLISTS...') # clear distributor jobsQueue @@ -367,8 +375,7 @@ def halt_machines(distributor, hard=False): MONITORLISTS = [] print('cleared') return - except ZeroDivisionError: - pass + print('not cleared.') return @@ -438,8 +445,6 @@ def main(initialExecution=True): add_job(distributor=distributor, numberOfTasksets=PC.maxAllowedNumberOfMachines, tasksetSize=CURRENTTASKSETSIZE) add_job(distributor=distributor, numberOfTasksets=PC.maxAllowedNumberOfMachines, tasksetSize=CURRENTTASKSETSIZE) - # creating a signal for alarm - will be called upton alarm - signal(SIGALRM, lambda x, y: 1 / 0) # have output to explain controll options inputMessage = 'options are (d)ebug, show status(ss), (h)alt/(k)ill machines, (r)esume machines, (s)ave current progress, e(x)it.' print(inputMessage) @@ -456,11 +461,10 @@ def main(initialExecution=True): aORb = not aORb # main programm loop # wait for input: - try: - alarm(10) # argument should be a variable - option = input() - alarm(0) - except ZeroDivisionError: + i, _, _ = select.select( [sys.stdin], [], [], 10 ) + if i: + option = sys.stdin.readline().strip() + else: option = '' # act depending on option provided if option == 'ss': # show status diff --git a/parameter_config.py b/parameter_config.py index 88b157a..12dfd5b 100644 --- a/parameter_config.py +++ b/parameter_config.py @@ -7,8 +7,8 @@ availableSessions = ['QemuSession','PandaSession'] -numberOfMachinesToStartWith = 9 -maxAllowedNumberOfMachines = 9 +numberOfMachinesToStartWith = 1 +maxAllowedNumberOfMachines = 1 loggingLevel = logging.DEBUG delayAfterStartOfGenode = 60 timesTasksetIsTriedBeforeLabeldBad = 2 @@ -16,7 +16,7 @@ savedEveryNLaps = 90 # times 10s, so one save every 15 min -sessionType = availableSessions[1] +sessionType = availableSessions[0] taskTypes = ['hey', 'pi', 'tumatmul', 'cond_mod'] # to use all available task types use the following list instead:['hey', 'pi', 'tumatmul', 'cond_mod', 'cond_42'] tasksPerLine = 100 # number of tasks put in one list @@ -126,4 +126,4 @@ def make_tasks(pkg): for i in range(linesPerCall): tasks += str([get_task_hash(task) for task in VI.generate_tasks_of_type(tasksPerLine, pkg, taskParameters)[pkg]])+'\n' with open('./data/new_tasks_'+pkg, 'a') as file: - file.write(tasks) \ No newline at end of file + file.write(tasks)