From 640fbe59ec4385a64baf688e1676f3094e3f9ac9 Mon Sep 17 00:00:00 2001 From: AJRubio-Montero <AJRubio-Montero@github.com> Date: Fri, 11 Jun 2021 11:20:52 +0200 Subject: [PATCH] Stable release of onedataSim tested during the EOSC-SYNERGY simulation Challenge in 21-27 May 2021 --- Dockerfile | 9 +++++++-- arti | 2 +- wrappers/arguments.py | 8 ++++---- wrappers/do_sims_onedata.py | 39 +++++++++++++++++++++++++++++++++---- 4 files changed, 47 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3dfd027..dee7050 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,13 +70,18 @@ RUN yum -y install acl attr # xattr (this is python2 but I had found the command only in python2) RUN yum -y install python2-pip python-devel libffi-devel -# sometimes pip's upgrade fails -#RUN pip install --upgrade pip +# sometimes pip's upgrade fails and doesn't find "typing" module +# RUN pip install --upgrade pip +# RUN pip install typing +RUN pip install cffi RUN pip install xattr #python3 and libraries for Lago processing with onedata RUN yum -y install python3 python36-pyxattr +# utils for debugging +RUN yum -y install screen nano + WORKDIR /opt/corsika-77402-lago/run #ENTRYPOINT /opt/arti/sims/do_datahub.sh CMD bash diff --git a/arti b/arti index ae6d90f..d8f8caa 160000 --- a/arti +++ b/arti @@ -1 +1 @@ -Subproject commit ae6d90f66a978e0ac3e874ac887229d8801e7617 +Subproject commit d8f8caa280550078fad90fc2dfc9b1c4c06ca543 diff --git a/wrappers/arguments.py b/wrappers/arguments.py index b4654c5..cc472b4 100644 --- a/wrappers/arguments.py +++ b/wrappers/arguments.py @@ -138,10 +138,10 @@ def get_sys_args(): # echo -e " -s <site> : \ # Location (several options)" parser.add_argument('-s', dest='s', required=True, - choices=[ "QUIE","and","asu","ber","bga","brc","bue", - "cha","chia","cpv","cuz","gua","kna","lim", - "lpb","lsc","mapi","mge","pam","sac","sao", - "sawb","serb","sng","tac","tuc","vcp" ], + #choices=[ "QUIE","and","asu","ber","bga","brc","bue", + # "cha","chia","cpv","cuz","gua","kna","lim", + # "lpb","lsc","mapi","mge","pam","sac","sao", + # "sawb","serb","sng","tac","tuc","vcp" ], help='Predefined LAGO site') # echo -e " -j <procs> : \ # Number of processors to use" diff --git a/wrappers/do_sims_onedata.py b/wrappers/do_sims_onedata.py index 89d0c74..086ba6d 100755 --- a/wrappers/do_sims_onedata.py +++ b/wrappers/do_sims_onedata.py @@ -95,7 +95,8 @@ def get_first_catalog_metadata_json(catcodename, arti_params_dict): return json.loads(s) -def get_catalog_metadata_activity(startdate, enddate, arti_params_dict): +def get_catalog_metadata_activity(startdate, enddate, catcodename, + arti_params_dict): with open(onedataSimPath+'/json_tpl/catalog_corsika_activity.json', 'r') as file1: @@ -218,6 +219,10 @@ def _consumer_onedata_mv(onedata_path): print(id + ': copy queued again') q_onedata.put(md) time.sleep(2) + # we have to substract 1 to queue lenght because q.put + # always add 1 to lenght but really we are re-queing and + # size remains the same + q_onedata.task_done() def _run_check_and_copy_results(catcodename, filecode, task, onedata_path, @@ -266,7 +271,27 @@ def _producer(catcodename, arti_params): # clean a possible previous simulation if os.path.exists(catcodename): shutil.rmtree(catcodename, ignore_errors=True) + + # PATCH: correct the creation of tasks, which is based on (-j) in ARTI. + # ARTI tries fit the number of tasks (NRUN) to the number of procs + # for being correct in terms of physics, however was not implemented + # for fit the output sizes vs flux-time (arti_params[t]) + + params_aux_flux = arti_params[arti_params.find("-t")+3:] + flux_time = int(params_aux_flux[:params_aux_flux.find("-")]) + params_aux = arti_params[arti_params.find("-j")+3:] + old_j = int(params_aux[:params_aux.find("-")]) + + aux_j = int(int(flux_time)/900) + if aux_j == 0 : + aux_j = 1 + if aux_j > 12 : + aux_j = 12 + arti_params = arti_params[:arti_params.find("-j")] + "-j " + str(aux_j) +" " + params_aux[params_aux.find("-"):] + + print("PATCH: change -j : " + str(old_j) + " by :" + str(aux_j) + " to generate tasks") + cmd = 'do_sims.sh ' + arti_params _run_Popen_interactive(cmd) @@ -276,8 +301,9 @@ def _producer(catcodename, arti_params): _run_Popen(cmd) # WARNING, I HAD TO PATCH rain.pl FOR AVOID .long files !!! - cmd = "sed 's/\$llongi /F /' rain.pl -i" - _run_Popen(cmd) + # 20210519 not necessary since arti@d8f8caa + # cmd = "sed 's/\$llongi /F /' rain.pl -i" + # _run_Popen(cmd) # -g only creates .input's # cmd="sed 's/\.\/rain.pl/echo \$i: \.\/rain.pl -g /' go-*.sh -i" @@ -314,13 +340,17 @@ def _consumer(catcodename, onedata_path, arti_params_dict): q.task_done() except Exception as inst: q.put((filecode, task)) + # we have to substract 1 to queue lenght because q.put + # always add 1 to lenght but really we are re-queing and + # size remains the same + q.task_done() # ------------ main stuff --------- (arti_params, arti_params_dict, arti_params_json_md) = get_sys_args() catcodename = arti_params_dict["p"] -# onedata_path = '/mnt/datahub.egi.eu/LAGOsim' onedata_path = '/mnt/datahub.egi.eu/LAGOsim' +#onedata_path = '/mnt/datahub.egi.eu/test8/LAGOSIM' catalog_path = onedata_path + '/' + catcodename print(arti_params, arti_params_dict, arti_params_json_md) @@ -375,6 +405,7 @@ md['dataset'] = ["/" + catcodename + "/" + s for s in md = _add_json(md, json.loads(get_catalog_metadata_activity(main_start_date, _xsd_dateTime(), + catcodename, arti_params_dict))) _write_file(catalog_path + '/.metadata/.' + catcodename + '.jsonld', -- GitLab