Merge pull request #29 from lagoproject/dev-asoreyh

ARTI is now compatible with CORSIKA+FLUKA and a new Docker compatible mode working with OneData was added

eas/do_sims.sh

@@ -55,7 +55,8 @@ showhelp() {
   echo -e "  -w <working dir>                   : Working directory, where bin (run) files are located"
   echo -e "  -p <project name>                  : Project name (suggested format: NAMEXX)"
   echo -e "  -v <CORSIKA version>               : CORSIKA version"
-  echo -e "  -h <HE Int Model (EPOS|QGSII)>     : Define the high interaction model to be used"
+  echo -e "  -h <HE Int Model (EPOS|QGSII)>     : Define the high energy interaction model to be used. Default: QGSJET-II-04"
+  echo -e "  -f <LE Int Model (gheisha|fluka)>  : Define the lown energy interaction model to be used. Default: gheisha"
   echo -e "  -u <user name>                     : User Name."
   echo -e "  -j <procs>                         : Number of processors to use"
   echo -e
@@ -79,8 +80,8 @@ showhelp() {
   echo -e
   echo -e "Modifiers"
   echo -e "  -l                                 : Enables SLURM cluster compatibility (with sbatch)."
+  echo -e "  -d                                 : Enable DOCKER compatible mode"
   echo -e "  -e                                 : Enable CHERENKOV mode"
-  echo -e "  -d                                 : Enable DEBUG mode"
   echo -e "  -x                                 : Enable other defaults (It doesn't prompt user for unset parameters)"
   echo -e "  -?                                 : Shows this help and exit."
   echo
@@ -89,7 +90,7 @@ cta=false
 debug=false
 highsec=false
 sites=false
-usr="LAGO";
+usr="LAGO"
 vol=false
 alt=false
 altitude=0.
@@ -105,9 +106,12 @@ BZcomp=false
 defaults=false
 ecut=800
 slurm=false
+onedataBase="/mnt/datahub.egi.eu/test8/fluka"; # need to change also at rain.pl
+nprocs=$(/usr/bin/nproc)   # number of simultaneous process for paralllel local processing
+docker=false
 
 echo
-while getopts 'w:k:p:t:v:u:h:s:j:c:b:m:n:r:i:o:q:a:?lydex' opt; do
+while getopts 'w:k:p:t:v:u:f:h:s:j:c:b:m:n:r:i:o:q:a:?lydex' opt; do
   case $opt in
     w)
       wdir=$OPTARG
@@ -138,6 +142,10 @@ while getopts 'w:k:p:t:v:u:h:s:j:c:b:m:n:r:i:o:q:a:?lydex' opt; do
       hig=$OPTARG
       echo -e "#  High Energy Interaction Model = $hig"
       ;;
+    f)
+      lemodel=$OPTARG
+      echo -e "#  Lew Energy Interaction Model = $lemodel"
+      ;;
     s)
       site=$OPTARG
       sites=true
@@ -204,7 +212,7 @@ while getopts 'w:k:p:t:v:u:h:s:j:c:b:m:n:r:i:o:q:a:?lydex' opt; do
       slurm=true
       ;;
     d)
-      debug=true
+	  docker=true
       ;;
     x)
       defaults=true
@@ -226,10 +234,21 @@ if [ "X$prj" == "X" ]; then
   exit 1;
 fi
 
-if [ "X$wdir" == "X" ]; then
-  echo; echo -e "ERROR: You have to set the working directory (where corsika bin files are located)"
-  showhelp
-  exit 1;
+if [ "X$ver" == "X" ]; then
+  ver="77402"
+  echo -e "#  INFO: CORSIKA version was not provided. Using default: $ver"
+fi
+
+
+if ! $docker; then
+	if [ "X$wdir" == "X" ]; then
+		echo; echo -e "ERROR: You have to set the working directory (where corsika bin files are located)"
+		showhelp
+		exit 1;
+	fi
+else
+	wdir="/opt/corsika-${ver}/run"
+	echo -e "#  INFO: Working directory fixed by DOCKER: $wdir"
 fi
 
 if [ "X$tim" == "X" ]; then
@@ -242,16 +261,16 @@ if [ "X$procs" == "X0" ]; then
   echo -e "#  INFO: Processors should be >0, using default: $procs"
 fi
 
-if [ "X$ver" == "X" ]; then
-  ver="77402"
-  echo -e "#  INFO: CORSIKA version was not provided. Using default: $ver"
-fi
-
 if [ "X$hig" == "X" ]; then
   hig="QGSII"
   echo -e "#  INFO: High energy interaction model was not provided. Using default: $hig"
 fi
 
+if [ "X$lemodel" == "X" ]; then
+  lemodel="gheisha"
+  echo -e "#  INFO: Low energy interaction model was not provided. Using default: $lemodel"
+fi
+
 if [ "X$atm_model" == "X" ]; then
   atm_model="E1"
   echo -e "#  INFO: Atmospheric Model was not provided. Using default: $atm_model"
@@ -320,18 +339,27 @@ if $highsec; then
 fi
 
 if $slurm; then
-	echo -e "#  INFO: SLURM mode is enable. Will not work in other environments."
+	echo -e "#  INFO: SLURM mode is enabled. Will not work in other environments."
+fi
+if $docker; then
+	if [ ! -d $onedataBase ]; then
+		echo; echo -e "ERROR: You are running in DOCKER enabled mode but onedata is not accessible"
+		showhelp
+		exit 1;
+	fi
+	echo -e "#  INFO: DOCKER mode is enabled."
 fi
 
-corsika_bin="corsika${ver}Linux_${hig}_gheisha"
+corsika_bin="corsika${ver}Linux_${hig}_${lemodel}"
 if [ ! -e $wdir/$corsika_bin ]; then
 	echo; echo -e "ERROR: Can't locate corsika executable file ($corsika_bin) in the working dir you provided. Please check."
     showhelp
     exit 1;
 fi
-echo -e "#  INFO   : Executable file is ($corsika_bin)"
-
+echo -e "#  INFO: Executable file is ($corsika_bin)"
 
+# It is important to now the total time in onedata. Adding total simulation time to the project name...
+prj="${prj}_$(printf "%06d" ${tim})"
 direct=$wdir/$prj
 basearti=${ARTI}
 #helium -> Usually for 1 hour flux, 4 different procces is enough
@@ -409,7 +437,7 @@ if $sites; then
   rain="$rain -s ${site}"
 fi
 
-if $debug; then
+if $docker; then
   rain="$rain -d"
 fi
 
@@ -421,9 +449,18 @@ if $slurm; then
 	rain="$rain -l"
 fi
 
-rain="$rain -r $wdir -v $ver -h $hig -b $prj/\$i-*.run"
-# echo -e "#  INFO   : rain command: $rain"
+rain="$rain -r $wdir -v $ver -h $hig -f $lemodel -b $prj/\$i-*.run"
+echo -e "#  INFO   : rain command: $rain"
+if $docker; then
+	oneout="$onedataBase/S3_${prj}_${site}_${lemodel}"
+	[[ ! -d $oneout ]] && mkdir $oneout
+	while ! cp -v $wdir/$prj/inject $oneout; do 
+		sleep 5
+	done
+	echo -e "#  INFO   : Results will be transferred to $oneout"
+fi
 echo -e "#  INFO   : Calculations done. Now run the go_${prj}_* scripts in $wdir/"
+
 basenice=19
 if $slurm; then
   basenice=0;
@@ -435,6 +472,9 @@ if $slurm; then
 	echo -e "" >> $wdir/go-slurm-$prj.sh
     chmod 744 $wdir/go-slurm-$prj.sh
 fi
+if $docker; then
+	echo -n > $wdir/go-docker-$prj.run
+fi
 
 stuff=(001206 001608 000703 002412 001105 002814 001407 002010 005626 000904 003216 002713 002311 004020 001909 005224 004018 004822 005525 003919 005123 003115 003517 004521)
 t=0
@@ -459,12 +499,18 @@ mv $wdir/go-$prj-all-$n.sh $wdir/go-$prj-all-$n.run
 		echo $wdir/go-$prj-all-$n.sh >> $wdir/go-slurm-$prj.sh
 	fi
 done
-
+if $docker; then
+	for i in $(seq 0 $[${#stuff[@]}-1]); do
+		nuc=${stuff[$i]}
+		r=${rain/\$i/${nuc}}
+		echo -e "$r" >> $wdir/go-docker-$prj.run
+	done
+	rm $wdir/go-$prj-all-*.sh
+fi
 #helium
 b=$(basename $(ls -1 $wdir/$prj/000402-*) .run | sed -e "s/000402-//" | awk '{print $1*1.0}')
 c=$(basename $(ls -1 $wdir/$prj/000402-*) .run | sed -e "s/000402-//" | awk -v p=${prcHe} '{print int($1/p+0.5)}')
 printf -v k "%011d" $c
-
 for i in $(seq 1 $prcHe); do
   u="0${i}0402"
   cat $wdir/$prj/000402-*.run | sed -e "s/000402/$u/" | sed -e "s/$b/$c/" > $wdir/$prj/$u-$k.run
@@ -485,6 +531,14 @@ rm $wdir/$prj/000402-*.run
 if $slurm; then
 	echo $wdir/go-${prj}-he.sh >> $wdir/go-slurm-$prj.sh
 fi
+if $docker; then
+	for i in $(seq 1 $prcHe); do
+		u="0${i}0402"
+        r=${rain/\$i/${u}}
+        echo -e "$r" >> $wdir/go-docker-$prj.run
+	done
+	rm $wdir/go-$prj-he.sh
+fi
 
 #protons
 b=$(basename $(ls -1 $wdir/$prj/000014-*) .run | sed -e "s/000014-//" | awk '{print $1*1.0}')
@@ -512,9 +566,17 @@ for j in \$(seq $ii $ff); do
 done
 chmod 644 $wdir/go-$prj-pr-$i.sh
 mv $wdir/go-$prj-pr-$i.sh $wdir/go-$prj-pr-$i.run" > $wdir/go-${prj}-pr-$i.sh
-if $slurm; then
-    echo $wdir/go-${prj}-pr-$i.sh >> $wdir/go-slurm-$prj.sh
-fi
+	if $slurm; then
+		echo $wdir/go-${prj}-pr-$i.sh >> $wdir/go-slurm-$prj.sh
+	fi
+	if $docker; then
+		for j in $(seq $ii $ff); do
+			printf -v n "%02d" $j
+			u="${n}0014"
+			r=${rain/\$i/${u}}
+			echo -e "$r" >> $wdir/go-docker-$prj.run
+		done
+	fi
 done
 rm $wdir/$prj/000014-*.run
 for i in $(seq 1 $multPr); do
@@ -525,3 +587,31 @@ if $slurm; then
     echo -e "mv $wdir/go-slurm-$prj.sh $wdir/go-slurm-$prj.run" >> $wdir/go-slurm-$prj.sh
     echo -e "chmod 644 $wdir/go-slurm-$prj.run" >> $wdir/go-slurm-$prj.sh
 fi
+if $docker; then
+	rm $wdir/go-${prj}-pr-*.sh
+	rm -r 
+ 	echo -e "#!/bin/bash" > $wdir/go-docker-$prj.sh
+ 	echo -e "# go docker $prj" >> $wdir/go-docker-$prj.sh
+ 	echo -e "" >> $wdir/go-docker-$prj.sh
+	echo -e "N=\$(/usr/bin/nproc)" >> $wdir/go-docker-$prj.sh
+ 	echo -e "nl=\$(cat $wdir/go-docker-$prj.run | wc -l)" >> $wdir/go-docker-$prj.sh
+	echo -e "np=0" >> $wdir/go-docker-$prj.sh
+	echo -e "nr=0" >> $wdir/go-docker-$prj.sh
+	echo -e "while IFS= read -r line; do" >> $wdir/go-docker-$prj.sh
+    echo -e "    eval \${line}" >> $wdir/go-docker-$prj.sh
+	echo -e "    ((nr++))" >> $wdir/go-docker-$prj.sh
+    echo -e "    np=\$(ps aux | grep corsika77402 | grep -v grep | wc -l)" >> $wdir/go-docker-$prj.sh
+    echo -e "    echo \$np: \$nr/\$nl" >> $wdir/go-docker-$prj.sh
+    echo -e "    while [ \$np -ge \$N ]; do" >> $wdir/go-docker-$prj.sh
+	echo -e "        sleep 3;" >> $wdir/go-docker-$prj.sh
+    echo -e "        np=\$(ps aux | grep corsika77402 | grep -v grep | wc -l)" >> $wdir/go-docker-$prj.sh
+    echo -e "    done" >> $wdir/go-docker-$prj.sh
+	echo -e "done < $wdir/go-docker-$prj.run" >> $wdir/go-docker-$prj.sh
+    echo -e "mv $wdir/go-dockerslurm-$prj.sh $wdir/go-slurm-$prj.run" >> $wdir/go-slurm-$prj.sh
+    echo -e "chmod 644 $wdir/go-slurm-$prj.run" >> $wdir/go-slurm-$prj.sh
+	chmod 744 $wdir/go-docker-$prj.sh
+	cd $wdir
+	eval $wdir/go-docker-$prj.sh
+	ls -l $oneout/
+	cd ${HOME}
+fi

eas/get_atm_from_repo.sh

+#!/bin/bash
+# /************************************************************************/
+# /* Package:  ARTI                                                       */
+# /* Module:   get_atm_from_repo.sh                                       */
+# /************************************************************************/
+# /* Authors:  Hernán Asorey                                              */
+# /* e-mail:   hernan.asoreyh@iteda.cnea.gov.ar                           */
+# /************************************************************************/
+# /* Comments: Get GDAS monthly averaged atmopsheric profiles produced by */ 
+# /*           LAGO foreach of the sites described at in the LAGO DMP     */
+# /************************************************************************/
+# /*
+# LICENSE BSD-3-Clause
+# Copyright (c) 2015
+# The LAGO Collaboration
+# https://lagoproject.net
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    1. Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#
+#    2. Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
+# NO EVENT SHALL LAB DPR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and documentation are
+# those of the authors and should not be interpreted as representing
+# official policies, either expressed or implied, of the LAGO Collaboration.
+#
+# */
+# /************************************************************************/
+
+url_base="https://raw.githubusercontent.com/lagoproject/atmospheres/main/atmprof"
+[ -z "$1" ] && { echo "Usage: $0 <siteId> <year>" ; exit 99; }
+[ -z "$2" ] && { echo "Usage: $0 <siteId> <year (YY)>" ; exit 99; }
+if [ "$2" -lt 0 ] || [ $2 -gt 99 ]; then
+	echo "Year should be given in YY format (00 to 99). Usage: $0 <siteId> <year (YY)>" 
+	exit 99
+fi
+site=$1
+year=$(printf "%02d" $2)
+n=0
+for m in $(seq -w 1 12); do
+	file="atmprof${site}${year}${m}.dat"
+	url_file="$url_base/${site}/${file}"
+	if $(wget -q $url_file); then
+		echo "File $file retrived"
+		n=$(( n + 1 ))
+	else
+		echo "Failed to retrieve $file"
+	fi
+done
+if [ "$n" -eq 12 ]; then
+	echo "Success retrieving site:${site} for year:$year"
+else
+	echo "Failed to retrieve some files. $n files extracted"
+	echo "Perhaps files does not be already produced. Please check at https://github.com/lagoproject/atmospheres/"
+	exit 99
+fi

eas/rain.pl

@@ -60,6 +60,7 @@ my $wdir = "x";
 my $crk_ver = "77402";
 my $heim = "QGSII";
 my $debug = 0;
+my $docker = 0;
 my $help = 0;
 my $slurm = 0;
 my $highsec = 0;
@@ -89,6 +90,8 @@ my $grid = 0;
 my $imuaddi = 0;
 my $nofruns = 1;
 my $ecutshe = 800.;
+my $lemodel = "gheisha";
+my $onedataBase = "/mnt/datahub.egi.eu/test8/fluka"; # need to change also at do_sims.sh
 
 sub get_answer {
   my $question = $_[0];
@@ -167,7 +170,7 @@ while ($_ = $ARGV[0]) {
     $help++;
   }
   if (/-d$/i) {
-    $debug++;
+    $docker++;
   }
   if (/-mu$/i) {
     $imuaddi=1;
@@ -197,36 +200,42 @@ while ($_ = $ARGV[0]) {
     $heim = $ARGV[0];
     shift;
   }
+  if (/-f$/i) {
+    $lemodel = $ARGV[0];
+    shift;
+  }
 }
 
-my $package="corsika".$crk_ver."Linux_".$heim."_gheisha";
+my $package="corsika".$crk_ver."Linux_".$heim."_".$lemodel;
 $package = $package . "_thin" if ($ithin != 0);
 my $usage="
        $0 $VERSION\n
        Usage: $0 options\n
-       -b                                  Activates batch mode
-       -i                                  Disable PLOTSH and PLOTSH2 modes (usual simms production)
-       -d                                  Debug mode: only shows what it should do. Don't start simulation
        -r  <working directory>             Specify where corsika bin files are located
        -v  <version>                       Corsika version number
        -h  <high energy interaction model> High energy interaction model used for compilation of CORSIKA (EPOS|QGSII|SIBYLL)
-       -l                                  Enables SLURM cluster compatibility (with sbatch). 
+       -f  <low energy interaction model>  Low energy interaction model used for compilation of CORSIKA (gheisha|fluka)
+       -s <site>                           Choice site for simulation (some predefined sites: hess|sac|etn|ber|bga|lim|glr|mch|mge|and|mpc|cha|cid|mor|ccs|lsc|mbo)
+       -a <high energy ecuts (GeV)>        Enables and set high energy cuts for ECUTS
+       -m <energy>                         Defines energy (in GeV) for monoenergetic showers (CHERENKOV)
+       -q <theta>                          Defines zenith angle (in degs) for fixed angle showers (CHERENKOV)
+       -p <prmpar>                         Defines primary particle (see table 4 pg 87) (CHERENKOV)
        -t  <EFRCTHN> <WMAX> <RMAX>         Enables THIN Mode (see manual for pg 62 for values)
        -th <THINRAT> <WEITRAT>             If THIN Mode, select different thining levels for Hadronic (THINH) ...
        -te <THINRAT> <WEITRAT>             ... and electromagnetic particles (THINEM)
-       -a <high energy ecuts (GeV)>        Enables and set high energy cuts for ECUTS
+       -b                                  Activates batch mode
+       -i                                  Disable PLOTSH and PLOTSH2 modes (usual simms production)
+       -l                                  Enables SLURM cluster compatibility (with sbatch). 
+       -d                                  Enables DOCKER mode (oneclient should be running)
        -z                                  Enables CHERENKOV mode
        -mu                                 Enables additional information from muons and EM particles
        -g                                  Enables GRID mode
-       -s <site>                           Choice site for simulation (some predefined sites: hess|sac|etn|ber|bga|lim|glr|mch|mge|and|mpc|cha|cid|mor|ccs|lsc|mbo)
-       -m <energy>                         Defines energy (in GeV) for monoenergetic showers (CHERENKOV)
-       -q <theta>                          Defines zenith angle (in degs) for fixed angle showers (CHERENKOV)
-       -p <prmpar>                         Defines primary particle (see table 4 pg 87) (CHERENKOV)
        \n";
 die "$usage\n" if ($help != 0);
 
 print STDERR "\nWARNING! You are running in DEBUG mode. I'll only show what I should do\n\n" if ($debug != 0);
 print STDERR "\nWARNING! CHERENKOV mode is enabled.\n\n" if ($cherenkov != 0);
+print STDERR "\nWARNING! DOCKER mode is enabled.\n\n" if ($docker != 0);
 print STDERR "\nWARNING! Site selected for simulation: $site.\n\n" unless ($site eq "");
 
 if ($runmode != 0) {
@@ -438,6 +447,9 @@ for (my $i=0; $i < $nofruns; $i++) {
   }
   # LAGO ECUTS, minimum possible values as for the current corsika version
   my @ecuts=(0.05, 0.01, 0.00005, 0.00005);
+  if ($lemodel eq "fluka") {
+    $ecuts[0]=0.02;
+  }
   if ($highsec != 0) {
     @ecuts=($ecutshe, $ecutshe, $ecutshe, $ecutshe);
     $e_low = $ecutshe if ($e_low < $ecutshe);
@@ -584,6 +596,14 @@ EXIT
       print $fh "echo \"compressing output files...\"\n";
       print $fh "bzip2 -9v $binout\n";
       print $fh "bzip2 -9v $out\n";
+	  if ($docker != 0) {
+		  my $oneout = "$onedataBase/S3_${prj}_${site}_${lemodel}";
+		  print $fh "echo \"tranferring to onedata...\"\n";
+	      print $fh "while ! cp -v $binout.bz2 $oneout; do sleep 5; done\n";
+		  print $fh "while ! cp -v $file $oneout; do sleep 5; done\n";
+	      print $fh "while ! cp -v $out.bz2 $oneout; do sleep 5; done\n";
+	      print $fh "rm $binout.bz2 $file $out.bz2\n";
+	  }
       print $fh "rm $script\n";
       close($fh);
       system("chmod 777 $script");
@@ -597,7 +617,7 @@ EXIT
     print "# $getjson\n";
     if ($slurm != 0) {
       print "###################################################################\n";
-      $cmd="sbatch -p highpri2 -o ${name}_srun_%j.log ${script}";
+      $cmd="sbatch -p cpu36c -o ${name}_srun_%j.log ${script}";
     }
     else {
       print "# in screen $name\n";
@@ -612,8 +632,3 @@ EXIT
     }
   }
 }
-unless ($grid != 0) {
-  print "###################################################################\n";
-  print "# BYE BYE\n";
-  print "###################################################################\n";
-}