diff --git a/CTA.mainRunScriptsReduced.sh b/CTA.mainRunScriptsReduced.sh index 651b6ab..aa145b5 100755 --- a/CTA.mainRunScriptsReduced.sh +++ b/CTA.mainRunScriptsReduced.sh @@ -24,7 +24,7 @@ if [ $# -lt 2 ]; then prod6-South-20deg run modes: - MAKETABLES PREPAREFILELISTS DISPBDT ANATABLES PREPARETMVA TRAIN ANGRES QC CUTS PHYS CLEANUP + MAKETABLES PREPAREFILELISTS DISPBDT ANATABLES PREPARETMVA PREPAREANA XGBSTEREOTRAIN XGBSTEREOANA TRAIN ANGRES QC CUTS PHYS CLEANUP optional run modes: TRAIN_RECO_QUALITY TRAIN_RECO_METHOD @@ -49,7 +49,7 @@ if [[ ${RUN} != "CLEANUP" ]] && [[ ${RUN} != "PREPAREFILELISTS" ]]; then mkdir -p ${RUNSCRIPTDIR} fi -if [[ ${RUN} == "MAKETABLES" ]] || [[ ${RUN} == "DISPBDT" ]] || [[ ${RUN} == "ANATABLES" ]] || [[ ${RUN} == "PREPARETMVA" ]]; then +if [[ ${RUN} == "MAKETABLES" ]] || [[ ${RUN} == "DISPBDT" ]] || [[ ${RUN} == "ANATABLES" ]] || [[ ${RUN} == "PREPARETMVA" ]] || [[ ${RUN} == "PREPAREANA" ]] || [[ ${RUN} == "XGBSTEREOTRAIN" ]] || [[ ${RUN} == "XGBSTEREOANA" ]]; then ./CTA.runAnalysis.sh ${P2} ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR} if [[ $SITE == "South" ]] || [[ $P2 == *"prod6"* ]]; then ./CTA.runAnalysis.sh ${P2}-sub ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR} diff --git a/CTA.runAnalysis.sh b/CTA.runAnalysis.sh index 921a664..d2c0d02 100755 --- a/CTA.runAnalysis.sh +++ b/CTA.runAnalysis.sh @@ -28,7 +28,7 @@ then prod6-North-20deg prod6-North-40deg prod6-North-52deg prod6-North-60deg prod6-South-20deg - possible run modes are EVNDISP MAKETABLES PREPAREFILELISTS DISPBDT ANATABLES PREPARETMVA TRAIN ANGRES QC CUTS PHYS CLEANUP + possible run modes are EVNDISP MAKETABLES PREPAREFILELISTS DISPBDT ANATABLES XGBSTEREOTRAIN XGBSTEREOANA PREPARETMVA TRAIN PREPAREANA ANGRES QC CUTS PHYS CLEANUP optional run modes: TRAIN_RECO_QUALITY TRAIN_RECO_METHOD @@ -51,7 +51,7 @@ RUN="$2" [[ "$5" ]] && MST=$5 || MST="2" [[ "$6" ]] && SST=$6 || SST="2" [[ "$7" ]] && SCMST=$7 || SCMST="2" -[[ "$8" ]] && PDIR=${8} || PDIR="${CTA_USER_LOG_DIR%/}/" +[[ "$8" ]] && PDIR=${8} || PDIR="${CTA_USER_LOG_DIR%/}" echo "Telescope multiplicities: LST ${LST} MST ${MST} SST ${SST} SCMST ${SCMST}" ##################################### @@ -334,10 +334,10 @@ then EDM="-sq50-LL-DL2plus" fi ARRAY=( "subArray.prod6.${NS}ML${SCT}.list" ) - ARRAY=( "subArray.prod6.${NS}Alphab${SCT}.list" ) + ARRAY=( "subArray.prod6.${NS}Alpha${SCT}.list" ) if [[ $P2 == *"sub"* ]]; then ARRAY=( "subArray.prod6.${NS}ML-sub.list" ) - ARRAY=( "subArray.prod6.${NS}Alphab-sub.list" ) + ARRAY=( "subArray.prod6.${NS}Alpha-sub.list" ) fi if [[ $P2 == *"Hyper"* ]] || [[ $P2 == *"hyper"* ]]; then ARRAY=( "subArray.prod6.NorthHyper.list" ) @@ -345,6 +345,7 @@ then ARRAYDIR="prod6" TDATE="g20251021" ANADATE="${TDATE}" + ANADATE="g20260127" TMVADATE="${ANADATE}" EFFDATE="${ANADATE}" PHYSDATE="${EFFDATE}" @@ -424,7 +425,7 @@ then fi # remove from PHYS directory any unreasonable files (e.g. LST4 requirement for 2 LST array) if [[ $RUN == "CLEANUP" ]]; then - PHYSDIR="${CTA_USER_DATA_DIR}/analysis/AnalysisData/${SITE}${EDM}/Phys-${PHYSDATE}" + PHYSDIR="${CTA_USER_DATA_DIR%/}/analysis/AnalysisData/${SITE}${EDM}/Phys-${PHYSDATE}" ./utilities/removeUnreasonablePhysFiles.sh ${PHYSDIR} exit fi @@ -457,7 +458,7 @@ then BDTDIR="BDTdisp." RUNPAR="${CTA_EVNDISP_AUX_DIR}/ParameterFiles/TMVA.BDTDisp.runparameter" QCPAR="${CTA_EVNDISP_AUX_DIR}/ParameterFiles/TMVA.BDTDispQualityCuts.runparameter" - DDIR="${CTA_USER_DATA_DIR}/analysis/AnalysisData/${SITE}${EDM}/" + DDIR="${CTA_USER_DATA_DIR%/}/analysis/AnalysisData/${SITE}${EDM}/" for A in $NXARRAY do cd ./analysis/ @@ -567,7 +568,7 @@ do touch "$PARA" echo "WRITING PARAMETERFILE $PARA" EFFDIR=EffectiveArea-"$OOTIME"-ID$ID$AZ-$ETYPF-$EFFDATE-$EFFVERSION - EFFFULLDIR="${CTA_USER_DATA_DIR}/analysis/AnalysisData/${SITE}${EDM}/EffectiveAreas/${EFFDIR}/" + EFFFULLDIR="${CTA_USER_DATA_DIR%/}/analysis/AnalysisData/${SITE}${EDM}/EffectiveAreas/${EFFDIR}/" echo "MSCWSUBDIRECTORY ${MSCWSUBDIRECTORY}" >> "$PARA" echo "TMVASUBDIR BDT-${TMVAVERSION}-ID$ID$AZ-$TMVATYPF-$TMVADATE" >> "$PARA" echo "TMVA_RECO_METHOD BDT-RECO-METHOD-${TMVAVERSION}-ID$ID$AZ-$TMVATYPF-$TMVADATE" >> "$PARA" @@ -604,8 +605,33 @@ do fi cd ./analysis/ ########################################## +# XGB stereo analysis training and analysis + if [[ $RUN == "XGBSTEREOTRAIN" ]] + then + # Train XGB independently of AZ + if [ ${o} -eq 0 ] && [[ -z ${AZ} ]] + then + ./CTA.XGBSTEREO.sub_train.sh \ + "$NFILARRAY" \ + ${SITE}${EDM} \ + "$PARA" \ + $QSUBOPT + fi + elif [[ $RUN == "XGBSTEREOANA" ]] + then + if [ ${o} -eq 0 ] && [[ ! -z ${AZ} ]] + then + ./CTA.XGBSTEREO.sub_analyse.sh \ + "$NFILARRAY" \ + ${SITE}${EDM} \ + "$PARA" \ + $QSUBOPT \ + $AZ \ + ${PDIR}/${RUN} + fi +########################################## # prepare train BDTs - if [[ $RUN == "PREPARETMVA" ]] + elif [[ $RUN == "PREPARETMVA" ]] then if [ ${o} -eq 0 ] && [[ ! -z ${AZ} ]] then @@ -616,13 +642,24 @@ do $QSUBOPT \ $AZ \ ${PDIR}/${RUN} - fi + fi +########################################## +# prepare analysis files to separate training / evaluation +# (already done for non-empty Az in PREPARETMVA stage + elif [[ $RUN == "PREPAREANA" ]] + then + if [ ${o} -eq 0 ] && [[ -z ${AZ} ]] + then + ./CTA.prepareAnalysis_no_sub.sh \ + "$NFILARRAY" \ + ${SITE}${EDM} \ + "$PARA" + fi ########################################## # train BDTs # (note: BDT training does not need to be done for all observing periods) elif [[ $RUN == TRAIN* ]] || [[ $RUN == "TMVA" ]] then - if [ $RUN == "TRAIN_RECO_METHOD" ]; then TMVA_RUN_MODE="TrainAngularReconstructionMethod" elif [ $RUN == "TRAIN_RECO_QUALITY" ]; then @@ -640,7 +677,7 @@ do $QSUBOPT \ $AZ \ ${PDIR}/${RUN} - fi + fi ########################################## # IRFs: angular resolution elif [[ $RUN == "ANGRES" ]] diff --git a/analysis/CTA.EFFAREA.qsub_analyse_list.sh b/analysis/CTA.EFFAREA.qsub_analyse_list.sh index 6e2fc81..6d6efa5 100644 --- a/analysis/CTA.EFFAREA.qsub_analyse_list.sh +++ b/analysis/CTA.EFFAREA.qsub_analyse_list.sh @@ -28,6 +28,12 @@ MCAZ="PPPMCAZ" PARTID="PARTIDNOTSET" ###################################################################### +if [[ ! -n "$TMPDIR" ]]; +then + TMPDIR="$CTA_USER_DATA_DIR/tmp" + mkdir "$TMPDIR" +fi + # Choose PARTICLE type from job id for SGE qsub system re='^[0-9]+$' if ! [[ $PARTID =~ $re ]] ; then @@ -125,7 +131,7 @@ if [ -z $EFFAREABASEDIR ] then EFFAREABASEDIR=$EFFAREADIR fi -# see if strict separation of training/testing events if possible +# see if strict separation of training/testing events is possible # (mscw files would be in a directory ....EFF) if [ -e ${PRODBASEDIR}/$ARRAY/${ANADIR}.EFFAREA.MCAZ${MCAZ} ] then @@ -505,6 +511,7 @@ do echo "* FILLINGMODE 3" >> $MSCF fi # fill IRFs only + echo "* RECONSTRUCTIONTYPE XGBSTEREO" >> $MSCF echo "* ENERGYRECONSTRUCTIONMETHOD 1" >> $MSCF echo "* ENERGYAXISBINS 60 -2. 4." >> $MSCF echo "* ENERGYAXISBINHISTOS 25 -1.9 3.1" >> $MSCF diff --git a/analysis/CTA.EFFAREA.sub_analyse_list.sh b/analysis/CTA.EFFAREA.sub_analyse_list.sh index cc22436..4a17975 100755 --- a/analysis/CTA.EFFAREA.sub_analyse_list.sh +++ b/analysis/CTA.EFFAREA.sub_analyse_list.sh @@ -159,7 +159,6 @@ do chmod u+x $QSHELLDIR/$FNAM.sh - echo $QSHELLDIR/$FNAM.sh ########################################### # submit the job script @@ -173,6 +172,7 @@ do sed -e "s|PARTIDNOTSET|$PARTICLEID|" "${QSHELLDIR}/${FNAM}.sh" > "${QSHELLDIR}/${FNAM}-${PARTICLEID}.sh" chmod u+x "${QSHELLDIR}/${FNAM}-${PARTICLEID}.sh" ./condorSubmission.sh "${QSHELLDIR}/${FNAM}-${PARTICLEID}.sh" $h_vmem $tmpdir_size + echo "$QSHELLDIR/${FNAM}-${PARTICLEID}.sh" done rm -f "$QSHELLDIR/$FNAM.sh" fi @@ -187,6 +187,7 @@ do sed -e "s|PARTIDNOTSET|$PARTICLEID|" "${QSHELLDIR}/${FNAM}.sh" > "${QSHELLDIR}/${FNAM}-${PARTICLEID}.sh" chmod u+x "${QSHELLDIR}/${FNAM}-${PARTICLEID}.sh" ./condorSubmission.sh "${QSHELLDIR}/${FNAM}-${PARTICLEID}.sh" $h_vmem $tmpdir_size + echo "$QSHELLDIR/${FNAM}-${PARTICLEID}.sh" done rm -f "$QSHELLDIR/$FNAM.sh" fi diff --git a/analysis/CTA.XGBSTEREO.qsub_analyse.sh b/analysis/CTA.XGBSTEREO.qsub_analyse.sh new file mode 100755 index 0000000..7c68d67 --- /dev/null +++ b/analysis/CTA.XGBSTEREO.qsub_analyse.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# train XGB stereo for CTA +# + +MSCW_FILE="FFILE" +XGBDIR="DIRXGB" +XGB="xgb_stereo" +DSET="DATASET" +env_name="eventdisplay_ml_cta" +MAXCORES=48 + +# set environmental variables +source $EVNDISPSYS/setObservatory.sh CTA + + +check_conda_installation() +{ + if command -v conda &> /dev/null; then + echo "Found conda installation." + else + echo "Error: found no conda installation." + echo "exiting..." + exit + fi + env_info=$(conda info --envs) + if [[ "$env_info" == *"$env_name"* ]]; then + echo "Found conda environment '$env_name'" + else + echo "Error: the conda environment '$env_name' does not exist." + echo "exiting..." + exit + fi +} + +check_conda_installation + +source activate base +conda activate $env_name + +PREFIX="${XGBDIR}/dispdir_bdt" + +if [[ $DSET == *"LaPalma"* ]]; then + site="CTAO-NORTH" +else + site="CTAO-SOUTH" +fi + +OFIL=$(basename $MSCW_FILE .root) +ODIR=$(dirname $MSCW_FILE) +OFIL="${ODIR}/${OFIL}.${XGB}" +LOGFILE="${OFIL}.log" +rm -f "$LOGFILE" + +echo "LOG FILE: $LOGFILE" + +eventdisplay-ml-apply-xgb-stereo \ + --input_file "$MSCW_FILE" \ + --model_prefix "${PREFIX}" \ + --output_file "${OFIL}.root" \ + --max_cores $MAXCORES \ + --observatory $site >| "${LOGFILE}" 2>&1 + +python --version >> "${LOGFILE}" +conda list -n $env_name >> "${LOGFILE}" + +conda deactivate diff --git a/analysis/CTA.XGBSTEREO.qsub_train.sh b/analysis/CTA.XGBSTEREO.qsub_train.sh new file mode 100755 index 0000000..8a16afb --- /dev/null +++ b/analysis/CTA.XGBSTEREO.qsub_train.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# train XGB stereo for CTA +# + +ODIR=OUTPUTDIR +LLIST=MSCWLIST +DSET="DATASET" +env_name="eventdisplay_ml_cta" +P="0.90" +N="500000000" +MAXCORES=48 + +# set environmental variables +source $EVNDISPSYS/setObservatory.sh CTA + +# output data files are written to this directory +mkdir -p "${ODIR}" +echo -e "Output files will be written to:\n ${ODIR}" + +check_conda_installation() +{ + if command -v conda &> /dev/null; then + echo "Found conda installation." + else + echo "Error: found no conda installation." + echo "exiting..." + exit + fi + env_info=$(conda info --envs) + if [[ "$env_info" == *"$env_name"* ]]; then + echo "Found conda environment '$env_name'" + else + echo "Error: the conda environment '$env_name' does not exist." + echo "exiting..." + exit + fi +} + +check_conda_installation + +source activate base +conda activate $env_name + +PREFIX="${ODIR}/dispdir_bdt" +LOGFILE="${PREFIX}.log" +rm -f "$LOGFILE" + +if [[ $DSET == *"LaPalma"* ]]; then + site="CTAO-NORTH" +else + site="CTAO-SOUTH" +fi + +eventdisplay-ml-train-xgb-stereo \ + --input_file_list "$LLIST" \ + --model_prefix "${PREFIX}" \ + --max_cores $MAXCORES \ + --observatory $site \ + --train_test_fraction $P --max_events $N >| "${LOGFILE}" 2>&1 + +python --version >> "${LOGFILE}" +conda list -n $env_name >> "${LOGFILE}" + +conda deactivate diff --git a/analysis/CTA.XGBSTEREO.sub_analyse.sh b/analysis/CTA.XGBSTEREO.sub_analyse.sh new file mode 100755 index 0000000..80d5134 --- /dev/null +++ b/analysis/CTA.XGBSTEREO.sub_analyse.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# +# XGB stereo analysis training +# +# +# + +SUBC="condor" +h_cpu="0:29:00" +h_vmem="4000M" +tmpdir_size="1G" + +if [ $# -lt 4 ] +then + echo + echo "/CTA.XGBSTEREO.sub_analyse [qsub options] [direction (e.g. _180deg)] [job_dir]" + echo "" + echo " text file with list of subarray IDs" + echo + echo " e.g. cta-ultra3, ISDC3700, ... " + echo + echo " e.g. for north: \"_180deg\", for south: \"_0deg\", for all directions: no option" + echo + exit +fi + +####################################### +# read values from parameter file +ANAPAR=$3 +if [ ! -e "$ANAPAR" ] +then + echo "error: analysis parameter file not found: $ANAPAR" + exit +fi +echo "reading analysis parameter from $ANAPAR" +NIMAGESMIN=$(grep NIMAGESMIN "$ANAPAR" | awk {'print $2'}) +NCUTLST=$(grep NLST "$ANAPAR" | awk {'print $2'}) +NCUTMST=$(grep NMST "$ANAPAR" | awk {'print $2'}) +NCUTSST=$(grep NSST "$ANAPAR" | awk {'print $2'}) +NCUTMSCT=$(grep NSCMST "$ANAPAR" | awk {'print $2'}) +ANADIR=$(grep MSCWSUBDIRECTORY "$ANAPAR" | awk {'print $2'}) +RECID=$(grep RECID "$ANAPAR" | awk {'print $2'}) +DSET=$2 +echo "Analysis parameter: " "$NIMAGESMIN" "$ANADIR" "$DSET" +VARRAY=$(awk '{printf "%s ",$0} END {print ""}' "$1") + +MCAZ=${5:-$MCAZ} +# batch farm submission options +QSUBOPT=${4:-$QSUBOPT} +QSUBOPT=${QSUBOPT//_X_/ } +QSUBOPT=${QSUBOPT//_M_/-} +QSUBOPT=${QSUBOPT//\"/} +# log dir +DATE=$(date +"%y%m%d") +LDIR=$CTA_USER_LOG_DIR/$DATE/XGBSTEREOTRAINING/ +LDIR=${6:-$LDIR} +# Evaluation mscw files only +ANADIR=${ANADIR}.EFFAREA.MCAZ${MCAZ} + +###################################### +# software paths +source ../setSoftwarePaths.sh "$DSET" +# checking the path for binary +if [ -z "$EVNDISPSYS" ] +then + echo "no EVNDISPSYS env variable defined" + exit +fi + +###################################### +# log files +QLOG=$LDIR +mkdir -p "$LDIR" +echo "Log directory: " "$LDIR" + +########################### +# particle types +VPART=( "gamma_onSource" "gamma_cone" "electron" "proton" ) +NPART=${#VPART[@]} + +###################################### +# script name template +FSCRIPT="CTA.XGBSTEREO.qsub_analyse" + +############################################################### +# loop over all arrays +for ARRAY in $VARRAY +do + XGBDIR=$CTA_USER_DATA_DIR/analysis/AnalysisData/$DSET/$ARRAY/XGB_stereo + + for ((m = 0; m < $NPART; m++ )) + do + PART=${VPART[$m]} + echo "STARTING $DSET PARTICLE $PART ARRAY $ARRAY MCAZ $MCAZ" + + FILES=$(ls -1 $CTA_USER_DATA_DIR/analysis/AnalysisData/$DSET/$ARRAY/$ANADIR/$PART."$ARRAY"_ID"$RECID$MCAZ"*.mscw.root) + + i=0 + for FILE in $FILES + do + ((i++)) + + FNAM=$LDIR/$FSCRIPT.$DSET.$ARRAY.AZ${MCAZ}.ID${RECID}.${PART}-${i} + sed -e "s|FFILE|$FILE|" \ + -e "s|DATASET|$DSET|" \ + -e "s|DIRXGB|$XGBDIR|" $FSCRIPT.sh > $FNAM.sh + chmod u+x $FNAM.sh + echo "SCRIPT $FNAM.sh" + + # submit job to queue + if [[ $SUBC == *qsub* ]]; then + qsub $QSUBOPT -V -l h_cpu=${h_cpu} -l h_rss=${h_vmem} -l tmpdir_size=${tmpdir_size} -o $QLOG -e $QLOG "$FNAM.sh" + elif [[ $SUBC == *condor* ]]; then + ./condorSubmission.sh ${FNAM}.sh $h_vmem $tmpdir_size + fi + done + done +done diff --git a/analysis/CTA.XGBSTEREO.sub_train.sh b/analysis/CTA.XGBSTEREO.sub_train.sh new file mode 100755 index 0000000..1bc45ce --- /dev/null +++ b/analysis/CTA.XGBSTEREO.sub_train.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# +# XGB stereo analysis training +# +# +# + +SUBC="condor" +h_cpu="8:29:00" +h_vmem="24000M" +tmpdir_size="1G" + +if [ $# -lt 4 ] +then + echo + echo "CTA.XGBSTEREO.sub_train.sh [qsub options] [direction (e.g. _180deg)] [job_dir]" + echo "" + echo " text file with list of subarray IDs" + echo + echo " e.g. cta-ultra3, ISDC3700, ... " + echo + echo " e.g. for north: \"_180deg\", for south: \"_0deg\", for all directions: no option" + echo + exit +fi + +####################################### +# read values from parameter file +ANAPAR=$3 +if [ ! -e "$ANAPAR" ] +then + echo "error: analysis parameter file not found: $ANAPAR" + exit +fi +echo "reading analysis parameter from $ANAPAR" +NIMAGESMIN=$(grep NIMAGESMIN "$ANAPAR" | awk {'print $2'}) +NCUTLST=$(grep NLST "$ANAPAR" | awk {'print $2'}) +NCUTMST=$(grep NMST "$ANAPAR" | awk {'print $2'}) +NCUTSST=$(grep NSST "$ANAPAR" | awk {'print $2'}) +NCUTMSCT=$(grep NSCMST "$ANAPAR" | awk {'print $2'}) +ANADIR=$(grep MSCWSUBDIRECTORY "$ANAPAR" | awk {'print $2'}) +RECID=$(grep RECID "$ANAPAR" | awk {'print $2'}) +DSET=$2 +echo "Analysis parameter: " "$NIMAGESMIN" "$ANADIR" "$DSET" +VARRAY=$(awk '{printf "%s ",$0} END {print ""}' "$1") + +# batch farm submission options +QSUBOPT=${5:-$QSUBOPT} +QSUBOPT=${QSUBOPT//_X_/ } +QSUBOPT=${QSUBOPT//_M_/-} +QSUBOPT=${QSUBOPT//\"/} +# log dir +DATE=$(date +"%y%m%d") +LDIR=$CTA_USER_LOG_DIR/$DATE/XGBSTEREOTRAINING/ +LDIR=${6:-$LDIR} + +###################################### +# software paths +source ../setSoftwarePaths.sh "$DSET" +# checking the path for binary +if [ -z "$EVNDISPSYS" ] +then + echo "no EVNDISPSYS env variable defined" + exit +fi + +###################################### +# log files +QLOG=$LDIR +mkdir -p "$LDIR" +echo "Log directory: " "$LDIR" + +###################################### +# script name template +FSCRIPT="CTA.XGBSTEREO.qsub_train" + +############################################################### +# loop over all arrays +for ARRAY in $VARRAY +do + echo "STARTING $DSET ARRAY $ARRAY" + + ODIR=$CTA_USER_DATA_DIR/analysis/AnalysisData/$DSET/$ARRAY/XGB_stereo + mkdir -p "$ODIR" + # training list identical to TMVA gamma/hadron signal training + SIGNALTRAINLIST=${ODIR}/training_files.list + rm -f "${SIGNALTRAINLIST}" + ls -1 $CTA_USER_DATA_DIR/analysis/AnalysisData/$DSET/$ARRAY/$ANADIR/gamma_cone."$ARRAY"_ID"$RECID"*.mscw.root | sort -g | awk 'NR % 3 != 0' > "${SIGNALTRAINLIST}" + + FNAM=$LDIR/$FSCRIPT.$DSET.$ARRAY.ID${RECID} + sed -e "s|MSCWLIST|$SIGNALTRAINLIST|" \ + -e "s|DATASET|$DSET|" \ + -e "s|OUTPUTDIR|$ODIR|" $FSCRIPT.sh > $FNAM.sh + chmod u+x $FNAM.sh + echo "SCRIPT $FNAM.sh" + +# submit job to queue + if [[ $SUBC == *qsub* ]]; then + qsub $QSUBOPT -V -l h_cpu=${h_cpu} -l h_rss=${h_vmem} -l tmpdir_size=${tmpdir_size} -o $QLOG -e $QLOG "$FNAM.sh" + elif [[ $SUBC == *condor* ]]; then + ./condorSubmission.sh ${FNAM}.sh $h_vmem $tmpdir_size + fi +done diff --git a/analysis/CTA.prepareAnalysis_no_sub.sh b/analysis/CTA.prepareAnalysis_no_sub.sh new file mode 100755 index 0000000..ecd2955 --- /dev/null +++ b/analysis/CTA.prepareAnalysis_no_sub.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# +# script to prepare event files for analysis (average AZ only) +# +# +# + +if [ $# -lt 3 ] +then + echo + echo "CTA.prepareAnalysis_no_sub.sh " + echo "" + echo " text file with list of subarray IDs" + echo + echo " e.g. cta-ultra3, ISDC3700, ... " + echo + exit 1 +fi + +####################################### +# read values from parameter file +ANAPAR=$3 +if [ ! -e "$ANAPAR" ] +then + echo "error: analysis parameter file not found: $ANAPAR" + exit +fi +echo "reading analysis parameter from $ANAPAR" +ANADIR=$(grep MSCWSUBDIRECTORY "$ANAPAR" | awk {'print $2'}) +DSET=$2 +echo "Analysis parameter: " "$ANADIR" "$DSET" +VARRAY=$(awk '{printf "%s ",$0} END {print ""}' "$1") + +###################################### +# software paths +source ../setSoftwarePaths.sh "$DSET" +# checking the path for binary +if [ -z "$EVNDISPSYS" ] +then + echo "no EVNDISPSYS env variable defined" + exit +fi + +############################################################### +# loop over all arrays +for ARRAY in $VARRAY +do + echo "STARTING $DSET ARRAY $ARRAY" + ########################################################## + # set links for events used in effective area calculation + # (separate training and events used for analysis) + ANAEFF_noAZ="$CTA_USER_DATA_DIR/analysis/AnalysisData/$DSET/$ARRAY/${ANADIR}.EFFAREA.MCAZ" + rm -rf "$ANAEFF_noAZ" + mkdir -p "$ANAEFF_noAZ" + for MCAZ in "_0deg" "_180deg" + do + ANAEFF="$CTA_USER_DATA_DIR/analysis/AnalysisData/$DSET/$ARRAY/${ANADIR}.EFFAREA.MCAZ${MCAZ}" + echo "ORG $ANAEFF" + echo "TARG $ANAEFF_noAZ" + find "$ANAEFF" -name "*.root" -exec ln -s -t "$ANAEFF_noAZ" {} + + done +done diff --git a/setSoftwarePaths.sh b/setSoftwarePaths.sh index 0c0fd28..c12e6f7 100755 --- a/setSoftwarePaths.sh +++ b/setSoftwarePaths.sh @@ -65,8 +65,8 @@ then fi export ROOT_INCLUDE_PATH=${EVNDISPSYS}/inc -export CTA_EVNDISP_AUX_DIR=${WORKDIR}/Eventdisplay_AnalysisFiles_CTA/ +export CTA_EVNDISP_AUX_DIR=${WORKDIR}/Eventdisplay_AnalysisFiles_CTA export OBS_EVNDISP_AUX_DIR=${CTA_EVNDISP_AUX_DIR} -export CTA_USER_LOG_DIR="${WORKDIR}/LOGS/" +export CTA_USER_LOG_DIR="${WORKDIR}/LOGS" export SOFASYS=${EVNDISPSYS}/sofa