Xgb

CTA.mainRunScriptsReduced.sh

@@ -24,7 +24,7 @@ if [ $# -lt 2 ]; then
             prod6-South-20deg
 
     run modes:
-        MAKETABLES PREPAREFILELISTS DISPBDT ANATABLES PREPARETMVA TRAIN ANGRES QC CUTS PHYS CLEANUP
+        MAKETABLES PREPAREFILELISTS DISPBDT ANATABLES PREPARETMVA XGBSTEREOTRAIN XGBSTEREOANA TRAIN ANGRES QC CUTS PHYS CLEANUP
 
     optional run modes: TRAIN_RECO_QUALITY TRAIN_RECO_METHOD
 
@@ -49,7 +49,7 @@ if [[ ${RUN} != "CLEANUP" ]] && [[ ${RUN} != "PREPAREFILELISTS" ]]; then
     mkdir -p ${RUNSCRIPTDIR}
 fi
 
-if [[ ${RUN} == "MAKETABLES" ]] || [[ ${RUN} == "DISPBDT" ]] || [[ ${RUN} == "ANATABLES" ]] || [[ ${RUN} == "PREPARETMVA" ]]; then
+if [[ ${RUN} == "MAKETABLES" ]] || [[ ${RUN} == "DISPBDT" ]] || [[ ${RUN} == "ANATABLES" ]] || [[ ${RUN} == "PREPARETMVA" ]] || [[ ${RUN} == "XGBSTEREOTRAIN" ]] || [[ ${RUN} == "XGBSTEREOANA" ]]; then
    ./CTA.runAnalysis.sh ${P2} ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR}
    if [[ $SITE == "South" ]] || [[ $P2 == *"prod6"* ]]; then
        ./CTA.runAnalysis.sh ${P2}-sub ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR}

CTA.runAnalysis.sh

@@ -28,7 +28,7 @@ then
         prod6-North-20deg prod6-North-40deg prod6-North-52deg prod6-North-60deg
         prod6-South-20deg
 
-    possible run modes are EVNDISP MAKETABLES PREPAREFILELISTS DISPBDT ANATABLES PREPARETMVA TRAIN ANGRES QC CUTS PHYS CLEANUP
+    possible run modes are EVNDISP MAKETABLES PREPAREFILELISTS DISPBDT ANATABLES XGBSTEREOTRAIN XGBSTEREOANA PREPARETMVA TRAIN ANGRES QC CUTS PHYS CLEANUP
 
     optional run modes: TRAIN_RECO_QUALITY TRAIN_RECO_METHOD
 
@@ -51,7 +51,7 @@ RUN="$2"
 [[ "$5" ]] && MST=$5 || MST="2"
 [[ "$6" ]] && SST=$6 || SST="2"
 [[ "$7" ]] && SCMST=$7 || SCMST="2"
-[[ "$8" ]] && PDIR=${8} || PDIR="${CTA_USER_LOG_DIR%/}/"
+[[ "$8" ]] && PDIR=${8} || PDIR="${CTA_USER_LOG_DIR%/}"
 echo "Telescope multiplicities: LST ${LST} MST ${MST} SST ${SST} SCMST ${SCMST}"
 
 #####################################
@@ -334,17 +334,18 @@ then
        EDM="-sq50-LL-DL2plus"
    fi
    ARRAY=( "subArray.prod6.${NS}ML${SCT}.list" )
-   ARRAY=( "subArray.prod6.${NS}Alphab${SCT}.list" )
+   ARRAY=( "subArray.prod6.${NS}Alpha${SCT}.list" )
    if [[ $P2 == *"sub"* ]]; then
        ARRAY=( "subArray.prod6.${NS}ML-sub.list" )
-       ARRAY=( "subArray.prod6.${NS}Alphab-sub.list" )
+       ARRAY=( "subArray.prod6.${NS}Alpha-sub.list" )
    fi
    if [[ $P2 == *"Hyper"* ]] || [[ $P2 == *"hyper"* ]]; then
        ARRAY=( "subArray.prod6.NorthHyper.list" )
    fi
    ARRAYDIR="prod6"
    TDATE="g20251021"
    ANADATE="${TDATE}"
+   ANADATE="g20260127"
    TMVADATE="${ANADATE}"
    EFFDATE="${ANADATE}"
    PHYSDATE="${EFFDATE}"
@@ -424,7 +425,7 @@ then
 fi
 # remove from PHYS directory any unreasonable files (e.g. LST4 requirement for 2 LST array)
 if [[ $RUN == "CLEANUP" ]]; then
-    PHYSDIR="${CTA_USER_DATA_DIR}/analysis/AnalysisData/${SITE}${EDM}/Phys-${PHYSDATE}"
+    PHYSDIR="${CTA_USER_DATA_DIR%/}/analysis/AnalysisData/${SITE}${EDM}/Phys-${PHYSDATE}"
     ./utilities/removeUnreasonablePhysFiles.sh ${PHYSDIR}
     exit
 fi
@@ -457,7 +458,7 @@ then
     BDTDIR="BDTdisp."
     RUNPAR="${CTA_EVNDISP_AUX_DIR}/ParameterFiles/TMVA.BDTDisp.runparameter"
     QCPAR="${CTA_EVNDISP_AUX_DIR}/ParameterFiles/TMVA.BDTDispQualityCuts.runparameter"
-    DDIR="${CTA_USER_DATA_DIR}/analysis/AnalysisData/${SITE}${EDM}/"
+    DDIR="${CTA_USER_DATA_DIR%/}/analysis/AnalysisData/${SITE}${EDM}/"
     for A in $NXARRAY
     do
         cd ./analysis/
@@ -567,7 +568,7 @@ do
                   touch "$PARA"
                   echo "WRITING PARAMETERFILE $PARA"
                   EFFDIR=EffectiveArea-"$OOTIME"-ID$ID$AZ-$ETYPF-$EFFDATE-$EFFVERSION
-                  EFFFULLDIR="${CTA_USER_DATA_DIR}/analysis/AnalysisData/${SITE}${EDM}/EffectiveAreas/${EFFDIR}/"
+                  EFFFULLDIR="${CTA_USER_DATA_DIR%/}/analysis/AnalysisData/${SITE}${EDM}/EffectiveAreas/${EFFDIR}/"
                   echo "MSCWSUBDIRECTORY ${MSCWSUBDIRECTORY}" >> "$PARA"
                   echo "TMVASUBDIR BDT-${TMVAVERSION}-ID$ID$AZ-$TMVATYPF-$TMVADATE" >> "$PARA"
                   echo "TMVA_RECO_METHOD BDT-RECO-METHOD-${TMVAVERSION}-ID$ID$AZ-$TMVATYPF-$TMVADATE" >> "$PARA"
@@ -604,8 +605,33 @@ do
                   fi
                   cd ./analysis/
 ##########################################
+# XGB stereo analysis training and analysis
+                  if [[ $RUN == "XGBSTEREOTRAIN" ]]
+                  then
+                     # Train XGB independently of AZ
+                     if [ ${o} -eq 0 ] && [[ -z ${AZ} ]]
+                     then
+                         ./CTA.XGBSTEREO.sub_train.sh \
+                         "$NFILARRAY" \
+                         ${SITE}${EDM} \
+                         "$PARA" \
+                         $QSUBOPT
+                  fi
+                  elif [[ $RUN == "XGBSTEREOANA" ]]
+                  then
+                     if [ ${o} -eq 0 ] && [[ ! -z ${AZ} ]]
+                     then
+                         ./CTA.XGBSTEREO.sub_analyse.sh \
+                         "$NFILARRAY" \
+                         ${SITE}${EDM} \
+                         "$PARA" \
+                         $QSUBOPT \
+                         $AZ \
+                         ${PDIR}/${RUN}
+                  fi
+##########################################
 # prepare train BDTs
-                  if [[ $RUN == "PREPARETMVA" ]]
+                  elif [[ $RUN == "PREPARETMVA" ]]
                   then
                      if [ ${o} -eq 0 ] && [[ ! -z ${AZ} ]]
                      then
@@ -622,7 +648,6 @@ do
 # (note: BDT training does not need to be done for all observing periods)
                   elif [[ $RUN == TRAIN* ]] || [[ $RUN == "TMVA" ]]
                   then
-
                      if [ $RUN == "TRAIN_RECO_METHOD" ]; then
                          TMVA_RUN_MODE="TrainAngularReconstructionMethod"
                      elif [ $RUN == "TRAIN_RECO_QUALITY" ]; then

analysis/CTA.EFFAREA.qsub_analyse_list.sh

@@ -28,6 +28,12 @@ MCAZ="PPPMCAZ"
 PARTID="PARTIDNOTSET"
 ######################################################################
 
+if [[ ! -n "$TMPDIR" ]];
+then
+    TMPDIR="$CTA_USER_DATA_DIR/tmp"
+    mkdir "$TMPDIR"
+fi
+
 # Choose PARTICLE type from job id for SGE qsub system
 re='^[0-9]+$'
 if ! [[ $PARTID =~ $re ]] ; then
@@ -505,6 +511,7 @@ do
          echo "* FILLINGMODE 3" >> $MSCF
       fi
 # fill IRFs only
+      echo "* RECONSTRUCTIONTYPE XGBSTEREO" >> $MSCF
       echo "* ENERGYRECONSTRUCTIONMETHOD 1" >> $MSCF
       echo "* ENERGYAXISBINS 60 -2. 4." >> $MSCF
       echo "* ENERGYAXISBINHISTOS 25 -1.9 3.1" >> $MSCF

analysis/CTA.EFFAREA.sub_analyse_list.sh

@@ -159,7 +159,6 @@ do
 
       chmod u+x $QSHELLDIR/$FNAM.sh
 
-      echo $QSHELLDIR/$FNAM.sh
 
 ###########################################
 # submit the job script
@@ -173,6 +172,7 @@ do
                  sed -e "s|PARTIDNOTSET|$PARTICLEID|" "${QSHELLDIR}/${FNAM}.sh" > "${QSHELLDIR}/${FNAM}-${PARTICLEID}.sh"
                  chmod u+x "${QSHELLDIR}/${FNAM}-${PARTICLEID}.sh"
                  ./condorSubmission.sh "${QSHELLDIR}/${FNAM}-${PARTICLEID}.sh" $h_vmem $tmpdir_size
+                 echo "$QSHELLDIR/${FNAM}-${PARTICLEID}.sh"
              done
              rm -f "$QSHELLDIR/$FNAM.sh"
          fi
@@ -187,6 +187,7 @@ do
                  sed -e "s|PARTIDNOTSET|$PARTICLEID|" "${QSHELLDIR}/${FNAM}.sh" > "${QSHELLDIR}/${FNAM}-${PARTICLEID}.sh"
                  chmod u+x "${QSHELLDIR}/${FNAM}-${PARTICLEID}.sh"
                  ./condorSubmission.sh "${QSHELLDIR}/${FNAM}-${PARTICLEID}.sh" $h_vmem $tmpdir_size
+                 echo "$QSHELLDIR/${FNAM}-${PARTICLEID}.sh"
              done
              rm -f "$QSHELLDIR/$FNAM.sh"
          fi

analysis/CTA.XGBSTEREO.qsub_analyse.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+#
+# train XGB stereo for CTA
+#
+
+MSCW_FILE="FFILE"
+XGBDIR="DIRXGB"
+XGB="xgb_stereo"
+DSET="DATASET"
+env_name="eventdisplay_ml_cta"
+MAXCORES=48
+
+# set environmental variables
+source $EVNDISPSYS/setObservatory.sh CTA
+
+
+check_conda_installation()
+{
+    if command -v conda &> /dev/null; then
+        echo "Found conda installation."
+    else
+        echo "Error: found no conda installation."
+        echo "exiting..."
+        exit
+    fi
+    env_info=$(conda info --envs)
+    if [[ "$env_info" == *"$env_name"* ]]; then
+        echo "Found conda environment '$env_name'"
+    else
+        echo "Error: the conda environment '$env_name' does not exist."
+        echo "exiting..."
+        exit
+    fi
+}
+
+check_conda_installation
+
+source activate base
+conda activate $env_name
+
+PREFIX="${XGBDIR}/dispdir_bdt"
+rm -f "$LOGFILE"
+
+if [[ $DSET == *"LaPalma"* ]]; then
+    site="CTAO-NORTH"
+else
+    site="CTAO-SOUTH"
+fi
+
+OFIL=$(basename $MSCW_FILE .root)
+ODIR=$(dirname $MSCW_FILE)
+OFIL="${ODIR}/${OFIL}.${XGB}"
+LOGFILE="${OFIL}.log"
+
+echo "LOG FILE: $LOGFILE"
+
+eventdisplay-ml-apply-xgb-stereo \
+    --input_file "$MSCW_FILE" \
+    --model_prefix "${PREFIX}" \
+    --output_file "${OFIL}.root" \
+    --max_cores $MAXCORES \
+    --observatory $site  >| "${LOGFILE}" 2>&1
+
+python --version >> "${LOGFILE}"
+conda list -n $env_name >> "${LOGFILE}"
+
+conda deactivate

analysis/CTA.XGBSTEREO.qsub_train.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+# train XGB stereo for CTA
+#
+
+ODIR=OUTPUTDIR
+LLIST=MSCWLIST
+DSET="DATASET"
+env_name="eventdisplay_ml_cta"
+P="0.90"
+N="500000000"
+MAXCORES=48
+
+# set environmental variables
+source $EVNDISPSYS/setObservatory.sh CTA
+
+# output data files are written to this directory
+mkdir -p "${ODIR}"
+echo -e "Output files will be written to:\n ${ODIR}"
+
+check_conda_installation()
+{
+    if command -v conda &> /dev/null; then
+        echo "Found conda installation."
+    else
+        echo "Error: found no conda installation."
+        echo "exiting..."
+        exit
+    fi
+    env_info=$(conda info --envs)
+    if [[ "$env_info" == *"$env_name"* ]]; then
+        echo "Found conda environment '$env_name'"
+    else
+        echo "Error: the conda environment '$env_name' does not exist."
+        echo "exiting..."
+        exit
+    fi
+}
+
+check_conda_installation
+
+source activate base
+conda activate $env_name
+
+PREFIX="${ODIR}/dispdir_bdt"
+LOGFILE="${PREFIX}.log"
+rm -f "$LOGFILE"
+
+if [[ $DSET == *"LaPalma"* ]]; then
+    site="CTAO-NORTH"
+else
+    site="CTAO-SOUTH"
+fi
+
+eventdisplay-ml-train-xgb-stereo \
+    --input_file_list "$LLIST" \
+    --model_prefix "${PREFIX}" \
+    --max_cores $MAXCORES \
+    --observatory $site \
+    --train_test_fraction $P --max_events $N >| "${LOGFILE}" 2>&1
+
+python --version >> "${LOGFILE}"
+conda list -n $env_name >> "${LOGFILE}"
+
+conda deactivate