From 85f224f4e59137dd11fb5bc9ff62ab064906adbb Mon Sep 17 00:00:00 2001 From: Anand Joseph Date: Tue, 23 Sep 2025 15:36:29 +0530 Subject: [PATCH 1/9] Fix Jenkinsfile for CI Signed-off-by: Anand Joseph --- Jenkinsfile | 113 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 66 insertions(+), 47 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 51ce37a10..61b07eca1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,8 +1,8 @@ pipeline { agent { docker { - image 'tnitn_ci:py310' - args '--user 0:128 -v /home/jenkinsci:/home/jenkinsci -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""' + image 'tnitn_ci_3:py312' + args '-v /mnt/jenkins/jenkinsci:/home/jenkins -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""' } } options { @@ -11,33 +11,27 @@ pipeline { } environment { - AR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-24-24-0' - DE_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-23-24-0' - EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/09-04-24-0' - ES_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/09-25-24-0' - ES_EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-30-24-0' - FR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-07-25-0' - HU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/07-16-24-0' - PT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' - RU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' - VI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' - SV_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' - ZH_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/11-13-24-0' - IT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-22-24-0' - HY_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-0' - MR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-1' - JA_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-17-24-1' - HI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-22-25-0' - DEFAULT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' + AR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-24-24-0' + DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-23-24-0' + EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-04-24-0' + ES_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-25-24-0' + ES_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-30-24-0' + FR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-07-25-0' + HU_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-16-24-0' + PT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' + RU_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' + VI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' + SV_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' + ZH_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/11-13-24-0' + IT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-22-24-0' + HY_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-0' + MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' + JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-22-25-0' + DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { - stage('Add git safe directory'){ - steps{ - sh 'git config --global --add safe.directory /var/lib/jenkins/workspace/NTP_$GIT_BRANCH' - sh 'git config --global --add safe.directory /home/jenkinsci/workspace/NTP_$GIT_BRANCH' - } - } stage('PyTorch version') { steps { @@ -65,7 +59,10 @@ pipeline { when { anyOf { branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' + } } failFast true @@ -97,6 +94,8 @@ pipeline { when { anyOf { branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } @@ -120,6 +119,8 @@ pipeline { when { anyOf { branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } @@ -156,7 +157,9 @@ pipeline { stage('L0: Create AR TN/ITN Grammars') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } @@ -179,7 +182,9 @@ pipeline { stage('L0: Create FR TN/ITN & VI ITN & HU TN & IT TN') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } @@ -216,7 +221,9 @@ pipeline { stage('L0: Create RU TN/ITN Grammars & SV & PT') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } @@ -258,7 +265,9 @@ pipeline { stage('L0: Create HY TN/ITN Grammars & MR') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } @@ -284,7 +293,9 @@ pipeline { stage('L0: Create ZH TN/ITN Grammar') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } @@ -305,7 +316,9 @@ pipeline { stage('L0: Create JA ITN Grammars') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } @@ -325,7 +338,9 @@ pipeline { stage('L1: TN/ITN Tests CPU') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } @@ -412,7 +427,9 @@ pipeline { stage('L2: Sparrowhawk Tests') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } @@ -445,7 +462,9 @@ pipeline { stage('L2: NeMo text processing') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } @@ -453,23 +472,23 @@ pipeline { parallel { stage('L2: Eng TN') { steps { - sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && \ + sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkins/TestData/text_norm/output_${TIME} && \ cd tools/text_processing_deployment && python pynini_export.py --output=$NORM_OUTPUT_DIR --grammars=tn_grammars --cache_dir ${EN_TN_CACHE} --language=en && ls -R $NORM_OUTPUT_DIR && echo ".far files created "|| exit 1' - sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \ - cd nemo_text_processing/text_normalization/ && python normalize.py --input_file=/home/jenkinsci/TestData/text_norm/ci/test.txt --input_case="lower_cased" --language=en --output_file=$NORM_OUTPUT_DIR/test.pynini.txt --verbose && \ + sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkins/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \ + cd nemo_text_processing/text_normalization/ && python normalize.py --input_file=/home/jenkins/TestData/text_norm/ci/test.txt --input_case="lower_cased" --language=en --output_file=$NORM_OUTPUT_DIR/test.pynini.txt --verbose && \ cat $NORM_OUTPUT_DIR/test.pynini.txt && \ - cmp --silent $NORM_OUTPUT_DIR/test.pynini.txt /home/jenkinsci/TestData/text_norm/ci/test_goal_py.txt || exit 1 && \ + cmp --silent $NORM_OUTPUT_DIR/test.pynini.txt /home/jenkins/TestData/text_norm/ci/test_goal_py.txt || exit 1 && \ rm -rf $NORM_OUTPUT_DIR' } } stage('L2: Eng ITN export') { steps { - sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && \ + sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkins/TestData/text_denorm/output_${TIME} && \ cd tools/text_processing_deployment && python pynini_export.py --output=$DENORM_OUTPUT_DIR --grammars=itn_grammars --cache_dir ${EN_TN_CACHE} --language=en && ls -R $DENORM_OUTPUT_DIR && echo ".far files created "|| exit 1' - sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \ - cd nemo_text_processing/inverse_text_normalization/ && python inverse_normalize.py --input_file=/home/jenkinsci/TestData/text_denorm/ci/test.txt --language=en --output_file=$DENORM_OUTPUT_DIR/test.pynini.txt --verbose && \ - cmp --silent $DENORM_OUTPUT_DIR/test.pynini.txt /home/jenkinsci/TestData/text_denorm/ci/test_goal_py.txt || exit 1 && \ + sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkins/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \ + cd nemo_text_processing/inverse_text_normalization/ && python inverse_normalize.py --input_file=/home/jenkins/TestData/text_denorm/ci/test.txt --language=en --output_file=$DENORM_OUTPUT_DIR/test.pynini.txt --verbose && \ + cmp --silent $DENORM_OUTPUT_DIR/test.pynini.txt /home/jenkins/TestData/text_denorm/ci/test_goal_py.txt || exit 1 && \ rm -rf $DENORM_OUTPUT_DIR' } } @@ -477,18 +496,18 @@ pipeline { stage('L2: Eng alignment TN') { steps { - sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \ + sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkins/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \ cd nemo_text_processing/fst_alignment && python alignment.py --text="2615 Forest Av, 90501 CA, Santa Clara. 10kg, 12/16/2018" --grammar=tn --rule=tokenize_and_classify --fst=${EN_TN_CACHE}/en_tn_True_deterministic_cased__tokenize.far 2>&1 | tee $NORM_OUTPUT_DIR/pred.txt && \ - cmp --silent $NORM_OUTPUT_DIR/pred.txt /home/jenkinsci/TestData/text_norm/ci/alignment_gold.txt || exit 1 && \ + cmp --silent $NORM_OUTPUT_DIR/pred.txt /home/jenkins/TestData/text_norm/ci/alignment_gold.txt || exit 1 && \ rm -rf $NORM_OUTPUT_DIR' } } stage('L2: Eng alignment ITN') { steps { - sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \ + sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkins/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \ cd nemo_text_processing/fst_alignment && python alignment.py --text="one million twenty three thousand two hundred eleven ten kilograms one hundred twenty three dollars and twenty five cents" --grammar=itn --rule=tokenize_and_classify --fst=${EN_TN_CACHE}/en_itn_lower_cased.far 2>&1 | tee $DENORM_OUTPUT_DIR/pred.txt && \ - cmp --silent $DENORM_OUTPUT_DIR/pred.txt /home/jenkinsci/TestData/text_denorm/ci/alignment_gold.txt || exit 1 && \ + cmp --silent $DENORM_OUTPUT_DIR/pred.txt /home/jenkins/TestData/text_denorm/ci/alignment_gold.txt || exit 1 && \ rm -rf $DENORM_OUTPUT_DIR' } } From 182657f553e2fd4e9ba271636cee92ef4b200895 Mon Sep 17 00:00:00 2001 From: Anand Joseph Date: Tue, 23 Sep 2025 15:42:34 +0530 Subject: [PATCH 2/9] Fix requirements for test Signed-off-by: Anand Joseph --- requirements/requirements_test.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/requirements_test.txt b/requirements/requirements_test.txt index a3e90e5dc..aacfde319 100644 --- a/requirements/requirements_test.txt +++ b/requirements/requirements_test.txt @@ -1,6 +1,6 @@ -black==19.10b0 -click==8.0.2 -isort[requirements]>5.1.0,<6.0.0 +black==25.1.0 +click>=8.0.2 +isort[requirements]>5.1.0,<=6.0.1 parameterized pynini==2.1.6.post1 pytest From b720871777fed3f0e24f1f8895a416f7f98c57b3 Mon Sep 17 00:00:00 2001 From: Anand Joseph Date: Tue, 23 Sep 2025 19:57:42 +0530 Subject: [PATCH 3/9] Update paths and docker Signed-off-by: Anand Joseph --- Jenkinsfile | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 61b07eca1..619582ab5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ pipeline { agent { docker { - image 'tnitn_ci_3:py312' + image 'tnitn_ci_4:py312' args '-v /mnt/jenkins/jenkinsci:/home/jenkins -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""' } } @@ -40,14 +40,6 @@ pipeline { } } - stage('Install test requirements') { - steps { - sh 'apt-get update && apt-get install -y bc' - } - } - - - stage('NeMo Installation') { steps { sh './reinstall.sh release' From 98db87069129d4b77e88c42920c6793faa8693b4 Mon Sep 17 00:00:00 2001 From: Anand Joseph Date: Tue, 23 Sep 2025 20:06:29 +0530 Subject: [PATCH 4/9] Fix docker name Signed-off-by: Anand Joseph --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 619582ab5..6a2b451a4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ pipeline { agent { docker { - image 'tnitn_ci_4:py312' + image 'tnitn_ci_py312:25.03' args '-v /mnt/jenkins/jenkinsci:/home/jenkins -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""' } } From 4c900957b1155b03e9603824825b18c7dd654298 Mon Sep 17 00:00:00 2001 From: Anand Joseph Date: Tue, 23 Sep 2025 20:09:29 +0530 Subject: [PATCH 5/9] Fix click version Signed-off-by: Anand Joseph --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cbc636f1a..a2886d56e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,4 +50,4 @@ repos: - id: black name: Format code args: [--skip-string-normalization, --line-length=119] - additional_dependencies: ['click==8.0.2'] + additional_dependencies: ['click>=8.0.2'] From 368a766d1603d51de3aea93fef1a5b00b059d378 Mon Sep 17 00:00:00 2001 From: Anand Joseph Date: Tue, 23 Sep 2025 20:54:15 +0530 Subject: [PATCH 6/9] Change path of grammars for sparrowhawk tests Signed-off-by: Anand Joseph --- Jenkinsfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6a2b451a4..7ca1f971c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -426,25 +426,25 @@ pipeline { } } failFast true - parallel { + stages { stage('L2: EN ITN Run Sparrowhawk test - Lower Cased Input') { steps { - sh 'CUDA_VISIBLE_DEVICES="" cp -r /workspace/sparrowhawk/documentation/grammars /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_lower_cased && cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_itn_grammars" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN --LANGUAGE="en"' - sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_inverse_text_normalization.sh /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_lower_cased `pwd`' + sh 'CUDA_VISIBLE_DEVICES="" bash sh_test.sh --MODE="test_itn_grammars" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN --LANGUAGE="en"' + sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_inverse_text_normalization.sh /workspace/sparrowhawk/documentation/grammars `pwd`' } } stage('L2: EN ITN Run Sparrowhawk test - Cased Input') { steps { - sh 'CUDA_VISIBLE_DEVICES="" cp -r /workspace/sparrowhawk/documentation/grammars /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_cased && cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_itn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN_cased --LANGUAGE="en"' - sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_inverse_text_normalization_cased.sh /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_cased `pwd`' + sh 'CUDA_VISIBLE_DEVICES="" bash sh_test.sh --MODE="test_itn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN_cased --LANGUAGE="en"' + sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_inverse_text_normalization_cased.sh /workspace/sparrowhawk/documentation/grammars `pwd`' } } stage('L2: EN TN Run Sparrowhawk test') { steps { - sh 'CUDA_VISIBLE_DEVICES="" cp -r /workspace/sparrowhawk/documentation/grammars /workspace/sparrowhawk/documentation/grammars_en_tn_grammars_cased && cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_tn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_TN --GRAMMARS="tn_grammars" --LANGUAGE="en" ' - sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_normalization.sh /workspace/sparrowhawk/documentation/grammars_en_tn_grammars_cased `pwd`' + sh 'CUDA_VISIBLE_DEVICES="" bash sh_test.sh --MODE="test_tn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_TN --GRAMMARS="tn_grammars" --LANGUAGE="en" ' + sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_normalization.sh /workspace/sparrowhawk/documentation/grammars `pwd`' } } From fe63bae1bf2d80a519b4aaf5efc09146f00861a9 Mon Sep 17 00:00:00 2001 From: Anand Joseph Date: Tue, 23 Sep 2025 21:14:01 +0530 Subject: [PATCH 7/9] Update paths in sh_test.sh Signed-off-by: Anand Joseph --- tools/text_processing_deployment/sh_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/text_processing_deployment/sh_test.sh b/tools/text_processing_deployment/sh_test.sh index 3e31de37c..88fb207e5 100644 --- a/tools/text_processing_deployment/sh_test.sh +++ b/tools/text_processing_deployment/sh_test.sh @@ -62,5 +62,5 @@ VERBALIZE_FAR=${CACHE_DIR}_${GRAMMARS}_${INPUT_CASE}/verbalize/verbalize.far CONFIG=${LANGUAGE}_${GRAMMARS}_${INPUT_CASE} -cp $CLASSIFY_FAR /workspace/sparrowhawk/documentation/grammars_${CONFIG}/en_toy/classify/ -cp $VERBALIZE_FAR /workspace/sparrowhawk/documentation/grammars_${CONFIG}/en_toy/verbalize/ \ No newline at end of file +cp $CLASSIFY_FAR /workspace/sparrowhawk/documentation/grammars/en_toy/classify/ +cp $VERBALIZE_FAR /workspace/sparrowhawk/documentation/grammars/en_toy/verbalize/ \ No newline at end of file From 4a7481dc119fe7d33ee99846dd956b7ab0eeab65 Mon Sep 17 00:00:00 2001 From: Anand Joseph Date: Tue, 23 Sep 2025 21:15:15 +0530 Subject: [PATCH 8/9] Update paths Signed-off-by: Anand Joseph --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7ca1f971c..3c5aa1de3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -416,7 +416,7 @@ pipeline { } } - stage('L2: Sparrowhawk Tests') { + stage('L2: EN Sparrowhawk Tests') { when { anyOf { branch 'main' @@ -429,21 +429,21 @@ pipeline { stages { stage('L2: EN ITN Run Sparrowhawk test - Lower Cased Input') { steps { - sh 'CUDA_VISIBLE_DEVICES="" bash sh_test.sh --MODE="test_itn_grammars" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN --LANGUAGE="en"' + sh 'CUDA_VISIBLE_DEVICES="" cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_itn_grammars" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN --LANGUAGE="en"' sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_inverse_text_normalization.sh /workspace/sparrowhawk/documentation/grammars `pwd`' } } stage('L2: EN ITN Run Sparrowhawk test - Cased Input') { steps { - sh 'CUDA_VISIBLE_DEVICES="" bash sh_test.sh --MODE="test_itn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN_cased --LANGUAGE="en"' + sh 'CUDA_VISIBLE_DEVICES="" cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_itn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN_cased --LANGUAGE="en"' sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_inverse_text_normalization_cased.sh /workspace/sparrowhawk/documentation/grammars `pwd`' } } stage('L2: EN TN Run Sparrowhawk test') { steps { - sh 'CUDA_VISIBLE_DEVICES="" bash sh_test.sh --MODE="test_tn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_TN --GRAMMARS="tn_grammars" --LANGUAGE="en" ' + sh 'CUDA_VISIBLE_DEVICES="" cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_tn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_TN --GRAMMARS="tn_grammars" --LANGUAGE="en" ' sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_normalization.sh /workspace/sparrowhawk/documentation/grammars `pwd`' } } From 73adea8ebd5e48672858e66543b2001e99315563 Mon Sep 17 00:00:00 2001 From: Anand Joseph Date: Tue, 23 Sep 2025 22:56:25 +0530 Subject: [PATCH 9/9] Revert paths Signed-off-by: Anand Joseph --- Jenkinsfile | 18 +++++++++--------- tools/text_processing_deployment/sh_test.sh | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3c5aa1de3..3781a171d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ pipeline { agent { docker { - image 'tnitn_ci_py312:25.03' + image 'tnitn_ci_py310:24.07' args '-v /mnt/jenkins/jenkinsci:/home/jenkins -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""' } } @@ -426,31 +426,31 @@ pipeline { } } failFast true - stages { + parallel { stage('L2: EN ITN Run Sparrowhawk test - Lower Cased Input') { steps { - sh 'CUDA_VISIBLE_DEVICES="" cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_itn_grammars" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN --LANGUAGE="en"' - sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_inverse_text_normalization.sh /workspace/sparrowhawk/documentation/grammars `pwd`' + sh 'CUDA_VISIBLE_DEVICES="" cp -r /workspace/sparrowhawk/documentation/grammars /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_lower_cased && cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_itn_grammars" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN --LANGUAGE="en"' + sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_inverse_text_normalization.sh /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_lower_cased `pwd`' } } stage('L2: EN ITN Run Sparrowhawk test - Cased Input') { steps { - sh 'CUDA_VISIBLE_DEVICES="" cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_itn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN_cased --LANGUAGE="en"' - sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_inverse_text_normalization_cased.sh /workspace/sparrowhawk/documentation/grammars `pwd`' + sh 'CUDA_VISIBLE_DEVICES="" cp -r /workspace/sparrowhawk/documentation/grammars /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_cased && cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_itn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN_cased --LANGUAGE="en"' + sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_inverse_text_normalization_cased.sh /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_cased `pwd`' } } stage('L2: EN TN Run Sparrowhawk test') { steps { - sh 'CUDA_VISIBLE_DEVICES="" cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_tn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_TN --GRAMMARS="tn_grammars" --LANGUAGE="en" ' - sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_normalization.sh /workspace/sparrowhawk/documentation/grammars `pwd`' + sh 'CUDA_VISIBLE_DEVICES="" cp -r /workspace/sparrowhawk/documentation/grammars /workspace/sparrowhawk/documentation/grammars_en_tn_grammars_cased && cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_tn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_TN --GRAMMARS="tn_grammars" --LANGUAGE="en" ' + sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_normalization.sh /workspace/sparrowhawk/documentation/grammars_en_tn_grammars_cased `pwd`' } } } } - + stage('L2: NeMo text processing') { when { anyOf { diff --git a/tools/text_processing_deployment/sh_test.sh b/tools/text_processing_deployment/sh_test.sh index 88fb207e5..3e31de37c 100644 --- a/tools/text_processing_deployment/sh_test.sh +++ b/tools/text_processing_deployment/sh_test.sh @@ -62,5 +62,5 @@ VERBALIZE_FAR=${CACHE_DIR}_${GRAMMARS}_${INPUT_CASE}/verbalize/verbalize.far CONFIG=${LANGUAGE}_${GRAMMARS}_${INPUT_CASE} -cp $CLASSIFY_FAR /workspace/sparrowhawk/documentation/grammars/en_toy/classify/ -cp $VERBALIZE_FAR /workspace/sparrowhawk/documentation/grammars/en_toy/verbalize/ \ No newline at end of file +cp $CLASSIFY_FAR /workspace/sparrowhawk/documentation/grammars_${CONFIG}/en_toy/classify/ +cp $VERBALIZE_FAR /workspace/sparrowhawk/documentation/grammars_${CONFIG}/en_toy/verbalize/ \ No newline at end of file