From 7b5549e9e86551bab41a13a29851f62869a60562 Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Thu, 15 Feb 2024 09:45:43 +0800
Subject: [PATCH 01/22] update

---
 main.py   |  9 ++++++++-
 runpod.sh | 25 ++++++++++++++++++++++++-
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/main.py b/main.py
index 5096915..9805170 100644
--- a/main.py
+++ b/main.py
@@ -14,8 +14,14 @@
 BENCHMARK = os.getenv("BENCHMARK")
 GITHUB_API_TOKEN = os.getenv("GITHUB_API_TOKEN")
 
-
 def main(directory: str, elapsed_time: float) -> None:
+    file_path = f"{directory}/result.log"
+    summary = open(file_path, "r").read()
+    upload_to_github_gist(
+        summary, f"{MODEL.split('/')[-1]}-MedTasks.md", GITHUB_API_TOKEN
+    )
+
+    '''
     # Variables
     tables = []
     averages = []
@@ -68,6 +74,7 @@ def main(directory: str, elapsed_time: float) -> None:
     upload_to_github_gist(
         summary, f"{MODEL.split('/')[-1]}-{BENCHMARK.capitalize()}.md", GITHUB_API_TOKEN
     )
+    '''
 
 
 if __name__ == "__main__":
diff --git a/runpod.sh b/runpod.sh
index a3c57e0..3c7a604 100644
--- a/runpod.sh
+++ b/runpod.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+#main func start here
 
 start=$(date +%s)
 
@@ -29,9 +30,30 @@ if [ "$DEBUG" == "True" ]; then
     echo "Launch LLM AutoEval in debug mode"
 fi
 
+# Run evaluation
+git clone https://github.com/chenhaodev/lm-evaluation-harness
+cd lm-evaluation-harness
+pip install -e .
+
+# Call example: lm_eval --model hf --model_args pretrained=/path-to-model,parallelize=True,load_in_4bit=True --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine --device cuda:0 --batch_size auto --limit 100
+
+lm_eval --model hf \
+    --model_args pretrained=$MODEL,trust_remote_code=$TRUST_REMOTE_CODE,parallelize=True \
+    --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine \
+    --device cuda:$cuda_devices \
+    --batch_size auto \
+    --limit 100 \
+    --output_path ./result.log
+
+end=$(date +%s)
+echo "Elapsed Time: $(($end-$start)) seconds" >> ./result.log
+
+python ../llm-autoeval/main.py . $(($end-$start))
+
+'''
 # Run evaluation
 if [ "$BENCHMARK" == "nous" ]; then
-    git clone -b add-agieval https://github.com/dmahan93/lm-evaluation-harness
+    git clone https://github.com/chenhaodev/lm-evaluation-harness
     cd lm-evaluation-harness
     pip install -e .
 
@@ -138,6 +160,7 @@ elif [ "$BENCHMARK" == "openllm" ]; then
 else
     echo "Error: Invalid BENCHMARK value. Please set BENCHMARK to 'nous' or 'openllm'."
 fi
+'''
 
 if [ "$DEBUG" == "False" ]; then
     runpodctl remove pod $RUNPOD_POD_ID

From 560871459e3772a00ec342407fc45e2203cb5627 Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Thu, 15 Feb 2024 10:33:59 +0800
Subject: [PATCH 02/22] update to support HF token

---
 create_pods.sh | 44 ++++++++++++++++++++++++++++++++++++++++++++
 runpod.sh      | 13 +++++--------
 2 files changed, 49 insertions(+), 8 deletions(-)
 create mode 100644 create_pods.sh

diff --git a/create_pods.sh b/create_pods.sh
new file mode 100644
index 0000000..848cd31
--- /dev/null
+++ b/create_pods.sh
@@ -0,0 +1,44 @@
+!pip install -qqq runpod --progress-bar off
+
+import runpod
+
+BENCHMARK = "medtasks"
+MODEL = sys.argv[1] #"mlabonne/NeuralMarcoro14-7B" # @param {type:"string"}
+GPU = sys.argv[2] #"NVIDIA GeForce RTX 3090" # @param ["NVIDIA A100 80GB PCIe", "NVIDIA A100-SXM4-80GB", "NVIDIA A30", "NVIDIA A40", "NVIDIA GeForce RTX 3070", "NVIDIA GeForce RTX 3080", "NVIDIA GeForce RTX 3080 Ti", "NVIDIA GeForce RTX 3090", "NVIDIA GeForce RTX 3090 Ti", "NVIDIA GeForce RTX 4070 Ti", "NVIDIA GeForce RTX 4080", "NVIDIA GeForce RTX 4090", "NVIDIA H100 80GB HBM3", "NVIDIA H100 PCIe", "NVIDIA L4", "NVIDIA L40", "NVIDIA RTX 4000 Ada Generation", "NVIDIA RTX 4000 SFF Ada Generation", "NVIDIA RTX 5000 Ada Generation", "NVIDIA RTX 6000 Ada Generation", "NVIDIA RTX A2000", "NVIDIA RTX A4000", "NVIDIA RTX A4500", "NVIDIA RTX A5000", "NVIDIA RTX A6000", "Tesla V100-FHHL-16GB", "Tesla V100-PCIE-16GB", "Tesla V100-SXM2-16GB", "Tesla V100-SXM2-32GB"]
+NUMBER_OF_GPUS = 1 # @param {type:"slider", min:1, max:8, step:1}
+CONTAINER_DISK = 150 # @param {type:"slider", min:50, max:500, step:25}
+CLOUD_TYPE = "COMMUNITY" # @param ["COMMUNITY", "SECURE"]
+REPO = "https://github.com/chenhaodev/llm-autoeval.git" # @param {type:"string"}
+TRUST_REMOTE_CODE = False # @param {type:"boolean"}
+DEBUG = False # @param {type:"boolean"}
+
+# @markdown ---
+RUNPOD_TOKEN = sys.argv[3] #"runpod" # @param {type:"string"}
+GITHUB_TOKEN = sys.argv[4] #"github" # @param {type:"string"}
+HF_TOKEN = sys.argv[5]
+
+# Environment variables
+runpod.api_key = RUNPOD_TOKEN
+GITHUB_API_TOKEN = GITHUB_TOKEN
+
+# Create a pod
+pod = runpod.create_pod(
+    name=f"Eval {MODEL.split('/')[-1]} on {BENCHMARK.capitalize()}",
+    image_name="runpod/pytorch:2.0.1-py3.10-cuda11.8.0-devel-ubuntu22.04",
+    gpu_type_id=GPU,
+    cloud_type=CLOUD_TYPE,
+    gpu_count=NUMBER_OF_GPUS,
+    volume_in_gb=0,
+    container_disk_in_gb=CONTAINER_DISK,
+    env={
+        "BENCHMARK": BENCHMARK,
+        "MODEL": MODEL,
+        "REPO": REPO,
+        "TRUST_REMOTE_CODE": TRUST_REMOTE_CODE,
+        "DEBUG": DEBUG,
+        "GITHUB_API_TOKEN": GITHUB_API_TOKEN,
+        "HF_TOKEN": HF_TOKEN,
+    }
+)
+
+print("Pod started: https://www.runpod.io/console/pods")
diff --git a/runpod.sh b/runpod.sh
index 3c7a604..2eee249 100644
--- a/runpod.sh
+++ b/runpod.sh
@@ -24,21 +24,18 @@ apt install -y screen vim git-lfs
 screen
 
 # Install common libraries
-pip install -q requests accelerate sentencepiece pytablewriter einops protobuf
+#pip install -q requests accelerate sentencepiece pytablewriter einops protobuf
 
 if [ "$DEBUG" == "True" ]; then
     echo "Launch LLM AutoEval in debug mode"
 fi
 
-# Run evaluation
-git clone https://github.com/chenhaodev/lm-evaluation-harness
-cd lm-evaluation-harness
-pip install -e .
-
-# Call example: lm_eval --model hf --model_args pretrained=/path-to-model,parallelize=True,load_in_4bit=True --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine --device cuda:0 --batch_size auto --limit 100
+# Run evaluation; e.g. apt-get update; apt-get install -y tmux vim git-lfs; tmux new -s ssh-download-llm; cd /workspace/; mkdir -p cache model; pip install huggingface_hub; huggingface-cli login --token xxx; huggingface-cli download --resume-download xxx/xxx --local-dir /workspace/model --local-dir-use-symlinks False --cache-dir /workspace/cache; lm_eval --model hf --model_args pretrained=/path-to-model,parallelize=True,load_in_4bit=True --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine --device cuda:0 --batch_size auto --limit 100
+cd /workspace/; mkdir -p cache model; git clone https://github.com/chenhaodev/lm-evaluation-harness; cd lm-evaluation-harness; pip install -e .;
+pip install huggingface_hub; huggingface-cli login --token $HF_TOKEN; huggingface-cli download --resume-download $MODEL --local-dir /workspace/model --local-dir-use-symlinks False --cache-dir /workspace/cache;
 
 lm_eval --model hf \
-    --model_args pretrained=$MODEL,trust_remote_code=$TRUST_REMOTE_CODE,parallelize=True \
+    --model_args pretrained=/workspace/model,trust_remote_code=$TRUST_REMOTE_CODE,parallelize=True \
     --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine \
     --device cuda:$cuda_devices \
     --batch_size auto \

From df3b5c975b04bb676b8a2a85efa2c9630dae08c4 Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Thu, 15 Feb 2024 11:05:30 +0800
Subject: [PATCH 03/22] rename files

---
 create_pods.sh => main.sh   | 0
 runpod.sh                   | 2 +-
 main.py => upload-result.py | 0
 3 files changed, 1 insertion(+), 1 deletion(-)
 rename create_pods.sh => main.sh (100%)
 rename main.py => upload-result.py (100%)

diff --git a/create_pods.sh b/main.sh
similarity index 100%
rename from create_pods.sh
rename to main.sh
diff --git a/runpod.sh b/runpod.sh
index 2eee249..259b1b0 100644
--- a/runpod.sh
+++ b/runpod.sh
@@ -45,7 +45,7 @@ lm_eval --model hf \
 end=$(date +%s)
 echo "Elapsed Time: $(($end-$start)) seconds" >> ./result.log
 
-python ../llm-autoeval/main.py . $(($end-$start))
+python ../llm-autoeval/upload-result.py . $(($end-$start))
 
 '''
 # Run evaluation
diff --git a/main.py b/upload-result.py
similarity index 100%
rename from main.py
rename to upload-result.py

From f7120e915bbfd063d935cfc0889471fe8262f562 Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Thu, 15 Feb 2024 11:14:11 +0800
Subject: [PATCH 04/22] rename files

---
 main.sh => main.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)
 rename main.sh => main.py (92%)

diff --git a/main.sh b/main.py
similarity index 92%
rename from main.sh
rename to main.py
index 848cd31..7da0898 100644
--- a/main.sh
+++ b/main.py
@@ -1,12 +1,14 @@
-!pip install -qqq runpod --progress-bar off
+#pip install -qqq runpod --progress-bar off
 
 import runpod
+import sys
 
 BENCHMARK = "medtasks"
 MODEL = sys.argv[1] #"mlabonne/NeuralMarcoro14-7B" # @param {type:"string"}
 GPU = sys.argv[2] #"NVIDIA GeForce RTX 3090" # @param ["NVIDIA A100 80GB PCIe", "NVIDIA A100-SXM4-80GB", "NVIDIA A30", "NVIDIA A40", "NVIDIA GeForce RTX 3070", "NVIDIA GeForce RTX 3080", "NVIDIA GeForce RTX 3080 Ti", "NVIDIA GeForce RTX 3090", "NVIDIA GeForce RTX 3090 Ti", "NVIDIA GeForce RTX 4070 Ti", "NVIDIA GeForce RTX 4080", "NVIDIA GeForce RTX 4090", "NVIDIA H100 80GB HBM3", "NVIDIA H100 PCIe", "NVIDIA L4", "NVIDIA L40", "NVIDIA RTX 4000 Ada Generation", "NVIDIA RTX 4000 SFF Ada Generation", "NVIDIA RTX 5000 Ada Generation", "NVIDIA RTX 6000 Ada Generation", "NVIDIA RTX A2000", "NVIDIA RTX A4000", "NVIDIA RTX A4500", "NVIDIA RTX A5000", "NVIDIA RTX A6000", "Tesla V100-FHHL-16GB", "Tesla V100-PCIE-16GB", "Tesla V100-SXM2-16GB", "Tesla V100-SXM2-32GB"]
 NUMBER_OF_GPUS = 1 # @param {type:"slider", min:1, max:8, step:1}
-CONTAINER_DISK = 150 # @param {type:"slider", min:50, max:500, step:25}
+CONTAINER_DISK = 50 # @param {type:"slider", min:50, max:500, step:25}
+VOLUME_IN_GB = 150
 CLOUD_TYPE = "COMMUNITY" # @param ["COMMUNITY", "SECURE"]
 REPO = "https://github.com/chenhaodev/llm-autoeval.git" # @param {type:"string"}
 TRUST_REMOTE_CODE = False # @param {type:"boolean"}
@@ -28,7 +30,7 @@
     gpu_type_id=GPU,
     cloud_type=CLOUD_TYPE,
     gpu_count=NUMBER_OF_GPUS,
-    volume_in_gb=0,
+    volume_in_gb=VOLUME_IN_GB,
     container_disk_in_gb=CONTAINER_DISK,
     env={
         "BENCHMARK": BENCHMARK,

From 98464abbd34f50a9916bc9b77c9d93af92c21efe Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Thu, 15 Feb 2024 11:25:58 +0800
Subject: [PATCH 05/22] update

---
 main.py   |   3 +-
 runpod.sh | 116 +-----------------------------------------------------
 2 files changed, 3 insertions(+), 116 deletions(-)

diff --git a/main.py b/main.py
index 7da0898..2e339e0 100644
--- a/main.py
+++ b/main.py
@@ -8,7 +8,7 @@
 GPU = sys.argv[2] #"NVIDIA GeForce RTX 3090" # @param ["NVIDIA A100 80GB PCIe", "NVIDIA A100-SXM4-80GB", "NVIDIA A30", "NVIDIA A40", "NVIDIA GeForce RTX 3070", "NVIDIA GeForce RTX 3080", "NVIDIA GeForce RTX 3080 Ti", "NVIDIA GeForce RTX 3090", "NVIDIA GeForce RTX 3090 Ti", "NVIDIA GeForce RTX 4070 Ti", "NVIDIA GeForce RTX 4080", "NVIDIA GeForce RTX 4090", "NVIDIA H100 80GB HBM3", "NVIDIA H100 PCIe", "NVIDIA L4", "NVIDIA L40", "NVIDIA RTX 4000 Ada Generation", "NVIDIA RTX 4000 SFF Ada Generation", "NVIDIA RTX 5000 Ada Generation", "NVIDIA RTX 6000 Ada Generation", "NVIDIA RTX A2000", "NVIDIA RTX A4000", "NVIDIA RTX A4500", "NVIDIA RTX A5000", "NVIDIA RTX A6000", "Tesla V100-FHHL-16GB", "Tesla V100-PCIE-16GB", "Tesla V100-SXM2-16GB", "Tesla V100-SXM2-32GB"]
 NUMBER_OF_GPUS = 1 # @param {type:"slider", min:1, max:8, step:1}
 CONTAINER_DISK = 50 # @param {type:"slider", min:50, max:500, step:25}
-VOLUME_IN_GB = 150
+VOLUME_IN_GB = 100
 CLOUD_TYPE = "COMMUNITY" # @param ["COMMUNITY", "SECURE"]
 REPO = "https://github.com/chenhaodev/llm-autoeval.git" # @param {type:"string"}
 TRUST_REMOTE_CODE = False # @param {type:"boolean"}
@@ -32,6 +32,7 @@
     gpu_count=NUMBER_OF_GPUS,
     volume_in_gb=VOLUME_IN_GB,
     container_disk_in_gb=CONTAINER_DISK,
+    template_id="au6nz6emhk",
     env={
         "BENCHMARK": BENCHMARK,
         "MODEL": MODEL,
diff --git a/runpod.sh b/runpod.sh
index 259b1b0..747897c 100644
--- a/runpod.sh
+++ b/runpod.sh
@@ -1,5 +1,4 @@
 #!/bin/bash
-#main func start here
 
 start=$(date +%s)
 
@@ -24,13 +23,12 @@ apt install -y screen vim git-lfs
 screen
 
 # Install common libraries
-#pip install -q requests accelerate sentencepiece pytablewriter einops protobuf
 
 if [ "$DEBUG" == "True" ]; then
     echo "Launch LLM AutoEval in debug mode"
 fi
 
-# Run evaluation; e.g. apt-get update; apt-get install -y tmux vim git-lfs; tmux new -s ssh-download-llm; cd /workspace/; mkdir -p cache model; pip install huggingface_hub; huggingface-cli login --token xxx; huggingface-cli download --resume-download xxx/xxx --local-dir /workspace/model --local-dir-use-symlinks False --cache-dir /workspace/cache; lm_eval --model hf --model_args pretrained=/path-to-model,parallelize=True,load_in_4bit=True --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine --device cuda:0 --batch_size auto --limit 100
+# Run evaluation
 cd /workspace/; mkdir -p cache model; git clone https://github.com/chenhaodev/lm-evaluation-harness; cd lm-evaluation-harness; pip install -e .;
 pip install huggingface_hub; huggingface-cli login --token $HF_TOKEN; huggingface-cli download --resume-download $MODEL --local-dir /workspace/model --local-dir-use-symlinks False --cache-dir /workspace/cache;
 
@@ -47,118 +45,6 @@ echo "Elapsed Time: $(($end-$start)) seconds" >> ./result.log
 
 python ../llm-autoeval/upload-result.py . $(($end-$start))
 
-'''
-# Run evaluation
-if [ "$BENCHMARK" == "nous" ]; then
-    git clone https://github.com/chenhaodev/lm-evaluation-harness
-    cd lm-evaluation-harness
-    pip install -e .
-
-    benchmark="agieval"
-    python main.py \
-        --model hf-causal \
-        --model_args pretrained=$MODEL,trust_remote_code=$TRUST_REMOTE_CODE \
-        --tasks agieval_aqua_rat,agieval_logiqa_en,agieval_lsat_ar,agieval_lsat_lr,agieval_lsat_rc,agieval_sat_en,agieval_sat_en_without_passage,agieval_sat_math \
-        --device cuda:$cuda_devices \
-        --batch_size auto \
-        --output_path ./${benchmark}.json
-
-    benchmark="gpt4all"
-    python main.py \
-        --model hf-causal \
-        --model_args pretrained=$MODEL,trust_remote_code=$TRUST_REMOTE_CODE \
-        --tasks hellaswag,openbookqa,winogrande,arc_easy,arc_challenge,boolq,piqa \
-        --device cuda:$cuda_devices \
-        --batch_size auto \
-        --output_path ./${benchmark}.json
-
-    benchmark="truthfulqa"
-    python main.py \
-        --model hf-causal \
-        --model_args pretrained=$MODEL,trust_remote_code=$TRUST_REMOTE_CODE \
-        --tasks truthfulqa_mc \
-        --device cuda:$cuda_devices \
-        --batch_size auto \
-        --output_path ./${benchmark}.json
-
-    benchmark="bigbench"
-    python main.py \
-        --model hf-causal \
-        --model_args pretrained=$MODEL,trust_remote_code=$TRUST_REMOTE_CODE \
-        --tasks bigbench_causal_judgement,bigbench_date_understanding,bigbench_disambiguation_qa,bigbench_geometric_shapes,bigbench_logical_deduction_five_objects,bigbench_logical_deduction_seven_objects,bigbench_logical_deduction_three_objects,bigbench_movie_recommendation,bigbench_navigate,bigbench_reasoning_about_colored_objects,bigbench_ruin_names,bigbench_salient_translation_error_detection,bigbench_snarks,bigbench_sports_understanding,bigbench_temporal_sequences,bigbench_tracking_shuffled_objects_five_objects,bigbench_tracking_shuffled_objects_seven_objects,bigbench_tracking_shuffled_objects_three_objects \
-        --device cuda:$cuda_devices \
-        --batch_size auto \
-        --output_path ./${benchmark}.json
-
-    end=$(date +%s)
-    echo "Elapsed Time: $(($end-$start)) seconds"
-    
-    python ../llm-autoeval/main.py . $(($end-$start))
-
-elif [ "$BENCHMARK" == "openllm" ]; then
-    git clone https://github.com/EleutherAI/lm-evaluation-harness
-    cd lm-evaluation-harness
-    pip install -e ".[vllm,promptsource]"
-    pip install langdetect immutabledict
-
-    benchmark="arc"
-    lm_eval --model vllm \
-        --model_args pretrained=${MODEL},dtype=auto,gpu_memory_utilization=0.8,trust_remote_code=$TRUST_REMOTE_CODE \
-        --tasks arc_challenge \
-        --num_fewshot 25 \
-        --batch_size auto \
-        --output_path ./${benchmark}.json
-
-    benchmark="hellaswag"
-    lm_eval --model vllm \
-        --model_args pretrained=${MODEL},dtype=auto,gpu_memory_utilization=0.8,trust_remote_code=$TRUST_REMOTE_CODE \
-        --tasks hellaswag \
-        --num_fewshot 10 \
-        --batch_size auto \
-        --output_path ./${benchmark}.json
-
-    # benchmark="mmlu"
-    # lm_eval --model vllm \
-    #     --model_args pretrained=${MODEL},dtype=auto,gpu_memory_utilization=0.8,trust_remote_code=$TRUST_REMOTE_CODE \
-    #     --tasks mmlu \
-    #     --num_fewshot 5 \
-    #     --batch_size auto \
-    #     --verbosity DEBUG \
-    #     --output_path ./${benchmark}.json
-    
-    benchmark="truthfulqa"
-    lm_eval --model vllm \
-        --model_args pretrained=${MODEL},dtype=auto,gpu_memory_utilization=0.8,trust_remote_code=$TRUST_REMOTE_CODE \
-        --tasks truthfulqa \
-        --num_fewshot 0 \
-        --batch_size auto \
-        --output_path ./${benchmark}.json
-    
-    benchmark="winogrande"
-    lm_eval --model vllm \
-        --model_args pretrained=${MODEL},dtype=auto,gpu_memory_utilization=0.8,trust_remote_code=$TRUST_REMOTE_CODE \
-        --tasks winogrande \
-        --num_fewshot 5 \
-        --batch_size auto \
-        --output_path ./${benchmark}.json
-    
-    benchmark="gsm8k"
-    lm_eval --model vllm \
-        --model_args pretrained=${MODEL},dtype=auto,gpu_memory_utilization=0.8,trust_remote_code=$TRUST_REMOTE_CODE \
-        --tasks gsm8k \
-        --num_fewshot 5 \
-        --batch_size auto \
-        --output_path ./${benchmark}.json
-
-    end=$(date +%s)
-    echo "Elapsed Time: $(($end-$start)) seconds"
-    
-    python ../llm-autoeval/main.py . $(($end-$start))
-else
-    echo "Error: Invalid BENCHMARK value. Please set BENCHMARK to 'nous' or 'openllm'."
-fi
-'''
-
 if [ "$DEBUG" == "False" ]; then
     runpodctl remove pod $RUNPOD_POD_ID
 fi

From 0132d1d8911c00bf580ff65aedac7e9d35a59ce0 Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Thu, 15 Feb 2024 11:43:28 +0800
Subject: [PATCH 06/22] update

---
 main.py   |  8 ++++----
 runpod.sh | 10 +---------
 2 files changed, 5 insertions(+), 13 deletions(-)

diff --git a/main.py b/main.py
index 2e339e0..4a5524e 100644
--- a/main.py
+++ b/main.py
@@ -7,8 +7,8 @@
 MODEL = sys.argv[1] #"mlabonne/NeuralMarcoro14-7B" # @param {type:"string"}
 GPU = sys.argv[2] #"NVIDIA GeForce RTX 3090" # @param ["NVIDIA A100 80GB PCIe", "NVIDIA A100-SXM4-80GB", "NVIDIA A30", "NVIDIA A40", "NVIDIA GeForce RTX 3070", "NVIDIA GeForce RTX 3080", "NVIDIA GeForce RTX 3080 Ti", "NVIDIA GeForce RTX 3090", "NVIDIA GeForce RTX 3090 Ti", "NVIDIA GeForce RTX 4070 Ti", "NVIDIA GeForce RTX 4080", "NVIDIA GeForce RTX 4090", "NVIDIA H100 80GB HBM3", "NVIDIA H100 PCIe", "NVIDIA L4", "NVIDIA L40", "NVIDIA RTX 4000 Ada Generation", "NVIDIA RTX 4000 SFF Ada Generation", "NVIDIA RTX 5000 Ada Generation", "NVIDIA RTX 6000 Ada Generation", "NVIDIA RTX A2000", "NVIDIA RTX A4000", "NVIDIA RTX A4500", "NVIDIA RTX A5000", "NVIDIA RTX A6000", "Tesla V100-FHHL-16GB", "Tesla V100-PCIE-16GB", "Tesla V100-SXM2-16GB", "Tesla V100-SXM2-32GB"]
 NUMBER_OF_GPUS = 1 # @param {type:"slider", min:1, max:8, step:1}
-CONTAINER_DISK = 50 # @param {type:"slider", min:50, max:500, step:25}
-VOLUME_IN_GB = 100
+CONTAINER_DISK = 100 # @param {type:"slider", min:50, max:500, step:25}
+VOLUME_IN_GB = 0
 CLOUD_TYPE = "COMMUNITY" # @param ["COMMUNITY", "SECURE"]
 REPO = "https://github.com/chenhaodev/llm-autoeval.git" # @param {type:"string"}
 TRUST_REMOTE_CODE = False # @param {type:"boolean"}
@@ -32,7 +32,6 @@
     gpu_count=NUMBER_OF_GPUS,
     volume_in_gb=VOLUME_IN_GB,
     container_disk_in_gb=CONTAINER_DISK,
-    template_id="au6nz6emhk",
     env={
         "BENCHMARK": BENCHMARK,
         "MODEL": MODEL,
@@ -41,7 +40,8 @@
         "DEBUG": DEBUG,
         "GITHUB_API_TOKEN": GITHUB_API_TOKEN,
         "HF_TOKEN": HF_TOKEN,
-    }
+    },
+    template_id='6p59tg6cln', #bash -c 'cd /workspace/; git clone https://github.com/chenhaodev/llm-autoeval; cd /workspace/llm-autoeval/; sh runpod.sh' #template_id="au6nz6emhk",
 )
 
 print("Pod started: https://www.runpod.io/console/pods")
diff --git a/runpod.sh b/runpod.sh
index 747897c..1b6edf2 100644
--- a/runpod.sh
+++ b/runpod.sh
@@ -8,14 +8,6 @@ if [ $gpu_count -eq 0 ]; then
     echo "No NVIDIA GPUs detected. Exiting."
     exit 1
 fi
-# Construct the CUDA device string
-cuda_devices=""
-for ((i=0; i<gpu_count; i++)); do
-    if [ $i -gt 0 ]; then
-        cuda_devices+=","
-    fi
-    cuda_devices+="$i"
-done
 
 # Install dependencies
 apt update
@@ -35,7 +27,7 @@ pip install huggingface_hub; huggingface-cli login --token $HF_TOKEN; huggingfac
 lm_eval --model hf \
     --model_args pretrained=/workspace/model,trust_remote_code=$TRUST_REMOTE_CODE,parallelize=True \
     --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine \
-    --device cuda:$cuda_devices \
+    --device cuda:0 \
     --batch_size auto \
     --limit 100 \
     --output_path ./result.log

From 1b01c95914595dc916263b12c1c532291559440b Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Thu, 15 Feb 2024 12:02:12 +0800
Subject: [PATCH 07/22] fix bugs

---
 runpod.sh        | 2 +-
 upload-result.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/runpod.sh b/runpod.sh
index 1b6edf2..febe512 100644
--- a/runpod.sh
+++ b/runpod.sh
@@ -35,7 +35,7 @@ lm_eval --model hf \
 end=$(date +%s)
 echo "Elapsed Time: $(($end-$start)) seconds" >> ./result.log
 
-python ../llm-autoeval/upload-result.py . $(($end-$start))
+cd /workspace/; python ../llm-autoeval/upload-result.py . $(($end-$start))
 
 if [ "$DEBUG" == "False" ]; then
     runpodctl remove pod $RUNPOD_POD_ID
diff --git a/upload-result.py b/upload-result.py
index 9805170..73910f7 100644
--- a/upload-result.py
+++ b/upload-result.py
@@ -15,7 +15,7 @@
 GITHUB_API_TOKEN = os.getenv("GITHUB_API_TOKEN")
 
 def main(directory: str, elapsed_time: float) -> None:
-    file_path = f"{directory}/result.log"
+    file_path = "/workspace/model/result.log"
     summary = open(file_path, "r").read()
     upload_to_github_gist(
         summary, f"{MODEL.split('/')[-1]}-MedTasks.md", GITHUB_API_TOKEN

From 6fc3c721f66d4ee4aae7322fa7ee1f7efa951545 Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Thu, 15 Feb 2024 13:55:36 +0800
Subject: [PATCH 08/22] update

---
 main.py   | 4 ++--
 runpod.sh | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/main.py b/main.py
index 4a5524e..ff45742 100644
--- a/main.py
+++ b/main.py
@@ -6,8 +6,8 @@
 BENCHMARK = "medtasks"
 MODEL = sys.argv[1] #"mlabonne/NeuralMarcoro14-7B" # @param {type:"string"}
 GPU = sys.argv[2] #"NVIDIA GeForce RTX 3090" # @param ["NVIDIA A100 80GB PCIe", "NVIDIA A100-SXM4-80GB", "NVIDIA A30", "NVIDIA A40", "NVIDIA GeForce RTX 3070", "NVIDIA GeForce RTX 3080", "NVIDIA GeForce RTX 3080 Ti", "NVIDIA GeForce RTX 3090", "NVIDIA GeForce RTX 3090 Ti", "NVIDIA GeForce RTX 4070 Ti", "NVIDIA GeForce RTX 4080", "NVIDIA GeForce RTX 4090", "NVIDIA H100 80GB HBM3", "NVIDIA H100 PCIe", "NVIDIA L4", "NVIDIA L40", "NVIDIA RTX 4000 Ada Generation", "NVIDIA RTX 4000 SFF Ada Generation", "NVIDIA RTX 5000 Ada Generation", "NVIDIA RTX 6000 Ada Generation", "NVIDIA RTX A2000", "NVIDIA RTX A4000", "NVIDIA RTX A4500", "NVIDIA RTX A5000", "NVIDIA RTX A6000", "Tesla V100-FHHL-16GB", "Tesla V100-PCIE-16GB", "Tesla V100-SXM2-16GB", "Tesla V100-SXM2-32GB"]
-NUMBER_OF_GPUS = 1 # @param {type:"slider", min:1, max:8, step:1}
-CONTAINER_DISK = 100 # @param {type:"slider", min:50, max:500, step:25}
+NUMBER_OF_GPUS = 2 # @param {type:"slider", min:1, max:8, step:1}
+CONTAINER_DISK = 200 # @param {type:"slider", min:50, max:500, step:25}
 VOLUME_IN_GB = 0
 CLOUD_TYPE = "COMMUNITY" # @param ["COMMUNITY", "SECURE"]
 REPO = "https://github.com/chenhaodev/llm-autoeval.git" # @param {type:"string"}
diff --git a/runpod.sh b/runpod.sh
index febe512..39cea29 100644
--- a/runpod.sh
+++ b/runpod.sh
@@ -29,13 +29,13 @@ lm_eval --model hf \
     --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine \
     --device cuda:0 \
     --batch_size auto \
-    --limit 100 \
-    --output_path ./result.log
+    --limit 100 > result.log
+    #--output_path ./result
 
 end=$(date +%s)
 echo "Elapsed Time: $(($end-$start)) seconds" >> ./result.log
 
-cd /workspace/; python ../llm-autoeval/upload-result.py . $(($end-$start))
+cd /workspace/; python /workspace/llm-autoeval/upload-result.py . $(($end-$start))
 
 if [ "$DEBUG" == "False" ]; then
     runpodctl remove pod $RUNPOD_POD_ID

From f4437e122b9b5aa8872a71e9ac245a700b22230d Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Thu, 15 Feb 2024 13:57:16 +0800
Subject: [PATCH 09/22] rename files

---
 runpod.sh => auto-eval.sh | 0
 main.py => create_pods.py | 4 ++--
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename runpod.sh => auto-eval.sh (100%)
 rename main.py => create_pods.py (94%)

diff --git a/runpod.sh b/auto-eval.sh
similarity index 100%
rename from runpod.sh
rename to auto-eval.sh
diff --git a/main.py b/create_pods.py
similarity index 94%
rename from main.py
rename to create_pods.py
index ff45742..4a5524e 100644
--- a/main.py
+++ b/create_pods.py
@@ -6,8 +6,8 @@
 BENCHMARK = "medtasks"
 MODEL = sys.argv[1] #"mlabonne/NeuralMarcoro14-7B" # @param {type:"string"}
 GPU = sys.argv[2] #"NVIDIA GeForce RTX 3090" # @param ["NVIDIA A100 80GB PCIe", "NVIDIA A100-SXM4-80GB", "NVIDIA A30", "NVIDIA A40", "NVIDIA GeForce RTX 3070", "NVIDIA GeForce RTX 3080", "NVIDIA GeForce RTX 3080 Ti", "NVIDIA GeForce RTX 3090", "NVIDIA GeForce RTX 3090 Ti", "NVIDIA GeForce RTX 4070 Ti", "NVIDIA GeForce RTX 4080", "NVIDIA GeForce RTX 4090", "NVIDIA H100 80GB HBM3", "NVIDIA H100 PCIe", "NVIDIA L4", "NVIDIA L40", "NVIDIA RTX 4000 Ada Generation", "NVIDIA RTX 4000 SFF Ada Generation", "NVIDIA RTX 5000 Ada Generation", "NVIDIA RTX 6000 Ada Generation", "NVIDIA RTX A2000", "NVIDIA RTX A4000", "NVIDIA RTX A4500", "NVIDIA RTX A5000", "NVIDIA RTX A6000", "Tesla V100-FHHL-16GB", "Tesla V100-PCIE-16GB", "Tesla V100-SXM2-16GB", "Tesla V100-SXM2-32GB"]
-NUMBER_OF_GPUS = 2 # @param {type:"slider", min:1, max:8, step:1}
-CONTAINER_DISK = 200 # @param {type:"slider", min:50, max:500, step:25}
+NUMBER_OF_GPUS = 1 # @param {type:"slider", min:1, max:8, step:1}
+CONTAINER_DISK = 100 # @param {type:"slider", min:50, max:500, step:25}
 VOLUME_IN_GB = 0
 CLOUD_TYPE = "COMMUNITY" # @param ["COMMUNITY", "SECURE"]
 REPO = "https://github.com/chenhaodev/llm-autoeval.git" # @param {type:"string"}

From 1f6e8e26a5d8d528491ae78720678fb02e36a2ab Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Thu, 15 Feb 2024 13:58:49 +0800
Subject: [PATCH 10/22] update

---
 create_pods.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/create_pods.py b/create_pods.py
index 4a5524e..f22335f 100644
--- a/create_pods.py
+++ b/create_pods.py
@@ -41,7 +41,7 @@
         "GITHUB_API_TOKEN": GITHUB_API_TOKEN,
         "HF_TOKEN": HF_TOKEN,
     },
-    template_id='6p59tg6cln', #bash -c 'cd /workspace/; git clone https://github.com/chenhaodev/llm-autoeval; cd /workspace/llm-autoeval/; sh runpod.sh' #template_id="au6nz6emhk",
+    template_id='6p59tg6cln', #bash -c 'cd /workspace/; git clone https://github.com/chenhaodev/llm-autoeval; cd /workspace/llm-autoeval/; sh auto-eval.sh' #template_id="au6nz6emhk",
 )
 
 print("Pod started: https://www.runpod.io/console/pods")

From a49ab2b44064db506805051b609893195ec2f6a6 Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Thu, 15 Feb 2024 14:21:49 +0800
Subject: [PATCH 11/22] update

---
 upload-result.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/upload-result.py b/upload-result.py
index 73910f7..2ecf488 100644
--- a/upload-result.py
+++ b/upload-result.py
@@ -15,7 +15,7 @@
 GITHUB_API_TOKEN = os.getenv("GITHUB_API_TOKEN")
 
 def main(directory: str, elapsed_time: float) -> None:
-    file_path = "/workspace/model/result.log"
+    file_path = "/workspace/lm-evaluation-harness/result.log"
     summary = open(file_path, "r").read()
     upload_to_github_gist(
         summary, f"{MODEL.split('/')[-1]}-MedTasks.md", GITHUB_API_TOKEN

From c56e19ad19983818b7852317023b732089ffbc98 Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Thu, 15 Feb 2024 17:47:48 +0800
Subject: [PATCH 12/22] update create_pods.py, add arg for num_of_gpu

---
 create_pods.py                          |  10 +--
 scripts/.DS_Store                       | Bin 0 -> 6148 bytes
 scripts/submit-result/result.log        |  17 +++++
 scripts/submit-result/submit-results.py |  90 ++++++++++++++++++++++++
 4 files changed, 112 insertions(+), 5 deletions(-)
 create mode 100644 scripts/.DS_Store
 create mode 100644 scripts/submit-result/result.log
 create mode 100644 scripts/submit-result/submit-results.py

diff --git a/create_pods.py b/create_pods.py
index f22335f..ebded6c 100644
--- a/create_pods.py
+++ b/create_pods.py
@@ -6,8 +6,8 @@
 BENCHMARK = "medtasks"
 MODEL = sys.argv[1] #"mlabonne/NeuralMarcoro14-7B" # @param {type:"string"}
 GPU = sys.argv[2] #"NVIDIA GeForce RTX 3090" # @param ["NVIDIA A100 80GB PCIe", "NVIDIA A100-SXM4-80GB", "NVIDIA A30", "NVIDIA A40", "NVIDIA GeForce RTX 3070", "NVIDIA GeForce RTX 3080", "NVIDIA GeForce RTX 3080 Ti", "NVIDIA GeForce RTX 3090", "NVIDIA GeForce RTX 3090 Ti", "NVIDIA GeForce RTX 4070 Ti", "NVIDIA GeForce RTX 4080", "NVIDIA GeForce RTX 4090", "NVIDIA H100 80GB HBM3", "NVIDIA H100 PCIe", "NVIDIA L4", "NVIDIA L40", "NVIDIA RTX 4000 Ada Generation", "NVIDIA RTX 4000 SFF Ada Generation", "NVIDIA RTX 5000 Ada Generation", "NVIDIA RTX 6000 Ada Generation", "NVIDIA RTX A2000", "NVIDIA RTX A4000", "NVIDIA RTX A4500", "NVIDIA RTX A5000", "NVIDIA RTX A6000", "Tesla V100-FHHL-16GB", "Tesla V100-PCIE-16GB", "Tesla V100-SXM2-16GB", "Tesla V100-SXM2-32GB"]
-NUMBER_OF_GPUS = 1 # @param {type:"slider", min:1, max:8, step:1}
-CONTAINER_DISK = 100 # @param {type:"slider", min:50, max:500, step:25}
+NUMBER_OF_GPUS = sys.argv[3] # @param {type:"slider", min:1, max:8, step:1}
+CONTAINER_DISK = 200 # @param {type:"slider", min:50, max:500, step:25}
 VOLUME_IN_GB = 0
 CLOUD_TYPE = "COMMUNITY" # @param ["COMMUNITY", "SECURE"]
 REPO = "https://github.com/chenhaodev/llm-autoeval.git" # @param {type:"string"}
@@ -15,9 +15,9 @@
 DEBUG = False # @param {type:"boolean"}
 
 # @markdown ---
-RUNPOD_TOKEN = sys.argv[3] #"runpod" # @param {type:"string"}
-GITHUB_TOKEN = sys.argv[4] #"github" # @param {type:"string"}
-HF_TOKEN = sys.argv[5]
+RUNPOD_TOKEN = sys.argv[4] #"runpod" # @param {type:"string"}
+GITHUB_TOKEN = sys.argv[5] #"github" # @param {type:"string"}
+HF_TOKEN = sys.argv[6]
 
 # Environment variables
 runpod.api_key = RUNPOD_TOKEN
diff --git a/scripts/.DS_Store b/scripts/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6
GIT binary patch
literal 6148
zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3
zem<@ulZcFPQ@L2!n>{z**<q8>++&mCkOWA81W14cNZ<zv;LbK1Poaz?KmsK2CSc!(
z0ynLxE!0092;Krf2c+FF_Fe*7ECH>lEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ
zLs35+`xjp>T0<F0fCPF1$Cyrb|F7^5{eNG?83~ZUUlGt@xh*qZDeu<Z%US-OSsOPv
j)R!Z4KLME7ReXlK;d!wEw5GODWMKRea10D2@KpjYNUI8I

literal 0
HcmV?d00001

diff --git a/scripts/submit-result/result.log b/scripts/submit-result/result.log
new file mode 100644
index 0000000..7b5e05a
--- /dev/null
+++ b/scripts/submit-result/result.log
@@ -0,0 +1,17 @@
+Passed argument batch_size = auto:1. Detecting largest batch size
+Determined largest batch size: 4
+hf (pretrained=/workspace/model,trust_remote_code=False,parallelize=True), gen_kwargs: (None), limit: 100.0, num_fewshot: None, batch_size: auto (4)
+|        Tasks        |Version|Filter|n-shot| Metric |Value|   |Stderr|
+|---------------------|-------|------|------|--------|----:|---|-----:|
+|pubmedqa             |      1|none  |None  |acc     | 0.97|±  |0.0171|
+|ocn                  |Yaml   |none  |None  |acc     | 0.78|±  |0.0416|
+|                     |       |none  |None  |acc_norm| 0.78|±  |0.0416|
+|professional_medicine|      0|none  |None  |acc     | 0.89|±  |0.0314|
+|college_medicine     |      0|none  |None  |acc     | 0.80|±  |0.0402|
+|clinical_knowledge   |      0|none  |None  |acc     | 0.84|±  |0.0368|
+|medmcqa              |Yaml   |none  |None  |acc     | 0.54|±  |0.0501|
+|                     |       |none  |None  |acc_norm| 0.54|±  |0.0501|
+|aocnp                |Yaml   |none  |None  |acc     | 0.74|±  |0.0441|
+|                     |       |none  |None  |acc_norm| 0.74|±  |0.0441|
+
+Elapsed Time: 1417 seconds
diff --git a/scripts/submit-result/submit-results.py b/scripts/submit-result/submit-results.py
new file mode 100644
index 0000000..4e2b21f
--- /dev/null
+++ b/scripts/submit-result/submit-results.py
@@ -0,0 +1,90 @@
+import json
+import logging
+import os
+import argparse
+import time
+
+from llm_autoeval.table import make_table, make_final_table
+from llm_autoeval.upload import upload_to_github_gist
+
+MODEL = 'abacusai/Smaug-72B-v0.1'
+GITHUB_API_TOKEN = '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'
+
+def main(directory: str, elapsed_time: float) -> None:
+    file_path = "result.log"
+    summary = open(file_path, "r").read()
+    upload_to_github_gist(
+        summary, f"{MODEL.split('/')[-1]}-MedTasks.md", GITHUB_API_TOKEN
+    )
+
+    '''
+    # Variables
+    tables = []
+    averages = []
+
+    # Tasks
+    if BENCHMARK == "openllm":
+        tasks = ["ARC", "HellaSwag", "MMLU", "TruthfulQA", "Winogrande", "GSM8K"]
+    elif BENCHMARK == "nous":
+        tasks = ["AGIEval", "GPT4All", "TruthfulQA", "Bigbench"]
+    else:
+        raise NotImplementedError(f"BENCHMARK should be 'openllm' or 'nous' (current value = {BENCHMARK})")
+
+    # Load results
+    for task in tasks:
+        file_path = f"{directory}/{task.lower()}.json"
+        if os.path.exists(file_path):
+            json_data = open(file_path, "r").read()
+            data = json.loads(json_data, strict=False)
+            table, average = make_table(data, task)
+        else:
+            table = ""
+            average = "Error: File does not exist"
+
+        tables.append(table)
+        averages.append(average)
+
+    # Generate tables
+    summary = ""
+    for index, task in enumerate(tasks):
+        summary += f"### {task}\n{tables[index]}\nAverage: {averages[index]}%\n\n"
+    result_dict = {k: v for k, v in zip(tasks, averages)}
+
+    # Calculate the final average, excluding strings
+    if all(isinstance(e, float) for e in averages):
+        final_average = round(sum(averages) / len(averages), 2)
+        summary += f"Average score: {final_average}%"
+        result_dict.update({"Average": final_average})
+    else:
+        summary += "Average score: Not available due to errors"
+
+    # Add elapsed time
+    convert = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
+    summary += f"\n\nElapsed time: {convert}"
+
+    # Generate final table
+    final_table = make_final_table(result_dict, MODEL)
+    summary = final_table + "\n" + summary
+
+    # Upload to GitHub Gist
+    upload_to_github_gist(
+        summary, f"{MODEL.split('/')[-1]}-{BENCHMARK.capitalize()}.md", GITHUB_API_TOKEN
+    )
+    '''
+
+
+if __name__ == "__main__":
+    # Create the parser
+    parser = argparse.ArgumentParser(description="Summarize results and upload them.")
+    parser.add_argument("directory", type=str, help="The path to the directory with the JSON results")
+    parser.add_argument("elapsed_time", type=float, help="Elapsed time since the start of the evaluation")
+
+    # Parse the arguments
+    args = parser.parse_args()
+
+    # Check if the directory exists
+    if not os.path.isdir(args.directory):
+        raise ValueError(f"The directory {args.directory} does not exist.")
+
+    # Call the main function with the directory argument
+    main(args.directory, args.elapsed_time)

From d6d4755da147ea7b56904e6be228b9328577f763 Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Thu, 15 Feb 2024 18:34:05 +0800
Subject: [PATCH 13/22] update

---
 auto-eval.sh | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/auto-eval.sh b/auto-eval.sh
index 39cea29..07ac693 100644
--- a/auto-eval.sh
+++ b/auto-eval.sh
@@ -14,12 +14,6 @@ apt update
 apt install -y screen vim git-lfs
 screen
 
-# Install common libraries
-
-if [ "$DEBUG" == "True" ]; then
-    echo "Launch LLM AutoEval in debug mode"
-fi
-
 # Run evaluation
 cd /workspace/; mkdir -p cache model; git clone https://github.com/chenhaodev/lm-evaluation-harness; cd lm-evaluation-harness; pip install -e .;
 pip install huggingface_hub; huggingface-cli login --token $HF_TOKEN; huggingface-cli download --resume-download $MODEL --local-dir /workspace/model --local-dir-use-symlinks False --cache-dir /workspace/cache;
@@ -37,7 +31,8 @@ echo "Elapsed Time: $(($end-$start)) seconds" >> ./result.log
 
 cd /workspace/; python /workspace/llm-autoeval/upload-result.py . $(($end-$start))
 
-if [ "$DEBUG" == "False" ]; then
-    runpodctl remove pod $RUNPOD_POD_ID
-fi
+#if [ "$DEBUG" == "False" ]; then
+#    runpodctl remove pod $RUNPOD_POD_ID
+#fi
+
 sleep infinity

From 853ec0138a53cf7470c50fa0c9c877c84c55912c Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Thu, 15 Feb 2024 21:30:37 +0800
Subject: [PATCH 14/22] tee result.log

---
 auto-eval.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/auto-eval.sh b/auto-eval.sh
index 07ac693..d3718e9 100644
--- a/auto-eval.sh
+++ b/auto-eval.sh
@@ -23,7 +23,7 @@ lm_eval --model hf \
     --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine \
     --device cuda:0 \
     --batch_size auto \
-    --limit 100 > result.log
+    --limit 100 | tee result.log
     #--output_path ./result
 
 end=$(date +%s)

From 24f8967b23cac597ec62445a12e30f69af9350ff Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Sun, 18 Feb 2024 16:56:56 +0800
Subject: [PATCH 15/22] update

---
 auto-eval.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/auto-eval.sh b/auto-eval.sh
index d3718e9..d49ed17 100644
--- a/auto-eval.sh
+++ b/auto-eval.sh
@@ -12,7 +12,6 @@ fi
 # Install dependencies
 apt update
 apt install -y screen vim git-lfs
-screen
 
 # Run evaluation
 cd /workspace/; mkdir -p cache model; git clone https://github.com/chenhaodev/lm-evaluation-harness; cd lm-evaluation-harness; pip install -e .;
@@ -29,7 +28,7 @@ lm_eval --model hf \
 end=$(date +%s)
 echo "Elapsed Time: $(($end-$start)) seconds" >> ./result.log
 
-cd /workspace/; python /workspace/llm-autoeval/upload-result.py . $(($end-$start))
+echo "cd /workspace/; python /workspace/llm-autoeval/upload-result.py . $(($end-$start))"
 
 #if [ "$DEBUG" == "False" ]; then
 #    runpodctl remove pod $RUNPOD_POD_ID

From e4d594b324b6c5df32f0acae794ca9491960d5e4 Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Sun, 18 Feb 2024 17:13:21 +0800
Subject: [PATCH 16/22] add pre-auto-eval script

---
 auto-eval.sh     |  2 +-
 create_pods.py   |  3 ++-
 pre-auto-eval.sh | 22 ++++++++++++++++++++++
 3 files changed, 25 insertions(+), 2 deletions(-)
 create mode 100644 pre-auto-eval.sh

diff --git a/auto-eval.sh b/auto-eval.sh
index d49ed17..8a961cb 100644
--- a/auto-eval.sh
+++ b/auto-eval.sh
@@ -28,7 +28,7 @@ lm_eval --model hf \
 end=$(date +%s)
 echo "Elapsed Time: $(($end-$start)) seconds" >> ./result.log
 
-echo "cd /workspace/; python /workspace/llm-autoeval/upload-result.py . $(($end-$start))"
+cd /workspace/; python /workspace/llm-autoeval/upload-result.py . $(($end-$start))
 
 #if [ "$DEBUG" == "False" ]; then
 #    runpodctl remove pod $RUNPOD_POD_ID
diff --git a/create_pods.py b/create_pods.py
index ebded6c..81f95d6 100644
--- a/create_pods.py
+++ b/create_pods.py
@@ -18,6 +18,7 @@
 RUNPOD_TOKEN = sys.argv[4] #"runpod" # @param {type:"string"}
 GITHUB_TOKEN = sys.argv[5] #"github" # @param {type:"string"}
 HF_TOKEN = sys.argv[6]
+RUNPOD_TEMPLATE = sys.argv[7] #if template id==6p59tg6cln, one-template-for-all-steps, e.g. download model, auto-eval, upload-results; if id==6rhltjf914, the script only download model; #bash -c 'cd /workspace/; git clone https://github.com/chenhaodev/llm-autoeval; cd /workspace/llm-autoeval/; sh auto-eval.sh' #template_id="au6nz6emhk",
 
 # Environment variables
 runpod.api_key = RUNPOD_TOKEN
@@ -41,7 +42,7 @@
         "GITHUB_API_TOKEN": GITHUB_API_TOKEN,
         "HF_TOKEN": HF_TOKEN,
     },
-    template_id='6p59tg6cln', #bash -c 'cd /workspace/; git clone https://github.com/chenhaodev/llm-autoeval; cd /workspace/llm-autoeval/; sh auto-eval.sh' #template_id="au6nz6emhk",
+    template_id=RUNPOD_TEMPLATE, 
 )
 
 print("Pod started: https://www.runpod.io/console/pods")
diff --git a/pre-auto-eval.sh b/pre-auto-eval.sh
new file mode 100644
index 0000000..9460d89
--- /dev/null
+++ b/pre-auto-eval.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Detect the number of NVIDIA GPUs and create a device string
+gpu_count=$(nvidia-smi -L | wc -l)
+if [ $gpu_count -eq 0 ]; then
+    echo "No NVIDIA GPUs detected. Exiting."
+    exit 1
+fi
+
+# Install dependencies
+apt update
+apt install -y screen vim git-lfs
+
+# Run evaluation
+cd /workspace/; mkdir -p cache model; git clone https://github.com/chenhaodev/lm-evaluation-harness; cd lm-evaluation-harness; pip install -e .;
+pip install huggingface_hub; huggingface-cli login --token $HF_TOKEN; huggingface-cli download --resume-download $MODEL --local-dir /workspace/model --local-dir-use-symlinks False --cache-dir /workspace/cache;
+
+echo "start=$(date +%s); lm_eval --model hf --model_args pretrained=/workspace/model,trust_remote_code=$TRUST_REMOTE_CODE,parallelize=True  --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine  --device cuda:0  --batch_size auto  --limit 100 | tee result.log; end=$(date +%s)"
+echo "Elapsed Time: $(($end-$start)) seconds" >> ./result.log
+echo "cd /workspace/; python /workspace/llm-autoeval/upload-result.py . $(($end-$start))"
+
+sleep infinity

From 2d0fa19ff37818b10c5cdc9689b28d5bebce53eb Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Sun, 18 Feb 2024 23:20:03 +0800
Subject: [PATCH 17/22] update for load-in-4bit

---
 auto-eval.sh     | 2 +-
 pre-auto-eval.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/auto-eval.sh b/auto-eval.sh
index 8a961cb..0d4f7f0 100644
--- a/auto-eval.sh
+++ b/auto-eval.sh
@@ -18,7 +18,7 @@ cd /workspace/; mkdir -p cache model; git clone https://github.com/chenhaodev/lm
 pip install huggingface_hub; huggingface-cli login --token $HF_TOKEN; huggingface-cli download --resume-download $MODEL --local-dir /workspace/model --local-dir-use-symlinks False --cache-dir /workspace/cache;
 
 lm_eval --model hf \
-    --model_args pretrained=/workspace/model,trust_remote_code=$TRUST_REMOTE_CODE,parallelize=True \
+    --model_args pretrained=/workspace/model,trust_remote_code=$TRUST_REMOTE_CODE,parallelize=True,load_in_4bit=True \
     --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine \
     --device cuda:0 \
     --batch_size auto \
diff --git a/pre-auto-eval.sh b/pre-auto-eval.sh
index 9460d89..f16e015 100644
--- a/pre-auto-eval.sh
+++ b/pre-auto-eval.sh
@@ -15,7 +15,7 @@ apt install -y screen vim git-lfs
 cd /workspace/; mkdir -p cache model; git clone https://github.com/chenhaodev/lm-evaluation-harness; cd lm-evaluation-harness; pip install -e .;
 pip install huggingface_hub; huggingface-cli login --token $HF_TOKEN; huggingface-cli download --resume-download $MODEL --local-dir /workspace/model --local-dir-use-symlinks False --cache-dir /workspace/cache;
 
-echo "start=$(date +%s); lm_eval --model hf --model_args pretrained=/workspace/model,trust_remote_code=$TRUST_REMOTE_CODE,parallelize=True  --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine  --device cuda:0  --batch_size auto  --limit 100 | tee result.log; end=$(date +%s)"
+echo "start=$(date +%s); lm_eval --model hf --model_args pretrained=/workspace/model,trust_remote_code=$TRUST_REMOTE_CODE,parallelize=True,load_in_4bit=True --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine  --device cuda:0  --batch_size auto  --limit 100 | tee result.log; end=$(date +%s)"
 echo "Elapsed Time: $(($end-$start)) seconds" >> ./result.log
 echo "cd /workspace/; python /workspace/llm-autoeval/upload-result.py . $(($end-$start))"
 

From d18bfde4460cacf8ba501b7a68fb027b26549780 Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Mon, 19 Feb 2024 00:01:32 +0800
Subject: [PATCH 18/22] update

---
 auto-eval.sh       |  2 +
 base-auto-train.sh | 91 ++++++++++++++++++++++++++++++++++++++++++++++
 pre-auto-eval.sh   | 10 ++---
 3 files changed, 98 insertions(+), 5 deletions(-)
 create mode 100644 base-auto-train.sh

diff --git a/auto-eval.sh b/auto-eval.sh
index 0d4f7f0..f219aea 100644
--- a/auto-eval.sh
+++ b/auto-eval.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+sh base-auto-train.sh
+
 start=$(date +%s)
 
 # Detect the number of NVIDIA GPUs and create a device string
diff --git a/base-auto-train.sh b/base-auto-train.sh
new file mode 100644
index 0000000..10d6957
--- /dev/null
+++ b/base-auto-train.sh
@@ -0,0 +1,91 @@
+#!/bin/bash
+set -e  # Exit the script if any statement returns a non-true return value
+
+# ---------------------------------------------------------------------------- #
+#                          Function Definitions                                #
+# ---------------------------------------------------------------------------- #
+
+# Start nginx service
+start_nginx() {
+    echo "Starting Nginx service..."
+    service nginx start
+}
+
+# Execute script if exists
+execute_script() {
+    if [[ -f ${script_path} ]]; then
+        echo "${script_msg}"
+        bash ${script_path}
+    fi
+}
+
+# Setup ssh
+setup_ssh() {
+    if [[ $PUBLIC_KEY ]]; then
+        echo "Setting up SSH..."
+        mkdir -p ~/.ssh
+        echo "$PUBLIC_KEY" >> ~/.ssh/authorized_keys
+        chmod 700 -R ~/.ssh
+
+         if [ ! -f /etc/ssh/ssh_host_rsa_key ]; then
+            ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -q -N ''
+        fi
+
+        if [ ! -f /etc/ssh/ssh_host_dsa_key ]; then
+            ssh-keygen -t dsa -f /etc/ssh/ssh_host_dsa_key -q -N ''
+        fi
+
+        if [ ! -f /etc/ssh/ssh_host_ecdsa_key ]; then
+            ssh-keygen -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key -q -N ''
+        fi
+
+        if [ ! -f /etc/ssh/ssh_host_ed25519_key ]; then
+            ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -q -N ''
+        fi
+
+        service ssh start
+
+        echo "SSH host keys:"
+        cat /etc/ssh/*.pub
+    fi
+}
+
+# Export env vars
+export_env_vars() {
+    echo "Exporting environment variables..."
+    printenv | grep -E '^RUNPOD_|^PATH=|^_=' | awk -F = '{ print "export " $1 "=\"" $2 "\"" }' >> /etc/rp_environment
+    echo 'source /etc/rp_environment' >> ~/.bashrc
+}
+
+# Start jupyter lab
+start_jupyter() {
+    if [[ $JUPYTER_PASSWORD ]]; then
+        echo "Starting Jupyter Lab..."
+        mkdir -p /workspace && \
+        cd / && \
+        nohup jupyter lab --allow-root --no-browser --port=8888 --ip=* --FileContentsManager.delete_to_trash=False --ServerApp.terminado_settings='{"shell_command":["/bin/bash"]}' --ServerApp.token=$JUPYTER_PASSWORD --ServerApp.allow_origin=* --ServerApp.preferred_dir=/workspace &> /jupyter.log &
+        echo "Jupyter Lab started"
+        nohup jupyter lab --allow-root --no-browser --port=8888 --ip=* --FileContentsManager.delete_to_trash=False --ServerApp.terminado_settings='{"shell_command":["/bin/bash"]}' --ServerApp.token=$JUPYTER_PASSWORD --ServerApp.allow_origin=* --ServerApp.preferred_dir=/workspace &> /jupyter.log &
+        echo "Jupyter Lab started"
+    fi
+}
+
+# ---------------------------------------------------------------------------- #
+#                               Main Program                                   #
+# ---------------------------------------------------------------------------- #
+
+start_nginx
+
+execute_script "/pre_start.sh" "Running pre-start script..."
+
+echo "Pod Started"
+
+setup_ssh
+start_jupyter
+export_env_vars
+
+execute_script "/post_start.sh" "Running post-start script..."
+
+echo "Start script(s) finished, pod is ready to use."
+
+#sleep infinity
diff --git a/pre-auto-eval.sh b/pre-auto-eval.sh
index f16e015..b1f4725 100644
--- a/pre-auto-eval.sh
+++ b/pre-auto-eval.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+sh base-auto-train.sh
+
 # Detect the number of NVIDIA GPUs and create a device string
 gpu_count=$(nvidia-smi -L | wc -l)
 if [ $gpu_count -eq 0 ]; then
@@ -11,12 +13,10 @@ fi
 apt update
 apt install -y screen vim git-lfs
 
-# Run evaluation
+echo "start=$(date +%s); lm_eval --model hf --model_args pretrained=/workspace/model,trust_remote_code=$TRUST_REMOTE_CODE,parallelize=True,load_in_4bit=True --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine  --device cuda:0  --batch_size auto  --limit 100 | tee result.log; end=$(date +%s)" > run-eval.sh 
+echo "cd /workspace/; python /workspace/llm-autoeval/upload-result.py . $(($end-$start))" > run-upload.sh 
+
 cd /workspace/; mkdir -p cache model; git clone https://github.com/chenhaodev/lm-evaluation-harness; cd lm-evaluation-harness; pip install -e .;
 pip install huggingface_hub; huggingface-cli login --token $HF_TOKEN; huggingface-cli download --resume-download $MODEL --local-dir /workspace/model --local-dir-use-symlinks False --cache-dir /workspace/cache;
 
-echo "start=$(date +%s); lm_eval --model hf --model_args pretrained=/workspace/model,trust_remote_code=$TRUST_REMOTE_CODE,parallelize=True,load_in_4bit=True --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine  --device cuda:0  --batch_size auto  --limit 100 | tee result.log; end=$(date +%s)"
-echo "Elapsed Time: $(($end-$start)) seconds" >> ./result.log
-echo "cd /workspace/; python /workspace/llm-autoeval/upload-result.py . $(($end-$start))"
-
 sleep infinity

From 6e15a98533e758c2e7e2a55de0393990b8240c11 Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Mon, 19 Feb 2024 00:06:23 +0800
Subject: [PATCH 19/22] update

---
 pre-auto-eval.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pre-auto-eval.sh b/pre-auto-eval.sh
index b1f4725..85ab0f8 100644
--- a/pre-auto-eval.sh
+++ b/pre-auto-eval.sh
@@ -13,8 +13,8 @@ fi
 apt update
 apt install -y screen vim git-lfs
 
-echo "start=$(date +%s); lm_eval --model hf --model_args pretrained=/workspace/model,trust_remote_code=$TRUST_REMOTE_CODE,parallelize=True,load_in_4bit=True --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine  --device cuda:0  --batch_size auto  --limit 100 | tee result.log; end=$(date +%s)" > run-eval.sh 
-echo "cd /workspace/; python /workspace/llm-autoeval/upload-result.py . $(($end-$start))" > run-upload.sh 
+echo "lm_eval --model hf --model_args pretrained=/workspace/model,trust_remote_code=$TRUST_REMOTE_CODE,parallelize=True,load_in_4bit=True --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine  --device cuda:0  --batch_size auto  --limit 100 | tee result.log; " > run-eval.sh 
+echo "cd /workspace/; python /workspace/llm-autoeval/upload-result.py . 9999" > run-upload.sh 
 
 cd /workspace/; mkdir -p cache model; git clone https://github.com/chenhaodev/lm-evaluation-harness; cd lm-evaluation-harness; pip install -e .;
 pip install huggingface_hub; huggingface-cli login --token $HF_TOKEN; huggingface-cli download --resume-download $MODEL --local-dir /workspace/model --local-dir-use-symlinks False --cache-dir /workspace/cache;

From 57affe883c56152e178045c7848262a0f90d32e4 Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Mon, 19 Feb 2024 00:36:00 +0800
Subject: [PATCH 20/22] fix bug

---
 auto-eval.sh     | 3 ++-
 pre-auto-eval.sh | 6 ++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/auto-eval.sh b/auto-eval.sh
index f219aea..080e02f 100644
--- a/auto-eval.sh
+++ b/auto-eval.sh
@@ -17,10 +17,11 @@ apt install -y screen vim git-lfs
 
 # Run evaluation
 cd /workspace/; mkdir -p cache model; git clone https://github.com/chenhaodev/lm-evaluation-harness; cd lm-evaluation-harness; pip install -e .;
+pip install transformers_stream_generator einops bitsandbyte tiktoken;
 pip install huggingface_hub; huggingface-cli login --token $HF_TOKEN; huggingface-cli download --resume-download $MODEL --local-dir /workspace/model --local-dir-use-symlinks False --cache-dir /workspace/cache;
 
 lm_eval --model hf \
-    --model_args pretrained=/workspace/model,trust_remote_code=$TRUST_REMOTE_CODE,parallelize=True,load_in_4bit=True \
+    --model_args pretrained=/workspace/model,trust_remote_code=True,parallelize=True,load_in_4bit=True \
     --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine \
     --device cuda:0 \
     --batch_size auto \
diff --git a/pre-auto-eval.sh b/pre-auto-eval.sh
index 85ab0f8..15266e6 100644
--- a/pre-auto-eval.sh
+++ b/pre-auto-eval.sh
@@ -13,10 +13,12 @@ fi
 apt update
 apt install -y screen vim git-lfs
 
-echo "lm_eval --model hf --model_args pretrained=/workspace/model,trust_remote_code=$TRUST_REMOTE_CODE,parallelize=True,load_in_4bit=True --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine  --device cuda:0  --batch_size auto  --limit 100 | tee result.log; " > run-eval.sh 
+cd /workspace/; mkdir -p cache model; git clone https://github.com/chenhaodev/lm-evaluation-harness; cd lm-evaluation-harness; pip install -e .;
+pip install transformers_stream_generator einops bitsandbyte tiktoken;
+
+echo "lm_eval --model hf --model_args pretrained=/workspace/model,trust_remote_code=True,parallelize=True,load_in_4bit=True --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine  --device cuda:0  --batch_size auto  --limit 100 | tee result.log; " > run-eval.sh 
 echo "cd /workspace/; python /workspace/llm-autoeval/upload-result.py . 9999" > run-upload.sh 
 
-cd /workspace/; mkdir -p cache model; git clone https://github.com/chenhaodev/lm-evaluation-harness; cd lm-evaluation-harness; pip install -e .;
 pip install huggingface_hub; huggingface-cli login --token $HF_TOKEN; huggingface-cli download --resume-download $MODEL --local-dir /workspace/model --local-dir-use-symlinks False --cache-dir /workspace/cache;
 
 sleep infinity

From b4302d04b1112227e442f0eca98b0e0c585b7c52 Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Mon, 19 Feb 2024 01:02:37 +0800
Subject: [PATCH 21/22] update

---
 scripts/submit-result/submit-results.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/submit-result/submit-results.py b/scripts/submit-result/submit-results.py
index 4e2b21f..361356f 100644
--- a/scripts/submit-result/submit-results.py
+++ b/scripts/submit-result/submit-results.py
@@ -8,7 +8,7 @@
 from llm_autoeval.upload import upload_to_github_gist
 
 MODEL = 'abacusai/Smaug-72B-v0.1'
-GITHUB_API_TOKEN = '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'
+GITHUB_API_TOKEN = 'ghp_xxxx'
 
 def main(directory: str, elapsed_time: float) -> None:
     file_path = "result.log"

From 5d7ca10b2d3c59e484ba31543d18fd2805a4b4a6 Mon Sep 17 00:00:00 2001
From: chenhao <chenhaomails@gmail.com>
Date: Thu, 29 Feb 2024 22:03:02 +0800
Subject: [PATCH 22/22] update pip install

---
 auto-eval.sh     | 2 +-
 pre-auto-eval.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/auto-eval.sh b/auto-eval.sh
index 080e02f..78fa1fe 100644
--- a/auto-eval.sh
+++ b/auto-eval.sh
@@ -17,7 +17,7 @@ apt install -y screen vim git-lfs
 
 # Run evaluation
 cd /workspace/; mkdir -p cache model; git clone https://github.com/chenhaodev/lm-evaluation-harness; cd lm-evaluation-harness; pip install -e .;
-pip install transformers_stream_generator einops bitsandbyte tiktoken;
+pip install transformers_stream_generator einops bitsandbytes tiktoken;
 pip install huggingface_hub; huggingface-cli login --token $HF_TOKEN; huggingface-cli download --resume-download $MODEL --local-dir /workspace/model --local-dir-use-symlinks False --cache-dir /workspace/cache;
 
 lm_eval --model hf \
diff --git a/pre-auto-eval.sh b/pre-auto-eval.sh
index 15266e6..9a99725 100644
--- a/pre-auto-eval.sh
+++ b/pre-auto-eval.sh
@@ -14,7 +14,7 @@ apt update
 apt install -y screen vim git-lfs
 
 cd /workspace/; mkdir -p cache model; git clone https://github.com/chenhaodev/lm-evaluation-harness; cd lm-evaluation-harness; pip install -e .;
-pip install transformers_stream_generator einops bitsandbyte tiktoken;
+pip install transformers_stream_generator einops bitsandbytes tiktoken;
 
 echo "lm_eval --model hf --model_args pretrained=/workspace/model,trust_remote_code=True,parallelize=True,load_in_4bit=True --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine  --device cuda:0  --batch_size auto  --limit 100 | tee result.log; " > run-eval.sh 
 echo "cd /workspace/; python /workspace/llm-autoeval/upload-result.py . 9999" > run-upload.sh