Ch dev

auto-eval.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+sh base-auto-train.sh
+
+start=$(date +%s)
+
+# Detect the number of NVIDIA GPUs and create a device string
+gpu_count=$(nvidia-smi -L | wc -l)
+if [ $gpu_count -eq 0 ]; then
+    echo "No NVIDIA GPUs detected. Exiting."
+    exit 1
+fi
+
+# Install dependencies
+apt update
+apt install -y screen vim git-lfs
+
+# Run evaluation
+cd /workspace/; mkdir -p cache model; git clone https://github.com/chenhaodev/lm-evaluation-harness; cd lm-evaluation-harness; pip install -e .;
+pip install transformers_stream_generator einops bitsandbytes tiktoken;
+pip install huggingface_hub; huggingface-cli login --token $HF_TOKEN; huggingface-cli download --resume-download $MODEL --local-dir /workspace/model --local-dir-use-symlinks False --cache-dir /workspace/cache;
+
+lm_eval --model hf \
+    --model_args pretrained=/workspace/model,trust_remote_code=True,parallelize=True,load_in_4bit=True \
+    --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine \
+    --device cuda:0 \
+    --batch_size auto \
+    --limit 100 | tee result.log
+    #--output_path ./result
+
+end=$(date +%s)
+echo "Elapsed Time: $(($end-$start)) seconds" >> ./result.log
+
+cd /workspace/; python /workspace/llm-autoeval/upload-result.py . $(($end-$start))
+
+#if [ "$DEBUG" == "False" ]; then
+#    runpodctl remove pod $RUNPOD_POD_ID
+#fi
+
+sleep infinity

base-auto-train.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+set -e  # Exit the script if any statement returns a non-true return value
+
+# ---------------------------------------------------------------------------- #
+#                          Function Definitions                                #
+# ---------------------------------------------------------------------------- #
+
+# Start nginx service
+start_nginx() {
+    echo "Starting Nginx service..."
+    service nginx start
+}
+
+# Execute script if exists
+execute_script() {
+    if [[ -f ${script_path} ]]; then
+        echo "${script_msg}"
+        bash ${script_path}
+    fi
+}
+
+# Setup ssh
+setup_ssh() {
+    if [[ $PUBLIC_KEY ]]; then
+        echo "Setting up SSH..."
+        mkdir -p ~/.ssh
+        echo "$PUBLIC_KEY" >> ~/.ssh/authorized_keys
+        chmod 700 -R ~/.ssh
+
+         if [ ! -f /etc/ssh/ssh_host_rsa_key ]; then
+            ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -q -N ''
+        fi
+
+        if [ ! -f /etc/ssh/ssh_host_dsa_key ]; then
+            ssh-keygen -t dsa -f /etc/ssh/ssh_host_dsa_key -q -N ''
+        fi
+
+        if [ ! -f /etc/ssh/ssh_host_ecdsa_key ]; then
+            ssh-keygen -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key -q -N ''
+        fi
+
+        if [ ! -f /etc/ssh/ssh_host_ed25519_key ]; then
+            ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -q -N ''
+        fi
+
+        service ssh start
+
+        echo "SSH host keys:"
+        cat /etc/ssh/*.pub
+    fi
+}
+
+# Export env vars
+export_env_vars() {
+    echo "Exporting environment variables..."
+    printenv | grep -E '^RUNPOD_|^PATH=|^_=' | awk -F = '{ print "export " $1 "=\"" $2 "\"" }' >> /etc/rp_environment
+    echo 'source /etc/rp_environment' >> ~/.bashrc
+}
+
+# Start jupyter lab
+start_jupyter() {
+    if [[ $JUPYTER_PASSWORD ]]; then
+        echo "Starting Jupyter Lab..."
+        mkdir -p /workspace && \
+        cd / && \
+        nohup jupyter lab --allow-root --no-browser --port=8888 --ip=* --FileContentsManager.delete_to_trash=False --ServerApp.terminado_settings='{"shell_command":["/bin/bash"]}' --ServerApp.token=$JUPYTER_PASSWORD --ServerApp.allow_origin=* --ServerApp.preferred_dir=/workspace &> /jupyter.log &
+        echo "Jupyter Lab started"
+        nohup jupyter lab --allow-root --no-browser --port=8888 --ip=* --FileContentsManager.delete_to_trash=False --ServerApp.terminado_settings='{"shell_command":["/bin/bash"]}' --ServerApp.token=$JUPYTER_PASSWORD --ServerApp.allow_origin=* --ServerApp.preferred_dir=/workspace &> /jupyter.log &
+        echo "Jupyter Lab started"
+    fi
+}
+
+# ---------------------------------------------------------------------------- #
+#                               Main Program                                   #
+# ---------------------------------------------------------------------------- #
+
+start_nginx
+
+execute_script "/pre_start.sh" "Running pre-start script..."
+
+echo "Pod Started"
+
+setup_ssh
+start_jupyter
+export_env_vars
+
+execute_script "/post_start.sh" "Running post-start script..."
+
+echo "Start script(s) finished, pod is ready to use."
+
+#sleep infinity

create_pods.py

@@ -0,0 +1,48 @@
+#pip install -qqq runpod --progress-bar off
+
+import runpod
+import sys
+
+BENCHMARK = "medtasks"
+MODEL = sys.argv[1] #"mlabonne/NeuralMarcoro14-7B" # @param {type:"string"}
+GPU = sys.argv[2] #"NVIDIA GeForce RTX 3090" # @param ["NVIDIA A100 80GB PCIe", "NVIDIA A100-SXM4-80GB", "NVIDIA A30", "NVIDIA A40", "NVIDIA GeForce RTX 3070", "NVIDIA GeForce RTX 3080", "NVIDIA GeForce RTX 3080 Ti", "NVIDIA GeForce RTX 3090", "NVIDIA GeForce RTX 3090 Ti", "NVIDIA GeForce RTX 4070 Ti", "NVIDIA GeForce RTX 4080", "NVIDIA GeForce RTX 4090", "NVIDIA H100 80GB HBM3", "NVIDIA H100 PCIe", "NVIDIA L4", "NVIDIA L40", "NVIDIA RTX 4000 Ada Generation", "NVIDIA RTX 4000 SFF Ada Generation", "NVIDIA RTX 5000 Ada Generation", "NVIDIA RTX 6000 Ada Generation", "NVIDIA RTX A2000", "NVIDIA RTX A4000", "NVIDIA RTX A4500", "NVIDIA RTX A5000", "NVIDIA RTX A6000", "Tesla V100-FHHL-16GB", "Tesla V100-PCIE-16GB", "Tesla V100-SXM2-16GB", "Tesla V100-SXM2-32GB"]
+NUMBER_OF_GPUS = sys.argv[3] # @param {type:"slider", min:1, max:8, step:1}
+CONTAINER_DISK = 200 # @param {type:"slider", min:50, max:500, step:25}
+VOLUME_IN_GB = 0
+CLOUD_TYPE = "COMMUNITY" # @param ["COMMUNITY", "SECURE"]
+REPO = "https://github.com/chenhaodev/llm-autoeval.git" # @param {type:"string"}
+TRUST_REMOTE_CODE = False # @param {type:"boolean"}
+DEBUG = False # @param {type:"boolean"}
+
+# @markdown ---
+RUNPOD_TOKEN = sys.argv[4] #"runpod" # @param {type:"string"}
+GITHUB_TOKEN = sys.argv[5] #"github" # @param {type:"string"}
+HF_TOKEN = sys.argv[6]
+RUNPOD_TEMPLATE = sys.argv[7] #if template id==6p59tg6cln, one-template-for-all-steps, e.g. download model, auto-eval, upload-results; if id==6rhltjf914, the script only download model; #bash -c 'cd /workspace/; git clone https://github.com/chenhaodev/llm-autoeval; cd /workspace/llm-autoeval/; sh auto-eval.sh' #template_id="au6nz6emhk",
+
+# Environment variables
+runpod.api_key = RUNPOD_TOKEN
+GITHUB_API_TOKEN = GITHUB_TOKEN
+
+# Create a pod
+pod = runpod.create_pod(
+    name=f"Eval {MODEL.split('/')[-1]} on {BENCHMARK.capitalize()}",
+    image_name="runpod/pytorch:2.0.1-py3.10-cuda11.8.0-devel-ubuntu22.04",
+    gpu_type_id=GPU,
+    cloud_type=CLOUD_TYPE,
+    gpu_count=NUMBER_OF_GPUS,
+    volume_in_gb=VOLUME_IN_GB,
+    container_disk_in_gb=CONTAINER_DISK,
+    env={
+        "BENCHMARK": BENCHMARK,
+        "MODEL": MODEL,
+        "REPO": REPO,
+        "TRUST_REMOTE_CODE": TRUST_REMOTE_CODE,
+        "DEBUG": DEBUG,
+        "GITHUB_API_TOKEN": GITHUB_API_TOKEN,
+        "HF_TOKEN": HF_TOKEN,
+    },
+    template_id=RUNPOD_TEMPLATE, 
+)
+
+print("Pod started: https://www.runpod.io/console/pods")

pre-auto-eval.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+sh base-auto-train.sh
+
+# Detect the number of NVIDIA GPUs and create a device string
+gpu_count=$(nvidia-smi -L | wc -l)
+if [ $gpu_count -eq 0 ]; then
+    echo "No NVIDIA GPUs detected. Exiting."
+    exit 1
+fi
+
+# Install dependencies
+apt update
+apt install -y screen vim git-lfs
+
+cd /workspace/; mkdir -p cache model; git clone https://github.com/chenhaodev/lm-evaluation-harness; cd lm-evaluation-harness; pip install -e .;
+pip install transformers_stream_generator einops bitsandbytes tiktoken;
+
+echo "lm_eval --model hf --model_args pretrained=/workspace/model,trust_remote_code=True,parallelize=True,load_in_4bit=True --tasks ocn,aocnp,medmcqa,pubmedqa,mmlu_clinical_knowledge,mmlu_college_medicine,mmlu_professional_medicine  --device cuda:0  --batch_size auto  --limit 100 | tee result.log; " > run-eval.sh 
+echo "cd /workspace/; python /workspace/llm-autoeval/upload-result.py . 9999" > run-upload.sh 
+
+pip install huggingface_hub; huggingface-cli login --token $HF_TOKEN; huggingface-cli download --resume-download $MODEL --local-dir /workspace/model --local-dir-use-symlinks False --cache-dir /workspace/cache;
+
+sleep infinity

scripts/submit-result/result.log

@@ -0,0 +1,17 @@
+Passed argument batch_size = auto:1. Detecting largest batch size
+Determined largest batch size: 4
+hf (pretrained=/workspace/model,trust_remote_code=False,parallelize=True), gen_kwargs: (None), limit: 100.0, num_fewshot: None, batch_size: auto (4)
+|        Tasks        |Version|Filter|n-shot| Metric |Value|   |Stderr|
+|---------------------|-------|------|------|--------|----:|---|-----:|
+|pubmedqa             |      1|none  |None  |acc     | 0.97|±  |0.0171|
+|ocn                  |Yaml   |none  |None  |acc     | 0.78|±  |0.0416|
+|                     |       |none  |None  |acc_norm| 0.78|±  |0.0416|
+|professional_medicine|      0|none  |None  |acc     | 0.89|±  |0.0314|
+|college_medicine     |      0|none  |None  |acc     | 0.80|±  |0.0402|
+|clinical_knowledge   |      0|none  |None  |acc     | 0.84|±  |0.0368|
+|medmcqa              |Yaml   |none  |None  |acc     | 0.54|±  |0.0501|
+|                     |       |none  |None  |acc_norm| 0.54|±  |0.0501|
+|aocnp                |Yaml   |none  |None  |acc     | 0.74|±  |0.0441|
+|                     |       |none  |None  |acc_norm| 0.74|±  |0.0441|
+
+Elapsed Time: 1417 seconds

main.py → scripts/submit-result/submit-results.py

@@ -4,20 +4,20 @@
 import argparse
 import time
 
-from lighteval.evaluator import make_results_table
-
 from llm_autoeval.table import make_table, make_final_table
 from llm_autoeval.upload import upload_to_github_gist
 
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-MODEL_ID = os.getenv("MODEL_ID")
-BENCHMARK = os.getenv("BENCHMARK")
-GITHUB_API_TOKEN = os.getenv("GITHUB_API_TOKEN")
+MODEL = 'abacusai/Smaug-72B-v0.1'
+GITHUB_API_TOKEN = 'ghp_xxxx'
 
+def main(directory: str, elapsed_time: float) -> None:
+    file_path = "result.log"
+    summary = open(file_path, "r").read()
+    upload_to_github_gist(
+        summary, f"{MODEL.split('/')[-1]}-MedTasks.md", GITHUB_API_TOKEN
+    )
 
-def _make_autoeval_summary(directory: str, elapsed_time: float) -> str:
+    '''
     # Variables
     tables = []
     averages = []
@@ -28,9 +28,7 @@ def _make_autoeval_summary(directory: str, elapsed_time: float) -> str:
     elif BENCHMARK == "nous":
         tasks = ["AGIEval", "GPT4All", "TruthfulQA", "Bigbench"]
     else:
-        raise NotImplementedError(
-            f"BENCHMARK should be 'openllm' or 'nous' (current value = {BENCHMARK})"
-        )
+        raise NotImplementedError(f"BENCHMARK should be 'openllm' or 'nous' (current value = {BENCHMARK})")
 
     # Load results
     for task in tasks:
@@ -60,64 +58,26 @@ def _make_autoeval_summary(directory: str, elapsed_time: float) -> str:
     else:
         summary += "Average score: Not available due to errors"
 
-    # Generate final table
-    final_table = make_final_table(result_dict, MODEL_ID)
-    summary = final_table + "\n" + summary
-    return summary
-
-
-def _get_result_dict(directory: str) -> dict:
-    """Walk down driectories to get JSON path"""
-
-    for root, dirs, files in os.walk(directory):
-        for file in files:
-            if file.endswith(".json"):
-                return json.load(open(os.path.join(root, file)))
-    raise FileNotFoundError(f"No JSON file found in {directory}")
-
-
-def _make_lighteval_summary(directory: str, elapsed_time: float) -> str:
-    result_dict = _get_result_dict(directory)
-    final_table = make_results_table(result_dict)
-    summary = f"## {MODEL_ID.split('/')[-1]} - {BENCHMARK.capitalize()}\n\n"
-    summary += final_table
-    return summary
-
-
-def main(directory: str, elapsed_time: float) -> None:
-    # Tasks
-    if BENCHMARK == "openllm" or BENCHMARK == "nous":
-        summary = _make_autoeval_summary(directory, elapsed_time)
-    elif BENCHMARK == "lighteval":
-        summary = _make_lighteval_summary(directory, elapsed_time)
-    else:
-        raise NotImplementedError(
-            f"BENCHMARK should be 'openllm' or 'nous' (current value = {BENCHMARK})"
-        )
-
     # Add elapsed time
     convert = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
     summary += f"\n\nElapsed time: {convert}"
 
+    # Generate final table
+    final_table = make_final_table(result_dict, MODEL)
+    summary = final_table + "\n" + summary
+
     # Upload to GitHub Gist
     upload_to_github_gist(
-        summary,
-        f"{MODEL_ID.split('/')[-1]}-{BENCHMARK.capitalize()}.md",
-        GITHUB_API_TOKEN,
+        summary, f"{MODEL.split('/')[-1]}-{BENCHMARK.capitalize()}.md", GITHUB_API_TOKEN
     )
+    '''
 
 
 if __name__ == "__main__":
     # Create the parser
     parser = argparse.ArgumentParser(description="Summarize results and upload them.")
-    parser.add_argument(
-        "directory", type=str, help="The path to the directory with the JSON results"
-    )
-    parser.add_argument(
-        "elapsed_time",
-        type=float,
-        help="Elapsed time since the start of the evaluation",
-    )
+    parser.add_argument("directory", type=str, help="The path to the directory with the JSON results")
+    parser.add_argument("elapsed_time", type=float, help="Elapsed time since the start of the evaluation")
 
     # Parse the arguments
     args = parser.parse_args()

upload-result.py

@@ -0,0 +1,46 @@
+import json
+import logging
+import os
+import argparse
+import time
+
+from lighteval.evaluator import make_results_table
+
+from llm_autoeval.table import make_table, make_final_table
+from llm_autoeval.upload import upload_to_github_gist
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+MODEL_ID = os.getenv("MODEL_ID")
+BENCHMARK = os.getenv("BENCHMARK")
+GITHUB_API_TOKEN = os.getenv("GITHUB_API_TOKEN")
+
+def main(directory: str, elapsed_time: float) -> None:
+    file_path = "/workspace/lm-evaluation-harness/result.log"
+    summary = open(file_path, "r").read()
+    upload_to_github_gist(
+        summary, f"{MODEL.split('/')[-1]}-MedTasks.md", GITHUB_API_TOKEN
+    )
+
+if __name__ == "__main__":
+    # Create the parser
+    parser = argparse.ArgumentParser(description="Summarize results and upload them.")
+    parser.add_argument(
+        "directory", type=str, help="The path to the directory with the JSON results"
+    )
+    parser.add_argument(
+        "elapsed_time",
+        type=float,
+        help="Elapsed time since the start of the evaluation",
+    )
+
+    # Parse the arguments
+    args = parser.parse_args()
+
+    # Check if the directory exists
+    if not os.path.isdir(args.directory):
+        raise ValueError(f"The directory {args.directory} does not exist.")
+
+    # Call the main function with the directory argument
+    main(args.directory, args.elapsed_time)