diff --git a/.github/scripts/dcperf_run_django.bash b/.github/scripts/dcperf_run_django.bash new file mode 100644 index 00000000..912255d4 --- /dev/null +++ b/.github/scripts/dcperf_run_django.bash @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + + +# shellcheck disable=SC1091,SC2128 +. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash" + + +################################################################################ +# Django Run Functions +################################################################################ + +run_django () { + local env_name="$1" + if [ "$env_name" == "" ]; then + echo "Usage: ${FUNCNAME[0]} ENV_NAME" + echo "Example(s):" + echo " ${FUNCNAME[0]} build_env" + return 1 + else + echo "################################################################################" + echo "# Run Django" + echo "#" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" + echo "################################################################################" + echo "" + fi + + + # shellcheck disable=SC2155 + local env_prefix=$(env_name_or_prefix "${env_name}") + + echo "[RUN] Running Django ..." + + # (print_exec conda run --no-capture-output ${env_prefix} \ + # python ./benchpress_cli.py run django_workload_default -r standalone -i "'{\"reps\": 1000, \"iterations\": 1}'") || return 1 + ./benchmarks/django_workload/bin/run.sh -r standalone -d 5M -i 1 p 1000 -l /siege.log -s urls.txt -c 127.0.0.1 -I cpython || true + echo "PWD" + echo $PWD + echo "Searching for django-uwsgi.log files:" + find_output=$(find . -name django-uwsgi.log) + echo "$find_output" + # cat django-workload/django-workload/django-uwsgi.log +} diff --git a/.github/scripts/dcperf_run_feedsim.bash b/.github/scripts/dcperf_run_feedsim.bash new file mode 100644 index 00000000..666df711 --- /dev/null +++ b/.github/scripts/dcperf_run_feedsim.bash @@ -0,0 +1,93 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + + +# shellcheck disable=SC1091,SC2128 +. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash" + + +################################################################################ +# Run FeedSim +################################################################################ + +run_feedsim () { + local env_name="$1" + if [ "$env_name" == "" ]; then + echo "Usage: ${FUNCNAME[0]} ENV_NAME" + echo "Example(s):" + echo " ${FUNCNAME[0]} build_env" + return 1 + else + echo "################################################################################" + echo "# Run FeedSim" + echo "#" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" + echo "################################################################################" + echo "" + fi + + # shellcheck disable=SC2155 + local env_prefix=$(env_name_or_prefix "${env_name}") + + echo "[RUN] Running FeedSim ..." + (print_exec conda run --no-capture-output ${env_prefix} \ + python ./benchpress_cli.py run feedsim_autoscale -i "'{\"extra_args\": \"-q 5 -d 20\"}'") || true + cat benchpress.log + + # Print all feedsim results and logs from benchmark_metrics_* directories + echo "" + echo "[INFO] Searching for FeedSim results in benchmark_metrics_* directories..." + for metrics_dir in benchmark_metrics_*/; do + if [ -d "$metrics_dir" ]; then + echo "" + echo "=== Found directory: $metrics_dir ===" + + # Print all feedsim_results*.txt files + for results_file in "${metrics_dir}"feedsim_results*.txt; do + if [ -f "$results_file" ]; then + echo "" + echo "--- Contents of $results_file ---" + cat "$results_file" + fi + done + + # Print all feedsim-multi-inst-*.log files + for log_file in "${metrics_dir}"feedsim-multi-inst-*.log; do + if [ -f "$log_file" ]; then + echo "" + echo "--- Contents of $log_file ---" + cat "$log_file" + fi + done + fi + done + + # Run benchpress and capture output + # if output=$(print_exec conda run --no-capture-output ${env_prefix} python ./benchpress_cli.py run feedsim_autoscale -i "'{\"extra_args\": \"-q 5 -d 20\"}'" 2>&1); then + # echo "$output" + # else + # echo "ERROR: Benchpress command failed to execute" + # return 1 + # fi + + # # Extract final_achieved_qps from the output and validate + # final_qps=$(echo "$output" | grep -A3 '"overall"' | grep -o '"final_achieved_qps":[^,]*' | grep -o '[0-9.]*' | head -1) + + # if [ -z "$final_qps" ]; then + # echo "ERROR: Could not extract final_achieved_qps from output" + # return 1 + # fi + + # # Check if final_achieved_qps is greater than zero + # if (( $(echo "$final_qps > 0" | bc -l) )); then + # echo "SUCCESS: final_achieved_qps ($final_qps) is greater than zero" + # return 0 + # else + # echo "FAILURE: final_achieved_qps ($final_qps) is not greater than zero" + # return 1 + # fi +} diff --git a/.github/scripts/dcperf_run_mediawiki.bash b/.github/scripts/dcperf_run_mediawiki.bash new file mode 100644 index 00000000..1e02be44 --- /dev/null +++ b/.github/scripts/dcperf_run_mediawiki.bash @@ -0,0 +1,61 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + + +# shellcheck disable=SC1091,SC2128 +. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash" + + +################################################################################ +# Run Mediawiki +################################################################################ + +run_mediawiki () { + local env_name="$1" + if [ "$env_name" == "" ]; then + echo "Usage: ${FUNCNAME[0]} ENV_NAME" + echo "Example(s):" + echo " ${FUNCNAME[0]} build_env" + return 1 + else + echo "################################################################################" + echo "# Run Mediawiki" + echo "#" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" + echo "################################################################################" + echo "" + fi + + # shellcheck disable=SC2155 + local env_prefix=$(env_name_or_prefix "${env_name}") + export DEBIAN_FRONTEND=noninteractive + echo "[RUN] Running Mediawiki ..." + # (print_exec conda run --no-capture-output ${env_prefix} \ + # python ./benchpress_cli.py run oss_performance_mediawiki_mini) || return 1 + + ./packages/mediawiki/run.sh \ + -r/usr/local/hphpi/legacy/bin/hhvm \ + -nnginx \ + -L wrk \ + -s benchmarks/oss_performance_mediawiki/wrk/wrk \ + -T default_no_temp_dir \ + -p -- \ + --mediawiki \ + --client-duration=4s \ + --client-timeout=60s \ + --run-as-root \ + --i-am-not-benchmarking \ + --shorten-health-check \ + --skip-single-request-warmup \ + --skip-sleep-between-warmups \ + --hhvm-extra-arguments="-vEval.JitRetranslateAllSeconds=5" \ + --num-multi-req-warmups=-1 \ + --no-load-if-pending-translate \ + --first-multi-warmup-duration=25 \ + --subseq-multi-warmup-duration=5 \ + --load-gen-seed=1000 +} diff --git a/.github/scripts/dcperf_run_spark.bash b/.github/scripts/dcperf_run_spark.bash new file mode 100644 index 00000000..647e9a86 --- /dev/null +++ b/.github/scripts/dcperf_run_spark.bash @@ -0,0 +1,68 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + + +# shellcheck disable=SC1091,SC2128 +. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash" + + +################################################################################ +# Run Spark +################################################################################ + +run_spark () { + local env_name="$1" + if [ "$env_name" == "" ]; then + echo "Usage: ${FUNCNAME[0]} ENV_NAME" + echo "Example(s):" + echo " ${FUNCNAME[0]} build_env" + return 1 + else + echo "################################################################################" + echo "# Run Spark" + echo "#" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" + echo "################################################################################" + echo "" + fi + + # shellcheck disable=SC2155 + local env_prefix=$(env_name_or_prefix "${env_name}") + + echo "[RUN] Running Spark ..." + wget https://github.com/facebookresearch/DCPerf-datasets/releases/download/v1.0/bpc_t93586_s2_synthetic_1GB.tar.gz + tar -xzf bpc_t93586_s2_synthetic_1GB.tar.gz + mkdir /flash23 + cp -rf bpc_t93586_s2_synthetic_1GB /flash23/ + (print_exec conda run --no-capture-output ${env_prefix} \ + python ./benchpress_cli.py run spark_standalone_remote_mini -i "'{\"dataset_name\":\"bpc_t93586_s2_synthetic_1GB\"}'") || return 1 + + # Print all files from benchmark_metrics_*/work directories + echo "" + echo "[INFO] Searching for files in benchmark_metrics_*/work directories..." + for metrics_dir in benchmark_metrics_*/; do + if [ -d "$metrics_dir" ]; then + echo "" + echo "=== Found directory: $metrics_dir ===" + + work_dir="${metrics_dir}work" + if [ -d "$work_dir" ]; then + echo "" + echo "=== Checking work directory: $work_dir ===" + + # Print all files in the work directory + for file in "$work_dir"/*; do + if [ -f "$file" ]; then + echo "" + echo "--- Contents of $file ---" + cat "$file" + fi + done + fi + fi + done +} diff --git a/.github/scripts/dcperf_run_syscall.bash b/.github/scripts/dcperf_run_syscall.bash new file mode 100644 index 00000000..7ef67f0c --- /dev/null +++ b/.github/scripts/dcperf_run_syscall.bash @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + + +# shellcheck disable=SC1091,SC2128 +. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash" + + +################################################################################ +# Run syscall +################################################################################ + +run_syscall () { + local env_name="$1" + if [ "$env_name" == "" ]; then + echo "Usage: ${FUNCNAME[0]} ENV_NAME" + echo "Example(s):" + echo " ${FUNCNAME[0]} build_env" + return 1 + else + echo "################################################################################" + echo "# Run Syscall" + echo "#" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" + echo "################################################################################" + echo "" + fi + + # shellcheck disable=SC2155 + local env_prefix=$(env_name_or_prefix "${env_name}") + + echo "[RUN] Running Syscall ..." + (print_exec conda run --no-capture-output ${env_prefix} \ + python ./benchpress_cli.py -b system run syscall_single_core) || return 1 +} diff --git a/.github/scripts/dcperf_run_tao_bench.bash b/.github/scripts/dcperf_run_tao_bench.bash new file mode 100644 index 00000000..21b23719 --- /dev/null +++ b/.github/scripts/dcperf_run_tao_bench.bash @@ -0,0 +1,63 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + + +# shellcheck disable=SC1091,SC2128 +. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash" + + +################################################################################ +# tao_bench Run Functions +################################################################################ + +run_tao_bench () { + local env_name="$1" + if [ "$env_name" == "" ]; then + echo "Usage: ${FUNCNAME[0]} ENV_NAME" + echo "Example(s):" + echo " ${FUNCNAME[0]} build_env" + return 1 + else + echo "################################################################################" + echo "# Run Tao_bench" + echo "#" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" + echo "################################################################################" + echo "" + fi + + + # shellcheck disable=SC2155 + local env_prefix=$(env_name_or_prefix "${env_name}") + + echo "[RUN] Running Tao_bench ..." + + # Run benchpress and capture output + if output=$(print_exec conda run --no-capture-output ${env_prefix} python ./benchpress_cli.py run tao_bench_standalone -i "'{\"memsize\": 0.5, \"stats_interval\":1000, \"warmup_time\": 60, \"test_time\": 60}'" 2>&1); then + echo "$output" + else + echo "ERROR: Benchpress command failed to execute" + return 1 + fi + + # Extract total_qps from the output and validate + total_qps=$(echo "$output" | grep -o '"total_qps":[^,]*' | grep -o '[0-9.]*' | head -1) + + if [ -z "$total_qps" ]; then + echo "ERROR: Could not extract total_qps from output" + return 1 + fi + + # Check if total_qps is greater than zero + if (( $(echo "$total_qps > 0" | bc -l) )); then + echo "SUCCESS: total_qps ($total_qps) is greater than zero" + return 0 + else + echo "FAILURE: total_qps ($total_qps) is not greater than zero" + return 1 + fi +} diff --git a/.github/scripts/setup_env.bash b/.github/scripts/setup_env.bash index 0523f91c..be784543 100755 --- a/.github/scripts/setup_env.bash +++ b/.github/scripts/setup_env.bash @@ -16,18 +16,30 @@ # shellcheck disable=SC1091,SC2128 . "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_django.bash" # shellcheck disable=SC1091,SC2128 +. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_django.bash" +# shellcheck disable=SC1091,SC2128 . "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_feedsim.bash" # shellcheck disable=SC1091,SC2128 +. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_feedsim.bash" +# shellcheck disable=SC1091,SC2128 . "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_tao_bench.bash" # shellcheck disable=SC1091,SC2128 +. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_tao_bench.bash" +# shellcheck disable=SC1091,SC2128 . "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_spark.bash" # shellcheck disable=SC1091,SC2128 +. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_spark.bash" +# shellcheck disable=SC1091,SC2128 . "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_video_transcode_bench.bash" # shellcheck disable=SC1091,SC2128 . "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_mediawiki.bash" # shellcheck disable=SC1091,SC2128 +. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_mediawiki.bash" +# shellcheck disable=SC1091,SC2128 . "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_wdl.bash" # shellcheck disable=SC1091,SC2128 . "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_health_check.bash" # shellcheck disable=SC1091,SC2128 . "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_syscall.bash" +# shellcheck disable=SC1091,SC2128 +. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_syscall.bash" diff --git a/.github/workflows/dcperf_ci_django.yml b/.github/workflows/dcperf_ci_django.yml index f97a611f..c437dd32 100644 --- a/.github/workflows/dcperf_ci_django.yml +++ b/.github/workflows/dcperf_ci_django.yml @@ -55,11 +55,11 @@ jobs: steps: - name: Setup Build Container (ubuntu-based) if: ${{ matrix.host-machine.type == 'ubuntu' }} - run: apt update -y; apt install -y build-essential git pciutils socat sudo wget libpython2-dev python3-dev + run: apt update -y; apt install -y build-essential git pciutils socat sudo wget libpython2-dev python3-dev numactl lshw dmidecode procps netcat bc - name: Setup Build Container (centos-based) if: ${{ matrix.host-machine.type == 'centos' }} - run: dnf update -y; dnf install -y git pciutils which python3-devel ; dnf group install -y "Development Tools" --exclude="texlive*" + run: dnf update -y; dnf install -y git pciutils which python3-devel numactl lshw dmidecode procps-ng nmap-ncat bc; dnf group install -y "Development Tools" --exclude="texlive*" - name: Checkout the Repository uses: actions/checkout@v4 @@ -86,3 +86,6 @@ jobs: - name: Install Django run: . $PRELUDE; install_django $BUILD_ENV + + - name: Run Django + run: . $PRELUDE; run_django $BUILD_ENV diff --git a/.github/workflows/dcperf_ci_feedsim.yml b/.github/workflows/dcperf_ci_feedsim.yml index e356843a..e54aba7c 100644 --- a/.github/workflows/dcperf_ci_feedsim.yml +++ b/.github/workflows/dcperf_ci_feedsim.yml @@ -55,11 +55,11 @@ jobs: steps: - name: Setup Build Container (ubuntu-based) if: ${{ matrix.host-machine.type == 'ubuntu' }} - run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode + run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode numactl lshw procps sysstat - name: Setup Build Container (centos-based) if: ${{ matrix.host-machine.type == 'centos' }} - run: dnf update -y; dnf install -y git pciutils which dmidecode + run: dnf update -y; dnf install -y git pciutils which dmidecode numactl lshw procps-ng sysstat - name: Checkout the Repository uses: actions/checkout@v4 @@ -86,3 +86,6 @@ jobs: - name: Install FeedSim run: . $PRELUDE; install_feedsim $BUILD_ENV + + - name: Run FeedSim + run: . $PRELUDE; run_feedsim $BUILD_ENV diff --git a/.github/workflows/dcperf_ci_mediawiki.yml b/.github/workflows/dcperf_ci_mediawiki.yml index 9c2579dc..3b053295 100644 --- a/.github/workflows/dcperf_ci_mediawiki.yml +++ b/.github/workflows/dcperf_ci_mediawiki.yml @@ -53,11 +53,11 @@ jobs: steps: - name: Setup Build Container (ubuntu-based) if: ${{ matrix.host-machine.type == 'ubuntu' }} - run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode curl + run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode curl numactl lshw procps bc - name: Setup Build Container (centos-based) if: ${{ matrix.host-machine.type == 'centos' }} - run: dnf update -y; dnf install -y git pciutils which dmidecode + run: dnf update -y; dnf install -y git pciutils which numactl lshw procps-ng bc - name: Checkout the Repository uses: actions/checkout@v4 @@ -87,3 +87,6 @@ jobs: - name: Install Mediawiki run: . $PRELUDE; install_mediawiki $BUILD_ENV + + - name: Run Mediawiki + run: . $PRELUDE; run_mediawiki $BUILD_ENV diff --git a/.github/workflows/dcperf_ci_spark.yml b/.github/workflows/dcperf_ci_spark.yml index 874bcfa1..7f0e64e2 100644 --- a/.github/workflows/dcperf_ci_spark.yml +++ b/.github/workflows/dcperf_ci_spark.yml @@ -55,11 +55,11 @@ jobs: steps: - name: Setup Build Container (ubuntu-based) if: ${{ matrix.host-machine.type == 'ubuntu' }} - run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode + run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode numactl lshw procps - name: Setup Build Container (centos-based) if: ${{ matrix.host-machine.type == 'centos' }} - run: dnf update -y; dnf install -y git pciutils which dmidecode + run: dnf update -y; dnf install -y git pciutils which dmidecode numactl lshw procps-ng - name: Checkout the Repository uses: actions/checkout@v4 @@ -86,3 +86,6 @@ jobs: - name: Install Spark run: . $PRELUDE; install_spark $BUILD_ENV + + - name: Run Spark + run: . $PRELUDE; run_spark $BUILD_ENV diff --git a/.github/workflows/dcperf_ci_syscall.yml b/.github/workflows/dcperf_ci_syscall.yml index 5eb08628..ca2afd2e 100644 --- a/.github/workflows/dcperf_ci_syscall.yml +++ b/.github/workflows/dcperf_ci_syscall.yml @@ -55,11 +55,11 @@ jobs: steps: - name: Setup Build Container (ubuntu-based) if: ${{ matrix.host-machine.type == 'ubuntu' }} - run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode pkgconf + run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode pkgconf numactl lshw procps - name: Setup Build Container (centos-based) if: ${{ matrix.host-machine.type == 'centos' }} - run: dnf update -y; dnf install -y git pciutils which dmidecode + run: dnf update -y; dnf install -y git pciutils which dmidecode numactl lshw procps-ng - name: Checkout the Repository uses: actions/checkout@v4 @@ -86,4 +86,6 @@ jobs: - name: Install Syscall run: . $PRELUDE; install_syscall $BUILD_ENV - continue-on-error: false + + - name: Run Syscall + run: . $PRELUDE; run_syscall $BUILD_ENV diff --git a/.github/workflows/dcperf_ci_tao_bench.yml b/.github/workflows/dcperf_ci_tao_bench.yml index 77cd1cd5..7264edee 100644 --- a/.github/workflows/dcperf_ci_tao_bench.yml +++ b/.github/workflows/dcperf_ci_tao_bench.yml @@ -55,11 +55,11 @@ jobs: steps: - name: Setup Build Container (ubuntu-based) if: ${{ matrix.host-machine.type == 'ubuntu' }} - run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode curl + run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode curl numactl lshw procps bc - name: Setup Build Container (centos-based) if: ${{ matrix.host-machine.type == 'centos' }} - run: dnf update -y; dnf install -y git pciutils which dmidecode + run: dnf update -y; dnf install -y git pciutils which dmidecode numactl lshw procps-ng bc - name: Checkout the Repository uses: actions/checkout@v4 @@ -86,3 +86,6 @@ jobs: - name: Install Tao_bench run: . $PRELUDE; install_tao_bench $BUILD_ENV + + - name: Run Tao_bench + run: . $PRELUDE; run_tao_bench $BUILD_ENV diff --git a/packages/django_workload/install_django_workload.sh b/packages/django_workload/install_django_workload.sh index 18b3b10a..0b9807bb 100755 --- a/packages/django_workload/install_django_workload.sh +++ b/packages/django_workload/install_django_workload.sh @@ -116,6 +116,9 @@ cp "${TEMPLATES_DIR}/jvm.options" "${OUT}/apache-cassandra/conf/jvm.options" || mkdir -p /data/cassandra/{commitlog,data,saved_caches,hints}/ chmod -R 0700 /data/cassandra +# Create logs directory for Cassandra JVM GC logging +mkdir -p "$OUT/apache-cassandra/logs" + # Copy configurations cp "${TEMPLATES_DIR}/cassandra.yaml" "${OUT}/apache-cassandra/conf/cassandra.yaml.template" || exit 1 diff --git a/packages/django_workload/install_django_workload_aarch64.sh b/packages/django_workload/install_django_workload_aarch64.sh index 3021dff8..13c42181 100755 --- a/packages/django_workload/install_django_workload_aarch64.sh +++ b/packages/django_workload/install_django_workload_aarch64.sh @@ -168,6 +168,9 @@ cp "${TEMPLATES_DIR}/jvm.options" "${CASSANDRA_ROOT}/conf/jvm.options" || exit 1 mkdir -p /data/cassandra/{commitlog,data,saved_caches,hints}/ chmod -R 0700 /data/cassandra +# Create logs directory for Cassandra JVM GC logging +mkdir -p "${CASSANDRA_ROOT}/logs" + # Copy configurations cp "${TEMPLATES_DIR}/cassandra.yaml" "${CASSANDRA_ROOT}/conf/cassandra.yaml.template" || exit 1 popd diff --git a/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh b/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh index 146841d4..2de5afb5 100755 --- a/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh +++ b/packages/django_workload/install_django_workload_aarch64_ubuntu22.sh @@ -170,6 +170,9 @@ cp "${TEMPLATES_DIR}/jvm.options" "${CASSANDRA_ROOT}/conf/jvm.options" || exit 1 mkdir -p /data/cassandra/{commitlog,data,saved_caches,hints}/ chmod -R 0700 /data/cassandra +# Create logs directory for Cassandra JVM GC logging +mkdir -p "${CASSANDRA_ROOT}/logs" + # Copy configurations cp "${TEMPLATES_DIR}/cassandra.yaml" "${CASSANDRA_ROOT}/conf/cassandra.yaml.template" || exit 1 popd diff --git a/packages/django_workload/install_django_workload_x86_64_centos9.sh b/packages/django_workload/install_django_workload_x86_64_centos9.sh index 23d0d7d1..5271374c 100755 --- a/packages/django_workload/install_django_workload_x86_64_centos9.sh +++ b/packages/django_workload/install_django_workload_x86_64_centos9.sh @@ -170,6 +170,9 @@ cp "${TEMPLATES_DIR}/jvm.options" "${CASSANDRA_ROOT}/conf/jvm.options" || exit 1 mkdir -p /data/cassandra/{commitlog,data,saved_caches,hints}/ chmod -R 0700 /data/cassandra +# Create logs directory for Cassandra JVM GC logging +mkdir -p "${CASSANDRA_ROOT}/logs" + # Copy configurations cp "${TEMPLATES_DIR}/cassandra.yaml" "${CASSANDRA_ROOT}/conf/cassandra.yaml.template" || exit 1 popd diff --git a/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh b/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh index 828736ad..4336ace1 100755 --- a/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh +++ b/packages/django_workload/install_django_workload_x86_64_ubuntu22.sh @@ -169,6 +169,9 @@ cp "${TEMPLATES_DIR}/jvm.options" "${CASSANDRA_ROOT}/conf/jvm.options" || exit 1 mkdir -p /data/cassandra/{commitlog,data,saved_caches,hints}/ chmod -R 0700 /data/cassandra +# Create logs directory for Cassandra JVM GC logging +mkdir -p "${CASSANDRA_ROOT}/logs" + # Copy configurations cp "${TEMPLATES_DIR}/cassandra.yaml" "${CASSANDRA_ROOT}/conf/cassandra.yaml.template" || exit 1 popd diff --git a/packages/django_workload/srcs/bin/run.sh b/packages/django_workload/srcs/bin/run.sh index af4c18ad..a0d34c6a 100755 --- a/packages/django_workload/srcs/bin/run.sh +++ b/packages/django_workload/srcs/bin/run.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -x # Copyright (c) Meta Platforms, Inc. and affiliates. # # This source code is licensed under the MIT license found in the @@ -35,6 +35,7 @@ cleanup() { [ -n "$MEMCACHED_PID" ] && { echo "Stopping memcached"; kill "$MEMCACHED_PID" || true; } # Stop Cassandra [ -f cassandra.pid ] && { echo "Stopping cassandra"; kill "$(cat cassandra.pid)" || true; } + cat cassandra.log # Kill Siege SIEGE_PID="$(pgrep siege)" [ -n "$SIEGE_PID" ] && { echo "Killing siege"; kill -9 "$SIEGE_PID" || true; } @@ -240,6 +241,9 @@ start_cassandra() { ${CASSANDRA_YAML}.tmp > ${CASSANDRA_YAML}.tmp2 mv -f "${CASSANDRA_YAML}.tmp2" "${CASSANDRA_YAML}" + # Create logs directory if it doesn't exist (required for JVM GC logging) + mkdir -p ./apache-cassandra/logs + ./apache-cassandra/bin/cassandra -R -f -p cassandra.pid > cassandra.log 2>&1 } diff --git a/packages/django_workload/templates/jvm.options b/packages/django_workload/templates/jvm.options index 73d511d4..b3abeb98 100644 --- a/packages/django_workload/templates/jvm.options +++ b/packages/django_workload/templates/jvm.options @@ -4,9 +4,9 @@ # LICENSE file in the root directory of this source tree. # Note: Make sure -Djava.net.preferIPv4Stack=true is not set or is set to false. Otherwise Cassandra won't bind to the IPv6 address --Xms16G --Xmx24G --Xmn4G +-Xms2G +-Xmx4G +-Xmn512M -XX:+HeapDumpOnOutOfMemoryError -Xss256k diff --git a/packages/mediawiki/run.sh b/packages/mediawiki/run.sh index 6716c40d..90eb1387 100755 --- a/packages/mediawiki/run.sh +++ b/packages/mediawiki/run.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -x # Copyright (c) Meta Platforms, Inc. and affiliates. # # This source code is licensed under the MIT license found in the @@ -35,7 +35,7 @@ restart_mariadb_docker() { sleep 1 done # Start MariaDB in the background with nohup to ensure it's fully detached - nohup mariadbd --user=mysql --socket=/var/lib/mysql/mysql.sock > /dev/null 2>&1 & + nohup mariadbd --user=mysql --socket=/var/lib/mysql/mysqld.sock & } # Function to restart MariaDB on bare-metal machine diff --git a/packages/spark_standalone/templates/proj_root/scripts/utils.py b/packages/spark_standalone/templates/proj_root/scripts/utils.py index ec662772..a49e01d5 100755 --- a/packages/spark_standalone/templates/proj_root/scripts/utils.py +++ b/packages/spark_standalone/templates/proj_root/scripts/utils.py @@ -86,6 +86,11 @@ def read_sys_configs() -> Dict[str, int]: sys_configs["sockets"] = int(item["data"]) if item["field"].startswith("Model name"): sys_configs["arch"] = item["data"] + + # Set default value for arch if not found + if "arch" not in sys_configs: + sys_configs["arch"] = platform.machine() + sys_configs["cores"] = ( sys_configs["threads_per_core"] * sys_configs["cores_per_socket"]