From e9ca878ce7fe89eddb028fad8682b37e77e7ff66 Mon Sep 17 00:00:00 2001 From: Dominic Tran Date: Thu, 25 Sep 2025 16:42:25 -0500 Subject: [PATCH 1/4] autogenerate script and sql --- .../autogenerate-system-tables.sh | 323 ++++++++++++++++++ scripts/system-tables/system-tables.sql | 17 + 2 files changed, 340 insertions(+) create mode 100755 scripts/system-tables/autogenerate-system-tables.sh create mode 100644 scripts/system-tables/system-tables.sql diff --git a/scripts/system-tables/autogenerate-system-tables.sh b/scripts/system-tables/autogenerate-system-tables.sh new file mode 100755 index 00000000000..517b1bbdbb9 --- /dev/null +++ b/scripts/system-tables/autogenerate-system-tables.sh @@ -0,0 +1,323 @@ +#!/usr/bin/env bash + +# --- Script Configuration --- +SCRIPT_NAME=$(basename "$0") +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +# Determine target directory assuming script is in project_root/scripts/system-tables/ +target_dir=$(dirname "$(dirname "$SCRIPT_DIR")") # Should be project root: .../clickhouse-docs +# Temporary directory within the target project structure +tmp_dir="$target_dir/scripts/tmp-system-tables" + +# --- Parse Command Line Arguments --- +CUSTOM_BINARY="" +while getopts "b:" opt; do + case $opt in + b) + CUSTOM_BINARY="$OPTARG" + echo "[$SCRIPT_NAME] Custom ClickHouse binary specified: $CUSTOM_BINARY" + ;; + \?) + echo "[$SCRIPT_NAME] Invalid option: -$OPTARG" >&2 + echo "[$SCRIPT_NAME] Usage: $0 [-b custom_clickhouse_binary_path]" >&2 + exit 1 + ;; + :) + echo "[$SCRIPT_NAME] Option -$OPTARG requires an argument." >&2 + echo "[$SCRIPT_NAME] Usage: $0 [-b custom_clickhouse_binary_path]" >&2 + exit 1 + ;; + esac +done + +# --- Ensure temporary directory exists --- +mkdir -p "$tmp_dir" || { echo "[$SCRIPT_NAME] Error: Failed to create temporary directory $tmp_dir"; exit 1; } +cd "$tmp_dir" || { echo "[$SCRIPT_NAME] Error: Failed to change directory to $tmp_dir"; exit 1; } +echo "[$SCRIPT_NAME] Changed working directory to: $(pwd)" + +# --- Target Binary Configuration --- +script_filename="clickhouse" +script_path="./$script_filename" # Relative path within tmp_dir + +# --- Helper Functions --- +command_exists() { + command -v "$1" &> /dev/null +} + +# Function to verify a ClickHouse binary +verify_clickhouse_binary() { + local binary_path="$1" + echo "[$SCRIPT_NAME] Verifying ClickHouse binary at '$binary_path'..." + + if [[ ! -f "$binary_path" ]]; then + echo "[$SCRIPT_NAME] Error: Binary file not found at '$binary_path'" + return 1 + fi + + if [[ ! -x "$binary_path" ]]; then + echo "[$SCRIPT_NAME] Setting executable permission on binary..." + chmod +x "$binary_path" || { + echo "[$SCRIPT_NAME] Error: Failed to set execute permission on '$binary_path'"; + return 1; + } + fi + + local version_output + version_output=$("$binary_path" --version 2> /dev/null) + local exit_code=$? + + if [ $exit_code -eq 0 ]; then + echo "[$SCRIPT_NAME] ClickHouse binary verified successfully at '$binary_path'" + echo "[$SCRIPT_NAME] Version info: $version_output" + return 0 # Success + else + echo "[$SCRIPT_NAME] Error: Binary at '$binary_path' is not a valid ClickHouse executable (Exit code: $exit_code)" + "$binary_path" --version # Show error output + return 1 # Failure + fi +} + +# Function to setup a custom binary +setup_custom_binary() { + local source_path="$1" + + # Verify the provided binary first + if ! verify_clickhouse_binary "$source_path"; then + echo "[$SCRIPT_NAME] Error: The specified custom binary is not valid." + return 1 + fi + + # Copy the binary to our tmp directory with the expected name + echo "[$SCRIPT_NAME] Copying custom binary to '$script_path'..." + cp "$source_path" "$script_path" || { + echo "[$SCRIPT_NAME] Error: Failed to copy custom binary to tmp directory." + return 1 + } + + chmod +x "$script_path" || { + echo "[$SCRIPT_NAME] Error: Failed to set execute permission on copied binary." + return 1 + } + + # Verify the copied binary + verify_clickhouse_binary "$script_path" + return $? +} + +# Function to install ClickHouse via official script +install_clickhouse_via_script() { + echo "[$SCRIPT_NAME] Attempting install via official script (curl | sh) into current directory ($tmp_dir)..." + local install_script_path="./install_clickhouse.sh" + local install_success=true + + # Download the installer script + if ! curl -fsSL -o "$install_script_path" https://clickhouse.com/; then + echo "[$SCRIPT_NAME] Error (Script): Failed to download the official install script from https://clickhouse.com/" + rm -f "$install_script_path" + return 1 + fi + + echo "[$SCRIPT_NAME] Downloaded install script. Executing..." + chmod +x "$install_script_path" || echo "[$SCRIPT_NAME] Warning (Script): Could not chmod install script." + + # Execute the downloaded script using bash + if ! bash "$install_script_path"; then + echo "[$SCRIPT_NAME] Error (Script): The official install script failed. See output above." + install_success=false + fi + rm -f "$install_script_path" # Clean up installer script + + if ! $install_success; then return 1; fi + echo "[$SCRIPT_NAME] Official install script finished." + + # --- Verification Steps --- + if [[ ! -f "$script_path" ]]; then # Checks for ./clickhouse + echo "[$SCRIPT_NAME] Error (Script): ClickHouse binary not found at '$script_path' after running install script." + ls -lA .; return 1; + fi + echo "[$SCRIPT_NAME] Found binary at '$script_path'." + chmod +x "$script_path" || { echo "[$SCRIPT_NAME] Error (Script): Failed to set execute permission on '$script_path'"; return 1; } + + echo "[$SCRIPT_NAME] Verifying downloaded binary '$script_path' (Script method)..." + local version_output + version_output=$("$script_path" --version 2> /dev/null) + local exit_code=$? + + if [ $exit_code -eq 0 ]; then + echo "[$SCRIPT_NAME] ClickHouse binary (Script method) installed and verified successfully at '$script_path'" + echo "[$SCRIPT_NAME] Installed version info: $version_output" + return 0 # Success + else + echo "[$SCRIPT_NAME] Error (Script): Downloaded binary '$script_path' is not functioning correctly (Exit code: $exit_code)." + "$script_path" --version # Show error output + rm -f "$script_path"; return 1; # Failure + fi +} + +# --- Dependency Checks --- +for cmd in bash curl uname mv chmod rm mkdir find cat grep sed ls pwd sort head tar; do + if ! command_exists $cmd; then + echo "[$SCRIPT_NAME] Error: Required command '$cmd' not found!" + exit 1 + fi +done +echo "[$SCRIPT_NAME] All basic dependencies found." + +# --- Main Script Logic --- +INSTALL_SUCCESS=false + +# Check if a custom binary was provided +if [[ -n "$CUSTOM_BINARY" ]]; then + echo "[$SCRIPT_NAME] Using custom ClickHouse binary: $CUSTOM_BINARY" + if setup_custom_binary "$CUSTOM_BINARY"; then + INSTALL_SUCCESS=true + else + echo "[$SCRIPT_NAME] Failed to setup custom binary. Exiting." + exit 1 + fi +else + # No custom binary provided, proceed with normal installation logic + # Check if the binary already exists and works + if [ -f "$script_path" ]; then + echo "[$SCRIPT_NAME] Found existing binary at $script_path. Verifying..." + if verify_clickhouse_binary "$script_path"; then + echo "[$SCRIPT_NAME] Using existing, working clickhouse binary at $script_path" + INSTALL_SUCCESS=true + else + echo "[$SCRIPT_NAME] Existing binary at $script_path is not working. Attempting installation..." + rm -f "$script_path" + if install_clickhouse_via_script; then + INSTALL_SUCCESS=true + else + INSTALL_SUCCESS=false + fi + fi + else + echo "[$SCRIPT_NAME] ClickHouse binary not found at $script_path. Attempting install via script..." + if install_clickhouse_via_script; then + INSTALL_SUCCESS=true + else + INSTALL_SUCCESS=false + fi + fi +fi + +# Check final install status +if ! $INSTALL_SUCCESS; then + echo "[$SCRIPT_NAME] FATAL: All installation methods failed." + cd .. || echo "[$SCRIPT_NAME] Warning: Failed to cd back to parent directory before exiting." + exit 1 +fi + +# --- Dynamically retrieve system tables list --- +echo "[$SCRIPT_NAME] Retrieving list of system tables from ClickHouse..." + +# Query to get all system tables +SYSTEM_TABLES_QUERY="SELECT name FROM system.tables WHERE database = 'system' ORDER BY name FORMAT TSV" + +# Execute the query and store results in an array (macOS compatible) +SYSTEM_TABLES=() +while IFS= read -r line; do + SYSTEM_TABLES+=("$line") +done < <("$script_path" --query "$SYSTEM_TABLES_QUERY" 2>/dev/null) + +if [ ${#SYSTEM_TABLES[@]} -eq 0 ]; then + echo "[$SCRIPT_NAME] Error: Failed to retrieve system tables list or no tables found." + exit 1 +fi + +echo "[$SCRIPT_NAME] Found ${#SYSTEM_TABLES[@]} system tables to process." + +# --- Auto-generate System Tables Documentation --- +echo "[$SCRIPT_NAME] Auto-generating system tables documentation..." + +# Generate system tables documentation using the consolidated SQL file +SYSTEM_TABLES_SQL_FILE="$SCRIPT_DIR/system-tables.sql" +if [ -f "$SYSTEM_TABLES_SQL_FILE" ]; then + echo "[$SCRIPT_NAME] Generating system tables documentation using consolidated SQL file..." + + for TABLE in "${SYSTEM_TABLES[@]}"; do + echo "[$SCRIPT_NAME] Processing system table: $TABLE" + "$script_path" --param_table="$TABLE" --queries-file "$SYSTEM_TABLES_SQL_FILE" || { + echo "[$SCRIPT_NAME] FATAL: Failed to execute queries for system.$TABLE table."; + echo "[$SCRIPT_NAME] Attempting to re-run with output:" + "$script_path" --param_table="$TABLE" --queries-file "$SYSTEM_TABLES_SQL_FILE" + exit 1; + } + + # Rename the temporary output file to the correct name + if [ -f "temp-system-table.md" ]; then + mv "temp-system-table.md" "${TABLE}.md" || { + echo "[$SCRIPT_NAME] Error: Failed to rename temp-system-table.md to ${TABLE}.md" + exit 1 + } + echo "[$SCRIPT_NAME] Generated ${TABLE}.md" + else + echo "[$SCRIPT_NAME] Warning: temp-system-table.md not found for system.$TABLE" + fi + done +else + echo "[$SCRIPT_NAME] Warning: System tables SQL file not found at $SYSTEM_TABLES_SQL_FILE" + exit 1 +fi + +# --- Insert generated content between AUTOGENERATED tags --- +echo "[$SCRIPT_NAME] Inserting generated markdown content between AUTOGENERATED_START and AUTOGENERATED_END tags" + +for TABLE in "${SYSTEM_TABLES[@]}"; do + src_file="${TABLE}.md" + # Construct full destination path using $target_dir (project root) + dest_full_path="$target_dir/docs/operations/system-tables/${TABLE}.md" + # Source file is relative to CWD ($tmp_dir) + src_rel_path="./$src_file" + + if [ -f "$src_rel_path" ]; then + # Ensure destination directory exists + mkdir -p "$(dirname "$dest_full_path")" || { + echo "[$SCRIPT_NAME] Error: Failed to create directory for $dest_full_path"; + exit 1; + } + + # Check if destination file exists and has AUTOGENERATED tags + if [ -f "$dest_full_path" ] && grep -q "" "$dest_full_path" && grep -q "" "$dest_full_path"; then + echo "[$SCRIPT_NAME] Updating content in existing file: $dest_full_path" + + # Create a temporary file + tmp_file=$(mktemp) + trap 'rm -f "$tmp_file"' EXIT + + # 1. Copy everything up to and including + awk '// {print; exit}; {print}' "$dest_full_path" > "$tmp_file" + + # 2. Add the content from the source file + cat "$src_rel_path" >> "$tmp_file" + + # 3. Add everything from to the end of the file + awk 'BEGIN {found=0}; // {found=1}; found {print}' "$dest_full_path" >> "$tmp_file" + + # Replace the original file with the modified content + mv "$tmp_file" "$dest_full_path" + else + echo "[$SCRIPT_NAME] Warning: File $dest_full_path does not exist or missing AUTOGENERATED tags. Skipping." + echo "[$SCRIPT_NAME] You may need to create the file with proper AUTOGENERATED_START and AUTOGENERATED_END tags." + fi + + # Remove source file after processing + rm -f "$src_rel_path" + else + echo "[$SCRIPT_NAME] Warning: Expected output file $src_rel_path not found in $tmp_dir. Skipping." + fi +done + +# --- Final Cleanup --- +echo "[$SCRIPT_NAME] Performing final cleanup..." +# Determine parent dir of tmp_dir for cd step +parent_of_tmp=$(dirname "$tmp_dir") +cd "$parent_of_tmp" || echo "[$SCRIPT_NAME] Warning: Failed to cd back to parent directory $parent_of_tmp before cleanup." +echo "[$SCRIPT_NAME] Removing contents of temporary directory: $tmp_dir" +# Use full path for removal just in case cd failed +rm -rf "$tmp_dir"/* +rmdir "$tmp_dir" 2>/dev/null || echo "[$SCRIPT_NAME] Unable to remove $tmp_dir" + +echo "[$SCRIPT_NAME] Auto-generation of system tables documentation completed successfully." +echo "[$SCRIPT_NAME] Processed ${#SYSTEM_TABLES[@]} system tables." +exit 0 \ No newline at end of file diff --git a/scripts/system-tables/system-tables.sql b/scripts/system-tables/system-tables.sql new file mode 100644 index 00000000000..298a007ac46 --- /dev/null +++ b/scripts/system-tables/system-tables.sql @@ -0,0 +1,17 @@ +-- Consolidated system table documentation generator +WITH table_columns AS ( + SELECT + name, + type, + comment, + position + FROM system.columns + WHERE database = 'system' AND table = {table:String} + ORDER BY position +) +SELECT + (SELECT groupConcat('- `' || name || '` ([' || type || '](../../sql-reference/data-types/)) — ' || comment, '\n') + FROM table_columns WHERE comment != '') +FROM table_columns +LIMIT 1 +INTO OUTFILE 'temp-system-table.md' TRUNCATE FORMAT LineAsString \ No newline at end of file From eaefe6ef258b51a0631499478aaf5f492478ffe2 Mon Sep 17 00:00:00 2001 From: Dominic Tran Date: Fri, 26 Sep 2025 10:29:33 -0500 Subject: [PATCH 2/4] ci and package update --- .github/workflows/build-search.yml | 3 +++ package.json | 1 + 2 files changed, 4 insertions(+) diff --git a/.github/workflows/build-search.yml b/.github/workflows/build-search.yml index 2521cf567af..27cf144147b 100644 --- a/.github/workflows/build-search.yml +++ b/.github/workflows/build-search.yml @@ -33,6 +33,9 @@ jobs: - name: Run Auto Generate Settings run: yarn autogenerate-settings + - name: Run Auto Generate System Tables + run: yarn autogenerate-system-tables + - name: Run Indexer run: yarn run-indexer env: diff --git a/package.json b/package.json index f2cee1d5b44..a40be909029 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "build-swagger": "yarn redocly build-docs https://api.clickhouse.cloud/v1 --output build/cloud/manage/api/swagger.html", "prep-from-local": "bash ./scripts/copy-clickhouse-repo-docs.sh -l $1", "autogenerate-settings": "bash ./scripts/settings/autogenerate-settings.sh", + "autogenerate-system-tables": "bash ./scripts/system-tables/autogenerate-system-tables.sh", "autogenerate-table-of-contents": "bash ./scripts/autogenerate-table-of-contents.sh", "generate-changelog": "bash ./scripts/generate-changelog.sh", "build-docs": "bash ./scripts/build.sh", From c50d468570997c207bd8aff929fd15c5902be1dd Mon Sep 17 00:00:00 2001 From: Dominic Tran Date: Tue, 14 Oct 2025 15:07:52 -0500 Subject: [PATCH 3/4] consolidating script, PR feedback --- .github/workflows/build-search.yml | 3 - package.json | 1 - scripts/settings/autogenerate-settings.sh | 186 ++++++---- .../system-tables.sql | 0 .../autogenerate-system-tables.sh | 323 ------------------ 5 files changed, 116 insertions(+), 397 deletions(-) rename scripts/{system-tables => settings}/system-tables.sql (100%) delete mode 100755 scripts/system-tables/autogenerate-system-tables.sh diff --git a/.github/workflows/build-search.yml b/.github/workflows/build-search.yml index 27cf144147b..2521cf567af 100644 --- a/.github/workflows/build-search.yml +++ b/.github/workflows/build-search.yml @@ -33,9 +33,6 @@ jobs: - name: Run Auto Generate Settings run: yarn autogenerate-settings - - name: Run Auto Generate System Tables - run: yarn autogenerate-system-tables - - name: Run Indexer run: yarn run-indexer env: diff --git a/package.json b/package.json index af6559cc881..59ac0d6ec89 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,6 @@ "build-swagger": "yarn redocly build-docs https://api.clickhouse.cloud/v1 --output build/cloud/manage/api/swagger.html", "prep-from-local": "bash ./scripts/copy-clickhouse-repo-docs.sh -l $1", "autogenerate-settings": "bash ./scripts/settings/autogenerate-settings.sh", - "autogenerate-system-tables": "bash ./scripts/system-tables/autogenerate-system-tables.sh", "autogenerate-table-of-contents": "bash ./scripts/autogenerate-table-of-contents.sh", "generate-changelog": "bash ./scripts/generate-changelog.sh", "build-docs": "bash ./scripts/build.sh", diff --git a/scripts/settings/autogenerate-settings.sh b/scripts/settings/autogenerate-settings.sh index 69262f95f97..ce01378d98f 100755 --- a/scripts/settings/autogenerate-settings.sh +++ b/scripts/settings/autogenerate-settings.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash - # --- Script Configuration --- SCRIPT_NAME=$(basename "$0") SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" @@ -7,7 +6,6 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" target_dir=$(dirname "$(dirname "$SCRIPT_DIR")") # Should be project root: .../clickhouse-docs # Temporary directory within the target project structure tmp_dir="$target_dir/scripts/tmp" - # --- Parse Command Line Arguments --- CUSTOM_BINARY="" while getopts "b:" opt; do @@ -28,21 +26,17 @@ while getopts "b:" opt; do ;; esac done - # --- Ensure temporary directory exists --- mkdir -p "$tmp_dir" || { echo "[$SCRIPT_NAME] Error: Failed to create temporary directory $tmp_dir"; exit 1; } cd "$tmp_dir" || { echo "[$SCRIPT_NAME] Error: Failed to change directory to $tmp_dir"; exit 1; } echo "[$SCRIPT_NAME] Changed working directory to: $(pwd)" - # --- Target Binary Configuration --- script_filename="clickhouse" script_path="./$script_filename" # Relative path within tmp_dir - # --- Helper Functions --- command_exists() { command -v "$1" &> /dev/null } - # Function to get architecture name for download URLs (handles macOS arm64) get_arch() { local arch_raw @@ -54,17 +48,14 @@ get_arch() { *) echo "" ;; # Return empty for other architectures esac } - # Function to verify a ClickHouse binary verify_clickhouse_binary() { local binary_path="$1" echo "[$SCRIPT_NAME] Verifying ClickHouse binary at '$binary_path'..." - if [[ ! -f "$binary_path" ]]; then echo "[$SCRIPT_NAME] Error: Binary file not found at '$binary_path'" return 1 fi - if [[ ! -x "$binary_path" ]]; then echo "[$SCRIPT_NAME] Setting executable permission on binary..." chmod +x "$binary_path" || { @@ -72,11 +63,9 @@ verify_clickhouse_binary() { return 1; } fi - local version_output version_output=$("$binary_path" --version 2> /dev/null) local exit_code=$? - if [ $exit_code -eq 0 ]; then echo "[$SCRIPT_NAME] ClickHouse binary verified successfully at '$binary_path'" echo "[$SCRIPT_NAME] Version info: $version_output" @@ -87,34 +76,50 @@ verify_clickhouse_binary() { return 1 # Failure fi } - # Function to setup a custom binary setup_custom_binary() { local source_path="$1" - # Verify the provided binary first if ! verify_clickhouse_binary "$source_path"; then echo "[$SCRIPT_NAME] Error: The specified custom binary is not valid." return 1 fi - # Copy the binary to our tmp directory with the expected name echo "[$SCRIPT_NAME] Copying custom binary to '$script_path'..." cp "$source_path" "$script_path" || { echo "[$SCRIPT_NAME] Error: Failed to copy custom binary to tmp directory." return 1 } - chmod +x "$script_path" || { echo "[$SCRIPT_NAME] Error: Failed to set execute permission on copied binary." return 1 } - # Verify the copied binary verify_clickhouse_binary "$script_path" return $? } +# Function to insert content between AUTOGENERATED tags +insert_content_between_tags() { + local src_file="$1" + local dest_file="$2" + + # Create a temporary file + local tmp_file=$(mktemp) + trap 'rm -f "$tmp_file"' EXIT + + # 1. Copy everything up to and including + awk '// {print; exit}; {print}' "$dest_file" > "$tmp_file" + + # 2. Add the content from the source file + cat "$src_file" >> "$tmp_file" + + # 3. Add everything from to the end of the file + awk 'BEGIN {found=0}; // {found=1}; found {print}' "$dest_file" >> "$tmp_file" + + # Replace the original file with the modified content + mv "$tmp_file" "$dest_file" +} # --- Dependency Checks --- for cmd in bash curl uname mv chmod rm mkdir find cat grep sed ls pwd sort head tar; do if ! command_exists $cmd; then @@ -123,33 +128,27 @@ for cmd in bash curl uname mv chmod rm mkdir find cat grep sed ls pwd sort head fi done echo "[$SCRIPT_NAME] All basic dependencies found." - # --- Installation Function 2 (Primary: Official Script) --- install_clickhouse_via_script() { echo "[$SCRIPT_NAME] Attempting install via official script (curl | sh) into current directory ($tmp_dir)..." local install_script_path="./install_clickhouse.sh" local install_success=true - # Download the installer script if ! curl -fsSL -o "$install_script_path" https://clickhouse.com/; then echo "[$SCRIPT_NAME] Error (Script): Failed to download the official install script from https://clickhouse.com/" rm -f "$install_script_path" return 1 fi - echo "[$SCRIPT_NAME] Downloaded install script. Executing..." chmod +x "$install_script_path" || echo "[$SCRIPT_NAME] Warning (Script): Could not chmod install script." - # Execute the downloaded script using bash if ! bash "$install_script_path"; then echo "[$SCRIPT_NAME] Error (Script): The official install script failed. See output above." install_success=false fi rm -f "$install_script_path" # Clean up installer script - if ! $install_success; then return 1; fi echo "[$SCRIPT_NAME] Official install script finished." - # --- Verification Steps --- if [[ ! -f "$script_path" ]]; then # Checks for ./clickhouse echo "[$SCRIPT_NAME] Error (Script): ClickHouse binary not found at '$script_path' after running install script." @@ -157,12 +156,10 @@ install_clickhouse_via_script() { fi echo "[$SCRIPT_NAME] Found binary at '$script_path'." chmod +x "$script_path" || { echo "[$SCRIPT_NAME] Error (Script): Failed to set execute permission on '$script_path'"; return 1; } - echo "[$SCRIPT_NAME] Verifying downloaded binary '$script_path' (Script method)..." local version_output version_output=$("$script_path" --version 2> /dev/null) local exit_code=$? - if [ $exit_code -eq 0 ]; then echo "[$SCRIPT_NAME] ClickHouse binary (Script method) installed and verified successfully at '$script_path'" echo "[$SCRIPT_NAME] Installed version info: $version_output" @@ -173,11 +170,8 @@ install_clickhouse_via_script() { rm -f "$script_path"; return 1; # Failure fi } - # --- Main Script Logic --- - INSTALL_SUCCESS=false - # Check if a custom binary was provided if [[ -n "$CUSTOM_BINARY" ]]; then echo "[$SCRIPT_NAME] Using custom ClickHouse binary: $CUSTOM_BINARY" @@ -215,24 +209,21 @@ else fi fi fi - # Check final install status if ! $INSTALL_SUCCESS; then echo "[$SCRIPT_NAME] FATAL: All installation methods failed." cd .. || echo "[$SCRIPT_NAME] Warning: Failed to cd back to parent directory before exiting." exit 1 fi - # --- Auto-generate Settings Documentation --- echo "[$SCRIPT_NAME] Auto-generating settings markdown pages..." - -# Process other SQL files first (non-function related) -sql_files_found=$(find "$SCRIPT_DIR" -maxdepth 1 -name '*.sql' ! -name 'generate-functions.sql' -print -quit) +# Process other SQL files first (non-function and non-system-tables related) +sql_files_found=$(find "$SCRIPT_DIR" -maxdepth 1 -name '*.sql' ! -name 'generate-functions.sql' ! -name 'system-tables.sql' -print -quit) if [ -z "$sql_files_found" ]; then - echo "[$SCRIPT_NAME] Warning: No non-function *.sql files found in $SCRIPT_DIR to process." + echo "[$SCRIPT_NAME] Warning: No settings *.sql files found in $SCRIPT_DIR to process." else for SQL_FILE in "$SCRIPT_DIR"/*.sql; do - if [ -f "$SQL_FILE" ] && [ "$(basename "$SQL_FILE")" != "generate-functions.sql" ]; then + if [ -f "$SQL_FILE" ] && [ "$(basename "$SQL_FILE")" != "generate-functions.sql" ] && [ "$(basename "$SQL_FILE")" != "system-tables.sql" ]; then echo "[$SCRIPT_NAME] Processing SQL file: $SQL_FILE" # Run ClickHouse from CWD ($tmp_dir), passing absolute path to SQL file "$script_path" --queries-file "$SQL_FILE" || { @@ -244,12 +235,10 @@ else fi done fi - # Generate function documentation using the consolidated SQL file FUNCTION_SQL_FILE="$SCRIPT_DIR/generate-functions.sql" if [ -f "$FUNCTION_SQL_FILE" ]; then echo "[$SCRIPT_NAME] Generating function documentation using consolidated SQL file..." - # Define function categories to process FUNCTION_CATEGORIES=( "Arithmetic" @@ -276,7 +265,6 @@ if [ -f "$FUNCTION_SQL_FILE" ]; then "Time Series" "Time Window" ) - for CATEGORY in "${FUNCTION_CATEGORIES[@]}"; do echo "[$SCRIPT_NAME] Processing $CATEGORY functions..." "$script_path" --param_category="$CATEGORY" --queries-file "$FUNCTION_SQL_FILE" || { @@ -285,7 +273,6 @@ if [ -f "$FUNCTION_SQL_FILE" ]; then "$script_path" --param_category="$CATEGORY" --queries-file "$FUNCTION_SQL_FILE" exit 1; } - # Rename the temporary output file to the correct name # Category can be "Dates and Times" for example, resulting in `dates and times-functions.md` so we change spaces to underscores CATEGORY_LOWER=$(echo "$CATEGORY" | tr '[:upper:]' '[:lower:]' | tr ' ' '_') @@ -303,7 +290,6 @@ if [ -f "$FUNCTION_SQL_FILE" ]; then else echo "[$SCRIPT_NAME] Warning: Consolidated function SQL file not found at $FUNCTION_SQL_FILE" fi - # --- temporary sed replacements --- sed -i.bak \ -e 's/Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited./Limit the max number of partitions that can be accessed in one query. `<=` 0 means unlimited./g' \ @@ -312,7 +298,6 @@ sed -i.bak \ -e 's#/#`/`#g' \ -e 's#(/columns and /checksums)#`(/columns and /checksums)`#g' \ "$tmp_dir"/generated_merge_tree_settings.md > /dev/null - # --- Move Generated Files --- # Define destination paths relative to the project root ($target_dir) move_src_files=("settings-formats.md" "settings.md" "server_settings.md") @@ -325,7 +310,6 @@ append_src_files=("generated_merge_tree_settings.md") append_dest_files=( "docs/operations/settings/merge-tree-settings.md" ) - echo "[$SCRIPT_NAME] Moving generated markdown files to documentation directories..." # Iterate using index; declare loop variables WITHOUT 'local' for i in "${!move_src_files[@]}"; do @@ -334,7 +318,6 @@ for i in "${!move_src_files[@]}"; do dest_full_path="$target_dir/${move_dest_files[i]}" # Source file is relative to CWD ($tmp_dir) src_rel_path="./$src_file" - if [ -f "$src_rel_path" ]; then # Ensure destination directory exists mkdir -p "$(dirname "$dest_full_path")" || { echo "[$SCRIPT_NAME] Error: Failed to create directory for $dest_full_path"; exit 1; } @@ -345,7 +328,6 @@ for i in "${!move_src_files[@]}"; do echo "[$SCRIPT_NAME] Warning: Expected output file $src_rel_path not found in $tmp_dir. Skipping move." fi done - echo "[$SCRIPT_NAME] Appending generated markdown files to documentation directories..." # Iterate using index; declare loop variables WITHOUT 'local' for i in "${!append_src_files[@]}"; do @@ -354,7 +336,6 @@ for i in "${!append_src_files[@]}"; do dest_full_path="$target_dir/${append_dest_files[i]}" # Source file is relative to CWD ($tmp_dir) src_rel_path="./$src_file" - if [ -f "$src_rel_path" ]; then # Ensure destination directory exists mkdir -p "$(dirname "$dest_full_path")" || { echo "[$SCRIPT_NAME] Error: Failed to create directory for $dest_full_path"; exit 1; } @@ -367,7 +348,6 @@ for i in "${!append_src_files[@]}"; do echo "[$SCRIPT_NAME] Warning: Expected output file $src_rel_path not found in $tmp_dir. Skipping append." fi done - # --- Append content between tags on the page insert_src_files=( "experimental-beta-settings.md" @@ -395,7 +375,6 @@ insert_src_files=( "time_series-functions.md" "time_window-functions.md" ) - insert_dest_files=( "docs/about-us/beta-and-experimental-features.md" "docs/sql-reference/functions/arithmetic-functions.md" @@ -422,45 +401,24 @@ insert_dest_files=( "docs/sql-reference/functions/time-series-functions.md" "docs/sql-reference/functions/time-window-functions.md" ) - echo "[$SCRIPT_NAME] Inserting generated markdown content between AUTOGENERATED_START and AUTOGENERATED_END tags" for i in "${!insert_src_files[@]}"; do src_file="${insert_src_files[i]}" - # Construct full destination path using $target_dir (project root) dest_full_path="$target_dir/${insert_dest_files[i]}" - # Source file is relative to CWD ($tmp_dir) src_rel_path="./$src_file" - if [ -f "$src_rel_path" ]; then - # Ensure destination directory exists mkdir -p "$(dirname "$dest_full_path")" || { echo "[$SCRIPT_NAME] Error: Failed to create directory for $dest_full_path"; exit 1; } echo "[$SCRIPT_NAME] Appending $src_rel_path to $dest_full_path" - # Replace the content between and tags if grep -q "" "$dest_full_path" && grep -q "" "$dest_full_path"; then - # Create a temporary file - tmp_file=$(mktemp) - trap 'rm -f "$tmp_file"' EXIT - - # 1. Copy everything up to and including - awk '// {print; exit}; {print}' "$dest_full_path" > "$tmp_file" - - # 2. Add the content from the source file - cat "$src_rel_path" >> "$tmp_file" - - # 3. Add everything from to the end of the file - awk 'BEGIN {found=0}; // {found=1}; found {print}' "$dest_full_path" >> "$tmp_file" - - # Replace the original file with the modified content - mv "$tmp_file" "$dest_full_path" + insert_content_between_tags "$src_rel_path" "$dest_full_path" else echo "[$SCRIPT_NAME] Error: Expected to find AUTOGENERATED_START and AUTOGENERATED_END tags in $dest_full_path, but did not" exit 1 fi - # Remove source file after successful append rm -f "$src_rel_path" else @@ -468,6 +426,94 @@ for i in "${!insert_src_files[@]}"; do fi done +# --- Dynamically retrieve system tables list --- +echo "[$SCRIPT_NAME] Retrieving list of system tables from ClickHouse..." + +# Query to get all system tables +SYSTEM_TABLES_QUERY="SELECT name FROM system.tables WHERE database = 'system' ORDER BY name FORMAT TSV" + +# Execute the query and store results in an array (macOS compatible) +SYSTEM_TABLES=() +while IFS= read -r line; do + SYSTEM_TABLES+=("$line") +done < <("$script_path" --query "$SYSTEM_TABLES_QUERY" 2>/dev/null) + +if [ ${#SYSTEM_TABLES[@]} -eq 0 ]; then + echo "[$SCRIPT_NAME] Error: Failed to retrieve system tables list or no tables found." + exit 1 +fi + +echo "[$SCRIPT_NAME] Found ${#SYSTEM_TABLES[@]} system tables to process." + +# --- Auto-generate System Tables Documentation --- +echo "[$SCRIPT_NAME] Auto-generating system tables documentation..." + +# Generate system tables documentation using the consolidated SQL file +SYSTEM_TABLES_SQL_FILE="$SCRIPT_DIR/system-tables.sql" +if [ -f "$SYSTEM_TABLES_SQL_FILE" ]; then + echo "[$SCRIPT_NAME] Generating system tables documentation using consolidated SQL file..." + + for TABLE in "${SYSTEM_TABLES[@]}"; do + echo "[$SCRIPT_NAME] Processing system table: $TABLE" + "$script_path" --param_table="$TABLE" --queries-file "$SYSTEM_TABLES_SQL_FILE" || { + echo "[$SCRIPT_NAME] FATAL: Failed to execute queries for system.$TABLE table."; + echo "[$SCRIPT_NAME] Attempting to re-run with output:" + "$script_path" --param_table="$TABLE" --queries-file "$SYSTEM_TABLES_SQL_FILE" + exit 1; + } + + # Rename the temporary output file to the correct name + if [ -f "temp-system-table.md" ]; then + mv "temp-system-table.md" "${TABLE}.md" || { + echo "[$SCRIPT_NAME] Error: Failed to rename temp-system-table.md to ${TABLE}.md" + exit 1 + } + echo "[$SCRIPT_NAME] Generated ${TABLE}.md" + else + echo "[$SCRIPT_NAME] Warning: temp-system-table.md not found for system.$TABLE" + fi + done +else + echo "[$SCRIPT_NAME] Warning: System tables SQL file not found at $SYSTEM_TABLES_SQL_FILE" + echo "[$SCRIPT_NAME] Skipping system tables generation." +fi + +# --- Insert generated content between AUTOGENERATED tags --- +echo "[$SCRIPT_NAME] Inserting system tables content between AUTOGENERATED_START and AUTOGENERATED_END tags" + +for TABLE in "${SYSTEM_TABLES[@]}"; do + src_file="${TABLE}.md" + # Construct full destination path using $target_dir (project root) + dest_full_path="$target_dir/docs/operations/system-tables/${TABLE}.md" + # Source file is relative to CWD ($tmp_dir) + src_rel_path="./$src_file" + + if [ -f "$src_rel_path" ]; then + # Ensure destination directory exists + mkdir -p "$(dirname "$dest_full_path")" || { + echo "[$SCRIPT_NAME] Error: Failed to create directory for $dest_full_path"; + exit 1; + } + + # Check if destination file exists and has AUTOGENERATED tags + if [ -f "$dest_full_path" ] && grep -q "" "$dest_full_path" && grep -q "" "$dest_full_path"; then + echo "[$SCRIPT_NAME] Updating content in existing file: $dest_full_path" + insert_content_between_tags "$src_rel_path" "$dest_full_path" + else + echo "[$SCRIPT_NAME] Warning: File $dest_full_path does not exist or missing AUTOGENERATED tags. Skipping." + echo "[$SCRIPT_NAME] You may need to create the file with proper AUTOGENERATED_START and AUTOGENERATED_END tags." + fi + + # Remove source file after processing + rm -f "$src_rel_path" + else + echo "[$SCRIPT_NAME] Warning: Expected output file $src_rel_path not found in $tmp_dir. Skipping." + fi +done + +echo "[$SCRIPT_NAME] System tables documentation completed." +echo "[$SCRIPT_NAME] Processed ${#SYSTEM_TABLES[@]} system tables." + # --- Final Cleanup --- echo "[$SCRIPT_NAME] Performing final cleanup..." # Determine parent dir of tmp_dir for cd step @@ -478,5 +524,5 @@ echo "[$SCRIPT_NAME] Removing contents of temporary directory: $tmp_dir" rm -rf "$tmp_dir"/* rmdir "$tmp_dir" 2>/dev/null || echo "[$SCRIPT_NAME] Unable to remove $tmp_dir" -echo "[$SCRIPT_NAME] Auto-generation of settings markdown pages completed successfully." -exit 0 +echo "[$SCRIPT_NAME] Auto-generation of settings and system tables documentation completed successfully." +exit 0 \ No newline at end of file diff --git a/scripts/system-tables/system-tables.sql b/scripts/settings/system-tables.sql similarity index 100% rename from scripts/system-tables/system-tables.sql rename to scripts/settings/system-tables.sql diff --git a/scripts/system-tables/autogenerate-system-tables.sh b/scripts/system-tables/autogenerate-system-tables.sh deleted file mode 100755 index 517b1bbdbb9..00000000000 --- a/scripts/system-tables/autogenerate-system-tables.sh +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env bash - -# --- Script Configuration --- -SCRIPT_NAME=$(basename "$0") -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -# Determine target directory assuming script is in project_root/scripts/system-tables/ -target_dir=$(dirname "$(dirname "$SCRIPT_DIR")") # Should be project root: .../clickhouse-docs -# Temporary directory within the target project structure -tmp_dir="$target_dir/scripts/tmp-system-tables" - -# --- Parse Command Line Arguments --- -CUSTOM_BINARY="" -while getopts "b:" opt; do - case $opt in - b) - CUSTOM_BINARY="$OPTARG" - echo "[$SCRIPT_NAME] Custom ClickHouse binary specified: $CUSTOM_BINARY" - ;; - \?) - echo "[$SCRIPT_NAME] Invalid option: -$OPTARG" >&2 - echo "[$SCRIPT_NAME] Usage: $0 [-b custom_clickhouse_binary_path]" >&2 - exit 1 - ;; - :) - echo "[$SCRIPT_NAME] Option -$OPTARG requires an argument." >&2 - echo "[$SCRIPT_NAME] Usage: $0 [-b custom_clickhouse_binary_path]" >&2 - exit 1 - ;; - esac -done - -# --- Ensure temporary directory exists --- -mkdir -p "$tmp_dir" || { echo "[$SCRIPT_NAME] Error: Failed to create temporary directory $tmp_dir"; exit 1; } -cd "$tmp_dir" || { echo "[$SCRIPT_NAME] Error: Failed to change directory to $tmp_dir"; exit 1; } -echo "[$SCRIPT_NAME] Changed working directory to: $(pwd)" - -# --- Target Binary Configuration --- -script_filename="clickhouse" -script_path="./$script_filename" # Relative path within tmp_dir - -# --- Helper Functions --- -command_exists() { - command -v "$1" &> /dev/null -} - -# Function to verify a ClickHouse binary -verify_clickhouse_binary() { - local binary_path="$1" - echo "[$SCRIPT_NAME] Verifying ClickHouse binary at '$binary_path'..." - - if [[ ! -f "$binary_path" ]]; then - echo "[$SCRIPT_NAME] Error: Binary file not found at '$binary_path'" - return 1 - fi - - if [[ ! -x "$binary_path" ]]; then - echo "[$SCRIPT_NAME] Setting executable permission on binary..." - chmod +x "$binary_path" || { - echo "[$SCRIPT_NAME] Error: Failed to set execute permission on '$binary_path'"; - return 1; - } - fi - - local version_output - version_output=$("$binary_path" --version 2> /dev/null) - local exit_code=$? - - if [ $exit_code -eq 0 ]; then - echo "[$SCRIPT_NAME] ClickHouse binary verified successfully at '$binary_path'" - echo "[$SCRIPT_NAME] Version info: $version_output" - return 0 # Success - else - echo "[$SCRIPT_NAME] Error: Binary at '$binary_path' is not a valid ClickHouse executable (Exit code: $exit_code)" - "$binary_path" --version # Show error output - return 1 # Failure - fi -} - -# Function to setup a custom binary -setup_custom_binary() { - local source_path="$1" - - # Verify the provided binary first - if ! verify_clickhouse_binary "$source_path"; then - echo "[$SCRIPT_NAME] Error: The specified custom binary is not valid." - return 1 - fi - - # Copy the binary to our tmp directory with the expected name - echo "[$SCRIPT_NAME] Copying custom binary to '$script_path'..." - cp "$source_path" "$script_path" || { - echo "[$SCRIPT_NAME] Error: Failed to copy custom binary to tmp directory." - return 1 - } - - chmod +x "$script_path" || { - echo "[$SCRIPT_NAME] Error: Failed to set execute permission on copied binary." - return 1 - } - - # Verify the copied binary - verify_clickhouse_binary "$script_path" - return $? -} - -# Function to install ClickHouse via official script -install_clickhouse_via_script() { - echo "[$SCRIPT_NAME] Attempting install via official script (curl | sh) into current directory ($tmp_dir)..." - local install_script_path="./install_clickhouse.sh" - local install_success=true - - # Download the installer script - if ! curl -fsSL -o "$install_script_path" https://clickhouse.com/; then - echo "[$SCRIPT_NAME] Error (Script): Failed to download the official install script from https://clickhouse.com/" - rm -f "$install_script_path" - return 1 - fi - - echo "[$SCRIPT_NAME] Downloaded install script. Executing..." - chmod +x "$install_script_path" || echo "[$SCRIPT_NAME] Warning (Script): Could not chmod install script." - - # Execute the downloaded script using bash - if ! bash "$install_script_path"; then - echo "[$SCRIPT_NAME] Error (Script): The official install script failed. See output above." - install_success=false - fi - rm -f "$install_script_path" # Clean up installer script - - if ! $install_success; then return 1; fi - echo "[$SCRIPT_NAME] Official install script finished." - - # --- Verification Steps --- - if [[ ! -f "$script_path" ]]; then # Checks for ./clickhouse - echo "[$SCRIPT_NAME] Error (Script): ClickHouse binary not found at '$script_path' after running install script." - ls -lA .; return 1; - fi - echo "[$SCRIPT_NAME] Found binary at '$script_path'." - chmod +x "$script_path" || { echo "[$SCRIPT_NAME] Error (Script): Failed to set execute permission on '$script_path'"; return 1; } - - echo "[$SCRIPT_NAME] Verifying downloaded binary '$script_path' (Script method)..." - local version_output - version_output=$("$script_path" --version 2> /dev/null) - local exit_code=$? - - if [ $exit_code -eq 0 ]; then - echo "[$SCRIPT_NAME] ClickHouse binary (Script method) installed and verified successfully at '$script_path'" - echo "[$SCRIPT_NAME] Installed version info: $version_output" - return 0 # Success - else - echo "[$SCRIPT_NAME] Error (Script): Downloaded binary '$script_path' is not functioning correctly (Exit code: $exit_code)." - "$script_path" --version # Show error output - rm -f "$script_path"; return 1; # Failure - fi -} - -# --- Dependency Checks --- -for cmd in bash curl uname mv chmod rm mkdir find cat grep sed ls pwd sort head tar; do - if ! command_exists $cmd; then - echo "[$SCRIPT_NAME] Error: Required command '$cmd' not found!" - exit 1 - fi -done -echo "[$SCRIPT_NAME] All basic dependencies found." - -# --- Main Script Logic --- -INSTALL_SUCCESS=false - -# Check if a custom binary was provided -if [[ -n "$CUSTOM_BINARY" ]]; then - echo "[$SCRIPT_NAME] Using custom ClickHouse binary: $CUSTOM_BINARY" - if setup_custom_binary "$CUSTOM_BINARY"; then - INSTALL_SUCCESS=true - else - echo "[$SCRIPT_NAME] Failed to setup custom binary. Exiting." - exit 1 - fi -else - # No custom binary provided, proceed with normal installation logic - # Check if the binary already exists and works - if [ -f "$script_path" ]; then - echo "[$SCRIPT_NAME] Found existing binary at $script_path. Verifying..." - if verify_clickhouse_binary "$script_path"; then - echo "[$SCRIPT_NAME] Using existing, working clickhouse binary at $script_path" - INSTALL_SUCCESS=true - else - echo "[$SCRIPT_NAME] Existing binary at $script_path is not working. Attempting installation..." - rm -f "$script_path" - if install_clickhouse_via_script; then - INSTALL_SUCCESS=true - else - INSTALL_SUCCESS=false - fi - fi - else - echo "[$SCRIPT_NAME] ClickHouse binary not found at $script_path. Attempting install via script..." - if install_clickhouse_via_script; then - INSTALL_SUCCESS=true - else - INSTALL_SUCCESS=false - fi - fi -fi - -# Check final install status -if ! $INSTALL_SUCCESS; then - echo "[$SCRIPT_NAME] FATAL: All installation methods failed." - cd .. || echo "[$SCRIPT_NAME] Warning: Failed to cd back to parent directory before exiting." - exit 1 -fi - -# --- Dynamically retrieve system tables list --- -echo "[$SCRIPT_NAME] Retrieving list of system tables from ClickHouse..." - -# Query to get all system tables -SYSTEM_TABLES_QUERY="SELECT name FROM system.tables WHERE database = 'system' ORDER BY name FORMAT TSV" - -# Execute the query and store results in an array (macOS compatible) -SYSTEM_TABLES=() -while IFS= read -r line; do - SYSTEM_TABLES+=("$line") -done < <("$script_path" --query "$SYSTEM_TABLES_QUERY" 2>/dev/null) - -if [ ${#SYSTEM_TABLES[@]} -eq 0 ]; then - echo "[$SCRIPT_NAME] Error: Failed to retrieve system tables list or no tables found." - exit 1 -fi - -echo "[$SCRIPT_NAME] Found ${#SYSTEM_TABLES[@]} system tables to process." - -# --- Auto-generate System Tables Documentation --- -echo "[$SCRIPT_NAME] Auto-generating system tables documentation..." - -# Generate system tables documentation using the consolidated SQL file -SYSTEM_TABLES_SQL_FILE="$SCRIPT_DIR/system-tables.sql" -if [ -f "$SYSTEM_TABLES_SQL_FILE" ]; then - echo "[$SCRIPT_NAME] Generating system tables documentation using consolidated SQL file..." - - for TABLE in "${SYSTEM_TABLES[@]}"; do - echo "[$SCRIPT_NAME] Processing system table: $TABLE" - "$script_path" --param_table="$TABLE" --queries-file "$SYSTEM_TABLES_SQL_FILE" || { - echo "[$SCRIPT_NAME] FATAL: Failed to execute queries for system.$TABLE table."; - echo "[$SCRIPT_NAME] Attempting to re-run with output:" - "$script_path" --param_table="$TABLE" --queries-file "$SYSTEM_TABLES_SQL_FILE" - exit 1; - } - - # Rename the temporary output file to the correct name - if [ -f "temp-system-table.md" ]; then - mv "temp-system-table.md" "${TABLE}.md" || { - echo "[$SCRIPT_NAME] Error: Failed to rename temp-system-table.md to ${TABLE}.md" - exit 1 - } - echo "[$SCRIPT_NAME] Generated ${TABLE}.md" - else - echo "[$SCRIPT_NAME] Warning: temp-system-table.md not found for system.$TABLE" - fi - done -else - echo "[$SCRIPT_NAME] Warning: System tables SQL file not found at $SYSTEM_TABLES_SQL_FILE" - exit 1 -fi - -# --- Insert generated content between AUTOGENERATED tags --- -echo "[$SCRIPT_NAME] Inserting generated markdown content between AUTOGENERATED_START and AUTOGENERATED_END tags" - -for TABLE in "${SYSTEM_TABLES[@]}"; do - src_file="${TABLE}.md" - # Construct full destination path using $target_dir (project root) - dest_full_path="$target_dir/docs/operations/system-tables/${TABLE}.md" - # Source file is relative to CWD ($tmp_dir) - src_rel_path="./$src_file" - - if [ -f "$src_rel_path" ]; then - # Ensure destination directory exists - mkdir -p "$(dirname "$dest_full_path")" || { - echo "[$SCRIPT_NAME] Error: Failed to create directory for $dest_full_path"; - exit 1; - } - - # Check if destination file exists and has AUTOGENERATED tags - if [ -f "$dest_full_path" ] && grep -q "" "$dest_full_path" && grep -q "" "$dest_full_path"; then - echo "[$SCRIPT_NAME] Updating content in existing file: $dest_full_path" - - # Create a temporary file - tmp_file=$(mktemp) - trap 'rm -f "$tmp_file"' EXIT - - # 1. Copy everything up to and including - awk '// {print; exit}; {print}' "$dest_full_path" > "$tmp_file" - - # 2. Add the content from the source file - cat "$src_rel_path" >> "$tmp_file" - - # 3. Add everything from to the end of the file - awk 'BEGIN {found=0}; // {found=1}; found {print}' "$dest_full_path" >> "$tmp_file" - - # Replace the original file with the modified content - mv "$tmp_file" "$dest_full_path" - else - echo "[$SCRIPT_NAME] Warning: File $dest_full_path does not exist or missing AUTOGENERATED tags. Skipping." - echo "[$SCRIPT_NAME] You may need to create the file with proper AUTOGENERATED_START and AUTOGENERATED_END tags." - fi - - # Remove source file after processing - rm -f "$src_rel_path" - else - echo "[$SCRIPT_NAME] Warning: Expected output file $src_rel_path not found in $tmp_dir. Skipping." - fi -done - -# --- Final Cleanup --- -echo "[$SCRIPT_NAME] Performing final cleanup..." -# Determine parent dir of tmp_dir for cd step -parent_of_tmp=$(dirname "$tmp_dir") -cd "$parent_of_tmp" || echo "[$SCRIPT_NAME] Warning: Failed to cd back to parent directory $parent_of_tmp before cleanup." -echo "[$SCRIPT_NAME] Removing contents of temporary directory: $tmp_dir" -# Use full path for removal just in case cd failed -rm -rf "$tmp_dir"/* -rmdir "$tmp_dir" 2>/dev/null || echo "[$SCRIPT_NAME] Unable to remove $tmp_dir" - -echo "[$SCRIPT_NAME] Auto-generation of system tables documentation completed successfully." -echo "[$SCRIPT_NAME] Processed ${#SYSTEM_TABLES[@]} system tables." -exit 0 \ No newline at end of file From 988a1765a6b2a9e40409778a0fae35fcc36aa7c6 Mon Sep 17 00:00:00 2001 From: Dominic Tran Date: Thu, 16 Oct 2025 15:07:13 -0500 Subject: [PATCH 4/4] nested enum fix --- scripts/settings/system-tables.sql | 43 +++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/scripts/settings/system-tables.sql b/scripts/settings/system-tables.sql index 298a007ac46..c8cd83294c1 100644 --- a/scripts/settings/system-tables.sql +++ b/scripts/settings/system-tables.sql @@ -1,4 +1,4 @@ --- Consolidated system table documentation generator +-- Consolidated system table documentation generator with nested enum handling WITH table_columns AS ( SELECT name, @@ -8,10 +8,45 @@ WITH table_columns AS ( FROM system.columns WHERE database = 'system' AND table = {table:String} ORDER BY position +), +formatted_columns AS ( + SELECT + CASE + -- Check if this is an Enum type with possible values listed + WHEN startsWith(type, 'Enum') AND position(comment, 'Possible values:') > 0 THEN + concat( + '- `', name, '` ([', type, '](../../sql-reference/data-types/)) — ', + -- Main description before "Possible values:" + trim(substring(comment, 1, position(comment, 'Possible values:') - 1)), + 'Possible values:\n', + -- Parse and format each enum value as nested bullet + arrayStringConcat( + arrayMap( + x -> concat(' - **', + trim(replaceRegexpOne(splitByString(' — ', x)[1], '^[''"]|[''"]$', '')), + '** — ', + trim(arrayElement(splitByString(' — ', x), 2))), + arrayFilter( + x -> length(trim(x)) > 0, + splitByString(', ', + trim(substring(comment, position(comment, 'Possible values:') + length('Possible values:')))) + ) + ), + char(10) + ) + ) + -- Regular column (non-enum or enum without parsed values) + ELSE + concat('- `', name, '` ([', type, '](../../sql-reference/data-types/)) — ', comment) + END as formatted_column + FROM table_columns + WHERE comment != '' ) SELECT - (SELECT groupConcat('- `' || name || '` ([' || type || '](../../sql-reference/data-types/)) — ' || comment, '\n') - FROM table_columns WHERE comment != '') -FROM table_columns + arrayStringConcat( + groupArray(formatted_column), + char(10) + ) +FROM formatted_columns LIMIT 1 INTO OUTFILE 'temp-system-table.md' TRUNCATE FORMAT LineAsString \ No newline at end of file