Skip to content

[None][infra] Fix OpenSearchDB env #9291

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

[None][infra] Fix OpenSearchDB env #9291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5757375
[None][infra] Fix OpenSearchDB env
ZhanruiSunCh Nov 19, 2025
3e63efc
Fix log
ZhanruiSunCh Nov 19, 2025
f757f65
Add log
ZhanruiSunCh Nov 19, 2025
f2c89e5
Fix
ZhanruiSunCh Nov 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions jenkins/L0_Test.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1570,6 +1570,7 @@ def launchTestListCheck(pipeline)
sh "tar -zxf ${tarName}"
def llmPath = sh (script: "realpath .", returnStdout: true).trim()
def llmSrc = "${llmPath}/TensorRT-LLM/src"
sh "python3 ${llmSrc}/jenkins/scripts/open_search_db.py"
sh "NVIDIA_TRITON_SERVER_VERSION=25.10 LLM_ROOT=${llmSrc} LLM_BACKEND_ROOT=${llmSrc}/triton_backend python3 ${llmSrc}/scripts/check_test_list.py --l0 --qa --waive"
} catch (InterruptedException e) {
throw e
Expand Down
47 changes: 26 additions & 21 deletions jenkins/scripts/open_search_db.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ def typeCheckForOpenSearchDB(json_data) -> bool:
OpenSearchDB.typeCheckForOpenSearchDB(item)
for item in json_data)
if not isinstance(json_data, dict):
OpenSearchDB.logger.info(
print(
f"OpenSearchDB type check failed! Expected dict, got {type(json_data).__name__}"
)
return False
Expand All @@ -120,15 +120,15 @@ def typeCheckForOpenSearchDB(json_data) -> bool:
for prefix, expected_type in type_map.items():
if key.startswith(prefix):
if not isinstance(value, expected_type):
OpenSearchDB.logger.info(
print(
f"OpenSearchDB type check failed! key:{key}, value:{value} value_type:{type(value)}"
)
return False
matched = True
break
if not matched:
if key not in allowed_keys:
OpenSearchDB.logger.info(
print(
f"Unknown key type! key:{key}, value_type:{type(value)}"
)
return False
Expand Down Expand Up @@ -173,27 +173,26 @@ def postToOpenSearchDB(json_data, project) -> bool:
"""
use_poc_db = "sandbox" in project
if not OPEN_SEARCH_DB_BASE_URL:
OpenSearchDB.logger.info("OPEN_SEARCH_DB_BASE_URL is not set")
print("OPEN_SEARCH_DB_BASE_URL is not set")
return False
if not use_poc_db and (not OPEN_SEARCH_DB_USERNAME
or not OPEN_SEARCH_DB_PASSWORD):
OpenSearchDB.logger.info(
print(
"OPEN_SEARCH_DB_USERNAME or OPEN_SEARCH_DB_PASSWORD is not set")
return False
if not use_poc_db and project not in WRITE_ACCESS_PROJECT_NAME:
OpenSearchDB.logger.info(
print(
f"project {project} is not in write access project list: {json.dumps(WRITE_ACCESS_PROJECT_NAME)}"
)
return False
if not OpenSearchDB.typeCheckForOpenSearchDB(json_data):
OpenSearchDB.logger.info(
f"OpenSearchDB type check failed! json_data:{json_data}")
print(f"OpenSearchDB type check failed! json_data:{json_data}")
return False

json_data_dump = json.dumps(json_data)

if DISABLE_OPEN_SEARCH_DB_FOR_LOCAL_TEST:
OpenSearchDB.logger.info(
print(
f"OpenSearchDB is disabled for local test, skip posting to OpenSearchDB: {json_data_dump}"
)
return True
Expand All @@ -218,19 +217,19 @@ def postToOpenSearchDB(json_data, project) -> bool:
res = requests.post(**args)
if res.status_code in (200, 201, 202):
if res.status_code != 200 and project == JOB_PROJECT_NAME:
OpenSearchDB.logger.info(
print(
f"OpenSearchDB post not 200, log:{res.status_code} {res.text}"
)
return True
else:
OpenSearchDB.logger.info(
print(
f"OpenSearchDB post failed, will retry, error:{res.status_code} {res.text}"
)
except Exception as e:
OpenSearchDB.logger.info(
print(
f"OpenSearchDB post exception, attempt {attempt + 1} error: {e}"
)
OpenSearchDB.logger.info(
print(
f"Fail to postToOpenSearchDB after {DEFAULT_RETRY_COUNT} tries: {url}, json: {json_data_dump}, last error: {getattr(res, 'text', 'N/A') if 'res' in locals() else ''}"
)
return False
Expand All @@ -246,10 +245,10 @@ def queryFromOpenSearchDB(json_data, project) -> dict:
"""
use_poc_db = "sandbox" in project
if not OPEN_SEARCH_DB_BASE_URL:
OpenSearchDB.logger.info("OPEN_SEARCH_DB_BASE_URL is not set")
print("OPEN_SEARCH_DB_BASE_URL is not set")
return None
if not use_poc_db and project not in READ_ACCESS_PROJECT_NAME:
OpenSearchDB.logger.info(
print(
f"project {project} is not in read access project list: {json.dumps(READ_ACCESS_PROJECT_NAME)}"
)
return None
Expand All @@ -270,18 +269,19 @@ def queryFromOpenSearchDB(json_data, project) -> dict:
timeout=QUERY_TIMEOUT_SECONDS)
if res.status_code in [200, 201, 202]:
return res
OpenSearchDB.logger.info(
print(
f"OpenSearchDB query failed, will retry, error:{res.status_code} {res.text}"
)
retry_time -= 1
OpenSearchDB.logger.info(
print(
f"Fail to queryFromOpenSearchDB after {retry_time} retry: {url}, json: {json_data_dump}, error: {res.text}"
)
return None

@staticmethod
def queryBuildIdFromOpenSearchDB(job_name, last_days=DEFAULT_LOOKBACK_DAYS):
if DISABLE_OPEN_SEARCH_DB_FOR_LOCAL_TEST:
print("Disabled OpenSearchDB for local test")
return []
if job_name in OpenSearchDB.query_build_id_cache:
return OpenSearchDB.query_build_id_cache[job_name]
Expand Down Expand Up @@ -310,8 +310,7 @@ def queryBuildIdFromOpenSearchDB(job_name, last_days=DEFAULT_LOOKBACK_DAYS):
OpenSearchDB.query_build_id_cache[job_name] = build_ids
return build_ids
except Exception as e:
OpenSearchDB.logger.warning(
f"Failed to query build IDs from OpenSearchDB: {e}")
print(f"WARNING: Failed to query build IDs from OpenSearchDB: {e}")
return []

@staticmethod
Expand All @@ -321,6 +320,7 @@ def queryPRIdsFromOpenSearchDB(repo_name, last_days=DEFAULT_LOOKBACK_DAYS):
Mirrors queryBuildIdFromOpenSearchDB for PR monitoring.
"""
if DISABLE_OPEN_SEARCH_DB_FOR_LOCAL_TEST:
print("Disabled OpenSearchDB for local test")
return []

cache_key = f"pr_{repo_name}"
Expand Down Expand Up @@ -368,6 +368,11 @@ def queryPRIdsFromOpenSearchDB(repo_name, last_days=DEFAULT_LOOKBACK_DAYS):
OpenSearchDB.query_build_id_cache[cache_key] = pr_numbers
return pr_numbers
except Exception as e:
OpenSearchDB.logger.warning(
f"Failed to query PR IDs from OpenSearchDB: {e}")
print(f"WARNING: Failed to query PR IDs from OpenSearchDB: {e}")
return []


print(OpenSearchDB.queryBuildIdFromOpenSearchDB("LLM/main/L0_MergeRequest_PR"))
print("OPEN_SEARCH_DB_BASE_URL: ", OPEN_SEARCH_DB_BASE_URL)
print("OPEN_SEARCH_DB_USERNAME: ", OPEN_SEARCH_DB_USERNAME)
print("OPEN_SEARCH_DB_PASSWORD: ", OPEN_SEARCH_DB_PASSWORD)
Loading