From 9e7c5d656bb06fc3341211271d7152b0d8b3445f Mon Sep 17 00:00:00 2001 From: Aryan Gosaliya Date: Mon, 24 Nov 2025 16:15:46 -0800 Subject: [PATCH 1/4] deployment params --- ads/aqua/modeldeployment/deployment.py | 42 +++++++++++++++++++------- models.json | 18 +++++++++++ test_inference.py | 42 ++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 11 deletions(-) create mode 100644 models.json create mode 100644 test_inference.py diff --git a/ads/aqua/modeldeployment/deployment.py b/ads/aqua/modeldeployment/deployment.py index 6f4ac2070..eeb330090 100644 --- a/ads/aqua/modeldeployment/deployment.py +++ b/ads/aqua/modeldeployment/deployment.py @@ -28,7 +28,6 @@ build_params_string, build_pydantic_error_message, find_restricted_params, - get_combined_params, get_container_env_type, get_container_params_type, get_ocid_substring, @@ -918,10 +917,31 @@ def _create( # The values provided by user will override the ones provided by default config env_var = {**config_env, **env_var} - # validate user provided params - user_params = env_var.get("PARAMS", UNKNOWN) + # SMM Parameter Resolution Logic + # Check the raw user input from create_deployment_details to determine intent. + # We cannot use the merged 'env_var' here because it may already contain defaults. + user_input_env = create_deployment_details.env_var or {} + user_input_params = user_input_env.get("PARAMS") + + deployment_params = "" + + if user_input_params is None: + # Case 1: None (CLI default) -> Load full defaults from config + logger.info("No PARAMS provided (None). Loading default SMM parameters.") + deployment_params = config_params + elif str(user_input_params).strip() == "": + # Case 2: Empty String (UI Clear) -> Explicitly use no parameters + logger.info("Empty PARAMS provided. Clearing all parameters.") + deployment_params = "" + else: + # Case 3: Value Provided -> Use exact user value (No merging) + logger.info( + f"User provided PARAMS. Using exact user values: {user_input_params}" + ) + deployment_params = user_input_params - if user_params: + # Validate the resolved parameters + if deployment_params: # todo: remove this check in the future version, logic to be moved to container_index if ( container_type_key.lower() @@ -935,7 +955,7 @@ def _create( ) restricted_params = find_restricted_params( - params, user_params, container_type_key + params, deployment_params, container_type_key ) if restricted_params: raise AquaValueError( @@ -943,8 +963,6 @@ def _create( f"and cannot be overridden or are invalid." ) - deployment_params = get_combined_params(config_params, user_params) - params = f"{params} {deployment_params}".strip() if isinstance(aqua_model, DataScienceModelGroup): @@ -1212,7 +1230,7 @@ def _create_deployment( # we arbitrarily choose last 8 characters of OCID to identify MD in telemetry deployment_short_ocid = get_ocid_substring(deployment_id, key_len=8) - + # Prepare telemetry kwargs telemetry_kwargs = {"ocid": deployment_short_ocid} @@ -2048,9 +2066,11 @@ def recommend_shape(self, **kwargs) -> Union[Table, ShapeRecommendationReport]: self.telemetry.record_event_async( category="aqua/deployment", action="recommend_shape", - detail=get_ocid_substring(model_id, key_len=8) - if is_valid_ocid(ocid=model_id) - else model_id, + detail=( + get_ocid_substring(model_id, key_len=8) + if is_valid_ocid(ocid=model_id) + else model_id + ), **kwargs, ) diff --git a/models.json b/models.json new file mode 100644 index 000000000..63075a0a5 --- /dev/null +++ b/models.json @@ -0,0 +1,18 @@ +[ + { + "model_id": "ocid1.datasciencemodel.oc1.iad.amaaaaaam3xyxziacjn3gsjl4mmvesis5pjeu43lj2vyzjxluoffuqm734da", + "gpu_count": 1, + "model_name": "llama3-8b-instruct", + "fine_tune_weights": [ + { + "model_id": "ocid1.datasciencemodel.oc1.iad.amaaaaaav66vvniabwlacbrsjukmrwk7mmec5ukpumcuefxmclz6suvygywq", + "model_name": "my-llama-v3.1-8b-instruct-ft" + }, + { + "model_id": "ocid1.datasciencemodel.oc1.iad.amaaaaaav66vvniabxhq42ft6ujro4vb5mwa5kelegkj6lle3g6hpaleomeq", + "model_name": "llama-oasst-ft" + } + ] + } +] + diff --git a/test_inference.py b/test_inference.py new file mode 100644 index 000000000..98fa3532c --- /dev/null +++ b/test_inference.py @@ -0,0 +1,42 @@ +import json +import requests +import ads + +# Set up OCI security token authentication +ads.set_auth("security_token") + +# Your Model Deployment OCID and endpoint URL +md_ocid = "ocid1.datasciencemodeldeploymentint.oc1.iad.amaaaaaav66vvniasakhgqe4hk6eqgci7jmj2nvxjzldaqlnb7ji7vjr5p6a" +endpoint = "https://modeldeployment-int.us-ashburn-1.oci.oc-test.com/ocid1.datasciencemodeldeploymentint.oc1.iad.amaaaaaav66vvniasakhgqe4hk6eqgci7jmj2nvxjzldaqlnb7ji7vjr5p6a/predict" + +# OCI request signer +auth = ads.common.auth.default_signer()["signer"] + + +def predict(model_name): + predict_data = { + "model": model_name, + "prompt": "[user] Write a SQL query to answer the question based on the table schema.\n\ncontext: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\nquestion: Name the ICAO for lilongwe international airport [/user] [assistant]", + "max_tokens": 100, + "temperature": 0, + } + predict_headers = {"cx": "application/json", "opc-request-id": "test-id"} + response = requests.post( + endpoint, + headers=predict_headers, + data=json.dumps(predict_data), + auth=auth, + verify=False, # Use verify=True in production! + ) + print("Status:", response.status_code) + try: + print(json.dumps(response.json(), indent=2)) + except Exception as e: + print("Error parsing JSON:", e) + print("Response.text:", response.text) + + +if __name__ == "__main__": + ft_model_name = "my-llama-v3.1-8b-instruct-ft" + print(f"Testing FT model: {ft_model_name}") + predict(ft_model_name) From e9b9a6b9251870a2508231908f8887499bf4c00f Mon Sep 17 00:00:00 2001 From: Aryan Gosaliya Date: Mon, 24 Nov 2025 17:08:46 -0800 Subject: [PATCH 2/4] Delete test_inference.py --- test_inference.py | 42 ------------------------------------------ 1 file changed, 42 deletions(-) delete mode 100644 test_inference.py diff --git a/test_inference.py b/test_inference.py deleted file mode 100644 index 98fa3532c..000000000 --- a/test_inference.py +++ /dev/null @@ -1,42 +0,0 @@ -import json -import requests -import ads - -# Set up OCI security token authentication -ads.set_auth("security_token") - -# Your Model Deployment OCID and endpoint URL -md_ocid = "ocid1.datasciencemodeldeploymentint.oc1.iad.amaaaaaav66vvniasakhgqe4hk6eqgci7jmj2nvxjzldaqlnb7ji7vjr5p6a" -endpoint = "https://modeldeployment-int.us-ashburn-1.oci.oc-test.com/ocid1.datasciencemodeldeploymentint.oc1.iad.amaaaaaav66vvniasakhgqe4hk6eqgci7jmj2nvxjzldaqlnb7ji7vjr5p6a/predict" - -# OCI request signer -auth = ads.common.auth.default_signer()["signer"] - - -def predict(model_name): - predict_data = { - "model": model_name, - "prompt": "[user] Write a SQL query to answer the question based on the table schema.\n\ncontext: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\nquestion: Name the ICAO for lilongwe international airport [/user] [assistant]", - "max_tokens": 100, - "temperature": 0, - } - predict_headers = {"cx": "application/json", "opc-request-id": "test-id"} - response = requests.post( - endpoint, - headers=predict_headers, - data=json.dumps(predict_data), - auth=auth, - verify=False, # Use verify=True in production! - ) - print("Status:", response.status_code) - try: - print(json.dumps(response.json(), indent=2)) - except Exception as e: - print("Error parsing JSON:", e) - print("Response.text:", response.text) - - -if __name__ == "__main__": - ft_model_name = "my-llama-v3.1-8b-instruct-ft" - print(f"Testing FT model: {ft_model_name}") - predict(ft_model_name) From f894725d41478321d3bf747a6bde88fa46413f88 Mon Sep 17 00:00:00 2001 From: Aryan Gosaliya Date: Mon, 24 Nov 2025 17:09:00 -0800 Subject: [PATCH 3/4] Delete models.json --- models.json | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 models.json diff --git a/models.json b/models.json deleted file mode 100644 index 63075a0a5..000000000 --- a/models.json +++ /dev/null @@ -1,18 +0,0 @@ -[ - { - "model_id": "ocid1.datasciencemodel.oc1.iad.amaaaaaam3xyxziacjn3gsjl4mmvesis5pjeu43lj2vyzjxluoffuqm734da", - "gpu_count": 1, - "model_name": "llama3-8b-instruct", - "fine_tune_weights": [ - { - "model_id": "ocid1.datasciencemodel.oc1.iad.amaaaaaav66vvniabwlacbrsjukmrwk7mmec5ukpumcuefxmclz6suvygywq", - "model_name": "my-llama-v3.1-8b-instruct-ft" - }, - { - "model_id": "ocid1.datasciencemodel.oc1.iad.amaaaaaav66vvniabxhq42ft6ujro4vb5mwa5kelegkj6lle3g6hpaleomeq", - "model_name": "llama-oasst-ft" - } - ] - } -] - From 27c1755f5c4c5a61e9602402083d84b3733a1cba Mon Sep 17 00:00:00 2001 From: Aryan Gosaliya Date: Mon, 24 Nov 2025 20:17:32 -0800 Subject: [PATCH 4/4] mmd --- .../modeldeployment/model_group_config.py | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/ads/aqua/modeldeployment/model_group_config.py b/ads/aqua/modeldeployment/model_group_config.py index ca8961620..cfd60a250 100644 --- a/ads/aqua/modeldeployment/model_group_config.py +++ b/ads/aqua/modeldeployment/model_group_config.py @@ -13,7 +13,6 @@ from ads.aqua.common.utils import ( build_params_string, find_restricted_params, - get_combined_params, get_container_params_type, get_params_dict, ) @@ -177,26 +176,35 @@ def _merge_gpu_count_params( model.model_id, AquaDeploymentConfig() ).configuration.get(deployment_details.instance_shape, ConfigurationItem()) + final_model_params = user_params params_found = False - for item in deployment_config.multi_model_deployment: - if model.gpu_count and item.gpu_count and item.gpu_count == model.gpu_count: - config_parameters = item.parameters.get( + + # If user DID NOT provide specific params (None or Empty), we look for defaults + if not user_params: + for item in deployment_config.multi_model_deployment: + if ( + model.gpu_count + and item.gpu_count + and item.gpu_count == model.gpu_count + ): + config_parameters = item.parameters.get( + get_container_params_type(container_type_key), UNKNOWN + ) + if config_parameters: + final_model_params = config_parameters + params_found = True + break + + if not params_found and deployment_config.parameters: + config_parameters = deployment_config.parameters.get( get_container_params_type(container_type_key), UNKNOWN ) - params = f"{params} {get_combined_params(config_parameters, user_params)}".strip() + if config_parameters: + final_model_params = config_parameters params_found = True - break - - if not params_found and deployment_config.parameters: - config_parameters = deployment_config.parameters.get( - get_container_params_type(container_type_key), UNKNOWN - ) - params = f"{params} {get_combined_params(config_parameters, user_params)}".strip() - params_found = True - # if no config parameters found, append user parameters directly. - if not params_found: - params = f"{params} {user_params}".strip() + # Combine Container System Defaults (params) + Model Params (final_model_params) + params = f"{params} {final_model_params}".strip() return params