Add placeholders for test configuration

Author: ycedres

test/acceptance_tests.py

@@ -98,18 +98,17 @@ def _run_mcphost_command(prompt, config_path, model):
         return f"UNEXPECTED_ERROR: {str(e)}"
 
 
-def run_test_case(test_case, config_path, model):
+def run_test_case(prompt, config_path, model):
     """Runs a single test case using the mcphost command.
 
     Args:
-        test_case (dict): The test case dictionary from the JSON file.
+        prompt (str): The prompt to send to the model.
         config_path (str): Path to the mcphost config file.
         model (str): The model to use for the test.
 
     Returns:
         str: The actual output from the command, or an error message.
     """
-    prompt = test_case.get("prompt")
     if not prompt:
         return "Error: 'prompt' not found in test case"
     return _run_mcphost_command(prompt, config_path, model)
@@ -162,6 +161,12 @@ def evaluate_test_case(expected, actual, config_path, judge_model):
         return "FAIL", f"LLM judge returned malformed JSON: '{judge_response_str}'"
 
 
+def _substitute_placeholders(text, placeholders):
+    """Substitutes placeholders in a string with their values."""
+    if not isinstance(text, str):
+        return text
+    return text.format(**placeholders)
+
 def main():
     """Main function to run acceptance tests."""
     parser = argparse.ArgumentParser(
@@ -179,6 +184,12 @@ def main():
         default=Path(__file__).parent / "test_results.json",
         help="Path to the output JSON file for test results. Defaults to 'test_results.json' in the same directory.",
     )
+    parser.add_argument(
+        "--test-config",
+        type=Path,
+        default=None,
+        help="Path to the JSON file with test configuration values (for placeholder substitution).",
+    )
     parser.add_argument(
         "--config",
         type=str,
@@ -207,6 +218,22 @@ def main():
         )
         sys.exit(1)
 
+    placeholders = {}
+    if args.test_config:
+        if not args.test_config.is_file():
+            print(
+                f"Error: Test config file not found at '{args.test_config}'",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+        with open(args.test_config, "r", encoding="utf-8") as f:
+            config_data = json.load(f)
+            if "systems" in config_data:
+                for sys_key, sys_values in config_data["systems"].items():
+                    for attr_key, attr_value in sys_values.items():
+                        placeholders[f"{sys_key}_{attr_key}"] = attr_value
+        print(f"Loaded {len(placeholders)} placeholders from '{args.test_config}'")
+
     judge_model = args.judge_model if args.judge_model else args.model
     print(f"Using model for tests: {args.model}")
     print(f"Using model for judging: {judge_model}\n")
@@ -225,11 +252,11 @@ def main():
     for i, tc in enumerate(test_cases, 1):
         test_start_time = time.monotonic()
         print(f"--- [{i}/{total_tests}] RUNNING: {Colors.BOLD}{tc.get('id', 'N/A')}{Colors.ENDC} ---")
-        prompt = tc.get("prompt")
-        expected_output = tc.get("expected_output")
+        prompt = _substitute_placeholders(tc.get("prompt"), placeholders)
+        expected_output = _substitute_placeholders(tc.get("expected_output"), placeholders)
 
         print(f"  PROMPT  : {prompt}")
-        actual_output = run_test_case(tc, args.config, args.model)
+        actual_output = run_test_case(prompt, args.config, args.model)
         print(f"  EXPECTED: {expected_output}")
         print(f"  ACTUAL  : {actual_output}")
 

test/test_cases_sys.json

@@ -1,32 +1,32 @@
 [
   {
     "id": "TC-SYS-001",
-    "prompt": "Can you get the name and system id of of the systems in the uyuni server?",
-    "expected_output": "The systems in the uyuni server are build-host. deblike-minion. proxy. rhlike-minion. suse-minion. suse-sshminion., with system ids 1000010005 1000010004 1000010000 1000010003 1000010001 1000010002"
+    "prompt": "Get the name and system id of all systems in the uyuni server.",
+    "expected_output": "The systems in the uyuni server are {build_host_name}, {deblike_minion_name}, {proxy_name}, {rhlike_minion_name}, {suse_minion_name} and {suse_ssh_minion_name}, with system ids {build_host_id}, {deblike_minion_id}, {proxy_id}, {rhlike_minion_id}, {suse_minion_id} and {suse_ssh_minion_id}."
   },
   {
     "id": "TC-SYS-002",
-    "prompt": "Get CPU details for system ID 1000010000.",
-    "expected_output": "Returns a message with CPU attributes with model name AMD EPYC-Milan Processor."
+    "prompt": "Get CPU details for system ID {proxy_id}.",
+    "expected_output": "Returns a message with CPU attributes with model name {proxy_cpu_model}."
   },
   {
     "id": "TC-SYS-003",
     "prompt": "Get CPU details for system ID 999999999.",
-    "expected_output": "Returns a message that this system does not exist."
+    "expected_output": "Returns a message that this system does not exist or an empty dictionary."
   },
   {
     "id": "TC-SYS-004",
     "prompt": "Show me the CPU information for all my systems.",
-    "expected_output": "Returns a message with the CPU information of each system. Except for the proxy., all the rest at QEMU Virtual CPU."
+    "expected_output": "Returns a message with the CPU information of each system. Except for the proxy ({proxy_cpu_model}), all the rest are {build_host_cpu_model}."
   },
   {
     "id": "TC-SYS-005",
-    "prompt": "Do all active servers have the same CPU?",
-    "expected_output": "No. All have QEMU Virtual CPU except proxy."
+    "prompt": "Do all active servers have the same CPU? Use available tools to answer.",
+    "expected_output": "No. All have {build_host_cpu_model} except proxy, which has {proxy_cpu_model}."
   },
   {
     "id": "TC-SYS-006",
-    "prompt": "Get CPU details for system 'build-host.'.",
-    "expected_output": "Returns a message with CPU attributes of model QEMU Virtual CPU."
+    "prompt": "Get CPU details for system '{build_host_name}'.",
+    "expected_output": "Returns a message with CPU attributes of model {build_host_cpu_model}."
   }
 ]

test/test_config.json

@@ -0,0 +1,34 @@
+{
+  "systems": {
+    "proxy": {
+      "id": "1000010000",
+      "name": "mlm-ref-head-proxy.mgr.suse.de",
+      "cpu_model": "AMD EPYC-Milan Processor"
+    },
+    "build_host": {
+      "id": "1000010005",
+      "name": "mlm-ref-head-build-host.mgr.suse.de",
+      "cpu_model": "QEMU Virtual CPU"
+    },
+    "deblike_minion": {
+      "id": "1000010004",
+      "name": "mlm-ref-head-deblike-minion.mgr.suse.de",
+      "cpu_model": "QEMU Virtual CPU"
+    },
+    "rhlike_minion": {
+      "id": "1000010003",
+      "name": "mlm-ref-head-rhlike-minion.mgr.suse.de",
+      "cpu_model": "QEMU Virtual CPU"
+    },
+    "suse_minion": {
+      "id": "1000010001",
+      "name": "mlm-ref-head-suse-minion.mgr.suse.de",
+      "cpu_model": "QEMU Virtual CPU"
+    },
+    "suse_ssh_minion": {
+      "id": "1000010002",
+      "name": "mlm-ref-head-suse-sshminion.mgr.suse.de",
+      "cpu_model": "QEMU Virtual CPU"
+    }
+  }
+}