⚡️ Speed up method LLMConfigRegistry.get_model_names by 5%

[codeflash-ai]

📄 5% (0.05x) speedup for LLMConfigRegistry.get_model_names in skyvern/forge/sdk/api/llm/config_registry.py

⏱️ Runtime : 40.5 microseconds → 38.6 microseconds (best of 250 runs)

📝 Explanation and details

The optimization removes the redundant .keys() method call when converting the dictionary to a list.

What changed:

• list(cls._configs.keys()) → list(cls._configs)

Why this is faster:
When iterating over a dictionary in Python, the default iteration is over its keys. Calling list(dict) directly creates a list from the dictionary's keys iterator, while list(dict.keys()) first creates a dictionary view object and then converts it to a list, adding an unnecessary intermediate step.

Performance impact:

• 5% overall speedup (40.5μs → 38.6μs)
• Per-call improvement from 2555.8ns to 2393.1ns average
• Test cases show consistent 4-24% improvements across different scenarios

Best performance gains are seen in:

• Small registries (17-20% faster for single/few entries)
• Registries with special characters (24% faster)
• The optimization is most effective for frequent calls with smaller dictionaries

This is a micro-optimization that maintains identical functionality while reducing the overhead of an unnecessary method call and intermediate object creation. Since get_model_names() appears to be a utility method that could be called frequently when retrieving available model configurations, even small per-call improvements can compound meaningfully.

✅ Correctness verification report:

Test	Status
⚙️ Existing Unit Tests	🔘 None Found
🌀 Generated Regression Tests	✅ 32 Passed
⏪ Replay Tests	🔘 None Found
🔎 Concolic Coverage Tests	🔘 None Found
📊 Tests Coverage	100.0%

🌀 Generated Regression Tests and Runtime

from typing import Dict, List, Union

# imports
import pytest
from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry

class LLMConfig:
    pass

class LLMRouterConfig:
    pass
from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry

# 1. Basic Test Cases

def test_get_model_names_empty_registry():
    """Test with an empty registry. Should return an empty list."""
    LLMConfigRegistry._configs = {}
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 574ns -> 478ns (20.1% faster)

def test_get_model_names_single_entry():
    """Test with a single model in the registry."""
    LLMConfigRegistry._configs = {"gpt-3.5": LLMConfig()}
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 561ns -> 472ns (18.9% faster)

def test_get_model_names_multiple_entries():
    """Test with multiple models in the registry."""
    LLMConfigRegistry._configs = {
        "gpt-3.5": LLMConfig(),
        "gpt-4": LLMRouterConfig(),
        "llama-2": LLMConfig()
    }
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 573ns -> 488ns (17.4% faster)

def test_get_model_names_returns_new_list_on_each_call():
    """Ensure that the returned list is a new object each time (not the same reference)."""
    LLMConfigRegistry._configs = {"gpt-3.5": LLMConfig()}
    codeflash_output = LLMConfigRegistry.get_model_names(); result1 = codeflash_output # 567ns -> 485ns (16.9% faster)
    codeflash_output = LLMConfigRegistry.get_model_names(); result2 = codeflash_output # 321ns -> 280ns (14.6% faster)
    # Mutating one result should not affect the other
    result1.append("new-model")

# 2. Edge Test Cases

def test_get_model_names_with_non_string_keys():
    """Test that only string keys are allowed. If non-string keys are present, they should appear in the result."""
    # This is not type safe but possible in Python; test for robustness
    LLMConfigRegistry._configs = {
        123: LLMConfig(),
        None: LLMConfig(),
        "gpt": LLMConfig()
    }
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 517ns -> 443ns (16.7% faster)

def test_get_model_names_with_special_characters():
    """Test with model names containing special characters."""
    LLMConfigRegistry._configs = {
        "gpt-3.5-turbo!": LLMConfig(),
        "llama_2@beta": LLMConfig(),
        "": LLMConfig(),  # empty string as key
        "中文模型": LLMConfig(),  # unicode key
    }
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 495ns -> 399ns (24.1% faster)

def test_get_model_names_does_not_modify_registry():
    """Ensure get_model_names does not modify the internal _configs dict."""
    LLMConfigRegistry._configs = {"gpt": LLMConfig(), "llama": LLMConfig()}
    before = LLMConfigRegistry._configs.copy()
    codeflash_output = LLMConfigRegistry.get_model_names(); _ = codeflash_output # 475ns -> 442ns (7.47% faster)
    after = LLMConfigRegistry._configs

def test_get_model_names_with_duplicate_keys():
    """Test that duplicate keys are not possible (dict property), only unique keys returned."""
    LLMConfigRegistry._configs = {"gpt": LLMConfig(), "gpt": LLMRouterConfig()}
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 524ns -> 449ns (16.7% faster)

def test_get_model_names_with_none_value():
    """Test that None as a value is allowed and does not affect the output."""
    LLMConfigRegistry._configs = {"gpt": None, "llama": LLMConfig()}
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 505ns -> 444ns (13.7% faster)

# 3. Large Scale Test Cases

def test_get_model_names_large_registry():
    """Test with a large number of models to check performance and correctness."""
    num_models = 1000
    LLMConfigRegistry._configs = {f"model_{i}": LLMConfig() for i in range(num_models)}
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 2.45μs -> 2.33μs (5.11% faster)

def test_get_model_names_large_registry_with_varied_types():
    """Test with a large registry with a mix of LLMConfig and LLMRouterConfig types."""
    num_models = 500
    LLMConfigRegistry._configs = {
        f"model_{i}": LLMConfig() if i % 2 == 0 else LLMRouterConfig()
        for i in range(num_models)
    }
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 1.59μs -> 1.52μs (4.73% faster)

def test_get_model_names_large_registry_with_long_keys():
    """Test with very long string keys."""
    num_models = 100
    long_key = "x" * 500  # 500 chars
    LLMConfigRegistry._configs = {f"{long_key}_{i}": LLMConfig() for i in range(num_models)}
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 776ns -> 716ns (8.38% faster)

# Mutation testing: If function is changed to return values, not keys, these tests will fail.
def test_get_model_names_returns_keys_not_values():
    """Ensure that get_model_names returns keys, not values."""
    LLMConfigRegistry._configs = {"gpt": LLMConfig(), "llama": LLMRouterConfig()}
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 579ns -> 488ns (18.6% faster)
    for item in result:
        pass

# Mutation testing: If function returns .keys() directly, it fails (should be list)
def test_get_model_names_returns_list_type():
    """Ensure that get_model_names returns a list, not a view or other type."""
    LLMConfigRegistry._configs = {"gpt": LLMConfig()}
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 560ns -> 465ns (20.4% faster)
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

from typing import Union

# imports
import pytest
from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry

class LLMConfig:
    pass

class LLMRouterConfig:
    pass
from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry

# ----------------------------
# Basic Test Cases
# ----------------------------

def test_empty_registry_returns_empty_list():
    # When there are no configs, should return an empty list
    codeflash_output = LLMConfigRegistry.get_model_names() # 508ns -> 458ns (10.9% faster)

def test_single_entry_returns_single_name():
    # Add one config and check if its key is returned
    LLMConfigRegistry._configs['gpt-3'] = LLMConfig()
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 526ns -> 452ns (16.4% faster)

def test_multiple_entries_return_all_names():
    # Add multiple configs and check if all keys are returned
    LLMConfigRegistry._configs['gpt-3'] = LLMConfig()
    LLMConfigRegistry._configs['gpt-4'] = LLMRouterConfig()
    LLMConfigRegistry._configs['llama-2'] = LLMConfig()
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 527ns -> 442ns (19.2% faster)

def test_keys_are_strings():
    # All keys must be strings
    LLMConfigRegistry._configs['gpt-3'] = LLMConfig()
    LLMConfigRegistry._configs['gpt-4'] = LLMRouterConfig()
    for name in LLMConfigRegistry.get_model_names():
        pass

# ----------------------------
# Edge Test Cases
# ----------------------------

def test_model_names_with_special_characters():
    # Keys with special characters should be returned as-is
    special_names = [
        'gpt-3.5-turbo',
        'llama@meta',
        'openai/gpt-4',
        'model with spaces',
        '中文模型',
        'model!$%^&*()',
        'model\nnewline'
    ]
    for name in special_names:
        LLMConfigRegistry._configs[name] = LLMConfig()
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 551ns -> 509ns (8.25% faster)

def test_model_names_with_empty_string_key():
    # Empty string as key should be handled
    LLMConfigRegistry._configs[''] = LLMConfig()
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 561ns -> 510ns (10.0% faster)

def test_model_names_with_very_long_string():
    # Very long string as key should be handled
    long_name = 'a' * 500
    LLMConfigRegistry._configs[long_name] = LLMConfig()
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 543ns -> 486ns (11.7% faster)

def test_registry_with_mixed_config_types():
    # Should handle both LLMConfig and LLMRouterConfig types
    LLMConfigRegistry._configs['gpt-3'] = LLMConfig()
    LLMConfigRegistry._configs['router'] = LLMRouterConfig()
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 583ns -> 497ns (17.3% faster)

def test_registry_with_duplicate_insertions():
    # Adding the same key twice should not duplicate names
    LLMConfigRegistry._configs['gpt-3'] = LLMConfig()
    LLMConfigRegistry._configs['gpt-3'] = LLMRouterConfig()
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 571ns -> 493ns (15.8% faster)

def test_registry_with_non_ascii_keys():
    # Non-ASCII (Unicode) keys should be handled
    names = ['模型一', 'модель2', 'モデル3']
    for name in names:
        LLMConfigRegistry._configs[name] = LLMConfig()
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 580ns -> 536ns (8.21% faster)

# ----------------------------
# Large Scale Test Cases
# ----------------------------

def test_large_number_of_model_names():
    # Add a large number of configs and check all names are returned
    num_models = 999
    names = [f'model_{i}' for i in range(num_models)]
    for name in names:
        LLMConfigRegistry._configs[name] = LLMConfig()
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 2.48μs -> 2.38μs (4.21% faster)

def test_large_number_of_long_names():
    # Add many configs with long names
    num_models = 500
    names = [f"model_{'x'*100}_{i}" for i in range(num_models)]
    for name in names:
        LLMConfigRegistry._configs[name] = LLMConfig()
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 3.36μs -> 3.33μs (0.901% faster)

def test_performance_with_large_registry(monkeypatch):
    # This test checks that the function completes quickly with many entries
    import time

    num_models = 999
    for i in range(num_models):
        LLMConfigRegistry._configs[f'model_{i}'] = LLMConfig()

    start = time.time()
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 4.08μs -> 4.16μs (1.85% slower)
    duration = time.time() - start

# ----------------------------
# Additional Edge Cases
# ----------------------------

def test_registry_isolation_between_tests():
    # Ensure registry is cleared between tests (enforced by fixture)
    codeflash_output = LLMConfigRegistry.get_model_names() # 4.02μs -> 3.96μs (1.54% faster)

def test_mutation_of_returned_list_does_not_affect_registry():
    # Mutating the returned list should not affect the underlying registry
    LLMConfigRegistry._configs['gpt-3'] = LLMConfig()
    codeflash_output = LLMConfigRegistry.get_model_names(); names = codeflash_output # 3.67μs -> 3.76μs (2.44% slower)
    names.append('new-model')
    # The registry should remain unchanged
    codeflash_output = set(LLMConfigRegistry.get_model_names()) # 3.00μs -> 2.96μs (1.08% faster)

def test_registry_with_keys_that_are_python_keywords():
    # Keys that are Python keywords should be handled
    keywords = ['class', 'def', 'return', 'for', 'while']
    for kw in keywords:
        LLMConfigRegistry._configs[kw] = LLMConfig()
    codeflash_output = LLMConfigRegistry.get_model_names(); result = codeflash_output # 3.91μs -> 3.74μs (4.52% faster)
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

To edit these changes git checkout codeflash/optimize-LLMConfigRegistry.get_model_names-mjav5q1j and push.