diff --git a/.codespellrc b/.codespellrc index 9087e5816..f91025d88 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,4 +1,4 @@ [codespell] # skipping auto generated folders skip = ./.tox,./.mypy_cache,./docs/_build,./target,*/LICENSE,./venv,*/cassettes -ignore-words-list = ot +ignore-words-list = asend,ot diff --git a/.github/workflows/loongsuite_lint_0.yml b/.github/workflows/loongsuite_lint_0.yml index 97d5fe968..dad9eff2a 100644 --- a/.github/workflows/loongsuite_lint_0.yml +++ b/.github/workflows/loongsuite_lint_0.yml @@ -84,7 +84,7 @@ jobs: shell: bash env: LOONGSUITE_ALL_JOBS: >- - [{"name": "lint-loongsuite-instrumentation-agentscope", "package": "loongsuite-instrumentation-agentscope", "tox_env": "lint-loongsuite-instrumentation-agentscope", "ui_name": "loongsuite-instrumentation-agentscope"}, {"name": "lint-loongsuite-instrumentation-dashscope", "package": "loongsuite-instrumentation-dashscope", "tox_env": "lint-loongsuite-instrumentation-dashscope", "ui_name": "loongsuite-instrumentation-dashscope"}, {"name": "lint-loongsuite-instrumentation-claude-agent-sdk", "package": "loongsuite-instrumentation-claude-agent-sdk", "tox_env": "lint-loongsuite-instrumentation-claude-agent-sdk", "ui_name": "loongsuite-instrumentation-claude-agent-sdk"}, {"name": "lint-loongsuite-instrumentation-google-adk", "package": "loongsuite-instrumentation-google-adk", "tox_env": "lint-loongsuite-instrumentation-google-adk", "ui_name": "loongsuite-instrumentation-google-adk"}, {"name": "lint-loongsuite-instrumentation-agno", "package": "loongsuite-instrumentation-agno", "tox_env": "lint-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno"}, {"name": "lint-loongsuite-instrumentation-langchain", "package": "loongsuite-instrumentation-langchain", "tox_env": "lint-loongsuite-instrumentation-langchain", "ui_name": "loongsuite-instrumentation-langchain"}, {"name": "lint-loongsuite-instrumentation-langgraph", "package": "loongsuite-instrumentation-langgraph", "tox_env": "lint-loongsuite-instrumentation-langgraph", "ui_name": "loongsuite-instrumentation-langgraph"}, {"name": "lint-loongsuite-instrumentation-autogen", "package": "loongsuite-instrumentation-autogen", "tox_env": "lint-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen"}, {"name": "lint-loongsuite-instrumentation-deepagents", "package": "loongsuite-instrumentation-deepagents", "tox_env": "lint-loongsuite-instrumentation-deepagents", "ui_name": "loongsuite-instrumentation-deepagents"}, {"name": "lint-loongsuite-instrumentation-microsoft-agent-framework", "package": "loongsuite-instrumentation-microsoft-agent-framework", "tox_env": "lint-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework"}, {"name": "lint-loongsuite-instrumentation-strands", "package": "loongsuite-instrumentation-strands", "tox_env": "lint-loongsuite-instrumentation-strands", "ui_name": "loongsuite-instrumentation-strands"}, {"name": "lint-loongsuite-instrumentation-qwen-agent", "package": "loongsuite-instrumentation-qwen-agent", "tox_env": "lint-loongsuite-instrumentation-qwen-agent", "ui_name": "loongsuite-instrumentation-qwen-agent"}, {"name": "lint-loongsuite-instrumentation-hermes-agent", "package": "loongsuite-instrumentation-hermes-agent", "tox_env": "lint-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent"}, {"name": "lint-loongsuite-instrumentation-mem0", "package": "loongsuite-instrumentation-mem0", "tox_env": "lint-loongsuite-instrumentation-mem0", "ui_name": "loongsuite-instrumentation-mem0"}, {"name": "lint-util-genai", "package": "util-genai", "tox_env": "lint-util-genai", "ui_name": "util-genai"}, {"name": "lint-loongsuite-instrumentation-litellm", "package": "loongsuite-instrumentation-litellm", "tox_env": "lint-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm"}, {"name": "lint-loongsuite-instrumentation-crewai", "package": "loongsuite-instrumentation-crewai", "tox_env": "lint-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai"}, {"name": "lint-loongsuite-instrumentation-qwenpaw", "package": "loongsuite-instrumentation-qwenpaw", "tox_env": "lint-loongsuite-instrumentation-qwenpaw", "ui_name": "loongsuite-instrumentation-qwenpaw"}, {"name": "lint-loongsuite-instrumentation-algotune", "package": "loongsuite-instrumentation-algotune", "tox_env": "lint-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune"}, {"name": "lint-loongsuite-instrumentation-bfclv4", "package": "loongsuite-instrumentation-bfclv4", "tox_env": "lint-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4"}, {"name": "lint-loongsuite-instrumentation-claw-eval", "package": "loongsuite-instrumentation-claw-eval", "tox_env": "lint-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval"}, {"name": "lint-loongsuite-instrumentation-minisweagent", "package": "loongsuite-instrumentation-minisweagent", "tox_env": "lint-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent"}, {"name": "lint-loongsuite-instrumentation-openhands", "package": "loongsuite-instrumentation-openhands", "tox_env": "lint-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands"}, {"name": "lint-loongsuite-instrumentation-slop-code", "package": "loongsuite-instrumentation-slop-code", "tox_env": "lint-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code"}, {"name": "lint-loongsuite-instrumentation-terminus2", "package": "loongsuite-instrumentation-terminus2", "tox_env": "lint-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2"}, {"name": "lint-loongsuite-instrumentation-vita", "package": "loongsuite-instrumentation-vita", "tox_env": "lint-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita"}, {"name": "lint-loongsuite-instrumentation-webarena", "package": "loongsuite-instrumentation-webarena", "tox_env": "lint-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena"}, {"name": "lint-loongsuite-instrumentation-widesearch", "package": "loongsuite-instrumentation-widesearch", "tox_env": "lint-loongsuite-instrumentation-widesearch", "ui_name": "loongsuite-instrumentation-widesearch"}, {"name": "lint-loongsuite-instrumentation-wildtool", "package": "loongsuite-instrumentation-wildtool", "tox_env": "lint-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool"}] + [{"name": "lint-loongsuite-instrumentation-agentscope", "package": "loongsuite-instrumentation-agentscope", "tox_env": "lint-loongsuite-instrumentation-agentscope", "ui_name": "loongsuite-instrumentation-agentscope"}, {"name": "lint-loongsuite-instrumentation-dashscope", "package": "loongsuite-instrumentation-dashscope", "tox_env": "lint-loongsuite-instrumentation-dashscope", "ui_name": "loongsuite-instrumentation-dashscope"}, {"name": "lint-loongsuite-instrumentation-claude-agent-sdk", "package": "loongsuite-instrumentation-claude-agent-sdk", "tox_env": "lint-loongsuite-instrumentation-claude-agent-sdk", "ui_name": "loongsuite-instrumentation-claude-agent-sdk"}, {"name": "lint-loongsuite-instrumentation-google-adk", "package": "loongsuite-instrumentation-google-adk", "tox_env": "lint-loongsuite-instrumentation-google-adk", "ui_name": "loongsuite-instrumentation-google-adk"}, {"name": "lint-loongsuite-instrumentation-google-genai", "package": "loongsuite-instrumentation-google-genai", "tox_env": "lint-loongsuite-instrumentation-google-genai", "ui_name": "loongsuite-instrumentation-google-genai"}, {"name": "lint-loongsuite-instrumentation-agno", "package": "loongsuite-instrumentation-agno", "tox_env": "lint-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno"}, {"name": "lint-loongsuite-instrumentation-langchain", "package": "loongsuite-instrumentation-langchain", "tox_env": "lint-loongsuite-instrumentation-langchain", "ui_name": "loongsuite-instrumentation-langchain"}, {"name": "lint-loongsuite-instrumentation-langgraph", "package": "loongsuite-instrumentation-langgraph", "tox_env": "lint-loongsuite-instrumentation-langgraph", "ui_name": "loongsuite-instrumentation-langgraph"}, {"name": "lint-loongsuite-instrumentation-autogen", "package": "loongsuite-instrumentation-autogen", "tox_env": "lint-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen"}, {"name": "lint-loongsuite-instrumentation-deepagents", "package": "loongsuite-instrumentation-deepagents", "tox_env": "lint-loongsuite-instrumentation-deepagents", "ui_name": "loongsuite-instrumentation-deepagents"}, {"name": "lint-loongsuite-instrumentation-microsoft-agent-framework", "package": "loongsuite-instrumentation-microsoft-agent-framework", "tox_env": "lint-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework"}, {"name": "lint-loongsuite-instrumentation-strands", "package": "loongsuite-instrumentation-strands", "tox_env": "lint-loongsuite-instrumentation-strands", "ui_name": "loongsuite-instrumentation-strands"}, {"name": "lint-loongsuite-instrumentation-qwen-agent", "package": "loongsuite-instrumentation-qwen-agent", "tox_env": "lint-loongsuite-instrumentation-qwen-agent", "ui_name": "loongsuite-instrumentation-qwen-agent"}, {"name": "lint-loongsuite-instrumentation-hermes-agent", "package": "loongsuite-instrumentation-hermes-agent", "tox_env": "lint-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent"}, {"name": "lint-loongsuite-instrumentation-mem0", "package": "loongsuite-instrumentation-mem0", "tox_env": "lint-loongsuite-instrumentation-mem0", "ui_name": "loongsuite-instrumentation-mem0"}, {"name": "lint-util-genai", "package": "util-genai", "tox_env": "lint-util-genai", "ui_name": "util-genai"}, {"name": "lint-loongsuite-instrumentation-litellm", "package": "loongsuite-instrumentation-litellm", "tox_env": "lint-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm"}, {"name": "lint-loongsuite-instrumentation-crewai", "package": "loongsuite-instrumentation-crewai", "tox_env": "lint-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai"}, {"name": "lint-loongsuite-instrumentation-qwenpaw", "package": "loongsuite-instrumentation-qwenpaw", "tox_env": "lint-loongsuite-instrumentation-qwenpaw", "ui_name": "loongsuite-instrumentation-qwenpaw"}, {"name": "lint-loongsuite-instrumentation-algotune", "package": "loongsuite-instrumentation-algotune", "tox_env": "lint-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune"}, {"name": "lint-loongsuite-instrumentation-bfclv4", "package": "loongsuite-instrumentation-bfclv4", "tox_env": "lint-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4"}, {"name": "lint-loongsuite-instrumentation-claw-eval", "package": "loongsuite-instrumentation-claw-eval", "tox_env": "lint-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval"}, {"name": "lint-loongsuite-instrumentation-minisweagent", "package": "loongsuite-instrumentation-minisweagent", "tox_env": "lint-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent"}, {"name": "lint-loongsuite-instrumentation-openhands", "package": "loongsuite-instrumentation-openhands", "tox_env": "lint-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands"}, {"name": "lint-loongsuite-instrumentation-slop-code", "package": "loongsuite-instrumentation-slop-code", "tox_env": "lint-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code"}, {"name": "lint-loongsuite-instrumentation-terminus2", "package": "loongsuite-instrumentation-terminus2", "tox_env": "lint-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2"}, {"name": "lint-loongsuite-instrumentation-vita", "package": "loongsuite-instrumentation-vita", "tox_env": "lint-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita"}, {"name": "lint-loongsuite-instrumentation-webarena", "package": "loongsuite-instrumentation-webarena", "tox_env": "lint-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena"}, {"name": "lint-loongsuite-instrumentation-widesearch", "package": "loongsuite-instrumentation-widesearch", "tox_env": "lint-loongsuite-instrumentation-widesearch", "ui_name": "loongsuite-instrumentation-widesearch"}, {"name": "lint-loongsuite-instrumentation-wildtool", "package": "loongsuite-instrumentation-wildtool", "tox_env": "lint-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool"}] LOONGSUITE_FULL: ${{ steps.detect.outputs.full }} LOONGSUITE_PACKAGES: ${{ steps.detect.outputs.packages }} run: | diff --git a/.github/workflows/loongsuite_test_0.yml b/.github/workflows/loongsuite_test_0.yml index bd5199e3b..092a82805 100644 --- a/.github/workflows/loongsuite_test_0.yml +++ b/.github/workflows/loongsuite_test_0.yml @@ -84,7 +84,7 @@ jobs: shell: bash env: LOONGSUITE_ALL_JOBS: >- - [{"name": "py310-test-loongsuite-instrumentation-agentscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-agentscope-oldest", "ui_name": "loongsuite-instrumentation-agentscope-oldest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-agentscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-agentscope-oldest", "ui_name": "loongsuite-instrumentation-agentscope-oldest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-agentscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-agentscope-oldest", "ui_name": "loongsuite-instrumentation-agentscope-oldest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-agentscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-agentscope-oldest", "ui_name": "loongsuite-instrumentation-agentscope-oldest 3.13 ARC"}, {"name": "py311-test-loongsuite-instrumentation-agentscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-agentscope-latest", "ui_name": "loongsuite-instrumentation-agentscope-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-agentscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-agentscope-latest", "ui_name": "loongsuite-instrumentation-agentscope-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-agentscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-agentscope-latest", "ui_name": "loongsuite-instrumentation-agentscope-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-claude-agent-sdk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-claude-agent-sdk-oldest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-claude-agent-sdk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-claude-agent-sdk-latest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-claude-agent-sdk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-claude-agent-sdk-oldest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-claude-agent-sdk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-claude-agent-sdk-latest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-claude-agent-sdk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-claude-agent-sdk-oldest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-claude-agent-sdk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-claude-agent-sdk-latest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-claude-agent-sdk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-claude-agent-sdk-oldest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-claude-agent-sdk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-claude-agent-sdk-latest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-google-adk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-google-adk-oldest", "ui_name": "loongsuite-instrumentation-google-adk-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-google-adk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-google-adk-latest", "ui_name": "loongsuite-instrumentation-google-adk-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-google-adk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-google-adk-oldest", "ui_name": "loongsuite-instrumentation-google-adk-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-google-adk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-google-adk-latest", "ui_name": "loongsuite-instrumentation-google-adk-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-google-adk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-google-adk-oldest", "ui_name": "loongsuite-instrumentation-google-adk-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-google-adk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-google-adk-latest", "ui_name": "loongsuite-instrumentation-google-adk-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-google-adk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-google-adk-oldest", "ui_name": "loongsuite-instrumentation-google-adk-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-google-adk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-google-adk-latest", "ui_name": "loongsuite-instrumentation-google-adk-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-autogen_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-autogen", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-autogen_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-autogen", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-autogen_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-autogen", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-autogen_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-autogen", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen 3.13 ARC"}, {"name": "py311-test-loongsuite-instrumentation-deepagents-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-deepagents", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-deepagents-latest", "ui_name": "loongsuite-instrumentation-deepagents-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-deepagents-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-deepagents", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-deepagents-latest", "ui_name": "loongsuite-instrumentation-deepagents-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-deepagents-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-deepagents", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-deepagents-latest", "ui_name": "loongsuite-instrumentation-deepagents-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-microsoft-agent-framework_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-microsoft-agent-framework", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-microsoft-agent-framework_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-microsoft-agent-framework", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-microsoft-agent-framework_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-microsoft-agent-framework", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-microsoft-agent-framework_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-microsoft-agent-framework", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-strands-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-strands-oldest", "ui_name": "loongsuite-instrumentation-strands-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-strands-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-strands-latest", "ui_name": "loongsuite-instrumentation-strands-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-strands-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-strands-oldest", "ui_name": "loongsuite-instrumentation-strands-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-strands-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-strands-latest", "ui_name": "loongsuite-instrumentation-strands-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-strands-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-strands-oldest", "ui_name": "loongsuite-instrumentation-strands-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-strands-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-strands-latest", "ui_name": "loongsuite-instrumentation-strands-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-strands-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-strands-oldest", "ui_name": "loongsuite-instrumentation-strands-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-strands-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-strands-latest", "ui_name": "loongsuite-instrumentation-strands-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-hermes-agent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-hermes-agent", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-hermes-agent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-hermes-agent", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-hermes-agent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-hermes-agent", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-hermes-agent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-hermes-agent", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-mem0-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-mem0-oldest", "ui_name": "loongsuite-instrumentation-mem0-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-mem0-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-mem0-latest", "ui_name": "loongsuite-instrumentation-mem0-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-mem0-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-mem0-oldest", "ui_name": "loongsuite-instrumentation-mem0-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-mem0-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-mem0-latest", "ui_name": "loongsuite-instrumentation-mem0-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-mem0-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-mem0-oldest", "ui_name": "loongsuite-instrumentation-mem0-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-mem0-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-mem0-latest", "ui_name": "loongsuite-instrumentation-mem0-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-mem0-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-mem0-oldest", "ui_name": "loongsuite-instrumentation-mem0-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-mem0-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-mem0-latest", "ui_name": "loongsuite-instrumentation-mem0-latest 3.13 ARC"}, {"name": "py39-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.9", "tox_env": "py39-test-util-genai", "ui_name": "util-genai 3.9 ARC"}, {"name": "py310-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.10", "tox_env": "py310-test-util-genai", "ui_name": "util-genai 3.10 ARC"}, {"name": "py311-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.11", "tox_env": "py311-test-util-genai", "ui_name": "util-genai 3.11 ARC"}, {"name": "py312-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.12", "tox_env": "py312-test-util-genai", "ui_name": "util-genai 3.12 ARC"}, {"name": "py313-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.13", "tox_env": "py313-test-util-genai", "ui_name": "util-genai 3.13 ARC"}, {"name": "py314-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.14", "tox_env": "py314-test-util-genai", "ui_name": "util-genai 3.14 ARC"}, {"name": "pypy3-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "pypy-3.9", "tox_env": "pypy3-test-util-genai", "ui_name": "util-genai pypy-3.9 ARC"}, {"name": "py311-test-detect-loongsuite-changes_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "detect-loongsuite-changes", "python_version": "3.11", "tox_env": "py311-test-detect-loongsuite-changes", "ui_name": "detect-loongsuite-changes 3.11 ARC"}, {"name": "py310-test-loongsuite-instrumentation-litellm_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-litellm", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-litellm_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-litellm", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-litellm_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-litellm", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-litellm_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-litellm", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-crewai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-crewai", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-crewai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-crewai", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-crewai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-crewai", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-crewai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-crewai", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai 3.13 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwenpaw-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwenpaw-latest", "ui_name": "loongsuite-instrumentation-qwenpaw-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwenpaw-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwenpaw-latest", "ui_name": "loongsuite-instrumentation-qwenpaw-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwenpaw-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwenpaw-latest", "ui_name": "loongsuite-instrumentation-qwenpaw-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-qwenpaw-v1_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-qwenpaw-v1", "ui_name": "loongsuite-instrumentation-qwenpaw-v1 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-qwenpaw-legacy_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-qwenpaw-legacy", "ui_name": "loongsuite-instrumentation-qwenpaw-legacy 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwenpaw-v1_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwenpaw-v1", "ui_name": "loongsuite-instrumentation-qwenpaw-v1 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwenpaw-legacy_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwenpaw-legacy", "ui_name": "loongsuite-instrumentation-qwenpaw-legacy 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwenpaw-v1_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwenpaw-v1", "ui_name": "loongsuite-instrumentation-qwenpaw-v1 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwenpaw-legacy_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwenpaw-legacy", "ui_name": "loongsuite-instrumentation-qwenpaw-legacy 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwenpaw-v1_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwenpaw-v1", "ui_name": "loongsuite-instrumentation-qwenpaw-v1 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwenpaw-legacy_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwenpaw-legacy", "ui_name": "loongsuite-instrumentation-qwenpaw-legacy 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-algotune_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-algotune", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-algotune_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-algotune", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-algotune_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-algotune", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-algotune_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-algotune", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-bfclv4_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-bfclv4", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-bfclv4_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-bfclv4", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-bfclv4_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-bfclv4", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-bfclv4_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-bfclv4", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-claw-eval_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claw-eval", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-claw-eval_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claw-eval", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-claw-eval_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claw-eval", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-claw-eval_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claw-eval", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-minisweagent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-minisweagent", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-minisweagent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-minisweagent", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-minisweagent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-minisweagent", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-minisweagent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-minisweagent", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-openhands_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-openhands", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-openhands_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-openhands", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-openhands_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-openhands", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-openhands_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-openhands", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-slop-code_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-slop-code", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-slop-code_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-slop-code", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-slop-code_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-slop-code", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-slop-code_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-slop-code", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-terminus2_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-terminus2", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-terminus2_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-terminus2", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-terminus2_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-terminus2", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-terminus2_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-terminus2", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-vita_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-vita", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-vita_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-vita", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-vita_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-vita", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-vita_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-vita", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-webarena_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-webarena", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-webarena_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-webarena", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-webarena_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-webarena", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-webarena_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-webarena", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena 3.13 ARC"}, {"name": "py311-test-loongsuite-instrumentation-widesearch_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-widesearch", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-widesearch", "ui_name": "loongsuite-instrumentation-widesearch 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-widesearch_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-widesearch", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-widesearch", "ui_name": "loongsuite-instrumentation-widesearch 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-widesearch_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-widesearch", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-widesearch", "ui_name": "loongsuite-instrumentation-widesearch 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-wildtool_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-wildtool", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-wildtool_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-wildtool", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-wildtool_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-wildtool", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-wildtool_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-wildtool", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool 3.13 ARC"}, {"name": "py39-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.9", "tox_env": "py39-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.9 ARC"}, {"name": "py310-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.10", "tox_env": "py310-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.10 ARC"}, {"name": "py311-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.11", "tox_env": "py311-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.11 ARC"}, {"name": "py312-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.12", "tox_env": "py312-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.12 ARC"}, {"name": "py313-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.13", "tox_env": "py313-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.13 ARC"}] + [{"name": "py310-test-loongsuite-instrumentation-agentscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-agentscope-oldest", "ui_name": "loongsuite-instrumentation-agentscope-oldest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-agentscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-agentscope-oldest", "ui_name": "loongsuite-instrumentation-agentscope-oldest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-agentscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-agentscope-oldest", "ui_name": "loongsuite-instrumentation-agentscope-oldest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-agentscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-agentscope-oldest", "ui_name": "loongsuite-instrumentation-agentscope-oldest 3.13 ARC"}, {"name": "py311-test-loongsuite-instrumentation-agentscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-agentscope-latest", "ui_name": "loongsuite-instrumentation-agentscope-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-agentscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-agentscope-latest", "ui_name": "loongsuite-instrumentation-agentscope-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-agentscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-agentscope-latest", "ui_name": "loongsuite-instrumentation-agentscope-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-claude-agent-sdk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-claude-agent-sdk-oldest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-claude-agent-sdk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-claude-agent-sdk-latest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-claude-agent-sdk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-claude-agent-sdk-oldest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-claude-agent-sdk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-claude-agent-sdk-latest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-claude-agent-sdk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-claude-agent-sdk-oldest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-claude-agent-sdk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-claude-agent-sdk-latest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-claude-agent-sdk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-claude-agent-sdk-oldest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-claude-agent-sdk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-claude-agent-sdk-latest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-google-adk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-google-adk-oldest", "ui_name": "loongsuite-instrumentation-google-adk-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-google-adk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-google-adk-latest", "ui_name": "loongsuite-instrumentation-google-adk-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-google-adk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-google-adk-oldest", "ui_name": "loongsuite-instrumentation-google-adk-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-google-adk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-google-adk-latest", "ui_name": "loongsuite-instrumentation-google-adk-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-google-adk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-google-adk-oldest", "ui_name": "loongsuite-instrumentation-google-adk-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-google-adk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-google-adk-latest", "ui_name": "loongsuite-instrumentation-google-adk-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-google-adk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-google-adk-oldest", "ui_name": "loongsuite-instrumentation-google-adk-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-google-adk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-google-adk-latest", "ui_name": "loongsuite-instrumentation-google-adk-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-google-genai-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-google-genai-oldest", "ui_name": "loongsuite-instrumentation-google-genai-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-google-genai-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-google-genai-latest", "ui_name": "loongsuite-instrumentation-google-genai-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-google-genai-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-google-genai-oldest", "ui_name": "loongsuite-instrumentation-google-genai-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-google-genai-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-google-genai-latest", "ui_name": "loongsuite-instrumentation-google-genai-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-google-genai-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-google-genai-oldest", "ui_name": "loongsuite-instrumentation-google-genai-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-google-genai-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-google-genai-latest", "ui_name": "loongsuite-instrumentation-google-genai-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-google-genai-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-google-genai-oldest", "ui_name": "loongsuite-instrumentation-google-genai-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-google-genai-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-google-genai-latest", "ui_name": "loongsuite-instrumentation-google-genai-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-google-genai-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-google-genai-oldest", "ui_name": "loongsuite-instrumentation-google-genai-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-google-genai-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-google-genai-latest", "ui_name": "loongsuite-instrumentation-google-genai-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-autogen_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-autogen", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-autogen_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-autogen", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-autogen_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-autogen", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-autogen_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-autogen", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen 3.13 ARC"}, {"name": "py311-test-loongsuite-instrumentation-deepagents-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-deepagents", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-deepagents-latest", "ui_name": "loongsuite-instrumentation-deepagents-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-deepagents-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-deepagents", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-deepagents-latest", "ui_name": "loongsuite-instrumentation-deepagents-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-deepagents-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-deepagents", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-deepagents-latest", "ui_name": "loongsuite-instrumentation-deepagents-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-microsoft-agent-framework_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-microsoft-agent-framework", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-microsoft-agent-framework_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-microsoft-agent-framework", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-microsoft-agent-framework_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-microsoft-agent-framework", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-microsoft-agent-framework_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-microsoft-agent-framework", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-strands-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-strands-oldest", "ui_name": "loongsuite-instrumentation-strands-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-strands-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-strands-latest", "ui_name": "loongsuite-instrumentation-strands-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-strands-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-strands-oldest", "ui_name": "loongsuite-instrumentation-strands-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-strands-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-strands-latest", "ui_name": "loongsuite-instrumentation-strands-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-strands-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-strands-oldest", "ui_name": "loongsuite-instrumentation-strands-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-strands-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-strands-latest", "ui_name": "loongsuite-instrumentation-strands-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-strands-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-strands-oldest", "ui_name": "loongsuite-instrumentation-strands-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-strands-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-strands-latest", "ui_name": "loongsuite-instrumentation-strands-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-hermes-agent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-hermes-agent", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-hermes-agent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-hermes-agent", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-hermes-agent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-hermes-agent", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-hermes-agent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-hermes-agent", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-mem0-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-mem0-oldest", "ui_name": "loongsuite-instrumentation-mem0-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-mem0-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-mem0-latest", "ui_name": "loongsuite-instrumentation-mem0-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-mem0-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-mem0-oldest", "ui_name": "loongsuite-instrumentation-mem0-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-mem0-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-mem0-latest", "ui_name": "loongsuite-instrumentation-mem0-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-mem0-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-mem0-oldest", "ui_name": "loongsuite-instrumentation-mem0-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-mem0-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-mem0-latest", "ui_name": "loongsuite-instrumentation-mem0-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-mem0-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-mem0-oldest", "ui_name": "loongsuite-instrumentation-mem0-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-mem0-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-mem0-latest", "ui_name": "loongsuite-instrumentation-mem0-latest 3.13 ARC"}, {"name": "py39-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.9", "tox_env": "py39-test-util-genai", "ui_name": "util-genai 3.9 ARC"}, {"name": "py310-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.10", "tox_env": "py310-test-util-genai", "ui_name": "util-genai 3.10 ARC"}, {"name": "py311-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.11", "tox_env": "py311-test-util-genai", "ui_name": "util-genai 3.11 ARC"}, {"name": "py312-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.12", "tox_env": "py312-test-util-genai", "ui_name": "util-genai 3.12 ARC"}, {"name": "py313-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.13", "tox_env": "py313-test-util-genai", "ui_name": "util-genai 3.13 ARC"}, {"name": "py314-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.14", "tox_env": "py314-test-util-genai", "ui_name": "util-genai 3.14 ARC"}, {"name": "pypy3-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "pypy-3.9", "tox_env": "pypy3-test-util-genai", "ui_name": "util-genai pypy-3.9 ARC"}, {"name": "py311-test-detect-loongsuite-changes_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "detect-loongsuite-changes", "python_version": "3.11", "tox_env": "py311-test-detect-loongsuite-changes", "ui_name": "detect-loongsuite-changes 3.11 ARC"}, {"name": "py310-test-loongsuite-instrumentation-litellm_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-litellm", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-litellm_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-litellm", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-litellm_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-litellm", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-litellm_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-litellm", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-crewai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-crewai", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-crewai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-crewai", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-crewai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-crewai", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-crewai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-crewai", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai 3.13 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwenpaw-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwenpaw-latest", "ui_name": "loongsuite-instrumentation-qwenpaw-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwenpaw-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwenpaw-latest", "ui_name": "loongsuite-instrumentation-qwenpaw-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwenpaw-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwenpaw-latest", "ui_name": "loongsuite-instrumentation-qwenpaw-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-qwenpaw-v1_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-qwenpaw-v1", "ui_name": "loongsuite-instrumentation-qwenpaw-v1 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-qwenpaw-legacy_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-qwenpaw-legacy", "ui_name": "loongsuite-instrumentation-qwenpaw-legacy 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwenpaw-v1_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwenpaw-v1", "ui_name": "loongsuite-instrumentation-qwenpaw-v1 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwenpaw-legacy_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwenpaw-legacy", "ui_name": "loongsuite-instrumentation-qwenpaw-legacy 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwenpaw-v1_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwenpaw-v1", "ui_name": "loongsuite-instrumentation-qwenpaw-v1 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwenpaw-legacy_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwenpaw-legacy", "ui_name": "loongsuite-instrumentation-qwenpaw-legacy 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwenpaw-v1_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwenpaw-v1", "ui_name": "loongsuite-instrumentation-qwenpaw-v1 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwenpaw-legacy_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwenpaw-legacy", "ui_name": "loongsuite-instrumentation-qwenpaw-legacy 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-algotune_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-algotune", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-algotune_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-algotune", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-algotune_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-algotune", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-algotune_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-algotune", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-bfclv4_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-bfclv4", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-bfclv4_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-bfclv4", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-bfclv4_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-bfclv4", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-bfclv4_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-bfclv4", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-claw-eval_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claw-eval", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-claw-eval_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claw-eval", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-claw-eval_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claw-eval", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-claw-eval_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claw-eval", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-minisweagent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-minisweagent", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-minisweagent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-minisweagent", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-minisweagent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-minisweagent", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-minisweagent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-minisweagent", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-openhands_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-openhands", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-openhands_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-openhands", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-openhands_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-openhands", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-openhands_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-openhands", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-slop-code_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-slop-code", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-slop-code_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-slop-code", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-slop-code_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-slop-code", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-slop-code_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-slop-code", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-terminus2_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-terminus2", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-terminus2_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-terminus2", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-terminus2_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-terminus2", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-terminus2_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-terminus2", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-vita_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-vita", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-vita_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-vita", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-vita_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-vita", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-vita_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-vita", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-webarena_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-webarena", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-webarena_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-webarena", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-webarena_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-webarena", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-webarena_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-webarena", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena 3.13 ARC"}, {"name": "py311-test-loongsuite-instrumentation-widesearch_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-widesearch", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-widesearch", "ui_name": "loongsuite-instrumentation-widesearch 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-widesearch_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-widesearch", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-widesearch", "ui_name": "loongsuite-instrumentation-widesearch 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-widesearch_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-widesearch", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-widesearch", "ui_name": "loongsuite-instrumentation-widesearch 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-wildtool_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-wildtool", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-wildtool_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-wildtool", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-wildtool_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-wildtool", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-wildtool_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-wildtool", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool 3.13 ARC"}, {"name": "py39-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.9", "tox_env": "py39-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.9 ARC"}, {"name": "py310-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.10", "tox_env": "py310-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.10 ARC"}, {"name": "py311-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.11", "tox_env": "py311-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.11 ARC"}, {"name": "py312-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.12", "tox_env": "py312-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.12 ARC"}, {"name": "py313-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.13", "tox_env": "py313-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.13 ARC"}] LOONGSUITE_FULL: ${{ steps.detect.outputs.full }} LOONGSUITE_PACKAGES: ${{ steps.detect.outputs.packages }} run: | diff --git a/instrumentation-loongsuite/README.md b/instrumentation-loongsuite/README.md index c173bb39b..8936c6c32 100644 --- a/instrumentation-loongsuite/README.md +++ b/instrumentation-loongsuite/README.md @@ -13,6 +13,7 @@ | [loongsuite-instrumentation-deepagents](./loongsuite-instrumentation-deepagents) | deepagents >= 0.6.0, < 0.7.0 | No | development | [loongsuite-instrumentation-dify](./loongsuite-instrumentation-dify) | dify | No | development | [loongsuite-instrumentation-google-adk](./loongsuite-instrumentation-google-adk) | google-adk >= 0.1.0 | No | development +| [loongsuite-instrumentation-google-genai](./loongsuite-instrumentation-google-genai) | google-genai >= 1.32.0, < 3 | Yes | development | [loongsuite-instrumentation-hermes-agent](./loongsuite-instrumentation-hermes-agent) | openai >= 1.0.0 | No | development | [loongsuite-instrumentation-langchain](./loongsuite-instrumentation-langchain) | langchain_core >= 0.1.0 | No | development | [loongsuite-instrumentation-langgraph](./loongsuite-instrumentation-langgraph) | langgraph >= 0.2 | No | development @@ -30,4 +31,4 @@ | [loongsuite-instrumentation-vita](./loongsuite-instrumentation-vita) | vita >= 0.0.1 | No | development | [loongsuite-instrumentation-webarena](./loongsuite-instrumentation-webarena) | webarena >= 0.0.1 | No | development | [loongsuite-instrumentation-widesearch](./loongsuite-instrumentation-widesearch) | widesearch >= 0.1.0 | No | development -| [loongsuite-instrumentation-wildtool](./loongsuite-instrumentation-wildtool) | openai >= 1.0.0 | No | development \ No newline at end of file +| [loongsuite-instrumentation-wildtool](./loongsuite-instrumentation-wildtool) | openai >= 1.0.0 | No | development diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/CHANGELOG.md b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/CHANGELOG.md new file mode 100644 index 000000000..d2b284717 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/CHANGELOG.md @@ -0,0 +1,52 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## Unreleased + +### Added + +- Align provider hooks and tests with the canonical + ``opentelemetry-instrumentation-google-genai==1.0b1`` baseline. +- Support ``interactions.create`` and automatic function-call + ``execute_tool`` spans in addition to generation, streaming, and embeddings. +- Retain LoongSuite ``ExtendedTelemetryHandler`` metrics, multimodal handling, + standard instrumentation suppression, reasoning-part capture, Python 3.9, + and completion-hook support through an isolated compatibility layer. +- Document that the provider runs on LoongSuite's shared GenAI util, while + Robin adds ARMS metrics and private lower-level SDK suppression as a + commercial overlay. +- Fix reusable-config mutation, stream-construction span leaks, Google async + stream closing, streaming TTFT, real Interactions SSE completion parsing, + and embedding raw-response state isolation found during the upstream delta + audit. +- Add local opt-in real Gemini API tests and redacted VCR coverage for public + CI without provider credentials. +- Apply the shared ``hook_advice`` fail-open contract to generation, + Interactions, embeddings, automatic tool calls, and sync/async streams so + probe failures cannot change SDK call count, results, chunks, cancellation, + ``GeneratorExit``, or original provider exceptions. +- Detach streaming context before returning SDK streams, make finalization + idempotent across close/aclose/error/GC paths, and cover cross-Context and + one-fault-among-many isolation. +- Keep OSS package metadata at Python 3.9 while avoiding Python 3.9/3.10-only + typing syntax so Robin can repackage the instrumentation wheel for its + Python 3.8 commercial install floor. + +### Fixed + +- Aggregate Google GenAI streaming deltas by candidate so + ``gen_ai.output.messages`` contains complete logical responses instead of + one message per SSE chunk. + +## Version 0.9.0.dev + +### Added + +- Add LoongSuite Google GenAI instrumentation backed by + `ExtendedTelemetryHandler`. +- Cover sync/async generation, streaming, and embeddings with extended GenAI + attributes, metrics, multimodal messages, response identity, and TTFT. diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/README.rst b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/README.rst new file mode 100644 index 000000000..7ab6ad716 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/README.rst @@ -0,0 +1,105 @@ +LoongSuite Google GenAI SDK Instrumentation +=========================================== + +This package aligns with +``opentelemetry-instrumentation-google-genai==1.0b1`` from the canonical +``open-telemetry/opentelemetry-python-genai`` repository, but it does not use +the upstream 1.x GenAI util at runtime. Provider hooks are adapted to +LoongSuite's own ``opentelemetry-util-genai`` source package and +``ExtendedTelemetryHandler``. LoongSuite release packaging renames that shared +util to ``loongsuite-otel-util-genai`` and replaces the renamed community +Google GenAI package with this implementation. + +Supported operations +-------------------- + +* synchronous and asynchronous ``generate_content``; +* synchronous and asynchronous ``generate_content_stream``; +* synchronous and asynchronous ``embed_content``; +* synchronous and asynchronous ``interactions.create``, including streams; +* SDK automatic function calling with nested ``execute_tool`` spans; +* Gemini Developer API and Vertex AI provider identification; +* text, reasoning, inline binary, URI, function-call, and function-response + message parts; +* request parameters, tool definitions, response IDs/models, token usage, + finish reasons, streaming TTFT, and ``hook_advice`` fail-open error handling; +* opt-in Google-specific request configuration attributes through + ``OTEL_GOOGLE_GENAI_GENERATE_CONTENT_CONFIG_INCLUDES`` and + ``OTEL_GOOGLE_GENAI_GENERATE_CONTENT_CONFIG_EXCLUDES``. + +The Interactions API is available in recent ``google-genai`` 2.x releases. +Generation and embedding remain supported from ``google-genai`` 1.32 onward. +Instrumentation-only failures are degraded to missing telemetry: the original +SDK call or application tool executes once, and its return object, streamed +chunks, cancellation, ``GeneratorExit``, and exception identity are preserved. + +Installation +------------ + +:: + + pip install loongsuite-instrumentation-google-genai google-genai + +Usage +----- + +:: + + from google import genai + from opentelemetry.instrumentation.google_genai import ( + GoogleGenAiSdkInstrumentor, + ) + + GoogleGenAiSdkInstrumentor().instrument() + client = genai.Client() + response = client.models.generate_content( + model="gemini-2.5-flash", + contents="Explain OpenTelemetry in one sentence.", + ) + +Message content follows the shared GenAI util content-capture configuration. +For example, use +``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=SPAN_ONLY`` to record +messages on spans. + +Metrics and commercial extensions +--------------------------------- + +Span lifecycle, standard ``gen_ai.client.*`` metrics, multimodal processing, +and completion hooks are owned by LoongSuite's shared GenAI util rather than +implemented again in this provider plugin. + +The Robin commercial package keeps the same provider hooks but supplies its +commercial GenAI util underneath ``ExtendedTelemetryHandler``. That util adds +ARMS metrics and other enterprise behavior. Robin also extends the +provider-level suppression check to honor its private +``_SUPPRESS_LLM_SDK_KEY`` so a high-level framework span can suppress only the +nested Google GenAI SDK spans. The private key is intentionally not part of +this OSS package. + +Upstream synchronization +------------------------ + +The upstream baseline and the intentionally retained LoongSuite delta are +documented in ``UPSTREAM.md``. Provider hook files preserve the upstream +layout, while ``_compat.py`` contains the removable adapter to LoongSuite's +extended GenAI util. + +Live verification +----------------- + +The regular test matrix does not require external credentials. To exercise the +latest supported SDK against the real Gemini Developer API, provide your own +restricted key through the environment and opt in explicitly: + +:: + + RUN_GOOGLE_GENAI_LIVE_TESTS=1 \ + tox -c tox-loongsuite.ini \ + -e py312-test-loongsuite-instrumentation-google-genai-latest \ + -- -k live_google_genai + +The local-only live test reads ``GEMINI_API_KEY`` (or ``GOOGLE_API_KEY``) and +uses ``gemini-3.5-flash-lite`` by default. Set the local +``GOOGLE_GENAI_LIVE_MODEL`` environment variable to change the model. Public CI +only replays redacted VCR cassettes and does not require provider credentials. diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/UPSTREAM.md b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/UPSTREAM.md new file mode 100644 index 000000000..11a4a6697 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/UPSTREAM.md @@ -0,0 +1,210 @@ +# Upstream synchronization + +This package follows the canonical Google GenAI instrumentation maintained in +[`open-telemetry/opentelemetry-python-genai`](https://github.com/open-telemetry/opentelemetry-python-genai/tree/main/instrumentation/opentelemetry-instrumentation-google-genai). + +## Baseline + +- Package release: `opentelemetry-instrumentation-google-genai==1.0b1` +- Repository tag: + `opentelemetry-instrumentation-google-genai==1.0b1` +- Repository commit: `d168d3e6bd280ae131fd56f60a15b449626ab336` +- Supported SDK range: `google-genai>=1.32,<3` +- Upstream `main` snapshot audited after the tag: + `e83bf7df909abb5c2e069ec6480b942638389706` +- OpenInference Google GenAI comparison snapshot: + `a1392c50d2d5b20fb805c195fb6006c80d5a6106` + +The provider hook modules keep the upstream file layout: + +- `allowlist_util.py` +- `client_info.py` +- `custom_semconv.py` +- `dict_util.py` +- `embeddings.py` +- `generate_content.py` +- `instrumentor.py` +- `interactions.py` +- `message.py` +- `tool_call_wrapper.py` + +Upstream tests for generation, streaming, embeddings, interactions, +configuration allowlists, finish reasons, and automatic function calls are +reused in this package. + +## LoongSuite delta + +The intentional product delta is kept small and explicit: + +- `_compat.py` adapts the upstream invocation-oriented handler API to + LoongSuite's own `opentelemetry-util-genai` implementation and + `ExtendedTelemetryHandler`; the upstream 1.x GenAI util is not a runtime + dependency of this package. +- `_stream.py` is the upstream GenAI stream lifecycle helper with Python 3.8 + source compatibility and LoongSuite fail-open stream ownership. +- provider source and tests avoid PEP 604 unions and PEP 585 built-in + generics. OSS metadata remains `requires-python >=3.9` because the supported + `google-genai>=1.32` SDK itself requires Python 3.9, while the source-level + compatibility lets Robin normalize the packaged instrumentation wheel to + its Python 3.8 commercial install floor without introducing import-time + syntax or annotation failures. +- provider-owned prepare, response mapping, failure reporting, stream wrapping, + chunk accumulation, finalization, embeddings, and automatic tool telemetry + run through `hook_advice`; the Google SDK call, SDK stream iteration, and + application tool callback remain outside advice and execute exactly once. +- standard OpenTelemetry suppression is honored without adding Robin's + commercial `_SUPPRESS_LLM_SDK_KEY`; +- `execute_tool` remains an INTERNAL span through the extended handler; +- Google thought parts map to the standard `Reasoning` message part; +- upstream 1.0 event/content behavior is preserved while the repository still + ships the 0.x shared GenAI util; +- LoongSuite completion hooks, metrics, and asynchronous multimodal processing + remain active. + +The audit also found lifecycle defects that are still present in the frozen +release and the audited upstream `main` snapshot. They are kept as isolated +provider fixes with regression tests: + +- tool wrapping copies `GenerateContentConfig` so a caller can safely reuse it; +- an invalid configuration dictionary is passed through unchanged instead of + being replaced by an empty validated configuration; +- synchronous and asynchronous stream-construction failures finish the + invocation instead of leaking its span/context; +- async stream close supports the Google SDK's `aclose()` contract; +- generation and Interactions streams record time to first token; +- Interactions streaming accepts both the upstream test fixture's + `interaction_completed` event and the dotted `interaction.completed` event + emitted by the real Google GenAI 2.x API; +- embedding raw-response state is reset with a `ContextVar` token even when a + request fails; +- explicit `OTEL_INSTRUMENTATION_GENAI_EMIT_EVENT=false` is preserved. +- probe failures cannot replace a Google SDK result, yielded chunk, application + tool result, cancellation, `GeneratorExit`, or original provider exception; +- stream context is detached before returning the stream, finalization is + idempotent, and cross-thread/cross-Context consumption does not reuse the + creation token; +- MCP tool metadata collection does not call application callbacks before the + real provider request. + +The upstream and OpenInference comparisons both provide useful partial +isolation, but neither snapshot satisfies this complete contract. +OpenInference protects many span mutation/finalization calls and falls back to +the raw stream when wrapper construction fails, while its request extraction +and stream chunk accumulator can still fail before or during business +delivery. The canonical upstream snapshot still mixes response mapping and +`invocation.stop()`/`fail()` with the provider-call `try` block. LoongSuite +keeps this lifecycle delta until an equivalent provider-neutral contract lands +upstream. + +Package metadata, release naming, bootstrap registration, local live tests, +and sanitized VCR cassettes are LoongSuite-owned. + +## OSS and Robin runtime boundary + +The provider hooks have one lifecycle path in both distributions: + +1. Google GenAI wrappers create the upstream-shaped invocation facade in + `_compat.py`. +2. The facade delegates start, stop, failure, embedding, and tool lifecycle to + LoongSuite's `ExtendedTelemetryHandler`. +3. The installed implementation of the shared GenAI util determines the + telemetry extensions. + +In OSS, the source dependency is named `opentelemetry-util-genai`. LoongSuite +release tooling publishes that source as `loongsuite-otel-util-genai` and +rewrites package dependencies without changing the +`opentelemetry.util.genai` import namespace. The shared util owns standard +`gen_ai.client.*` metrics, content capture, completion hooks, and multimodal +processing. + +After synchronization into Robin, the same imports resolve to Robin's +commercial GenAI util. Its `EnterpriseInvocationMetricsRecorder` adds ARMS +metrics through `ExtendedTelemetryHandler`; the Google provider plugin must not +duplicate those instruments locally. + +Robin also carries `_SUPPRESS_LLM_SDK_KEY` as a narrow provider overlay: + +- the high-level framework handler creates its own AGENT, LLM, and TOOL spans + and scopes the key around downstream SDK execution; +- Robin's Google GenAI `_compat._is_suppressed()` checks that private key in + addition to standard OpenTelemetry suppression; +- suppressed SDK invocations create no Google provider LLM, EMBEDDING, or TOOL + spans; +- context restoration is verified for success, failure, cancellation, and + stream close so a later direct Google SDK call remains instrumented. + +The private key must not be added to the OSS util or made a condition inside +the shared handler's `start_llm()`: high-level framework handlers use that same +method and must continue to create their own spans. + +## Updating the baseline + +For each upstream release: + +1. Freeze the release tag and commit in this file. +2. Diff the provider hook modules and upstream tests against the frozen + baseline. +3. Import upstream changes without mixing in framework instrumentation such as + Google ADK. +4. Reapply only the imports that route invocation, handler, stream, and + compatibility types through `_compat.py` and `_stream.py`. +5. Compare the new release with upstream `main` so unreleased fixes are tracked + separately from LoongSuite extensions. +6. Run oldest/latest SDK tests, sanitized VCR replay, real Gemini SDK tests, + Weaver validation, and ARMS/CMS readback before updating the pull request. + +Once LoongSuite's shared GenAI util adopts the upstream 1.x invocation API, +`_compat.py` should be removed and the provider modules should return to exact +upstream imports. + +## Changes already on upstream main + +Do not carry these as independent LoongSuite patches. They landed upstream +after `1.0b1` and should be consumed with the next upstream release: + +- interactions response ID + ([upstream #233](https://github.com/open-telemetry/opentelemetry-python-genai/pull/233)); +- interactions tool definitions and response parsing fixes + ([upstream #271](https://github.com/open-telemetry/opentelemetry-python-genai/pull/271)); +- INTERNAL span kind for `execute_tool` + ([upstream #274](https://github.com/open-telemetry/opentelemetry-python-genai/pull/274)); +- reasoning tokens included in the output-token count + ([upstream #283](https://github.com/open-telemetry/opentelemetry-python-genai/pull/283)); +- Google API error-code based `error.type` + ([upstream #304](https://github.com/open-telemetry/opentelemetry-python-genai/pull/304)). + +## Upstream contribution candidates + +The following deltas are provider-independent or useful to other +instrumentation consumers and should be proposed upstream as small, +independent changes: + +1. Map Google `Part(thought=True)` to the standard `Reasoning` message part + instead of `Text`, with sync, async, and stream tests. +2. Honor OpenTelemetry's standard instrumentation suppression context in the + invocation/handler path. +3. Respect an explicitly configured + `OTEL_INSTRUMENTATION_GENAI_EMIT_EVENT` value instead of unconditionally + overwriting it during Google GenAI instrumentation. +4. Make tool wrapping non-mutating and preserve invalid configuration + pass-through semantics. +5. Finalize invocations when sync or async stream construction raises. +6. Support SDK streams that expose `aclose()` and record TTFT on the first + generation or Interactions stream item. +7. Parse the real Interactions SSE `interaction.completed` event while + retaining compatibility with the underscore spelling used by existing + fixtures. +8. Isolate the embedding raw-response `ContextVar` across failed and nested + requests. +9. Discuss a supported handler/invocation factory injection point so vendors + can add completion hooks, upload handling, or extra metrics without copying + provider hooks. This needs an upstream design issue before a code change. +10. Contribute provider-neutral fail-open lifecycle patches in small steps: + keep the provider call outside telemetry advice, return raw streams when + wrapping fails, isolate chunk/final callbacks, detach stream context before + ownership transfer, and preserve cancellation/exception identity. + +Python 3.8 source compatibility, the 0.x-to-1.x compatibility facade, +LoongSuite-specific metrics and asynchronous multimodal processing, +release/bootstrap integration, and Robin's `_SUPPRESS_LLM_SDK_KEY` are product +or compatibility concerns and are not direct upstream contribution candidates. diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/examples/google_genai_genai_smoke.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/examples/google_genai_genai_smoke.py new file mode 100644 index 000000000..c92ac0ef1 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/examples/google_genai_genai_smoke.py @@ -0,0 +1,28 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google import genai + +from opentelemetry.instrumentation.google_genai import ( + GoogleGenAiSdkInstrumentor, +) + +GoogleGenAiSdkInstrumentor().instrument() +client = genai.Client() + +for chunk in client.models.generate_content_stream( + model="gemini-2.5-flash", + contents="Explain distributed tracing in three short bullets.", +): + print(chunk.text, end="") diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/pyproject.toml b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/pyproject.toml new file mode 100644 index 000000000..e0554197c --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/pyproject.toml @@ -0,0 +1,54 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "loongsuite-instrumentation-google-genai" +dynamic = ["version"] +description = "LoongSuite Google GenAI SDK instrumentation" +supports_metrics = true +readme = "README.rst" +license = "Apache-2.0" +requires-python = ">=3.9" +authors = [ + { name = "OpenTelemetry Authors", email = "cncf-opentelemetry-contributors@lists.cncf.io" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +dependencies = [ + "opentelemetry-api >= 1.37.0", + "opentelemetry-instrumentation >= 0.58b0", + "opentelemetry-semantic-conventions >= 0.58b0", + "opentelemetry-util-genai", + "wrapt >= 1.17.0, < 3.0.0", +] + +[project.optional-dependencies] +instruments = ["google-genai >= 1.32.0, < 3"] + +[project.entry-points.opentelemetry_instrumentor] +google-genai = "opentelemetry.instrumentation.google_genai:GoogleGenAiSdkInstrumentor" + +[project.urls] +Homepage = "https://github.com/alibaba/loongsuite-python/tree/main/instrumentation-loongsuite/loongsuite-instrumentation-google-genai" +Repository = "https://github.com/alibaba/loongsuite-python" +Upstream = "https://github.com/open-telemetry/opentelemetry-python-genai/tree/main/instrumentation/opentelemetry-instrumentation-google-genai" + +[tool.hatch.version] +path = "src/opentelemetry/instrumentation/google_genai/version.py" + +[tool.hatch.build.targets.sdist] +include = ["/examples", "/src", "/tests"] + +[tool.hatch.build.targets.wheel] +packages = ["src/opentelemetry"] diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/__init__.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/__init__.py new file mode 100644 index 000000000..58ca8e37a --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/__init__.py @@ -0,0 +1,55 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +""" +Google Gen AI SDK client instrumentation supporting the `google-genai` package. + +It can be enabled using ``GoogleGenAiSdkInstrumentor``. + +.. _google-genai: https://pypi.org/project/google-genai/ + +Usage +----- + +.. code:: python + + import os + import google.genai + from opentelemetry.instrumentation.google_genai import GoogleGenAiSdkInstrumentor + + GoogleGenAiSdkInstrumentor().instrument() + model = os.getenv('MODEL', 'gemini-2.0-flash-001') + client = google.genai.Client() + response = client.models.generate_content( + model=model, + contents='why is the sky blue?' + ) + print(response.text) + +API +--- +""" + +from .generate_content import GENERATE_CONTENT_EXTRA_ATTRIBUTES_CONTEXT_KEY +from .instrumentor import GoogleGenAiSdkInstrumentor +from .version import __version__ + +__all__ = [ + "GoogleGenAiSdkInstrumentor", + "GENERATE_CONTENT_EXTRA_ATTRIBUTES_CONTEXT_KEY", + "__version__", +] diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/_compat.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/_compat.py new file mode 100644 index 000000000..ef5e76e1f --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/_compat.py @@ -0,0 +1,612 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Bridge the upstream GenAI 1.0 invocation API to LoongSuite extensions. + +The canonical Google GenAI instrumentation uses the invocation-oriented API +introduced by ``opentelemetry-util-genai`` 1.0. LoongSuite still exposes the +older lifecycle API plus ``ExtendedTelemetryHandler``. Keeping that impedance +match in this module lets the provider instrumentation stay close to upstream +and makes this bridge removable after the shared util migrates to 1.x. +""" + +from __future__ import annotations + +import logging +import os +import timeit +from dataclasses import asdict, dataclass +from types import TracebackType +from typing import Any, Dict, List, Optional, Type, Union + +from opentelemetry import context as context_api +from opentelemetry._logs import LogRecord, get_logger +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY +from opentelemetry.trace import set_span_in_context +from opentelemetry.util.genai.completion_hook import ( + CompletionHook, + load_completion_hook, +) +from opentelemetry.util.genai.extended_handler import ( + ExtendedTelemetryHandler, +) +from opentelemetry.util.genai.extended_types import ( + EmbeddingInvocation as LegacyEmbeddingInvocation, +) +from opentelemetry.util.genai.extended_types import ( + ExecuteToolInvocation as LegacyToolInvocation, +) +from opentelemetry.util.genai.handler import _safe_detach +from opentelemetry.util.genai.span_utils import ( + _get_llm_common_attributes, + _get_llm_request_attributes, + _get_llm_response_attributes, +) +from opentelemetry.util.genai.types import ( + ContentCapturingMode, + Error, + LLMInvocation, +) +from opentelemetry.util.genai.utils import ( + gen_ai_json_dumps, + should_emit_event, +) + +_logger = logging.getLogger(__name__) +_REASONING_OUTPUT_TOKENS = "gen_ai.usage.reasoning.output_tokens" +_FINISH_REASONS = "gen_ai.response.finish_reasons" +_INPUT_MESSAGES = "gen_ai.input.messages" +_OUTPUT_MESSAGES = "gen_ai.output.messages" +_SYSTEM_INSTRUCTIONS = "gen_ai.system_instructions" +_TOOL_DEFINITIONS = "gen_ai.tool.definitions" + + +def _is_suppressed() -> bool: + # OSS honors only the standard OpenTelemetry suppression contract. Robin + # extends this provider-level check with its private + # _SUPPRESS_LLM_SDK_KEY after synchronization. Keep that check here rather + # than in ExtendedTelemetryHandler.start_llm(): high-level framework + # instrumentations use the same handler and must retain their own spans. + return bool(context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY)) + + +def _capture_mode() -> ContentCapturingMode: + configured = os.environ.get( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT" + ) + if not configured: + return ContentCapturingMode.NO_CONTENT + try: + return ContentCapturingMode[configured.upper()] + except KeyError: + return ContentCapturingMode.NO_CONTENT + + +def _captures_content() -> bool: + return _capture_mode() is not ContentCapturingMode.NO_CONTENT + + +def _captures_content_on_span() -> bool: + return _capture_mode() in ( + ContentCapturingMode.SPAN_ONLY, + ContentCapturingMode.SPAN_AND_EVENT, + ) + + +@dataclass +class GenericPart: + """Provider-specific message part used by the upstream interactions parser.""" + + value: Any + type: str = "generic" + + +class _InvocationLifecycle: + _finished: bool + + def __enter__(self): + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: + if exc_value is not None: + self.fail(exc_value) + else: + self.stop() + + +class InferenceInvocation(LLMInvocation, _InvocationLifecycle): + """Upstream-style inference invocation backed by the extended handler.""" + + def __init__( + self, + owner: TelemetryHandler, + provider: str, + *, + request_model: Optional[str] = None, + server_address: Optional[str] = None, + server_port: Optional[int] = None, + operation_name: Optional[str] = None, + **_: Any, + ) -> None: + super().__init__( + request_model=request_model, + provider=provider, + operation_name=operation_name or "chat", + server_address=server_address, + server_port=server_port, + ) + self._owner = owner + self._finished = False + self.thinking_tokens: Optional[int] = None + if not _is_suppressed(): + owner._handler.start_llm(self) + + @property + def request_choice_count(self) -> Optional[int]: + return self.choice_count + + @request_choice_count.setter + def request_choice_count(self, value: Optional[int]) -> None: + self.choice_count = value + + @property + def cache_creation_input_tokens(self) -> Optional[int]: + return self.usage_cache_creation_input_tokens + + @cache_creation_input_tokens.setter + def cache_creation_input_tokens(self, value: Optional[int]) -> None: + self.usage_cache_creation_input_tokens = value + + @property + def cache_read_input_tokens(self) -> Optional[int]: + return self.usage_cache_read_input_tokens + + @cache_read_input_tokens.setter + def cache_read_input_tokens(self, value: Optional[int]) -> None: + self.usage_cache_read_input_tokens = value + + def record_first_token(self) -> None: + if self.monotonic_first_token_s is None: + self.monotonic_first_token_s = timeit.default_timer() + + def _finish_reasons(self) -> List[str]: + if self.finish_reasons is not None: + return self.finish_reasons + return [ + message.finish_reason + for message in self.output_messages + if message.finish_reason + ] + + def _prepare_finish( + self, error: Optional[Union[Error, BaseException]] = None + ) -> None: + custom_attributes = dict(self.attributes) + if self.thinking_tokens is not None: + self.attributes[_REASONING_OUTPUT_TOKENS] = self.thinking_tokens + self.output_tokens = ( + self.output_tokens or 0 + ) + self.thinking_tokens + if finish_reasons := self._finish_reasons(): + self.attributes[_FINISH_REASONS] = finish_reasons + if self.tool_definitions: + self.attributes[_TOOL_DEFINITIONS] = gen_ai_json_dumps( + [asdict(tool) for tool in self.tool_definitions] + ) + if _captures_content_on_span(): + message_attributes = ( + (_INPUT_MESSAGES, self.input_messages), + (_OUTPUT_MESSAGES, self.output_messages), + (_SYSTEM_INSTRUCTIONS, self.system_instruction), + ) + for key, messages in message_attributes: + if messages: + self.attributes[key] = gen_ai_json_dumps( + [asdict(message) for message in messages] + ) + log_record = self._owner._create_event( + self, + custom_attributes=custom_attributes, + error=error, + ) + self._owner._notify_completion(self, log_record=log_record) + if log_record is not None: + self._owner._logger.emit(log_record) + + def stop(self) -> None: + if self._finished: + return + self._finished = True + if self.span is None: + return + try: + self._prepare_finish() + finally: + self._owner._handler.stop_llm(self) + + def fail(self, error: Union[Error, BaseException]) -> None: + if self._finished: + return + self._finished = True + if self.span is None: + return + converted_error: Optional[Error] = None + try: + converted_error = ( + error + if isinstance(error, Error) + else Error(message=str(error), type=type(error)) + ) + self._prepare_finish(converted_error) + finally: + if converted_error is None: + self._owner._handler.abandon_llm(self) + else: + self._owner._handler.fail_llm(self, converted_error) + + +class EmbeddingInvocation(LegacyEmbeddingInvocation, _InvocationLifecycle): + """Upstream-style embedding invocation backed by the extended handler.""" + + def __init__( + self, + owner: TelemetryHandler, + provider: str, + *, + request_model: Optional[str] = None, + server_address: Optional[str] = None, + server_port: Optional[int] = None, + ) -> None: + super().__init__( + request_model=request_model or "", + provider=provider, + server_address=server_address, + server_port=server_port, + ) + self._owner = owner + self._finished = False + self.metric_attributes: Dict[str, Any] = {} + if not _is_suppressed(): + try: + owner._handler.start_embedding(self) + except Exception: + owner.abandon_embedding(self) + raise + + def stop(self) -> None: + if self._finished: + return + self._finished = True + if self.span is not None: + self._owner._finish_embedding(self) + + def fail(self, error: Union[Error, BaseException]) -> None: + if self._finished: + return + self._finished = True + if self.span is None: + return + converted_error: Optional[Error] = None + try: + converted_error = ( + error + if isinstance(error, Error) + else Error(message=str(error), type=type(error)) + ) + finally: + if converted_error is None: + self._owner.abandon_embedding(self) + else: + self._owner._fail_embedding(self, converted_error) + + +class ToolInvocation(LegacyToolInvocation, _InvocationLifecycle): + """Upstream-style tool invocation backed by the extended handler.""" + + def __init__( + self, + owner: TelemetryHandler, + name: str, + *, + tool_call_id: Optional[str] = None, + tool_type: Optional[str] = None, + tool_description: Optional[str] = None, + ) -> None: + super().__init__( + tool_name=name, + tool_call_id=tool_call_id, + tool_type=tool_type, + tool_description=tool_description, + ) + self._owner = owner + self._finished = False + self.should_capture_content_on_span = _captures_content_on_span() + self.metric_attributes: Dict[str, Any] = {} + if not _is_suppressed(): + try: + owner._handler.start_execute_tool(self) + except Exception: + owner.abandon_tool(self) + raise + + @property + def arguments(self) -> Any: + return self.tool_call_arguments + + @arguments.setter + def arguments(self, value: Any) -> None: + self.tool_call_arguments = value + + @property + def tool_result(self) -> Any: + return self.tool_call_result + + @tool_result.setter + def tool_result(self, value: Any) -> None: + self.tool_call_result = value + + def stop(self) -> None: + if self._finished: + return + self._finished = True + if self.span is not None: + self._owner._finish_tool(self) + + def fail(self, error: Union[Error, BaseException]) -> None: + if self._finished: + return + self._finished = True + if self.span is None: + return + converted_error: Optional[Error] = None + try: + converted_error = ( + error + if isinstance(error, Error) + else Error(message=str(error), type=type(error)) + ) + finally: + if converted_error is None: + self._owner.abandon_tool(self) + else: + self._owner._fail_tool(self, converted_error) + + +class TelemetryHandler: + """Small 1.0-compatible facade over ``ExtendedTelemetryHandler``.""" + + def __init__( + self, + *, + tracer_provider=None, + meter_provider=None, + logger_provider=None, + completion_hook: Optional[CompletionHook] = None, + ) -> None: + self._handler = ExtendedTelemetryHandler( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + # LoongSuite 0.x util emits a legacy event shape. This facade owns the + # upstream 1.0 event so only one operation-details event is produced. + self._handler._logger = None # pylint: disable=protected-access + self._logger = get_logger( + __name__, + logger_provider=logger_provider, + ) + self._completion_hook = completion_hook or load_completion_hook() + + def inference( + self, + provider: str, + *, + request_model: Optional[str] = None, + server_address: Optional[str] = None, + server_port: Optional[int] = None, + operation_name: Optional[str] = None, + **kwargs: Any, + ) -> InferenceInvocation: + return InferenceInvocation( + self, + provider, + request_model=request_model, + server_address=server_address, + server_port=server_port, + operation_name=operation_name, + **kwargs, + ) + + def embedding( + self, + provider: str, + *, + request_model: Optional[str] = None, + server_address: Optional[str] = None, + server_port: Optional[int] = None, + ) -> EmbeddingInvocation: + return EmbeddingInvocation( + self, + provider, + request_model=request_model, + server_address=server_address, + server_port=server_port, + ) + + def tool( + self, + name: str, + *, + tool_call_id: Optional[str] = None, + tool_type: Optional[str] = None, + tool_description: Optional[str] = None, + ) -> ToolInvocation: + return ToolInvocation( + self, + name, + tool_call_id=tool_call_id, + tool_type=tool_type, + tool_description=tool_description, + ) + + def should_capture_content(self) -> bool: + return _captures_content() + + def detach_inference_context( + self, invocation: InferenceInvocation + ) -> None: + self._handler.detach_llm_context(invocation) + + def abandon_inference(self, invocation: InferenceInvocation) -> None: + self._handler.abandon_llm(invocation) + + def abandon_embedding(self, invocation: EmbeddingInvocation) -> None: + self._abandon_extended(invocation) + + def abandon_tool(self, invocation: ToolInvocation) -> None: + self._abandon_extended(invocation) + + @staticmethod + def _abandon_extended(invocation: Any) -> None: + token = invocation.context_token + invocation.context_token = None + try: + _safe_detach(token) + finally: + span = invocation.span + if span is not None and span.is_recording(): + span.end() + + def _finish_embedding(self, invocation: EmbeddingInvocation) -> None: + self._finalize_extended( + invocation, + self._handler.stop_embedding, + ) + + def _fail_embedding( + self, invocation: EmbeddingInvocation, error: Error + ) -> None: + self._finalize_extended( + invocation, + lambda current: self._handler.fail_embedding(current, error), + ) + + def _finish_tool(self, invocation: ToolInvocation) -> None: + self._finalize_extended( + invocation, + self._handler.stop_execute_tool, + ) + + def _fail_tool(self, invocation: ToolInvocation, error: Error) -> None: + self._finalize_extended( + invocation, + lambda current: self._handler.fail_execute_tool(current, error), + ) + + @staticmethod + def _finalize_extended(invocation: Any, callback: Any) -> None: + completed = False + token = invocation.context_token + try: + callback(invocation) + completed = True + finally: + invocation.context_token = None + if not completed: + _safe_detach(token) + span = invocation.span + if span is not None and span.is_recording(): + span.end() + + def _create_event( + self, + invocation: InferenceInvocation, + *, + custom_attributes: Dict[str, Any], + error: Optional[Union[Error, BaseException]], + ) -> Optional[LogRecord]: + if not should_emit_event(): + return None + attributes: Dict[str, Any] = {} + attributes.update(_get_llm_common_attributes(invocation)) + attributes.update(_get_llm_request_attributes(invocation)) + attributes.update(_get_llm_response_attributes(invocation)) + attributes.update(custom_attributes) + if invocation.thinking_tokens is not None: + attributes[_REASONING_OUTPUT_TOKENS] = invocation.thinking_tokens + if finish_reasons := invocation._finish_reasons(): + attributes[_FINISH_REASONS] = finish_reasons + if invocation.tool_definitions: + attributes[_TOOL_DEFINITIONS] = [ + asdict(tool) for tool in invocation.tool_definitions + ] + if _capture_mode() in ( + ContentCapturingMode.EVENT_ONLY, + ContentCapturingMode.SPAN_AND_EVENT, + ): + message_attributes = ( + (_INPUT_MESSAGES, invocation.input_messages), + (_OUTPUT_MESSAGES, invocation.output_messages), + (_SYSTEM_INSTRUCTIONS, invocation.system_instruction), + ) + for key, messages in message_attributes: + if messages: + attributes[key] = [asdict(message) for message in messages] + if error is not None: + error_type = ( + error.type if isinstance(error, Error) else type(error) + ) + attributes["error.type"] = error_type.__qualname__ + return LogRecord( + event_name="gen_ai.client.inference.operation.details", + attributes=attributes, + context=set_span_in_context(invocation.span), + ) + + def _notify_completion( + self, + invocation: InferenceInvocation, + *, + log_record: Optional[LogRecord], + ) -> None: + if invocation.span is None: + return + try: + self._completion_hook.on_completion( + inputs=invocation.input_messages, + outputs=invocation.output_messages, + system_instruction=invocation.system_instruction, + tool_definitions=invocation.tool_definitions, + span=invocation.span, + log_record=log_record, + ) + except Exception: + _logger.debug( + "Google GenAI completion hook failed", + exc_info=True, + ) + + +__all__ = [ + "EmbeddingInvocation", + "GenericPart", + "InferenceInvocation", + "TelemetryHandler", + "ToolInvocation", +] diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/_stream.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/_stream.py new file mode 100644 index 000000000..f4435e8dc --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/_stream.py @@ -0,0 +1,331 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import logging +from abc import ABCMeta, abstractmethod +from inspect import isawaitable +from threading import Lock +from types import TracebackType +from typing import ( + TYPE_CHECKING, + AsyncIterable, + Generic, + Iterable, + Literal, + Optional, + Protocol, + Type, + TypeVar, +) + +from opentelemetry.util.genai import hook_advice + +if TYPE_CHECKING: + + class _ObjectProxy: + def __init__(self, wrapped: object) -> None: ... + +else: + from wrapt import ObjectProxy as _ObjectProxy + + +ChunkT = TypeVar("ChunkT") +_ChunkT_co = TypeVar("_ChunkT_co", covariant=True) +_logger = logging.getLogger(__name__) + + +@hook_advice("google-genai", "stream_chunk") +def _process_stream_chunk_advice(wrapper: object, chunk: object) -> None: + wrapper._process_chunk(chunk) # type: ignore[attr-defined] + + +@hook_advice("google-genai", "stream_success") +def _finish_stream_success_advice(wrapper: object) -> None: + wrapper._on_stream_end() # type: ignore[attr-defined] + + +@hook_advice("google-genai", "stream_error") +def _finish_stream_error_advice(wrapper: object, error: BaseException) -> None: + wrapper._on_stream_error(error) # type: ignore[attr-defined] + + +class _StreamWrapperMeta(ABCMeta, type(_ObjectProxy)): + """Metaclass compatible with wrapt's proxy type and ABC hooks.""" + + +class _SyncStream(Iterable[_ChunkT_co], Protocol[_ChunkT_co]): + """Structural type for streams accepted by ``SyncStreamWrapper``.""" + + def close(self) -> None: ... + + +class _AsyncStream(AsyncIterable[_ChunkT_co], Protocol[_ChunkT_co]): + """Structural type for streams accepted by ``AsyncStreamWrapper``.""" + + +class SyncStreamWrapper( + _ObjectProxy, + Generic[ChunkT], + metaclass=_StreamWrapperMeta, +): + """Base class for synchronous instrumented stream wrappers. + + Subclass this when wrapping a provider SDK stream that is consumed with + normal iteration. The subclass should pass the SDK stream to + ``super().__init__(stream)`` and implement the three telemetry hooks: + ``_process_chunk`` for per-chunk state, ``_on_stream_end`` for successful + finalization, and ``_on_stream_error`` for failure finalization. + + Users should consume subclasses as normal streams, for example with + ``for chunk in wrapper`` or ``with wrapper``. The hook methods are called + internally by the wrapper lifecycle and are not part of the public API. + """ + + def __init__(self, stream: _SyncStream[ChunkT]): + super().__init__(stream) + self._self_stream = stream + self._self_iterator = iter(stream) + self._self_finalized = False + self._self_finalize_lock = Lock() + + def __enter__(self): + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Literal[False]: + if exc_val is not None: + self._finalize_failure(exc_val) + try: + self._self_stream.close() + except Exception: # pylint: disable=broad-exception-caught + _logger.debug( + "GenAI stream close error after user exception", + exc_info=True, + ) + return False + + self.close() + return False + + def close(self) -> None: + try: + self._self_stream.close() + except Exception: + _logger.debug( + "Google GenAI stream close failed", + exc_info=True, + ) + self._finalize_success() + + def __iter__(self): + # Override ``ObjectProxy.__iter__`` so iteration drives ``__next__`` + # below and runs ``_process_chunk`` per chunk; otherwise iteration + # would be forwarded to the wrapped stream and bypass instrumentation. + return self + + def __next__(self) -> ChunkT: + try: + chunk = next(self._self_iterator) + except StopIteration: + self._finalize_success() + raise + except GeneratorExit: + self._finalize_success() + raise + except BaseException as error: + self._finalize_failure(error) + raise + _process_stream_chunk_advice(self, chunk) + return chunk + + def _finalize_success(self) -> None: + with self._self_finalize_lock: + if self._self_finalized: + return + self._self_finalized = True + _finish_stream_success_advice(self) + + def _finalize_failure(self, error: BaseException) -> None: + with self._self_finalize_lock: + if self._self_finalized: + return + self._self_finalized = True + _finish_stream_error_advice(self, error) + + def __del__(self) -> None: + if getattr(self, "_self_finalized", True): + return + try: + self._finalize_success() + except Exception: + pass + + @abstractmethod + def _process_chunk(self, chunk: ChunkT) -> None: + """Process one stream chunk for telemetry.""" + + @abstractmethod + def _on_stream_end(self) -> None: + """Finalize the stream successfully.""" + + @abstractmethod + def _on_stream_error(self, error: BaseException) -> None: + """Finalize the stream with failure.""" + + +class AsyncStreamWrapper( + _ObjectProxy, + Generic[ChunkT], + metaclass=_StreamWrapperMeta, +): + """Base class for asynchronous instrumented stream wrappers. + + Subclass this when wrapping a provider SDK stream that is consumed with + async iteration. The subclass should pass the SDK stream to + ``super().__init__(stream)`` and implement the three telemetry hooks: + ``_process_chunk`` for per-chunk state, ``_on_stream_end`` for successful + finalization, and ``_on_stream_error`` for failure finalization. + + Users should consume subclasses as normal async streams, for example with + ``async for chunk in wrapper`` or ``async with wrapper``. The hook methods + remain synchronous telemetry hooks; async stream reads and close handling + are owned by this base class. + """ + + def __init__(self, stream: _AsyncStream[ChunkT]): + super().__init__(stream) + self._self_stream = stream + # LoongSuite supports Python 3.9, where aiter/anext are unavailable. + self._self_aiter = stream.__aiter__() + self._self_finalized = False + self._self_finalize_lock = Lock() + + async def __aenter__(self): + return self + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Literal[False]: + if exc_val is not None: + self._finalize_failure(exc_val) + try: + await self._close_wrapped_stream() + except Exception: # pylint: disable=broad-exception-caught + _logger.debug( + "GenAI stream close error after user exception", + exc_info=True, + ) + return False + + await self.close() + return False + + async def close(self) -> None: + try: + await self._close_wrapped_stream() + except Exception: + _logger.debug( + "Google GenAI async stream close failed", + exc_info=True, + ) + self._finalize_success() + + async def aclose(self) -> None: + """Close SDKs such as Google GenAI that expose ``aclose``.""" + + await self.close() + + async def _close_wrapped_stream(self) -> None: + close = getattr(self._self_stream, "aclose", None) + if close is None: + close = getattr(self._self_stream, "close", None) + if close is None: + return + result = close() + if isawaitable(result): + await result + + def __aiter__(self): + # Override ``ObjectProxy.__aiter__`` so iteration drives ``__anext__`` + # below and runs ``_process_chunk`` per chunk; otherwise iteration + # would be forwarded to the wrapped stream and bypass instrumentation. + return self + + async def __anext__(self) -> ChunkT: + try: + chunk = await self._self_aiter.__anext__() + except StopAsyncIteration: + self._finalize_success() + raise + except GeneratorExit: + self._finalize_success() + raise + except BaseException as error: + self._finalize_failure(error) + raise + + _process_stream_chunk_advice(self, chunk) + return chunk + + def _finalize_success(self) -> None: + with self._self_finalize_lock: + if self._self_finalized: + return + self._self_finalized = True + _finish_stream_success_advice(self) + + def _finalize_failure(self, error: BaseException) -> None: + with self._self_finalize_lock: + if self._self_finalized: + return + self._self_finalized = True + _finish_stream_error_advice(self, error) + + def __del__(self) -> None: + if getattr(self, "_self_finalized", True): + return + try: + self._finalize_success() + except Exception: + pass + + @abstractmethod + def _process_chunk(self, chunk: ChunkT) -> None: + """Process one stream chunk for telemetry.""" + + @abstractmethod + def _on_stream_end(self) -> None: + """Finalize the stream successfully.""" + + @abstractmethod + def _on_stream_error(self, error: BaseException) -> None: + """Finalize the stream with failure.""" + + +__all__ = [ + "AsyncStreamWrapper", + "SyncStreamWrapper", +] diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/allowlist_util.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/allowlist_util.py new file mode 100644 index 000000000..9a2e6d456 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/allowlist_util.py @@ -0,0 +1,97 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import os +import re +from typing import Iterable, Optional, Set + + +def _parse_env_list(s: str) -> Set[str]: + result = set() + for entry in s.split(","): + stripped_entry = entry.strip() + if not stripped_entry: + continue + result.add(stripped_entry) + return result + + +class _CompoundMatcher: + def __init__(self, entries: Set[str]): + self._match_all = "*" in entries + self._entries = entries + self._regex_matcher = None + regex_entries = [] + for entry in entries: + if "*" not in entry: + continue + if entry == "*": + continue + entry = entry.replace("[", "\\[") + entry = entry.replace("]", "\\]") + entry = entry.replace(".", "\\.") + entry = entry.replace("*", ".*") + regex_entries.append(f"({entry})") + if regex_entries: + joined_regex = "|".join(regex_entries) + regex_str = f"^({joined_regex})$" + self._regex_matcher = re.compile(regex_str) + + @property + def match_all(self): + return self._match_all + + def matches(self, x): + if self._match_all: + return True + if x in self._entries: + return True + if (self._regex_matcher is not None) and ( + self._regex_matcher.fullmatch(x) + ): + return True + return False + + +class AllowList: + def __init__( + self, + includes: Optional[Iterable[str]] = None, + excludes: Optional[Iterable[str]] = None, + ): + self._includes = _CompoundMatcher(set(includes or [])) + self._excludes = _CompoundMatcher(set(excludes or [])) + assert (not self._includes.match_all) or ( + not self._excludes.match_all + ), "Can't have '*' in both includes and excludes." + + def allowed(self, x: str): + if self._excludes.match_all: + return self._includes.matches(x) + if self._includes.match_all: + return not self._excludes.matches(x) + return self._includes.matches(x) and not self._excludes.matches(x) + + @staticmethod + def from_env( + includes_env_var: str, excludes_env_var: Optional[str] = None + ): + includes = _parse_env_list(os.getenv(includes_env_var) or "") + excludes = set() + if excludes_env_var: + excludes = _parse_env_list(os.getenv(excludes_env_var) or "") + return AllowList(includes=includes, excludes=excludes) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/client_info.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/client_info.py new file mode 100644 index 000000000..50f3449cc --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/client_info.py @@ -0,0 +1,66 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Any, Optional, Tuple +from urllib.parse import urlsplit + + +def get_client_info(instance: Any) -> Tuple[bool, Optional[str]]: + is_vertex = False + server_address = None + + if hasattr(instance, "_api_client"): + api_client = instance._api_client + is_vertex = getattr(api_client, "vertexai", False) + if hasattr(api_client, "_http_options"): + server_address = getattr( + api_client._http_options, "base_url", None + ) + elif hasattr(instance, "_client"): + client = instance._client + is_vertex = getattr(client, "_is_vertex", False) + server_address = getattr(client, "server", None) + elif hasattr(instance, "sdk_configuration"): + config = instance.sdk_configuration + server_url = getattr(config, "server_url", "") + if server_url: + server_address = server_url + server_url_string = str(server_url) + if "://" not in server_url_string: + server_url_string = "//" + server_url_string + server_hostname = urlsplit(server_url_string).hostname + if server_hostname and ( + server_hostname == "aiplatform.googleapis.com" + or server_hostname.endswith("-aiplatform.googleapis.com") + ): + is_vertex = True + + if server_address and "://" in str(server_address): + server_address = urlsplit(str(server_address)).hostname + elif server_address: + server_address = str(server_address).rstrip("/") + + if not server_address: + server_address = ( + "aiplatform.googleapis.com" + if is_vertex + else "generativelanguage.googleapis.com" + ) + + return bool(is_vertex), server_address diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/custom_semconv.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/custom_semconv.py new file mode 100644 index 000000000..c6ed4007b --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/custom_semconv.py @@ -0,0 +1,21 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + + +# Semantic Convention still being defined in: +# https://github.com/open-telemetry/semantic-conventions/pull/2125 +GCP_GENAI_OPERATION_CONFIG = "gcp.gen_ai.operation.config" diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/dict_util.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/dict_util.py new file mode 100644 index 000000000..31238a3a8 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/dict_util.py @@ -0,0 +1,133 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import json +import logging +from typing import ( + Any, + Dict, + List, + Set, + Union, +) + +Primitive = Union[bool, str, int, float] +BoolList = List[bool] +StringList = List[str] +IntList = List[int] +FloatList = List[float] +HomogenousPrimitiveList = Union[BoolList, StringList, IntList, FloatList] +FlattenedValue = Union[Primitive, HomogenousPrimitiveList] +FlattenedDict = Dict[str, FlattenedValue] + + +_logger = logging.getLogger(__name__) + + +def _is_homogenous_primitive_list(v): + if len(v) == 0: + return True + if not isinstance(v[0], (str, bool, int, float)): + return False + first_entry_value_type = type(v[0]) + for entry in v[1:]: + if not isinstance(entry, first_entry_value_type): + return False + return True + + +def _flatten_compound_value_using_json( + key: str, + value: Any, + exclude_keys: Set[str], + _from_json=False, +) -> FlattenedDict: + if _from_json: + _logger.debug( + "Cannot flatten value with key %s; value: %s", key, value + ) + return {} + try: + json_string = json.dumps(value) + except TypeError: + _logger.debug( + "Cannot flatten value with key %s; value: %s. Not JSON serializable.", + key, + value, + ) + return {} + return _flatten_value( + key, + json.loads(json_string), + exclude_keys, + # Ensure that we don't recurse indefinitely if "json.loads()" somehow returns + # a complex, compound object that does not get handled by the "primitive", "list", + # or "dict" cases. Prevents falling back on the JSON serialization fallback path. + True, + ) + + +def _flatten_compound_value( + key: str, + value: Any, + exclude_keys: Set[str], + _from_json=False, +) -> FlattenedDict: + if isinstance(value, dict): + return flatten_dict(value, key, exclude_keys) + if isinstance(value, list): + if _is_homogenous_primitive_list(value): + return {key: value} + result = {f"{key}.length": len(value)} + for idx, val in enumerate(value): + result.update(_flatten_value(f"{key}[{idx}]", val, exclude_keys)) + return result + if hasattr(value, "model_dump"): + try: + return flatten_dict(value.model_dump(), key, exclude_keys) + except TypeError: + return {key: str(value)} + return _flatten_compound_value_using_json( + key, value, exclude_keys, _from_json + ) + + +def _flatten_value( + key: str, + value: Any, + exclude_keys: Set[str], + _from_json=False, +) -> FlattenedDict: + if value is None or key in exclude_keys: + return {} + if isinstance(value, (str, bool, int, float)): + return {key: value} + return _flatten_compound_value(key, value, exclude_keys, _from_json) + + +def flatten_dict( + d: Dict[str, Any], + key_prefix: str, + exclude_keys: Set[str], +) -> FlattenedDict: + result = {} + for key, value in d.items(): + if key not in exclude_keys: + result.update( + _flatten_value(f"{key_prefix}.{key}", value, exclude_keys) + ) + return result diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/embeddings.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/embeddings.py new file mode 100644 index 000000000..42c93d9a6 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/embeddings.py @@ -0,0 +1,310 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from collections.abc import Callable +from contextvars import ContextVar +from dataclasses import dataclass +from typing import Any, Dict, Optional, Tuple, Union + +from google.genai._api_client import BaseApiClient +from google.genai.models import AsyncModels, Models +from google.genai.types import EmbedContentResponse +from wrapt import wrap_function_wrapper + +from opentelemetry.instrumentation.google_genai.client_info import ( + get_client_info as _get_client_info, +) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) +from opentelemetry.util.genai import hook_advice + +from ._compat import EmbeddingInvocation, TelemetryHandler + +_RAW_RESPONSE_BODY: ContextVar[Optional[str]] = ContextVar( + "raw_response_body", default=None +) +_CAPTURE_RAW_RESPONSE: ContextVar[bool] = ContextVar( + "capture_raw_response", default=False +) +_EMBEDDING_DIMENSION_COUNT = getattr( + GenAIAttributes, + "GEN_AI_EMBEDDINGS_DIMENSION_COUNT", + "gen_ai.embeddings.dimension.count", +) + + +class _EmbeddingMethodsSnapshot: + def __init__(self) -> None: + self._original_embed_content = Models.embed_content + self._original_embed_content_code = Models.embed_content.__code__ + self._original_async_embed_content = AsyncModels.embed_content + self._original_async_embed_content_code = ( + AsyncModels.embed_content.__code__ + ) + self._original_client_request = BaseApiClient.request + self._original_client_async_request = BaseApiClient.async_request + + def restore(self) -> None: + self._original_embed_content.__code__ = ( + self._original_embed_content_code + ) + self._original_async_embed_content.__code__ = ( + self._original_async_embed_content_code + ) + + Models.embed_content = self._original_embed_content + AsyncModels.embed_content = self._original_async_embed_content + BaseApiClient.request = self._original_client_request + BaseApiClient.async_request = self._original_client_async_request + + +# Magic incantation used by native Google ADK instrumentation to identify +# instrumented functions and suppress its own internal tracing when OTel is active. +def _set_co_filename(wrapped: object) -> None: + wrapped.__wrapped__.__code__ = wrapped.__wrapped__.__code__.replace( + co_filename=__file__.replace("\\", "/") + ) + + +def _apply_embedding_response_attributes( + response: EmbedContentResponse, + invocation: EmbeddingInvocation, +) -> None: + if response.embeddings: + first_embedding = response.embeddings[0] + if first_embedding.values: + invocation.dimension_count = len(first_embedding.values) + invocation.metric_attributes[_EMBEDDING_DIMENSION_COUNT] = ( + invocation.dimension_count + ) + + # In the future we can get rid of this and the monkey patching of the + # requests, and use the parsed SDK response instead. See: + # https://github.com/googleapis/python-genai/issues/2658 + if raw_body := _RAW_RESPONSE_BODY.get(): + try: + body_dict = json.loads(raw_body) + usage_metadata = body_dict.get("usageMetadata") + if isinstance(usage_metadata, dict): + invocation.input_tokens = usage_metadata.get( + "promptTokenCount" + ) + except Exception: + pass + + +@dataclass +class _EmbeddingAdviceState: + invocation: EmbeddingInvocation + raw_body_token: Any + capture_token: Any + + +@hook_advice("google-genai", "prepare_embedding") +def _prepare_embedding_advice( + telemetry_handler: TelemetryHandler, + instance: Union[Models, AsyncModels], + kwargs: Dict[str, Any], +) -> _EmbeddingAdviceState: + raw_body_token = None + capture_token = None + invocation = None + try: + raw_body_token = _RAW_RESPONSE_BODY.set(None) + capture_token = _CAPTURE_RAW_RESPONSE.set(True) + is_vertex, server_address = _get_client_info(instance) + invocation = telemetry_handler.embedding( + provider=( + GenAIAttributes.GenAiSystemValues.VERTEX_AI.value + if is_vertex + else GenAIAttributes.GenAiSystemValues.GEMINI.value + ), + request_model=kwargs.get("model"), + server_address=server_address, + ) + return _EmbeddingAdviceState( + invocation=invocation, + raw_body_token=raw_body_token, + capture_token=capture_token, + ) + except Exception: + if invocation is not None: + telemetry_handler.abandon_embedding(invocation) + if raw_body_token is not None: + _RAW_RESPONSE_BODY.reset(raw_body_token) + if capture_token is not None: + _CAPTURE_RAW_RESPONSE.reset(capture_token) + raise + + +@hook_advice("google-genai", "complete_embedding") +def _complete_embedding_advice( + state: _EmbeddingAdviceState, + response: EmbedContentResponse, +) -> None: + try: + try: + _apply_embedding_response_attributes(response, state.invocation) + finally: + state.invocation.stop() + finally: + _RAW_RESPONSE_BODY.reset(state.raw_body_token) + _CAPTURE_RAW_RESPONSE.reset(state.capture_token) + + +@hook_advice("google-genai", "fail_embedding") +def _fail_embedding_advice( + state: _EmbeddingAdviceState, + error: BaseException, +) -> None: + try: + state.invocation.fail(error) + finally: + _RAW_RESPONSE_BODY.reset(state.raw_body_token) + _CAPTURE_RAW_RESPONSE.reset(state.capture_token) + + +@hook_advice("google-genai", "capture_embedding_raw_response") +def _capture_embedding_raw_response_advice(response: Any) -> None: + if ( + _CAPTURE_RAW_RESPONSE.get() + and response + and getattr(response, "body", None) + ): + _RAW_RESPONSE_BODY.set(response.body) + + +def _create_instrumented_embed_content( + telemetry_handler: TelemetryHandler, +) -> Callable[ + [ + Callable[..., EmbedContentResponse], + Models, + Tuple[Any, ...], + Dict[str, Any], + ], + EmbedContentResponse, +]: + def instrumented_embed_content( + wrapped: Callable[..., EmbedContentResponse], + instance: Models, + args: Tuple[Any, ...], + kwargs: Dict[str, Any], + ) -> EmbedContentResponse: + state = _prepare_embedding_advice( + telemetry_handler, + instance, + kwargs, + ) + try: + response = wrapped(*args, **kwargs) + except BaseException as error: + if state is not None: + _fail_embedding_advice(state, error) + raise + if state is not None: + _complete_embedding_advice(state, response) + return response + + return instrumented_embed_content + + +def _create_instrumented_async_embed_content( + telemetry_handler: TelemetryHandler, +) -> Callable[ + [ + Callable[..., Any], + AsyncModels, + Tuple[Any, ...], + Dict[str, Any], + ], + Any, +]: + async def instrumented_embed_content( + wrapped: Callable[..., Any], + instance: AsyncModels, + args: Tuple[Any, ...], + kwargs: Dict[str, Any], + ) -> EmbedContentResponse: + state = _prepare_embedding_advice( + telemetry_handler, + instance, + kwargs, + ) + try: + response = await wrapped(*args, **kwargs) + except BaseException as error: + if state is not None: + _fail_embedding_advice(state, error) + raise + if state is not None: + _complete_embedding_advice(state, response) + return response + + return instrumented_embed_content + + +def uninstrument_embeddings(snapshot: object) -> None: + assert isinstance(snapshot, _EmbeddingMethodsSnapshot) + snapshot.restore() + + +def instrument_embeddings( + telemetry_handler: TelemetryHandler, +) -> object: + snapshot = _EmbeddingMethodsSnapshot() + + wrapped = wrap_function_wrapper( + "google.genai.models", + "Models.embed_content", + _create_instrumented_embed_content(telemetry_handler), + ) + wrapped2 = wrap_function_wrapper( + "google.genai.models", + "AsyncModels.embed_content", + _create_instrumented_async_embed_content(telemetry_handler), + ) + _set_co_filename(wrapped) + _set_co_filename(wrapped2) + + # Wrap BaseApiClient to capture raw responses + def instrumented_request(wrapped, instance, args, kwargs): + response = wrapped(*args, **kwargs) + _capture_embedding_raw_response_advice(response) + return response + + async def instrumented_async_request(wrapped, instance, args, kwargs): + response = await wrapped(*args, **kwargs) + _capture_embedding_raw_response_advice(response) + return response + + wrap_function_wrapper( + "google.genai._api_client", + "BaseApiClient.request", + instrumented_request, + ) + wrap_function_wrapper( + "google.genai._api_client", + "BaseApiClient.async_request", + instrumented_async_request, + ) + + return snapshot diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/generate_content.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/generate_content.py new file mode 100644 index 000000000..c323a1414 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/generate_content.py @@ -0,0 +1,1020 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import os +from collections.abc import AsyncIterable, Callable, Iterable +from copy import copy +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Union + +from google.genai.models import AsyncModels, Models +from google.genai.models import t as transformers +from google.genai.types import ( + ContentListUnion, + ContentListUnionDict, + GenerateContentConfig, + GenerateContentConfigOrDict, + GenerateContentResponse, + Tool, + ToolUnionDict, +) +from wrapt import wrap_function_wrapper + +from opentelemetry import context as context_api +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes, +) +from opentelemetry.util.genai import async_hook_advice, hook_advice +from opentelemetry.util.genai.types import ( + FunctionToolDefinition, + GenericToolDefinition, + ToolDefinition, +) +from opentelemetry.util.types import AttributeValue + +from ._compat import InferenceInvocation, TelemetryHandler +from ._stream import ( + AsyncStreamWrapper, + SyncStreamWrapper, +) +from .allowlist_util import AllowList +from .client_info import get_client_info as _get_client_info +from .custom_semconv import GCP_GENAI_OPERATION_CONFIG +from .dict_util import flatten_dict +from .message import ( + StreamOutputMessageAccumulator, + to_input_messages, + to_output_messages, + to_system_instructions, +) +from .tool_call_wrapper import wrapped_tool + +_is_mcp_imported = False +McpClientSession = McpTool = None +try: + from mcp import ClientSession as McpClientSession + from mcp import Tool as McpTool + + _is_mcp_imported = True +except ImportError: + pass + +GENERATE_CONTENT_EXTRA_ATTRIBUTES_CONTEXT_KEY = context_api.create_key( + "generate_content_extra_attributes_context_key" +) + + +class _MethodsSnapshot: + def __init__(self): + self._original_generate_content = Models.generate_content + self._original_generate_content_code = Models.generate_content.__code__ + self._original_generate_content_stream = Models.generate_content_stream + self._original_generate_content_stream_code = ( + Models.generate_content_stream.__code__ + ) + self._original_async_generate_content = AsyncModels.generate_content + self._original_async_generate_content_code = ( + AsyncModels.generate_content.__code__ + ) + self._original_async_generate_content_stream = ( + AsyncModels.generate_content_stream + ) + self._original_async_generate_content_stream_code = ( + AsyncModels.generate_content_stream.__code__ + ) + + @property + def generate_content(self): + return self._original_generate_content + + @property + def generate_content_stream(self): + return self._original_generate_content_stream + + @property + def async_generate_content(self): + return self._original_async_generate_content + + @property + def async_generate_content_stream(self): + return self._original_async_generate_content_stream + + def restore(self): + self._original_generate_content.__code__ = ( + self._original_generate_content_code + ) + self._original_generate_content_stream.__code__ = ( + self._original_generate_content_stream_code + ) + self._original_async_generate_content.__code__ = ( + self._original_async_generate_content_code + ) + self._original_async_generate_content_stream.__code__ = ( + self._original_async_generate_content_stream_code + ) + + Models.generate_content = self._original_generate_content + Models.generate_content_stream = self._original_generate_content_stream + AsyncModels.generate_content = self._original_async_generate_content + AsyncModels.generate_content_stream = ( + self._original_async_generate_content_stream + ) + + +# Magic incantation used by native Google ADK instrumentation to identify +# instrumented functions and suppress its own internal tracing when OTel is active. +def _set_co_filename(wrapped: object) -> None: + wrapped.__wrapped__.__code__ = wrapped.__wrapped__.__code__.replace( + co_filename=__file__.replace("\\", "/") + ) + + +def _guess_genai_system_from_env(): + if os.environ.get("GOOGLE_GENAI_USE_VERTEXAI", "0").lower() in [ + "true", + "1", + ]: + return gen_ai_attributes.GenAiSystemValues.VERTEX_AI.name.lower() + return gen_ai_attributes.GenAiSystemValues.GEMINI.name.lower() + + +def _get_is_vertexai(models_object: Union[Models, AsyncModels]): + # Since commit 8e561de04965bb8766db87ad8eea7c57c1040442 of "googleapis/python-genai", + # it is possible to obtain the information using a documented property. + if hasattr(models_object, "vertexai"): + vertexai_attr = getattr(models_object, "vertexai") + if vertexai_attr is not None: + return vertexai_attr + # For earlier revisions, it is necessary to deeply inspect the internals. + if hasattr(models_object, "_api_client"): + client = getattr(models_object, "_api_client") + if not client: + return None + if hasattr(client, "vertexai"): + return getattr(client, "vertexai") + return None + + +def _determine_genai_system(models_object: Union[Models, AsyncModels]): + vertexai_attr = _get_is_vertexai(models_object) + if vertexai_attr is None: + return _guess_genai_system_from_env() + if vertexai_attr: + return gen_ai_attributes.GenAiSystemValues.VERTEX_AI.name.lower() + return gen_ai_attributes.GenAiSystemValues.GEMINI.name.lower() + + +def _model_dump_to_tool_definition(tool: Any) -> ToolDefinition: + model_dump = tool.model_dump(exclude_none=True) + + name = ( + model_dump.get("name") + or getattr(tool, "name", None) + or type(tool).__name__ + ) + description = model_dump.get("description") or getattr( + tool, "description", None + ) + parameters = model_dump.get("parameters") or model_dump.get("inputSchema") + return FunctionToolDefinition( + name=name, + description=description, + parameters=parameters, + ) + + +def _clean_parameters(params: Any) -> Any: + """Converts parameter objects into plain dicts.""" + if params is None: + return None + if isinstance(params, dict): + return params + if hasattr(params, "to_dict"): + return params.to_dict() + if hasattr(params, "model_dump"): + return params.model_dump(exclude_none=True) + + try: + # Check if it's already a standard JSON type. + json.dumps(params) + return params + + except (TypeError, ValueError): + return { + "type": "object", + "properties": { + "serialization_error": { + "type": "string", + "description": f"Failed to serialize parameters: {type(params).__name__}", + } + }, + } + + +def _tool_to_tool_definition(tool: Tool) -> List[ToolDefinition]: + definitions = [] + if tool.function_declarations: + for fd in tool.function_declarations: + definitions.append( + FunctionToolDefinition( + name=getattr(fd, "name", type(fd).__name__), + description=getattr(fd, "description", None), + parameters=_clean_parameters( + getattr(fd, "parameters", None) + ), + ) + ) + + # Generic types + if hasattr(tool, "model_dump"): + exclude_fields = {"function_declarations"} + fields = { + k: v + for k, v in tool.model_dump().items() + if v is not None and k not in exclude_fields + } + + for tool_type, _ in fields.items(): + definitions.append( + GenericToolDefinition( + type=tool_type, + name=tool_type, + ) + ) + + return definitions + + +def _callable_tool_to_tool_definition(tool: Any) -> ToolDefinition: + doc = getattr(tool, "__doc__", "") or "" + return FunctionToolDefinition( + name=getattr(tool, "__name__", type(tool).__name__), + description=doc.strip(), + parameters=None, + ) + + +def _mcp_tool_to_tool_definition(tool: McpTool) -> ToolDefinition: + if hasattr(tool, "model_dump"): + return _model_dump_to_tool_definition(tool) + + return FunctionToolDefinition( + name=getattr(tool, "name", type(tool).__name__), + description=getattr(tool, "description", None), + parameters=getattr(tool, "input_schema", None), + ) + + +def _to_tool_definition_common(tool: ToolUnionDict) -> List[ToolDefinition]: + if isinstance(tool, Tool): + return _tool_to_tool_definition(tool) + + if callable(tool): + return [_callable_tool_to_tool_definition(tool)] + + if _is_mcp_imported and isinstance(tool, McpTool): + return [_mcp_tool_to_tool_definition(tool)] + + return [ + GenericToolDefinition( + name="UnserializableTool", + type=type(tool).__name__, + ) + ] + + +def _to_tool_definition(tool: ToolUnionDict) -> List[ToolDefinition]: + if _is_mcp_imported and isinstance(tool, McpClientSession): + return [] + + return _to_tool_definition_common(tool) + + +async def _to_tool_definition_async( + tool: ToolUnionDict, +) -> List[ToolDefinition]: + if _is_mcp_imported and isinstance(tool, McpClientSession): + result = await tool.list_tools() + return [_model_dump_to_tool_definition(t) for t in result.tools] + + return _to_tool_definition_common(tool) + + +def _apply_request_attributes( + config: GenerateContentConfig, + allow_list: AllowList, + invocation: InferenceInvocation, +) -> None: + invocation.temperature = config.temperature + invocation.top_p = config.top_p + invocation.top_k = config.top_k + invocation.request_choice_count = config.candidate_count + invocation.max_tokens = config.max_output_tokens + invocation.stop_sequences = config.stop_sequences + invocation.frequency_penalty = config.frequency_penalty + invocation.presence_penalty = config.presence_penalty + invocation.seed = config.seed + if config.response_mime_type == "text/plain": + invocation.output_type = "text" + elif config.response_mime_type == "application/json": + invocation.output_type = "json" + else: + invocation.output_type = config.response_mime_type + attributes = flatten_dict( + config.model_dump(exclude_none=True), + # A custom prefix is used, because the names/structure of the + # configuration is likely to be specific to Google Gen AI SDK. + key_prefix=GCP_GENAI_OPERATION_CONFIG, + # These are all captured above as semantic conventions. + exclude_keys={ + f"{GCP_GENAI_OPERATION_CONFIG}.system_instruction", + f"{GCP_GENAI_OPERATION_CONFIG}.temperature", + f"{GCP_GENAI_OPERATION_CONFIG}.top_k", + f"{GCP_GENAI_OPERATION_CONFIG}.top_p", + f"{GCP_GENAI_OPERATION_CONFIG}.candidate_count", + f"{GCP_GENAI_OPERATION_CONFIG}.max_output_tokens", + f"{GCP_GENAI_OPERATION_CONFIG}.stop_sequences", + f"{GCP_GENAI_OPERATION_CONFIG}.frequency_penalty", + f"{GCP_GENAI_OPERATION_CONFIG}.presence_penalty", + f"{GCP_GENAI_OPERATION_CONFIG}.seed", + f"{GCP_GENAI_OPERATION_CONFIG}.response_mime_type", + }, + ) + invocation.attributes.update( + {k: v for k, v in attributes.items() if allow_list.allowed(k)} + ) + + +def _get_response_property(response: GenerateContentResponse, path: str): + path_segments = path.split(".") + current_context = response + for path_segment in path_segments: + if current_context is None: + return None + if isinstance(current_context, dict): + current_context = current_context.get(path_segment) + else: + current_context = getattr(current_context, path_segment) + return current_context + + +def _wrapped_config_with_tools( + telemetry_handler: TelemetryHandler, + config: Optional[GenerateContentConfigOrDict], +) -> Tuple[GenerateContentConfig, bool]: + if config is None: + return GenerateContentConfig(), False + if isinstance(config, dict): + try: + config = GenerateContentConfig.model_validate(config) + except Exception: + return GenerateContentConfig(), False + if not config.tools: + return config, False + config = copy(config) + config.tools = [ + wrapped_tool(tool, telemetry_handler) for tool in config.tools + ] + return config, True + + +def _get_extra_generate_content_attributes() -> Dict[str, AttributeValue]: + attrs = context_api.get_value( + GENERATE_CONTENT_EXTRA_ATTRIBUTES_CONTEXT_KEY + ) + return dict(attrs or {}) + + +def _apply_response_attributes( + response: GenerateContentResponse, + finish_reasons: List[str], + invocation: InferenceInvocation, +): + if response.response_id: + invocation.response_id = response.response_id + if response.model_version: + invocation.response_model_name = response.model_version + for candidate in response.candidates or []: + if candidate.finish_reason: + finish_reasons.append(candidate.finish_reason.value.lower()) + invocation.finish_reasons = finish_reasons + input_tokens = _get_response_property( + response, "usage_metadata.prompt_token_count" + ) + output_tokens = _get_response_property( + response, "usage_metadata.candidates_token_count" + ) + cached_tokens = _get_response_property( + response, "usage_metadata.cached_content_token_count" + ) + thinking_tokens = _get_response_property( + response, "usage_metadata.thoughts_token_count" + ) + if cached_tokens is not None and isinstance(cached_tokens, int): + invocation.cache_read_input_tokens = cached_tokens + if input_tokens is not None and isinstance(input_tokens, int): + invocation.input_tokens = input_tokens + if output_tokens is not None and isinstance(output_tokens, int): + invocation.output_tokens = output_tokens + if thinking_tokens is not None and isinstance(thinking_tokens, int): + # The util library will add this total to output tokens. + invocation.thinking_tokens = thinking_tokens + + +def _maybe_get_tool_definitions( + config: GenerateContentConfig, +) -> List[ToolDefinition]: + return [ + de + for tool in config.tools or [] + for de in _to_tool_definition(tool) + if de + ] + + +async def _maybe_get_tool_definitions_async( + config: GenerateContentConfig, +) -> List[ToolDefinition]: + # Tool discovery is telemetry-only. Do not call MCP/application callbacks + # before the real SDK operation; losing those definitions is preferable to + # adding side effects or blocking the customer's request. + return [ + de + for tool in config.tools or [] + for de in _to_tool_definition(tool) + if de + ] + + +@dataclass +class _GenerateContentAdviceState: + invocation: InferenceInvocation + wrapped_config: GenerateContentConfig + has_wrapped_tools: bool + + +@hook_advice("google-genai", "prepare_generate_content") +def _prepare_generate_content_advice( + telemetry_handler: TelemetryHandler, + generate_content_config_key_allowlist: AllowList, + instance: Any, + model: str, + contents: Union[ContentListUnion, ContentListUnionDict], + config: Optional[GenerateContentConfigOrDict], +) -> _GenerateContentAdviceState: + invocation = None + try: + wrapped_config, has_wrapped_tools = _wrapped_config_with_tools( + telemetry_handler, + config, + ) + _, server_address = _get_client_info(instance) + invocation = telemetry_handler.inference( + provider=_determine_genai_system(instance), + request_model=model, + operation_name="generate_content", + server_address=server_address, + ) + _apply_request_attributes( + wrapped_config, + generate_content_config_key_allowlist, + invocation, + ) + invocation.attributes.update(_get_extra_generate_content_attributes()) + invocation.tool_definitions = _maybe_get_tool_definitions( + wrapped_config + ) + if telemetry_handler.should_capture_content(): + invocation.input_messages = to_input_messages( + contents=transformers.t_contents(contents) + ) + if wrapped_config.system_instruction: + invocation.system_instruction = to_system_instructions( + content=transformers.t_contents( + wrapped_config.system_instruction + )[0] + ) + return _GenerateContentAdviceState( + invocation=invocation, + wrapped_config=wrapped_config, + has_wrapped_tools=has_wrapped_tools, + ) + except Exception: + if invocation is not None: + telemetry_handler.abandon_inference(invocation) + raise + + +@async_hook_advice("google-genai", "prepare_async_generate_content") +async def _prepare_async_generate_content_advice( + telemetry_handler: TelemetryHandler, + generate_content_config_key_allowlist: AllowList, + instance: Any, + model: str, + contents: Union[ContentListUnion, ContentListUnionDict], + config: Optional[GenerateContentConfigOrDict], +) -> _GenerateContentAdviceState: + invocation = None + try: + wrapped_config, has_wrapped_tools = _wrapped_config_with_tools( + telemetry_handler, + config, + ) + _, server_address = _get_client_info(instance) + invocation = telemetry_handler.inference( + provider=_determine_genai_system(instance), + request_model=model, + operation_name="generate_content", + server_address=server_address, + ) + _apply_request_attributes( + wrapped_config, + generate_content_config_key_allowlist, + invocation, + ) + invocation.attributes.update(_get_extra_generate_content_attributes()) + invocation.tool_definitions = await _maybe_get_tool_definitions_async( + wrapped_config + ) + if telemetry_handler.should_capture_content(): + invocation.input_messages = to_input_messages( + contents=transformers.t_contents(contents) + ) + if wrapped_config.system_instruction: + invocation.system_instruction = to_system_instructions( + content=transformers.t_contents( + wrapped_config.system_instruction + )[0] + ) + return _GenerateContentAdviceState( + invocation=invocation, + wrapped_config=wrapped_config, + has_wrapped_tools=has_wrapped_tools, + ) + except Exception: + if invocation is not None: + telemetry_handler.abandon_inference(invocation) + raise + + +@hook_advice("google-genai", "complete_generate_content") +def _complete_generate_content_advice( + telemetry_handler: TelemetryHandler, + state: _GenerateContentAdviceState, + response: GenerateContentResponse, +) -> None: + finish_reasons: List[str] = [] + try: + _apply_response_attributes( + response, + finish_reasons, + state.invocation, + ) + if telemetry_handler.should_capture_content() and response.candidates: + state.invocation.output_messages = to_output_messages( + candidates=response.candidates + ) + finally: + state.invocation.stop() + + +@hook_advice("google-genai", "fail_generate_content") +def _fail_generate_content_advice( + state: _GenerateContentAdviceState, + error: BaseException, +) -> None: + state.invocation.fail(error) + + +@hook_advice("google-genai", "detach_stream_context") +def _detach_stream_context_advice( + telemetry_handler: TelemetryHandler, + state: _GenerateContentAdviceState, +) -> bool: + telemetry_handler.detach_inference_context(state.invocation) + return True + + +@hook_advice("google-genai", "abandon_generate_content") +def _abandon_generate_content_advice( + telemetry_handler: TelemetryHandler, + state: _GenerateContentAdviceState, +) -> None: + telemetry_handler.abandon_inference(state.invocation) + + +def _create_instrumented_generate_content( + telemetry_handler: TelemetryHandler, + generate_content_config_key_allowlist: AllowList, +): + def instrumented_generate_content( + wrapped: Callable[..., Any], + instance: Any, + args: Tuple[Any, ...], + kwargs: Dict[str, Any], + ): + def _execute( + model: str, + contents: Union[ContentListUnion, ContentListUnionDict], + config: Optional[GenerateContentConfigOrDict] = None, + *_args, + **_kwargs, + ): + state = _prepare_generate_content_advice( + telemetry_handler, + generate_content_config_key_allowlist, + instance, + model, + contents, + config, + ) + call_config = ( + state.wrapped_config + if state is not None and state.has_wrapped_tools + else config + ) + try: + response = wrapped( + model=model, + contents=contents, + config=call_config, + *_args, + **_kwargs, + ) + except BaseException as error: + if state is not None: + _fail_generate_content_advice(state, error) + raise + if state is not None: + _complete_generate_content_advice( + telemetry_handler, + state, + response, + ) + return response + + return _execute(*args, **kwargs) + + return instrumented_generate_content + + +class GenerateContentStreamWrapper(SyncStreamWrapper[GenerateContentResponse]): + def __init__( + self, + stream: Iterable[GenerateContentResponse], + invocation: InferenceInvocation, + telemetry_handler: TelemetryHandler, + ) -> None: + super().__init__(stream) + self._self_invocation = invocation + self._self_telemetry_handler = telemetry_handler + self._self_finish_reasons: List[str] = [] + self._self_output_accumulator = StreamOutputMessageAccumulator() + + def _process_chunk(self, chunk: GenerateContentResponse) -> None: + self._self_invocation.record_first_token() + _apply_response_attributes( + chunk, + self._self_finish_reasons, + self._self_invocation, + ) + if chunk.candidates: + self._self_output_accumulator.add_candidates(chunk.candidates) + + def _on_stream_end(self) -> None: + try: + if self._self_telemetry_handler.should_capture_content(): + self._self_invocation.output_messages = ( + self._self_output_accumulator.to_output_messages() + ) + finally: + self._self_invocation.stop() + + def _on_stream_error(self, error: BaseException) -> None: + self._self_invocation.fail(error) + + +class AsyncGenerateContentStreamWrapper( + AsyncStreamWrapper[GenerateContentResponse] +): + def __init__( + self, + stream: AsyncIterable[GenerateContentResponse], + invocation: InferenceInvocation, + telemetry_handler: TelemetryHandler, + ) -> None: + super().__init__(stream) + # _self_ is a naming convention used by the wrapt library to differentiate + # between attributes on the wrapped function and the original function. + self._self_invocation = invocation + self._self_telemetry_handler = telemetry_handler + self._self_finish_reasons: List[str] = [] + self._self_output_accumulator = StreamOutputMessageAccumulator() + + def _process_chunk(self, chunk: GenerateContentResponse) -> None: + self._self_invocation.record_first_token() + _apply_response_attributes( + chunk, + self._self_finish_reasons, + self._self_invocation, + ) + if chunk.candidates: + self._self_output_accumulator.add_candidates(chunk.candidates) + + def _on_stream_end(self) -> None: + try: + if self._self_telemetry_handler.should_capture_content(): + self._self_invocation.output_messages = ( + self._self_output_accumulator.to_output_messages() + ) + finally: + self._self_invocation.stop() + + def _on_stream_error(self, error: BaseException) -> None: + self._self_invocation.fail(error) + + +@hook_advice("google-genai", "wrap_generate_content_stream") +def _wrap_generate_content_stream_advice( + stream: Any, + state: _GenerateContentAdviceState, + telemetry_handler: TelemetryHandler, + *, + asynchronous: bool, +) -> Any: + wrapper_type = ( + AsyncGenerateContentStreamWrapper + if asynchronous + else GenerateContentStreamWrapper + ) + return wrapper_type( + stream, + state.invocation, + telemetry_handler, + ) + + +def _create_instrumented_generate_content_stream( + telemetry_handler: TelemetryHandler, + generate_content_config_key_allowlist: AllowList, +): + def instrumented_generate_content_stream( + wrapped: Callable[..., Any], + instance: Any, + args: Tuple[Any, ...], + kwargs: Dict[str, Any], + ): + def _execute( + model: str, + contents: Union[ContentListUnion, ContentListUnionDict], + config: Optional[GenerateContentConfigOrDict] = None, + *_args, + **_kwargs, + ): + state = _prepare_generate_content_advice( + telemetry_handler, + generate_content_config_key_allowlist, + instance, + model, + contents, + config, + ) + call_config = ( + state.wrapped_config + if state is not None and state.has_wrapped_tools + else config + ) + try: + stream = wrapped( + model=model, + contents=contents, + config=call_config, + *_args, + **_kwargs, + ) + except BaseException as error: + if state is not None: + _fail_generate_content_advice(state, error) + raise + if state is None: + return stream + if not _detach_stream_context_advice( + telemetry_handler, + state, + ): + _abandon_generate_content_advice( + telemetry_handler, + state, + ) + return stream + stream_wrapper = _wrap_generate_content_stream_advice( + stream, + state, + telemetry_handler, + asynchronous=False, + ) + if stream_wrapper is None: + _abandon_generate_content_advice( + telemetry_handler, + state, + ) + return stream + return stream_wrapper + + return _execute(*args, **kwargs) + + return instrumented_generate_content_stream + + +def _create_instrumented_async_generate_content( + telemetry_handler: TelemetryHandler, + generate_content_config_key_allowlist: AllowList, +): + async def instrumented_generate_content( + wrapped: Callable[..., Any], + instance: Any, + args: Tuple[Any, ...], + kwargs: Dict[str, Any], + ): + async def _execute( + model: str, + contents: Union[ContentListUnion, ContentListUnionDict], + config: Optional[GenerateContentConfigOrDict] = None, + *_args, + **_kwargs, + ): + state = await _prepare_async_generate_content_advice( + telemetry_handler, + generate_content_config_key_allowlist, + instance, + model, + contents, + config, + ) + call_config = ( + state.wrapped_config + if state is not None and state.has_wrapped_tools + else config + ) + try: + response = await wrapped( + model=model, + contents=contents, + config=call_config, + *_args, + **_kwargs, + ) + except BaseException as error: + if state is not None: + _fail_generate_content_advice(state, error) + raise + if state is not None: + _complete_generate_content_advice( + telemetry_handler, + state, + response, + ) + return response + + return await _execute(*args, **kwargs) + + return instrumented_generate_content + + +def _create_instrumented_async_generate_content_stream( # type: ignore + telemetry_handler: TelemetryHandler, + generate_content_config_key_allowlist: AllowList, +): + async def instrumented_generate_content_stream( + wrapped: Callable[..., Any], + instance: Any, + args: Tuple[Any, ...], + kwargs: Dict[str, Any], + ): + async def _execute( + model: str, + contents: Union[ContentListUnion, ContentListUnionDict], + config: Optional[GenerateContentConfigOrDict] = None, + *_args, + **_kwargs, + ): + state = await _prepare_async_generate_content_advice( + telemetry_handler, + generate_content_config_key_allowlist, + instance, + model, + contents, + config, + ) + call_config = ( + state.wrapped_config + if state is not None and state.has_wrapped_tools + else config + ) + try: + stream = await wrapped( + model=model, + contents=contents, + config=call_config, + *_args, + **_kwargs, + ) + except BaseException as error: + if state is not None: + _fail_generate_content_advice(state, error) + raise + if state is None: + return stream + if not _detach_stream_context_advice( + telemetry_handler, + state, + ): + _abandon_generate_content_advice( + telemetry_handler, + state, + ) + return stream + stream_wrapper = _wrap_generate_content_stream_advice( + stream, + state, + telemetry_handler, + asynchronous=True, + ) + if stream_wrapper is None: + _abandon_generate_content_advice( + telemetry_handler, + state, + ) + return stream + return stream_wrapper + + return await _execute(*args, **kwargs) + + return instrumented_generate_content_stream + + +def uninstrument_generate_content(snapshot: object): + assert isinstance(snapshot, _MethodsSnapshot) + snapshot.restore() + + +def instrument_generate_content( + telemetry_handler: TelemetryHandler, + generate_content_config_key_allowlist: AllowList, +) -> object: + os.environ.setdefault("OTEL_INSTRUMENTATION_GENAI_EMIT_EVENT", "true") + snapshot = _MethodsSnapshot() + wrapped = wrap_function_wrapper( + "google.genai.models", + "Models.generate_content", + _create_instrumented_generate_content( + telemetry_handler, + generate_content_config_key_allowlist, + ), + ) + wrapped2 = wrap_function_wrapper( + "google.genai.models", + "Models.generate_content_stream", + _create_instrumented_generate_content_stream( + telemetry_handler, + generate_content_config_key_allowlist, + ), + ) + wrapped3 = wrap_function_wrapper( + "google.genai.models", + "AsyncModels.generate_content", + _create_instrumented_async_generate_content( + telemetry_handler, + generate_content_config_key_allowlist, + ), + ) + wrapped4 = wrap_function_wrapper( + "google.genai.models", + "AsyncModels.generate_content_stream", + _create_instrumented_async_generate_content_stream( + telemetry_handler, + generate_content_config_key_allowlist, + ), + ) + _set_co_filename(wrapped) + _set_co_filename(wrapped2) + _set_co_filename(wrapped3) + _set_co_filename(wrapped4) + return snapshot diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/instrumentor.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/instrumentor.py new file mode 100644 index 000000000..815deb153 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/instrumentor.py @@ -0,0 +1,91 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from typing import Any, Collection, Optional + +from opentelemetry._logs import get_logger_provider +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.metrics import get_meter_provider +from opentelemetry.trace import get_tracer_provider +from opentelemetry.util.genai.completion_hook import load_completion_hook + +from ._compat import TelemetryHandler +from .allowlist_util import AllowList +from .embeddings import ( + instrument_embeddings, + uninstrument_embeddings, +) +from .generate_content import ( + instrument_generate_content, + uninstrument_generate_content, +) +from .interactions import ( + instrument_interactions, + uninstrument_interactions, +) + + +class GoogleGenAiSdkInstrumentor(BaseInstrumentor): + def __init__( + self, generate_content_config_key_allowlist: Optional[AllowList] = None + ): + self._generate_content_snapshot = None + self._interactions_snapshot = None + self._embedding_snapshot = None + self._generate_content_config_key_allowlist = ( + generate_content_config_key_allowlist + or AllowList.from_env( + "OTEL_GOOGLE_GENAI_GENERATE_CONTENT_CONFIG_INCLUDES", + excludes_env_var="OTEL_GOOGLE_GENAI_GENERATE_CONTENT_CONFIG_EXCLUDES", + ) + ) + + # Inherited, abstract function from 'BaseInstrumentor'. Even though 'self' is + # not used in the definition, a method is required per the API contract. + def instrumentation_dependencies(self) -> Collection[str]: # pylint: disable=no-self-use + return ["google-genai>=1.32.0,<3"] + + def _instrument(self, **kwargs: Any): + tracer_provider = ( + kwargs.get("tracer_provider") or get_tracer_provider() + ) + logger_provider = ( + kwargs.get("logger_provider") or get_logger_provider() + ) + meter_provider = kwargs.get("meter_provider") or get_meter_provider() + completion_hook = ( + kwargs.get("completion_hook") or load_completion_hook() + ) + telemetry_handler = TelemetryHandler( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + completion_hook=completion_hook, + ) + self._generate_content_snapshot = instrument_generate_content( + telemetry_handler, + generate_content_config_key_allowlist=self._generate_content_config_key_allowlist, + ) + self._interactions_snapshot = instrument_interactions( + telemetry_handler, + ) + self._embedding_snapshot = instrument_embeddings(telemetry_handler) + + def _uninstrument(self, **kwargs: Any): + uninstrument_generate_content(self._generate_content_snapshot) + uninstrument_interactions(self._interactions_snapshot) + uninstrument_embeddings(self._embedding_snapshot) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/interactions.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/interactions.py new file mode 100644 index 000000000..d8ab07cac --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/interactions.py @@ -0,0 +1,592 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from collections.abc import AsyncIterable, Callable, Iterable, Sequence +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Union, cast + +try: + # Google GenAI < 2.9.0 + from google.genai._interactions._streaming import Stream + from google.genai._interactions.resources.interactions import ( + AsyncInteractionsResource, + InteractionsResource, + ) + from google.genai._interactions.types.interaction import Interaction, Usage + from google.genai._interactions.types.interaction_create_params import ( + Input, + ) + from google.genai._interactions.types.interaction_sse_event import ( + InteractionSSEEvent, + ) + + _HAS_INTERACTIONS = True +except ImportError: + try: + # Google GenAI >= 2.9.0 + from google.genai._gaos.interactions import ( + AsyncInteractions as AsyncInteractionsResource, + ) + from google.genai._gaos.interactions import ( + Interactions as InteractionsResource, + ) + from google.genai._gaos.interactions import ( + Stream, + ) + from google.genai._gaos.types.interactions import ( + Interaction, + InteractionSSEEvent, + Usage, + ) + from google.genai._gaos.types.interactions import ( + InteractionsInput as Input, + ) + + _HAS_INTERACTIONS = True + except ImportError: + _HAS_INTERACTIONS = False + + # Placeholders for older versions where interactions are not supported + class InteractionsResource: + create = None + + class AsyncInteractionsResource: + create = None + + class Interaction: + model = None + usage = None + + class Usage: + total_input_tokens = None + total_output_tokens = None + total_thought_tokens = None + + class Input: + pass + + class InteractionSSEEvent: + pass + + class Stream: + pass + + +from wrapt import wrap_function_wrapper + +from opentelemetry.instrumentation.google_genai.client_info import ( + get_client_info as _get_client_info, +) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) +from opentelemetry.util.genai import hook_advice +from opentelemetry.util.genai.types import ( + InputMessage, + OutputMessage, + Text, + ToolCallResponse, + Uri, +) +from opentelemetry.util.genai.types import ( + ToolCall as ToolCallRequest, +) + +from ._compat import ( + GenericPart, + InferenceInvocation, + TelemetryHandler, +) +from ._stream import ( + AsyncStreamWrapper, + SyncStreamWrapper, +) + + +class _InteractionsMethodsSnapshot: + def __init__(self) -> None: + self._original_create = InteractionsResource.create + self._original_create_code = InteractionsResource.create.__code__ + self._original_async_create = AsyncInteractionsResource.create + self._original_async_create_code = ( + AsyncInteractionsResource.create.__code__ + ) + + def restore(self) -> None: + self._original_create.__code__ = self._original_create_code + self._original_async_create.__code__ = self._original_async_create_code + + InteractionsResource.create = self._original_create + AsyncInteractionsResource.create = self._original_async_create + + +# Magic incantation used by native Google ADK instrumentation to identify +# instrumented functions and suppress its own internal tracing when OTel is active. +def _set_co_filename(wrapped: object) -> None: + wrapped.__wrapped__.__code__ = wrapped.__wrapped__.__code__.replace( + co_filename=__file__.replace("\\", "/") + ) + + +def _apply_interaction_response_attributes( + response: Interaction, + invocation: InferenceInvocation, + telemetry_handler: TelemetryHandler, +) -> None: + invocation.response_model_name = response.model + + usage = response.usage or Usage() + + invocation.input_tokens = getattr(usage, "total_input_tokens", None) + invocation.output_tokens = getattr(usage, "total_output_tokens", None) + invocation.thinking_tokens = getattr(usage, "total_thought_tokens", None) + invocation.cache_read_input_tokens = getattr( + usage, "total_cached_tokens", None + ) + + if telemetry_handler.should_capture_content(): + invocation.output_messages = _interactions_response_to_messages( + response + ) + + +def _get_field(obj: Any, name: str) -> Any: + if isinstance(obj, dict): + return obj.get(name) + return getattr(obj, name, None) + + +# Logic for parsing Input is tricky: +# https://github.com/open-telemetry/donation-openinference/blob/6cdd644d79fccf50aedcb614187f924ddfcafb7b/python/instrumentation/openinference-instrumentation-google-genai/src/openinference/instrumentation/google_genai/interactions_attributes.py#L103 +# It doesn't make sense for this to be a List[InputMessage] (per semconv), +# because this API doesn't take conversation history as input (unlike the generate_content API). +# Conversation history is stored server-side and referenced via a interaction ID parameter. +def _interactions_input_to_messages( + input_data: Optional[Input], +) -> List[InputMessage]: + # None will end up raising an exception by the SDK + if input_data is None: + return [] + if isinstance(input_data, str): + return [InputMessage(role="user", parts=[Text(content=input_data)])] + + if not isinstance(input_data, Sequence): + input_data = [input_data] + + parts = [] + for item in input_data: + item_type = _get_field(item, "type") + if item_type == "function_call": + call_id = _get_field(item, "id") + name = _get_field(item, "name") + arguments = _get_field(item, "arguments") + part = ToolCallRequest( + id=call_id, name=name or "", arguments=arguments + ) + parts.append(part) + elif item_type == "function_result": + call_id = _get_field(item, "call_id") + result = _get_field(item, "result") + part = ToolCallResponse(id=call_id, response=result) + parts.append(part) + elif isinstance(item, str): + parts.append(Text(content=item)) + elif item_type == "text": + part = Text(content=_get_field(item, "text") or "") + parts.append(part) + elif item_type == "document": + part = Uri( + mime_type=_get_field(item, "mime_type"), + modality="document", + uri=_get_field(item, "uri") or "", + ) + parts.append(part) + elif item_type is not None: + part = GenericPart(value=type(item).__name__) + parts.append(part) + + return [InputMessage(role="user", parts=parts)] + + +def _get_interaction_output_text(interaction: Interaction) -> str: + if getattr(interaction, "output_text", None): + return interaction.output_text + + texts = [] + if interaction.steps: + for step in interaction.steps: + if getattr(step, "type", None) == "model_output": + content = getattr(step, "content", None) + if content: + for item in content: + if getattr(item, "type", None) == "text" and hasattr( + item, "text" + ): + texts.append(item.text) + return "".join(texts) + + +# It doesn't make sense for this to be a list of OutputMessage (per semconv), +# because this API doesn't return conversation history as output (unlike the generate_content API). +# Model's response is returned as a list of steps: +# https://ai.google.dev/gemini-api/docs/migrate-to-interactions#basic-input-output +# https://ai.google.dev/api/interactions-api#Resource:Step +def _interactions_response_to_messages( + interaction: Interaction, +) -> List[OutputMessage]: + output_text = _get_interaction_output_text(interaction) + return [ + OutputMessage( + role="assistant", + parts=[Text(content=output_text)], + finish_reason="stop", + ) + ] + + +class InteractionsStreamWrapper(SyncStreamWrapper[InteractionSSEEvent]): + def __init__( + self, + stream: Iterable[InteractionSSEEvent], + invocation: InferenceInvocation, + telemetry_handler: TelemetryHandler, + ) -> None: + super().__init__(stream) + self._self_invocation = invocation + self._self_telemetry_handler = telemetry_handler + self._self_last_interaction: Optional[Interaction] = None + + def _process_chunk(self, chunk: InteractionSSEEvent) -> None: + self._self_invocation.record_first_token() + event_type = _get_field(chunk, "event_type") + if event_type in ("interaction_completed", "interaction.completed"): + interaction = _get_field(chunk, "interaction") + if interaction: + self._self_last_interaction = interaction + + def _on_stream_end(self) -> None: + try: + if self._self_last_interaction: + _apply_interaction_response_attributes( + self._self_last_interaction, + self._self_invocation, + self._self_telemetry_handler, + ) + finally: + self._self_invocation.stop() + + def _on_stream_error(self, error: BaseException) -> None: + self._self_invocation.fail(error) + + +class AsyncInteractionsStreamWrapper(AsyncStreamWrapper[InteractionSSEEvent]): + def __init__( + self, + stream: AsyncIterable[InteractionSSEEvent], + invocation: InferenceInvocation, + telemetry_handler: TelemetryHandler, + ) -> None: + super().__init__(stream) + self._self_invocation = invocation + self._self_telemetry_handler = telemetry_handler + self._self_last_interaction: Optional[Interaction] = None + + def _process_chunk(self, chunk: InteractionSSEEvent) -> None: + self._self_invocation.record_first_token() + event_type = _get_field(chunk, "event_type") + if event_type in ("interaction_completed", "interaction.completed"): + interaction = _get_field(chunk, "interaction") + if interaction: + self._self_last_interaction = interaction + + def _on_stream_end(self) -> None: + try: + if self._self_last_interaction: + _apply_interaction_response_attributes( + self._self_last_interaction, + self._self_invocation, + self._self_telemetry_handler, + ) + finally: + self._self_invocation.stop() + + def _on_stream_error(self, error: BaseException) -> None: + self._self_invocation.fail(error) + + +@dataclass +class _InteractionsAdviceState: + invocation: InferenceInvocation + is_stream: bool + + +@hook_advice("google-genai", "prepare_interactions") +def _prepare_interactions_advice( + telemetry_handler: TelemetryHandler, + instance: Any, + kwargs: Dict[str, Any], +) -> _InteractionsAdviceState: + invocation = None + try: + is_vertex, server_address = _get_client_info(instance) + invocation = telemetry_handler.inference( + provider=( + GenAIAttributes.GenAiSystemValues.VERTEX_AI.value + if is_vertex + else GenAIAttributes.GenAiSystemValues.GEMINI.value + ), + request_model=kwargs.get("model") or kwargs.get("agent"), + operation_name="interactions.create", + server_address=server_address, + ) + if telemetry_handler.should_capture_content(): + invocation.input_messages = _interactions_input_to_messages( + kwargs.get("input") + ) + if system_instruction := kwargs.get("system_instruction"): + invocation.system_instruction = [ + Text(content=system_instruction) + ] + return _InteractionsAdviceState( + invocation=invocation, + is_stream=bool(kwargs.get("stream", False)), + ) + except Exception: + if invocation is not None: + telemetry_handler.abandon_inference(invocation) + raise + + +@hook_advice("google-genai", "complete_interactions") +def _complete_interactions_advice( + telemetry_handler: TelemetryHandler, + state: _InteractionsAdviceState, + response: Interaction, +) -> None: + try: + _apply_interaction_response_attributes( + response, + state.invocation, + telemetry_handler, + ) + finally: + state.invocation.stop() + + +@hook_advice("google-genai", "fail_interactions") +def _fail_interactions_advice( + state: _InteractionsAdviceState, + error: BaseException, +) -> None: + state.invocation.fail(error) + + +@hook_advice("google-genai", "detach_interactions_stream_context") +def _detach_interactions_stream_context_advice( + telemetry_handler: TelemetryHandler, + state: _InteractionsAdviceState, +) -> bool: + telemetry_handler.detach_inference_context(state.invocation) + return True + + +@hook_advice("google-genai", "abandon_interactions") +def _abandon_interactions_advice( + telemetry_handler: TelemetryHandler, + state: _InteractionsAdviceState, +) -> None: + telemetry_handler.abandon_inference(state.invocation) + + +@hook_advice("google-genai", "wrap_interactions_stream") +def _wrap_interactions_stream_advice( + stream: Any, + state: _InteractionsAdviceState, + telemetry_handler: TelemetryHandler, + *, + asynchronous: bool, +) -> Any: + wrapper_type = ( + AsyncInteractionsStreamWrapper + if asynchronous + else InteractionsStreamWrapper + ) + return wrapper_type( + stream, + state.invocation, + telemetry_handler, + ) + + +def _create_instrumented_interactions_create( + telemetry_handler: TelemetryHandler, +) -> Callable[ + [ + Callable[..., Union[Interaction, Stream[InteractionSSEEvent]]], + InteractionsResource, + Tuple[Any, ...], + Dict[str, Any], + ], + Union[Interaction, InteractionsStreamWrapper], +]: + def instrumented_interactions_create( + wrapped: Callable[ + ..., Union[Interaction, Stream[InteractionSSEEvent]] + ], + instance: InteractionsResource, + args: Tuple[Any, ...], + kwargs: Dict[str, Any], + ) -> Union[Interaction, InteractionsStreamWrapper]: + state = _prepare_interactions_advice( + telemetry_handler, + instance, + kwargs, + ) + try: + response = wrapped(*args, **kwargs) + except BaseException as error: + if state is not None: + _fail_interactions_advice(state, error) + raise + if state is None: + return response + if not state.is_stream: + _complete_interactions_advice( + telemetry_handler, + state, + cast(Interaction, response), + ) + return response + if not _detach_interactions_stream_context_advice( + telemetry_handler, + state, + ): + _abandon_interactions_advice(telemetry_handler, state) + return response + stream_wrapper = _wrap_interactions_stream_advice( + response, + state, + telemetry_handler, + asynchronous=False, + ) + if stream_wrapper is None: + _abandon_interactions_advice(telemetry_handler, state) + return response + return stream_wrapper + + return instrumented_interactions_create + + +def _create_instrumented_async_interactions_create( + telemetry_handler: TelemetryHandler, +) -> Callable[ + [ + Callable[..., Any], + AsyncInteractionsResource, + Tuple[Any, ...], + Dict[str, Any], + ], + Any, +]: + async def instrumented_interactions_create( + wrapped: Callable[..., Any], + instance: AsyncInteractionsResource, + args: Tuple[Any, ...], + kwargs: Dict[str, Any], + ) -> Union[Interaction, AsyncInteractionsStreamWrapper]: + state = _prepare_interactions_advice( + telemetry_handler, + instance, + kwargs, + ) + try: + response = await wrapped(*args, **kwargs) + except BaseException as error: + if state is not None: + _fail_interactions_advice(state, error) + raise + if state is None: + return response + if not state.is_stream: + _complete_interactions_advice( + telemetry_handler, + state, + cast(Interaction, response), + ) + return response + if not _detach_interactions_stream_context_advice( + telemetry_handler, + state, + ): + _abandon_interactions_advice(telemetry_handler, state) + return response + stream_wrapper = _wrap_interactions_stream_advice( + response, + state, + telemetry_handler, + asynchronous=True, + ) + if stream_wrapper is None: + _abandon_interactions_advice(telemetry_handler, state) + return response + return stream_wrapper + + return instrumented_interactions_create + + +def uninstrument_interactions(snapshot: object) -> None: + if snapshot is None: + return + assert isinstance(snapshot, _InteractionsMethodsSnapshot) + snapshot.restore() + + +def instrument_interactions( + telemetry_handler: TelemetryHandler, +) -> Optional[object]: + if not _HAS_INTERACTIONS: + return None + + snapshot = _InteractionsMethodsSnapshot() + + try: + import google.genai._interactions.resources.interactions # noqa: F401, PLC0415 + + module_path = "google.genai._interactions.resources.interactions" + sync_class = "InteractionsResource" + async_class = "AsyncInteractionsResource" + except ImportError: + # In version 2.9 of google-genai these were moved. + module_path = "google.genai._gaos.interactions" + sync_class = "Interactions" + async_class = "AsyncInteractions" + + wrapped = wrap_function_wrapper( + module_path, + f"{sync_class}.create", + _create_instrumented_interactions_create(telemetry_handler), + ) + _set_co_filename(wrapped) + wrapped2 = wrap_function_wrapper( + module_path, + f"{async_class}.create", + _create_instrumented_async_interactions_create(telemetry_handler), + ) + _set_co_filename(wrapped2) + return snapshot diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/message.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/message.py new file mode 100644 index 000000000..dd6bf2738 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/message.py @@ -0,0 +1,305 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Dict, List, Optional, Tuple, Union + +from google.genai import types as genai_types + +from opentelemetry.util.genai.types import ( + Blob, + FinishReason, + InputMessage, + MessagePart, + OutputMessage, + Reasoning, + Text, + ToolCallResponse, + Uri, +) +from opentelemetry.util.genai.types import ( + ToolCall as ToolCallRequest, +) + + +class Role(str, Enum): + SYSTEM = "system" + USER = "user" + ASSISTANT = "assistant" + TOOL = "tool" + + +_logger = logging.getLogger(__name__) + + +@dataclass +class _StreamCandidateState: + role: str = "" + finish_reason: Union[FinishReason, str] = "" + parts: List[MessagePart] = field(default_factory=list) + position_slots: Dict[int, Tuple[int, str]] = field(default_factory=dict) + + +class StreamOutputMessageAccumulator: + """Merge Google GenAI streaming candidates into logical output messages.""" + + def __init__(self) -> None: + self._candidate_states: Dict[int, _StreamCandidateState] = {} + + def add_candidates( + self, + candidates: List[genai_types.Candidate], + ) -> None: + for fallback_index, candidate in enumerate(candidates): + candidate_index = candidate.index + if not isinstance(candidate_index, int): + candidate_index = fallback_index + state = self._candidate_states.setdefault( + candidate_index, + _StreamCandidateState(), + ) + self._add_candidate(state, candidate) + + def to_output_messages(self) -> List[OutputMessage]: + messages = [] + for candidate_index in sorted(self._candidate_states): + state = self._candidate_states[candidate_index] + if not state.parts: + continue + messages.append( + OutputMessage( + role=state.role, + parts=state.parts, + finish_reason=state.finish_reason, + ) + ) + return messages + + @staticmethod + def _add_candidate( + state: _StreamCandidateState, + candidate: genai_types.Candidate, + ) -> None: + finish_reason = _to_finish_reason(candidate.finish_reason) + if finish_reason: + state.finish_reason = finish_reason + + content = candidate.content + if content is None: + return + role = _to_role(content.role) + if role: + state.role = role + + for position, provider_part in enumerate(content.parts or []): + part = _to_part(provider_part, position) + if part is None: + continue + part_kind = getattr(part, "type", type(part).__name__) + previous = state.position_slots.get(position) + if previous is not None: + slot, previous_kind = previous + if previous_kind == part_kind and _merge_stream_parts( + state.parts[slot], + part, + ): + continue + + slot = len(state.parts) + state.parts.append(part) + state.position_slots[position] = (slot, part_kind) + + +def _merge_stream_parts(existing: MessagePart, incoming: MessagePart) -> bool: + if isinstance(existing, Text) and isinstance(incoming, Text): + existing.content += incoming.content + return True + if isinstance(existing, Reasoning) and isinstance(incoming, Reasoning): + existing.content += incoming.content + return True + if isinstance(existing, ToolCallRequest) and isinstance( + incoming, ToolCallRequest + ): + if (existing.id and incoming.id and existing.id != incoming.id) or ( + existing.name and incoming.name and existing.name != incoming.name + ): + return False + existing.id = incoming.id or existing.id + existing.name = incoming.name or existing.name + existing.arguments = _merge_stream_values( + existing.arguments, + incoming.arguments, + ) + return True + if isinstance(existing, ToolCallResponse) and isinstance( + incoming, ToolCallResponse + ): + if existing.id and incoming.id and existing.id != incoming.id: + return False + existing.id = incoming.id or existing.id + existing.response = _merge_stream_values( + existing.response, + incoming.response, + ) + return True + if isinstance(existing, Blob) and isinstance(incoming, Blob): + return existing == incoming + if isinstance(existing, Uri) and isinstance(incoming, Uri): + return existing == incoming + return False + + +def _merge_stream_values(existing: Any, incoming: Any) -> Any: + if incoming in (None, "", {}, []): + return existing + if existing in (None, "", {}, []): + return incoming + if isinstance(existing, dict) and isinstance(incoming, dict): + return {**existing, **incoming} + if isinstance(existing, str) and isinstance(incoming, str): + return existing + incoming + return incoming + + +def to_input_messages( + *, + contents: List[genai_types.Content], +) -> List[InputMessage]: + return [_to_input_message(content) for content in contents] + + +def to_output_messages( + *, + candidates: List[genai_types.Candidate], +) -> List[OutputMessage]: + def content_to_output_message( + candidate: genai_types.Candidate, + ) -> Optional[OutputMessage]: + if not candidate.content: + return None + + message = _to_input_message(candidate.content) + return OutputMessage( + finish_reason=_to_finish_reason(candidate.finish_reason), + role=message.role, + parts=message.parts, + ) + + messages = ( + content_to_output_message(candidate) for candidate in candidates + ) + return [message for message in messages if message is not None] + + +def to_system_instructions( + *, + content: genai_types.Content, +) -> List[MessagePart]: + parts = ( + _to_part(part, idx) for idx, part in enumerate(content.parts or []) + ) + return [part for part in parts if part is not None] + + +def _to_input_message( + content: genai_types.Content, +) -> InputMessage: + parts = ( + _to_part(part, idx) for idx, part in enumerate(content.parts or []) + ) + return InputMessage( + role=_to_role(content.role), + # filter Nones + parts=[part for part in parts if part is not None], + ) + + +def _to_part(part: genai_types.Part, idx: int) -> Optional[MessagePart]: + def tool_call_id(name: Optional[str]) -> str: + if name: + return f"{name}_{idx}" + return f"{idx}" + + if (text := part.text) is not None: + if getattr(part, "thought", False): + return Reasoning(content=text) + return Text(content=text) + + if inline_data := part.inline_data: + mime_type = inline_data.mime_type or "" + modality = mime_type.split("/")[0] if mime_type else "" + return Blob( + mime_type=mime_type, + modality=modality, + content=inline_data.data or b"", + ) + + if file_data := part.file_data: + mime_type = file_data.mime_type or "" + modality = mime_type.split("/")[0] if mime_type else "" + return Uri( + mime_type=mime_type, + modality=modality, + uri=file_data.file_uri or "", + ) + + if call := part.function_call: + return ToolCallRequest( + id=call.id or tool_call_id(call.name), + name=call.name or "", + arguments=call.args, + ) + + if response := part.function_response: + return ToolCallResponse( + id=response.id or tool_call_id(response.name), + response=response.response, + ) + + _logger.info("Unknown part dropped from telemetry %s", part) + return None + + +def _to_role(role: Optional[str]) -> str: + if role == "user": + return Role.USER.value + if role == "model": + return Role.ASSISTANT.value + return "" + + +def _to_finish_reason( + finish_reason: Optional[genai_types.FinishReason], +) -> Union[FinishReason, str]: + if finish_reason is None: + return "" + if ( + finish_reason is genai_types.FinishReason.FINISH_REASON_UNSPECIFIED + or finish_reason is genai_types.FinishReason.OTHER + ): + return "error" + if finish_reason is genai_types.FinishReason.STOP: + return "stop" + if finish_reason is genai_types.FinishReason.MAX_TOKENS: + return "length" + + # If there is no 1:1 mapping to an OTel preferred enum value, use the exact vertex reason + return finish_reason.name diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/package.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/package.py new file mode 100644 index 000000000..7a2718750 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/package.py @@ -0,0 +1,16 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +_instruments = ("google-genai >= 1.32.0, < 3",) +_supports_metrics = True diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/tool_call_wrapper.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/tool_call_wrapper.py new file mode 100644 index 000000000..e1bd6ea85 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/tool_call_wrapper.py @@ -0,0 +1,201 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import functools +import inspect +import json +from dataclasses import dataclass +from typing import Any, Callable, Dict, Optional, Tuple, Union + +from google.genai.types import ( + ToolListUnion, + ToolListUnionDict, + ToolOrDict, +) + +from opentelemetry.util.genai import hook_advice + +from ._compat import TelemetryHandler, ToolInvocation + +ToolFunction = Callable[..., Any] + + +@dataclass +class _ToolAdviceState: + invocation: ToolInvocation + + +def _is_primitive(value): + return isinstance(value, (str, int, bool, float)) + + +def _to_otel_value(python_value): + """Coerces parameters to something representable with Open Telemetry.""" + if python_value is None or _is_primitive(python_value): + return python_value + if isinstance(python_value, list): + return [_to_otel_value(x) for x in python_value] + if isinstance(python_value, dict): + return { + key: _to_otel_value(val) for (key, val) in python_value.items() + } + if hasattr(python_value, "model_dump"): + return python_value.model_dump() + if hasattr(python_value, "__dict__"): + return _to_otel_value(python_value.__dict__) + return repr(python_value) + + +# There is no canonical way to serialize a Python object to a span attribute value. +# Span attribute values currently must be one of the primitive types, or a homogeneous list of primitive types. +# In the future the value will be expanded to include None, heterogeneous lists of primitive types, and a Map of these types. +# See https://github.com/open-telemetry/opentelemetry-specification/pull/4485 +def _get_function_args(wrapped_function, function_args, function_kwargs): + """Records the details about a function invocation as span attributes.""" + function_arg_attr = {} + signature = inspect.signature(wrapped_function) + params = list(signature.parameters.values()) + for index, entry in enumerate(function_args): + param_name = f"args[{index}]" + if index < len(params): + param_name = params[index].name + function_arg_attr[f"code.function.parameters.{param_name}.type"] = ( + type(entry).__name__ + ) + function_arg_attr[f"code.function.parameters.{param_name}.value"] = ( + _to_otel_value(entry) + ) + for key, value in function_kwargs.items(): + function_arg_attr[f"code.function.parameters.{key}.type"] = type( + value + ).__name__ + function_arg_attr[f"code.function.parameters.{key}.value"] = ( + _to_otel_value(value) + ) + return function_arg_attr + + +@hook_advice("google-genai", "prepare_tool") +def _prepare_tool_advice( + tool_function: ToolFunction, + telemetry_handler: TelemetryHandler, + args: Tuple[Any, ...], + kwargs: Dict[str, Any], +) -> _ToolAdviceState: + invocation = None + try: + invocation = telemetry_handler.tool( + tool_function.__name__, + tool_description=tool_function.__doc__, + ) + if invocation.should_capture_content_on_span: + invocation.arguments = json.dumps( + _get_function_args(tool_function, args, kwargs) + ) + return _ToolAdviceState(invocation=invocation) + except Exception: + if invocation is not None: + telemetry_handler.abandon_tool(invocation) + raise + + +@hook_advice("google-genai", "complete_tool") +def _complete_tool_advice( + state: _ToolAdviceState, + result: Any, +) -> None: + try: + if state.invocation.should_capture_content_on_span: + state.invocation.tool_result = json.dumps(_to_otel_value(result)) + finally: + state.invocation.stop() + + +@hook_advice("google-genai", "fail_tool") +def _fail_tool_advice( + state: _ToolAdviceState, + error: BaseException, +) -> None: + state.invocation.fail(error) + + +def _wrap_tool_function( + tool_function: ToolFunction, + telemetry_handler: TelemetryHandler, +): + if inspect.iscoroutinefunction(tool_function): + + @functools.wraps(tool_function) + async def wrapped_function(*args, **kwargs): + state = _prepare_tool_advice( + tool_function, + telemetry_handler, + args, + kwargs, + ) + try: + result = await tool_function(*args, **kwargs) + except BaseException as error: + if state is not None: + _fail_tool_advice(state, error) + raise + if state is not None: + _complete_tool_advice(state, result) + return result + else: + + @functools.wraps(tool_function) + def wrapped_function(*args, **kwargs): + state = _prepare_tool_advice( + tool_function, + telemetry_handler, + args, + kwargs, + ) + try: + result = tool_function(*args, **kwargs) + except BaseException as error: + if state is not None: + _fail_tool_advice(state, error) + raise + if state is not None: + _complete_tool_advice(state, result) + return result + + return wrapped_function + + +def wrapped_tool( + tool_or_tools: Optional[ + Union[ToolFunction, ToolOrDict, ToolListUnion, ToolListUnionDict] + ], + telemetry_handler: TelemetryHandler, +): + if tool_or_tools is None: + return None + if isinstance(tool_or_tools, list): + return [ + wrapped_tool(tool, telemetry_handler) for tool in tool_or_tools + ] + if isinstance(tool_or_tools, dict): + return { + key: wrapped_tool(tool, telemetry_handler) + for (key, tool) in tool_or_tools.items() + } + if callable(tool_or_tools): + return _wrap_tool_function(tool_or_tools, telemetry_handler) + return tool_or_tools diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/version.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/version.py new file mode 100644 index 000000000..13fdd7e2a --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/version.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.9.0.dev" diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/__init__.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/__init__.py new file mode 100644 index 000000000..b0a6f4284 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_async_embed_content.yaml b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_async_embed_content.yaml new file mode 100644 index 000000000..9d7d9cd93 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_async_embed_content.yaml @@ -0,0 +1,54 @@ +interactions: +- request: + body: '{"requests": [{"content": {"parts": [{"text": "LoongSuite async embedding + verification"}], "role": "user"}, "outputDimensionality": 8, "model": "models/gemini-embedding-001"}]}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '176' + content-type: + - application/json + host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/2.14.0 gl-python/3.12.10 + x-goog-api-client: + - google-genai-sdk/2.14.0 gl-python/3.12.10 + x-goog-api-key: + - test-google-key + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-001:batchEmbedContents + response: + body: + string: "{\n \"embeddings\": [\n {\n \"values\": [\n -0.012145887,\n + \ 0.027226992,\n -0.010417095,\n -0.10146105,\n -0.010073216,\n + \ -0.021136053,\n 0.003264262,\n 0.009064443\n ]\n + \ }\n ]\n}\n" + headers: + alt-svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + content-length: + - '235' + content-type: + - application/json; charset=UTF-8 + transfer-encoding: + - chunked + vary: + - Origin + - X-Origin + - Referer + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-xss-protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_async_generate_content.yaml b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_async_generate_content.yaml new file mode 100644 index 000000000..c7ec46874 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_async_generate_content.yaml @@ -0,0 +1,61 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "Reply with exactly: async ok"}], "role": + "user"}]}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '85' + content-type: + - application/json + host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/2.14.0 gl-python/3.12.10 + x-goog-api-client: + - google-genai-sdk/2.14.0 gl-python/3.12.10 + x-goog-api-key: + - test-google-key + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash-lite:generateContent + response: + body: + string: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": + [\n {\n \"text\": \"async ok\",\n \"thoughtSignature\": + \"dGVzdC10aG91Z2h0LXNpZ25hdHVyZQ==\"\n }\n ],\n \"role\": \"model\"\n + \ },\n \"finishReason\": \"STOP\",\n \"index\": 0\n }\n ],\n + \ \"usageMetadata\": {\n \"promptTokenCount\": 7,\n \"candidatesTokenCount\": + 2,\n \"totalTokenCount\": 9,\n \"promptTokensDetails\": [\n {\n + \ \"modality\": \"TEXT\",\n \"tokenCount\": 7\n }\n ],\n + \ \"serviceTier\": \"standard\"\n },\n \"modelVersion\": \"gemini-3.5-flash-lite\",\n + \ \"responseId\": \"XpJhatfQA5eHqtsPppLq8Qw\"\n}\n" + headers: + alt-svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + content-length: + - '660' + content-type: + - application/json; charset=UTF-8 + transfer-encoding: + - chunked + vary: + - Origin + - X-Origin + - Referer + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-gemini-service-tier: + - standard + x-xss-protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_async_generate_content_stream.yaml b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_async_generate_content_stream.yaml new file mode 100644 index 000000000..f1f6c3f54 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_async_generate_content_stream.yaml @@ -0,0 +1,65 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "Reply with exactly: async stream ok"}], + "role": "user"}]}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '92' + content-type: + - application/json + host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/2.14.0 gl-python/3.12.10 + x-goog-api-client: + - google-genai-sdk/2.14.0 gl-python/3.12.10 + x-goog-api-key: + - test-google-key + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash-lite:streamGenerateContent?alt=sse + response: + body: + string: "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"async + stream\"}],\"role\": \"model\"},\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": + 8,\"candidatesTokenCount\": 2,\"totalTokenCount\": 10,\"promptTokensDetails\": + [{\"modality\": \"TEXT\",\"tokenCount\": 8}],\"serviceTier\": \"standard\"},\"modelVersion\": + \"gemini-3.5-flash-lite\",\"responseId\": \"XpJhaoH0PPShz7IP15SpiQU\"}\r\n\r\ndata: + {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \" ok\"}],\"role\": + \"model\"},\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 8,\"candidatesTokenCount\": + 3,\"totalTokenCount\": 11,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": + 8}],\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-3.5-flash-lite\",\"responseId\": + \"XpJhaoH0PPShz7IP15SpiQU\"}\r\n\r\ndata: {\"candidates\": [{\"content\": + {\"parts\": [{\"text\": \"\",\"thoughtSignature\": \"dGVzdC10aG91Z2h0LXNpZ25hdHVyZQ==\"}],\"role\": + \"model\"},\"finishReason\": \"STOP\",\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": + 8,\"candidatesTokenCount\": 3,\"totalTokenCount\": 11,\"promptTokensDetails\": + [{\"modality\": \"TEXT\",\"tokenCount\": 8}],\"serviceTier\": \"standard\"},\"modelVersion\": + \"gemini-3.5-flash-lite\",\"responseId\": \"XpJhaoH0PPShz7IP15SpiQU\"}\r\n\r\n" + headers: + alt-svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + content-disposition: + - attachment + content-type: + - text/event-stream + transfer-encoding: + - chunked + vary: + - Origin + - X-Origin + - Referer + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-xss-protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_async_interactions_create.yaml b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_async_interactions_create.yaml new file mode 100644 index 000000000..f6f1630c4 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_async_interactions_create.yaml @@ -0,0 +1,52 @@ +interactions: +- request: + body: '{"model": "gemini-3.5-flash-lite", "input": "Reply with exactly: async + interaction ok"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '84' + content-type: + - application/json + host: + - generativelanguage.googleapis.com + user-agent: + - speakeasy-sdk/python 2.4.1-preview.5 2.918.3 v1beta google-genai + x-goog-api-client: + - google-genai-sdk/2.14.0 gl-python/3.12.10 + x-goog-api-key: + - test-google-key + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/interactions + response: + body: + string: '{"id":"v1_ChdCZEppYW9XQkp0bW1tdGtQdlpUS3lBcxIXQmRKaWFvV0JKdG1tbXRrUHZaVEt5QXM","status":"completed","usage":{"total_tokens":11,"total_input_tokens":8,"input_tokens_by_modality":[{"modality":"text","tokens":8}],"total_cached_tokens":0,"total_output_tokens":3,"total_tool_use_tokens":0,"total_thought_tokens":0},"created":"2026-07-24T02:46:29Z","updated":"2026-07-24T02:46:29Z","service_tier":"standard","steps":[{"signature":"dGVzdC10aG91Z2h0LXNpZ25hdHVyZQ==","type":"thought"},{"content":[{"text":"async + interaction ok","type":"text"}],"type":"model_output"}],"object":"interaction","model":"gemini-3.5-flash-lite"}' + headers: + alt-svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + content-length: + - '655' + content-type: + - application/json + transfer-encoding: + - chunked + vary: + - Origin + - X-Origin + - Referer + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-xss-protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_async_interactions_create_stream.yaml b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_async_interactions_create_stream.yaml new file mode 100644 index 000000000..3f8c830db --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_async_interactions_create_stream.yaml @@ -0,0 +1,106 @@ +interactions: +- request: + body: '{"model": "gemini-3.5-flash-lite", "input": "Reply with exactly: async + interaction stream ok", "stream": true}' + headers: + accept: + - text/event-stream + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '105' + content-type: + - application/json + host: + - generativelanguage.googleapis.com + user-agent: + - speakeasy-sdk/python 2.4.1-preview.5 2.918.3 v1beta google-genai + x-goog-api-client: + - google-genai-sdk/2.14.0 gl-python/3.12.10 + x-goog-api-key: + - test-google-key + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/interactions + response: + body: + string: 'event: interaction.created + + data: {"interaction":{"id":"v1_ChdwTnhpYXRuWkVwVGx6N0lQbHBMSWlBVRIXcE54aWF0blpFcFRsejdJUGxwTElpQVU","status":"in_progress","object":"interaction","model":"gemini-3.5-flash-lite"},"event_type":"interaction.created"} + + + event: interaction.status_update + + data: {"interaction_id":"v1_ChdwTnhpYXRuWkVwVGx6N0lQbHBMSWlBVRIXcE54aWF0blpFcFRsejdJUGxwTElpQVU","status":"in_progress","event_type":"interaction.status_update"} + + + event: step.start + + data: {"index":0,"step":{"type":"thought"},"event_type":"step.start"} + + + event: step.delta + + data: {"index":0,"delta":{"signature":"dGVzdC10aG91Z2h0LXNpZ25hdHVyZQ==","type":"thought_signature"},"event_type":"step.delta"} + + + event: step.stop + + data: {"index":0,"event_type":"step.stop"} + + + event: step.start + + data: {"index":1,"step":{"type":"model_output"},"event_type":"step.start"} + + + event: step.delta + + data: {"index":1,"delta":{"text":"async interaction","type":"text"},"event_type":"step.delta"} + + + event: step.delta + + data: {"index":1,"delta":{"text":" stream ok","type":"text"},"event_type":"step.delta"} + + + event: step.stop + + data: {"index":1,"event_type":"step.stop"} + + + event: interaction.completed + + data: {"interaction":{"id":"v1_ChdwTnhpYXRuWkVwVGx6N0lQbHBMSWlBVRIXcE54aWF0blpFcFRsejdJUGxwTElpQVU","status":"completed","usage":{"total_tokens":13,"total_input_tokens":9,"input_tokens_by_modality":[{"modality":"text","tokens":9}],"total_cached_tokens":0,"total_output_tokens":4,"total_tool_use_tokens":0,"total_thought_tokens":0},"created":"2026-07-24T03:31:48Z","updated":"2026-07-24T03:31:48Z","service_tier":"standard","object":"interaction","model":"gemini-3.5-flash-lite"},"event_type":"interaction.completed"} + + + event: done + + data: [DONE] + + + ' + headers: + alt-svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + content-length: + - '1725' + content-type: + - text/event-stream + transfer-encoding: + - chunked + vary: + - Origin + - X-Origin + - Referer + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-xss-protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_automatic_function_call.yaml b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_automatic_function_call.yaml new file mode 100644 index 000000000..bcb5df075 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_automatic_function_call.yaml @@ -0,0 +1,134 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "You must call get_temperature for Paris, + then reply with only the returned temperature."}], "role": "user"}], "tools": + [{"functionDeclarations": [{"description": "Return the current temperature for + a city.", "name": "get_temperature", "parameters": {"properties": {"city": {"type": + "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], "generationConfig": + {}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '397' + content-type: + - application/json + host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/2.14.0+afc gl-python/3.12.10 + x-goog-api-client: + - google-genai-sdk/2.14.0+afc gl-python/3.12.10 + x-goog-api-key: + - test-google-key + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash-lite:generateContent + response: + body: + string: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": + [\n {\n \"functionCall\": {\n \"name\": \"get_temperature\",\n + \ \"args\": {\n \"city\": \"Paris\"\n },\n + \ \"id\": \"G3pXycr5\"\n },\n \"thoughtSignature\": + \"dGVzdC10aG91Z2h0LXNpZ25hdHVyZQ==\"\n }\n ],\n \"role\": + \"model\"\n },\n \"finishReason\": \"STOP\",\n \"index\": 0,\n + \ \"finishMessage\": \"Model generated function call(s).\"\n }\n ],\n + \ \"usageMetadata\": {\n \"promptTokenCount\": 63,\n \"candidatesTokenCount\": + 16,\n \"totalTokenCount\": 79,\n \"promptTokensDetails\": [\n {\n + \ \"modality\": \"TEXT\",\n \"tokenCount\": 63\n }\n ],\n + \ \"serviceTier\": \"standard\"\n },\n \"modelVersion\": \"gemini-3.5-flash-lite\",\n + \ \"responseId\": \"p9FiauqAHtSGz7IPisOxqAU\"\n}\n" + headers: + alt-svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + content-length: + - '882' + content-type: + - application/json; charset=UTF-8 + transfer-encoding: + - chunked + vary: + - Origin + - X-Origin + - Referer + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-gemini-service-tier: + - standard + x-xss-protection: + - '0' + status: + code: 200 + message: OK +- request: + body: '{"contents": [{"parts": [{"text": "You must call get_temperature for Paris, + then reply with only the returned temperature."}], "role": "user"}, {"parts": + [{"functionCall": {"id": "G3pXycr5", "args": {"city": "Paris"}, "name": "get_temperature"}, + "thoughtSignature": "dGVzdC10aG91Z2h0LXNpZ25hdHVyZQ=="}], "role": "model"}, + {"parts": [{"functionResponse": {"name": "get_temperature", "response": {"result": + "17 C"}}}], "role": "user"}], "tools": [{"functionDeclarations": [{"description": + "Return the current temperature for a city.", "name": "get_temperature", "parameters": + {"properties": {"city": {"type": "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], + "generationConfig": {}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '727' + content-type: + - application/json + host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/2.14.0+afc gl-python/3.12.10 + x-goog-api-client: + - google-genai-sdk/2.14.0+afc gl-python/3.12.10 + x-goog-api-key: + - test-google-key + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash-lite:generateContent + response: + body: + string: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": + [\n {\n \"text\": \"17 C\",\n \"thoughtSignature\": + \"dGVzdC10aG91Z2h0LXNpZ25hdHVyZQ==\"\n }\n ],\n \"role\": + \"model\"\n },\n \"finishReason\": \"STOP\",\n \"index\": 0\n + \ }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": 95,\n \"candidatesTokenCount\": + 3,\n \"totalTokenCount\": 98,\n \"promptTokensDetails\": [\n {\n + \ \"modality\": \"TEXT\",\n \"tokenCount\": 95\n }\n ],\n + \ \"serviceTier\": \"standard\"\n },\n \"modelVersion\": \"gemini-3.5-flash-lite\",\n + \ \"responseId\": \"qNFiarnfCoDUz7IP64nL-QQ\"\n}\n" + headers: + alt-svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + content-length: + - '659' + content-type: + - application/json; charset=UTF-8 + transfer-encoding: + - chunked + vary: + - Origin + - X-Origin + - Referer + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-gemini-service-tier: + - standard + x-xss-protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_sync_embed_content.yaml b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_sync_embed_content.yaml new file mode 100644 index 000000000..d348cdd23 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_sync_embed_content.yaml @@ -0,0 +1,54 @@ +interactions: +- request: + body: '{"requests": [{"content": {"parts": [{"text": "LoongSuite sync embedding + verification"}], "role": "user"}, "outputDimensionality": 8, "model": "models/gemini-embedding-001"}]}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '175' + content-type: + - application/json + host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/2.14.0 gl-python/3.12.10 + x-goog-api-client: + - google-genai-sdk/2.14.0 gl-python/3.12.10 + x-goog-api-key: + - test-google-key + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-001:batchEmbedContents + response: + body: + string: "{\n \"embeddings\": [\n {\n \"values\": [\n 0.0016733545,\n + \ 0.030018326,\n -0.0071077654,\n -0.0957516,\n -0.0017130031,\n + \ 0.0017276846,\n -0.010695605,\n -0.0070412676\n ]\n + \ }\n ]\n}\n" + headers: + alt-svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + content-length: + - '239' + content-type: + - application/json; charset=UTF-8 + transfer-encoding: + - chunked + vary: + - Origin + - X-Origin + - Referer + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-xss-protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_sync_generate_content.yaml b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_sync_generate_content.yaml new file mode 100644 index 000000000..17586a55e --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_sync_generate_content.yaml @@ -0,0 +1,61 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "Reply with exactly: sync ok"}], "role": + "user"}]}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '84' + content-type: + - application/json + host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/2.14.0 gl-python/3.12.10 + x-goog-api-client: + - google-genai-sdk/2.14.0 gl-python/3.12.10 + x-goog-api-key: + - test-google-key + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash-lite:generateContent + response: + body: + string: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": + [\n {\n \"text\": \"sync ok\",\n \"thoughtSignature\": + \"dGVzdC10aG91Z2h0LXNpZ25hdHVyZQ==\"\n }\n ],\n \"role\": \"model\"\n + \ },\n \"finishReason\": \"STOP\",\n \"index\": 0\n }\n ],\n + \ \"usageMetadata\": {\n \"promptTokenCount\": 7,\n \"candidatesTokenCount\": + 2,\n \"totalTokenCount\": 9,\n \"promptTokensDetails\": [\n {\n + \ \"modality\": \"TEXT\",\n \"tokenCount\": 7\n }\n ],\n + \ \"serviceTier\": \"standard\"\n },\n \"modelVersion\": \"gemini-3.5-flash-lite\",\n + \ \"responseId\": \"W5JharzyDt6gz7IPx7qD2QI\"\n}\n" + headers: + alt-svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + content-length: + - '659' + content-type: + - application/json; charset=UTF-8 + transfer-encoding: + - chunked + vary: + - Origin + - X-Origin + - Referer + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-gemini-service-tier: + - standard + x-xss-protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_sync_generate_content_stream.yaml b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_sync_generate_content_stream.yaml new file mode 100644 index 000000000..9673f01e7 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_sync_generate_content_stream.yaml @@ -0,0 +1,65 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "Reply with exactly: sync stream ok"}], + "role": "user"}]}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '91' + content-type: + - application/json + host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/2.14.0 gl-python/3.12.10 + x-goog-api-client: + - google-genai-sdk/2.14.0 gl-python/3.12.10 + x-goog-api-key: + - test-google-key + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash-lite:streamGenerateContent?alt=sse + response: + body: + string: "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"sync + stream\"}],\"role\": \"model\"},\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": + 8,\"candidatesTokenCount\": 2,\"totalTokenCount\": 10,\"promptTokensDetails\": + [{\"modality\": \"TEXT\",\"tokenCount\": 8}],\"serviceTier\": \"standard\"},\"modelVersion\": + \"gemini-3.5-flash-lite\",\"responseId\": \"XJJhaoiZBNuLqtsPuMm82AI\"}\r\n\r\ndata: + {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \" ok\"}],\"role\": + \"model\"},\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 8,\"candidatesTokenCount\": + 3,\"totalTokenCount\": 11,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": + 8}],\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-3.5-flash-lite\",\"responseId\": + \"XJJhaoiZBNuLqtsPuMm82AI\"}\r\n\r\ndata: {\"candidates\": [{\"content\": + {\"parts\": [{\"text\": \"\",\"thoughtSignature\": \"dGVzdC10aG91Z2h0LXNpZ25hdHVyZQ==\"}],\"role\": + \"model\"},\"finishReason\": \"STOP\",\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": + 8,\"candidatesTokenCount\": 3,\"totalTokenCount\": 11,\"promptTokensDetails\": + [{\"modality\": \"TEXT\",\"tokenCount\": 8}],\"serviceTier\": \"standard\"},\"modelVersion\": + \"gemini-3.5-flash-lite\",\"responseId\": \"XJJhaoiZBNuLqtsPuMm82AI\"}\r\n\r\n" + headers: + alt-svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + content-disposition: + - attachment + content-type: + - text/event-stream + transfer-encoding: + - chunked + vary: + - Origin + - X-Origin + - Referer + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-xss-protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_sync_interactions_create.yaml b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_sync_interactions_create.yaml new file mode 100644 index 000000000..3f25b6a5c --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_sync_interactions_create.yaml @@ -0,0 +1,52 @@ +interactions: +- request: + body: '{"model": "gemini-3.5-flash-lite", "input": "Reply with exactly: interaction + ok"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '78' + content-type: + - application/json + host: + - generativelanguage.googleapis.com + user-agent: + - speakeasy-sdk/python 2.4.1-preview.5 2.918.3 v1beta google-genai + x-goog-api-client: + - google-genai-sdk/2.14.0 gl-python/3.12.10 + x-goog-api-key: + - test-google-key + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/interactions + response: + body: + string: '{"id":"v1_ChdBOUppYXY2SEw2bUlxdHNQN3BUQmdRYxIXQTlKaWF2NkhMNm1JcXRzUDdwVEJnUWM","status":"completed","usage":{"total_tokens":9,"total_input_tokens":7,"input_tokens_by_modality":[{"modality":"text","tokens":7}],"total_cached_tokens":0,"total_output_tokens":2,"total_tool_use_tokens":0,"total_thought_tokens":0},"created":"2026-07-24T02:46:27Z","updated":"2026-07-24T02:46:27Z","service_tier":"standard","steps":[{"signature":"dGVzdC10aG91Z2h0LXNpZ25hdHVyZQ==","type":"thought"},{"content":[{"text":"interaction + ok","type":"text"}],"type":"model_output"}],"object":"interaction","model":"gemini-3.5-flash-lite"}' + headers: + alt-svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + content-length: + - '648' + content-type: + - application/json + transfer-encoding: + - chunked + vary: + - Origin + - X-Origin + - Referer + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-xss-protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_sync_interactions_create_stream.yaml b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_sync_interactions_create_stream.yaml new file mode 100644 index 000000000..b857f09ab --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/cassettes/test_sync_interactions_create_stream.yaml @@ -0,0 +1,106 @@ +interactions: +- request: + body: '{"model": "gemini-3.5-flash-lite", "input": "Reply with exactly: interaction + stream ok", "stream": true}' + headers: + accept: + - text/event-stream + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '99' + content-type: + - application/json + host: + - generativelanguage.googleapis.com + user-agent: + - speakeasy-sdk/python 2.4.1-preview.5 2.918.3 v1beta google-genai + x-goog-api-client: + - google-genai-sdk/2.14.0 gl-python/3.12.10 + x-goog-api-key: + - test-google-key + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/interactions + response: + body: + string: 'event: interaction.created + + data: {"interaction":{"id":"v1_ChdvdHhpYXFmU0w4V0d6N0lQOTQzanVRaxIXb3R4aWFxZlNMOFdHejdJUDk0M2p1UWs","status":"in_progress","object":"interaction","model":"gemini-3.5-flash-lite"},"event_type":"interaction.created"} + + + event: interaction.status_update + + data: {"interaction_id":"v1_ChdvdHhpYXFmU0w4V0d6N0lQOTQzanVRaxIXb3R4aWFxZlNMOFdHejdJUDk0M2p1UWs","status":"in_progress","event_type":"interaction.status_update"} + + + event: step.start + + data: {"index":0,"step":{"type":"thought"},"event_type":"step.start"} + + + event: step.delta + + data: {"index":0,"delta":{"signature":"dGVzdC10aG91Z2h0LXNpZ25hdHVyZQ==","type":"thought_signature"},"event_type":"step.delta"} + + + event: step.stop + + data: {"index":0,"event_type":"step.stop"} + + + event: step.start + + data: {"index":1,"step":{"type":"model_output"},"event_type":"step.start"} + + + event: step.delta + + data: {"index":1,"delta":{"text":"interaction stream","type":"text"},"event_type":"step.delta"} + + + event: step.delta + + data: {"index":1,"delta":{"text":" ok","type":"text"},"event_type":"step.delta"} + + + event: step.stop + + data: {"index":1,"event_type":"step.stop"} + + + event: interaction.completed + + data: {"interaction":{"id":"v1_ChdvdHhpYXFmU0w4V0d6N0lQOTQzanVRaxIXb3R4aWFxZlNMOFdHejdJUDk0M2p1UWs","status":"completed","usage":{"total_tokens":11,"total_input_tokens":8,"input_tokens_by_modality":[{"modality":"text","tokens":8}],"total_cached_tokens":0,"total_output_tokens":3,"total_tool_use_tokens":0,"total_thought_tokens":0},"created":"2026-07-24T03:31:47Z","updated":"2026-07-24T03:31:47Z","service_tier":"standard","object":"interaction","model":"gemini-3.5-flash-lite"},"event_type":"interaction.completed"} + + + event: done + + data: [DONE] + + + ' + headers: + alt-svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + content-length: + - '1719' + content-type: + - text/event-stream + transfer-encoding: + - chunked + vary: + - Origin + - X-Origin + - Referer + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-xss-protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/common/auth.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/common/auth.py new file mode 100644 index 000000000..c96a1e020 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/common/auth.py @@ -0,0 +1,27 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import google.auth.credentials + + +class FakeCredentials(google.auth.credentials.AnonymousCredentials): + def __init__(self): + self.token = "a" + self._quota_project_id = "a" + + def refresh(self, request): + pass diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/common/base.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/common/base.py new file mode 100644 index 000000000..d7801fc5a --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/common/base.py @@ -0,0 +1,98 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import os +import unittest +from unittest.mock import patch + +import google.genai + +from .auth import FakeCredentials +from .instrumentation_context import InstrumentationContext +from .otel_mocker import OTelMocker + + +class TestCase(unittest.TestCase): + def setUp(self): + self.env_patcher = patch.dict( + os.environ, + { + "OTEL_SEMCONV_STABILITY_OPT_IN": "gen_ai_latest_experimental", + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "SPAN_AND_EVENT", + }, + ) + self.env_patcher.start() + self._otel = OTelMocker() + self._otel.install() + self._instrumentation_context = None + self._api_key = "test-api-key" + self._project = "test-project" + self._location = "test-location" + self._client = None + self._uses_vertex = False + self._credentials = FakeCredentials() + self._instrumentor_args = {} + + def tearDown(self): + if self._instrumentation_context is not None: + self._instrumentation_context.uninstall() + self._instrumentation_context = None + self._otel.uninstall() + self.env_patcher.stop() + + def _lazy_init(self): + self._instrumentation_context = InstrumentationContext( + **self._instrumentor_args + ) + self._instrumentation_context.install() + + def set_instrumentor_constructor_kwarg(self, key, value): + self._instrumentor_args[key] = value + + @property + def client(self): + if self._client is None: + self._client = self._create_client() + return self._client + + @property + def otel(self): + return self._otel + + def set_use_vertex(self, use_vertex): + self._uses_vertex = use_vertex + + def reset_client(self): + self._client = None + + def reset_instrumentation(self): + if self._instrumentation_context is None: + return + self._instrumentation_context.uninstall() + self._instrumentation_context = None + + def _create_client(self): + self._lazy_init() + if self._uses_vertex: + os.environ["GOOGLE_API_KEY"] = self._api_key + return google.genai.Client( + vertexai=True, + project=self._project, + location=self._location, + credentials=self._credentials, + ) + return google.genai.Client(vertexai=False, api_key=self._api_key) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/common/instrumentation_context.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/common/instrumentation_context.py new file mode 100644 index 000000000..2cfff2b44 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/common/instrumentation_context.py @@ -0,0 +1,31 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from opentelemetry.instrumentation.google_genai import ( + GoogleGenAiSdkInstrumentor, +) + + +class InstrumentationContext: + def __init__(self, **kwargs): + self._instrumentor = GoogleGenAiSdkInstrumentor(**kwargs) + + def install(self): + self._instrumentor.instrument() + + def uninstall(self): + self._instrumentor.uninstrument() diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/common/otel_mocker.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/common/otel_mocker.py new file mode 100644 index 000000000..68dca8743 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/common/otel_mocker.py @@ -0,0 +1,240 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + + +import opentelemetry._logs._internal +import opentelemetry.metrics._internal +import opentelemetry.trace +from opentelemetry._logs import ( + get_logger_provider, + set_logger_provider, +) +from opentelemetry.metrics import get_meter_provider, set_meter_provider +from opentelemetry.sdk._logs import LoggerProvider + +# Backward compatibility for InMemoryLogExporter -> InMemoryLogRecordExporter rename +try: + from opentelemetry.sdk._logs.export import ( # pylint: disable=no-name-in-module + InMemoryLogRecordExporter, + SimpleLogRecordProcessor, + ) +except ImportError: + # Fallback to old name for compatibility with older SDK versions + from opentelemetry.sdk._logs.export import ( + InMemoryLogExporter as InMemoryLogRecordExporter, + ) + from opentelemetry.sdk._logs.export import ( + SimpleLogRecordProcessor, + ) +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics._internal.export import InMemoryMetricReader +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.trace import get_tracer_provider, set_tracer_provider +from opentelemetry.util._once import Once + + +def _bypass_otel_once(): + opentelemetry.trace._TRACER_PROVIDER_SET_ONCE = Once() + opentelemetry._logs._internal._LOGGER_PROVIDER_SET_ONCE = Once() + opentelemetry.metrics._internal._METER_PROVIDER_SET_ONCE = Once() + + +class OTelProviderSnapshot: + def __init__(self): + self._tracer_provider = get_tracer_provider() + self._logger_provider = get_logger_provider() + self._meter_provider = get_meter_provider() + + def restore(self): + _bypass_otel_once() + opentelemetry.trace._TRACER_PROVIDER = None + opentelemetry._logs._internal._LOGGER_PROVIDER = None + opentelemetry.metrics._internal._METER_PROVIDER = None + + +class _LogWrapper: + def __init__(self, log_data): + self._log_data = log_data + + @property + def scope(self): + return self._log_data.instrumentation_scope + + @property + def resource(self): + return self._log_data.log_record.resource + + @property + def attributes(self): + return self._log_data.log_record.attributes + + @property + def body(self): + return self._log_data.log_record.body + + @property + def event_name(self): + return self._log_data.log_record.event_name + + def __str__(self): + return self._log_data.log_record.to_json() + + +class _MetricDataPointWrapper: + def __init__(self, resource, scope, metric): + self._resource = resource + self._scope = scope + self._metric = metric + + @property + def resource(self): + return self._resource + + @property + def scope(self): + return self._scope + + @property + def metric(self): + return self._metric + + @property + def name(self): + return self._metric.name + + @property + def data(self): + return self._metric.data + + +class OTelMocker: + def __init__(self): + self._snapshot = None + self._logs = InMemoryLogRecordExporter() + self._traces = InMemorySpanExporter() + self._metrics = InMemoryMetricReader() + self._spans = [] + self._finished_logs = [] + self._metrics_data = [] + + def install(self): + self._snapshot = OTelProviderSnapshot() + _bypass_otel_once() + self._install_logs() + self._install_metrics() + self._install_traces() + + def uninstall(self): + self._snapshot.restore() + + def get_finished_logs(self): + for log_data in self._logs.get_finished_logs(): + self._finished_logs.append(_LogWrapper(log_data)) + return self._finished_logs + + def get_finished_spans(self): + for span in self._traces.get_finished_spans(): + self._spans.append(span) + return self._spans + + def get_metrics_data(self): + data = self._metrics.get_metrics_data() + if data is not None: + for resource_metric in data.resource_metrics: + resource = resource_metric.resource + for scope_metrics in resource_metric.scope_metrics: + scope = scope_metrics.scope + for metric in scope_metrics.metrics: + wrapper = _MetricDataPointWrapper( + resource, scope, metric + ) + self._metrics_data.append(wrapper) + return self._metrics_data + + def get_span_named(self, name): + for span in self.get_finished_spans(): + if span.name == name: + return span + return None + + def assert_has_span_named(self, name): + span = self.get_span_named(name) + finished_spans = [span.name for span in self.get_finished_spans()] + assert span is not None, ( + f'Could not find span named "{name}"; finished spans: {finished_spans}' + ) + + def assert_does_not_have_span_named(self, name): + span = self.get_span_named(name) + assert span is None, f"Found unexpected span named {name}" + + def get_event_named(self, event_name): + return next( + ( + event + for event in self.get_finished_logs() + if event.event_name == event_name + ), + None, + ) + + def get_events_named(self, event_name): + return [ + event + for event in self.get_finished_logs() + if event.event_name == event_name + ] + + def assert_has_event_named(self, name): + event = self.get_event_named(name) + finished_logs = self.get_finished_logs() + assert event is not None, ( + f'Could not find event named "{name}"; finished logs: {finished_logs}' + ) + + def assert_does_not_have_event_named(self, name): + event = self.get_event_named(name) + assert event is None, f"Unexpected event: {event}" + + def get_metrics_data_named(self, name): + results = [] + for entry in self.get_metrics_data(): + if entry.name == name: + results.append(entry) + return results + + def assert_has_metrics_data_named(self, name): + data = self.get_metrics_data_named(name) + assert len(data) > 0 + + def _install_logs(self): + provider = LoggerProvider() + provider.add_log_record_processor(SimpleLogRecordProcessor(self._logs)) + set_logger_provider(provider) + + def _install_metrics(self): + provider = MeterProvider(metric_readers=[self._metrics]) + set_meter_provider(provider) + + def _install_traces(self): + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(self._traces)) + set_tracer_provider(provider) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/conftest.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/conftest.py new file mode 100644 index 000000000..7a6a772d6 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/conftest.py @@ -0,0 +1,94 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""VCR configuration for Google GenAI SDK integration tests.""" + +import re +from pathlib import Path + +import pytest + +_CASSETTES = Path(__file__).parent / "cassettes" +_THOUGHT_SIGNATURE = re.compile( + rb'("(?:thoughtSignature|thought_signature|signature)"\s*:\s*")[^"]+(")' +) + + +def _scrub_body(value): + if isinstance(value, str): + return _THOUGHT_SIGNATURE.sub( + rb"\1dGVzdC10aG91Z2h0LXNpZ25hdHVyZQ==\2", value.encode() + ).decode() + if isinstance(value, bytes): + return _THOUGHT_SIGNATURE.sub( + rb"\1dGVzdC10aG91Z2h0LXNpZ25hdHVyZQ==\2", value + ) + return value + + +def _scrub_request(request): + request.body = _scrub_body(request.body) + return request + + +def _scrub_response(response): + body = response.get("body", {}) + if "string" in body: + body["string"] = _scrub_body(body["string"]) + headers = response.get("headers", {}) + for name in list(headers): + if name.lower() in { + "date", + "server", + "server-timing", + "set-cookie", + "x-request-id", + }: + del headers[name] + return response + + +@pytest.fixture(scope="module") +def vcr_cassette_dir(): + return str(_CASSETTES) + + +@pytest.fixture(scope="module") +def vcr_config(): + return { + "filter_headers": [ + ("authorization", "Bearer test_google_key"), + ("x-goog-api-key", "test-google-key"), + ], + "filter_query_parameters": [ + "key", + "apiKey", + "quotaUser", + "userProject", + "access_token", + ], + "filter_post_data_parameters": ["key", "api_key", "apiKey"], + "before_record_request": _scrub_request, + "before_record_response": _scrub_response, + "decode_compressed_response": True, + "match_on": [ + "method", + "scheme", + "host", + "port", + "path", + "query", + "body", + ], + } diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/embeddings/__init__.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/embeddings/__init__.py new file mode 100644 index 000000000..cceb2a6af --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/embeddings/__init__.py @@ -0,0 +1,16 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/embeddings/test_embeddings.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/embeddings/test_embeddings.py new file mode 100644 index 000000000..65459acb9 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/embeddings/test_embeddings.py @@ -0,0 +1,181 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +from unittest.mock import AsyncMock, MagicMock + +from google.genai.models import AsyncModels, Models +from google.genai.types import ( + ContentEmbedding, + ContentEmbeddingStatistics, + EmbedContentResponse, +) + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) +from opentelemetry.trace import StatusCode + +from ..common.base import TestCase + +_EMBEDDING_DIMENSION_COUNT = getattr( + GenAIAttributes, + "GEN_AI_EMBEDDINGS_DIMENSION_COUNT", + "gen_ai.embeddings.dimension.count", +) + + +class TestEmbeddings(TestCase): + def setUp(self): + super().setUp() + self._original_embed_content = Models.embed_content + self._original_async_embed_content = AsyncModels.embed_content + + self.mock_response = EmbedContentResponse( + embeddings=[ + ContentEmbedding( + values=[0.1, 0.2, 0.3], + statistics=ContentEmbeddingStatistics( + token_count=5, + truncated=False, + ), + ) + ] + ) + + self.embed_content_mock = MagicMock(return_value=self.mock_response) + self.async_embed_content_mock = AsyncMock( + return_value=self.mock_response + ) + + def _sync_embed_wrapped(self_obj, *args, **kwargs): + return self.embed_content_mock(*args, **kwargs) + + async def _async_embed_wrapped(self_obj, *args, **kwargs): + return await self.async_embed_content_mock(*args, **kwargs) + + Models.embed_content = _sync_embed_wrapped + AsyncModels.embed_content = _async_embed_wrapped + + def tearDown(self): + super().tearDown() + Models.embed_content = self._original_embed_content + AsyncModels.embed_content = self._original_async_embed_content + + def test_sync_embed_content(self): + response = self.client.models.embed_content( + model="text-embedding-004", + contents="hello world", + ) + + self.assertEqual(response, self.mock_response) + self.embed_content_mock.assert_called_once_with( + model="text-embedding-004", + contents="hello world", + ) + + spans = self.otel.get_finished_spans() + self.assertEqual(len(spans), 1) + span = spans[0] + + self.assertEqual(span.name, "embeddings text-embedding-004") + self.assertEqual(span.status.status_code, StatusCode.UNSET) + + attrs = span.attributes + self.assertEqual( + attrs[GenAIAttributes.GEN_AI_OPERATION_NAME], "embeddings" + ) + self.assertEqual(attrs[GenAIAttributes.GEN_AI_PROVIDER_NAME], "gemini") + self.assertEqual( + attrs[GenAIAttributes.GEN_AI_REQUEST_MODEL], "text-embedding-004" + ) + self.assertEqual(attrs[_EMBEDDING_DIMENSION_COUNT], 3) + self.assertEqual( + attrs["server.address"], + "generativelanguage.googleapis.com", + ) + + def test_async_embed_content(self): + async def run_test(): + response = await self.client.aio.models.embed_content( + model="text-embedding-004", + contents="hello world", + ) + self.assertEqual(response, self.mock_response) + self.async_embed_content_mock.assert_called_once_with( + model="text-embedding-004", + contents="hello world", + ) + + asyncio.run(run_test()) + + spans = self.otel.get_finished_spans() + self.assertEqual(len(spans), 1) + span = spans[0] + + self.assertEqual(span.name, "embeddings text-embedding-004") + self.assertEqual(span.status.status_code, StatusCode.UNSET) + + attrs = span.attributes + self.assertEqual( + attrs[GenAIAttributes.GEN_AI_OPERATION_NAME], "embeddings" + ) + self.assertEqual(attrs[GenAIAttributes.GEN_AI_PROVIDER_NAME], "gemini") + self.assertEqual( + attrs[GenAIAttributes.GEN_AI_REQUEST_MODEL], "text-embedding-004" + ) + self.assertEqual(attrs[_EMBEDDING_DIMENSION_COUNT], 3) + self.assertEqual( + attrs["server.address"], + "generativelanguage.googleapis.com", + ) + + def test_embed_content_multiple_inputs(self): + _ = self.client.models.embed_content( + model="text-embedding-004", + contents=["hello", "world"], + ) + + spans = self.otel.get_finished_spans() + self.assertEqual(len(spans), 1) + span = spans[0] + + attrs = span.attributes + self.assertEqual( + attrs[GenAIAttributes.GEN_AI_OPERATION_NAME], "embeddings" + ) + self.assertEqual(attrs[GenAIAttributes.GEN_AI_PROVIDER_NAME], "gemini") + self.assertEqual( + attrs[GenAIAttributes.GEN_AI_REQUEST_MODEL], "text-embedding-004" + ) + self.assertEqual(attrs[_EMBEDDING_DIMENSION_COUNT], 3) + + def test_embed_content_error(self): + self.embed_content_mock.side_effect = ValueError("invalid model") + + with self.assertRaises(ValueError): + self.client.models.embed_content( + model="bad-model", + contents="test", + ) + + spans = self.otel.get_finished_spans() + self.assertEqual(len(spans), 1) + span = spans[0] + + self.assertEqual(span.status.status_code, StatusCode.ERROR) + self.assertEqual(span.status.description, "invalid model") diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/__init__.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/__init__.py new file mode 100644 index 000000000..b0a6f4284 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/__init__.py @@ -0,0 +1,13 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/base.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/base.py new file mode 100644 index 000000000..69db1d164 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/base.py @@ -0,0 +1,171 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import unittest +import unittest.mock + +from google.genai.models import AsyncModels, Models + +from ..common.base import TestCase as CommonTestCaseBase +from .util import convert_to_response, create_response + + +# Helper used in "_install_mocks" below. +def _wrap_output(mock_generate_content): + def _wrapped(*args, **kwargs): + return convert_to_response(mock_generate_content(*args, **kwargs)) + + return _wrapped + + +# Helper used in "_install_mocks" below. +def _wrap_output_stream(mock_generate_content_stream): + def _wrapped(*args, **kwargs): + for output in mock_generate_content_stream(*args, **kwargs): + yield convert_to_response(output) + + return _wrapped + + +# Helper used in "_install_mocks" below. +def _async_wrapper(mock_generate_content): + async def _wrapped(*args, **kwargs): + return mock_generate_content(*args, **kwargs) + + return _wrapped + + +# Helper used in "_install_mocks" below. +def _async_stream_wrapper(mock_generate_content_stream): + async def _wrapped(*args, **kwargs): + async def _internal_generator(): + for result in mock_generate_content_stream(*args, **kwargs): + yield result + + return _internal_generator() + + return _wrapped + + +class TestCase(CommonTestCaseBase): + # The "setUp" function is defined by "unittest.TestCase" and thus + # this name must be used. Uncertain why pylint doesn't seem to + # recognize that this is a unit test class for which this is inherited. + def setUp(self): # pylint: disable=invalid-name + super().setUp() + if self.__class__ == TestCase: + raise unittest.SkipTest("Skipping testcase base.") + self._generate_content_mock = None + self._generate_content_stream_mock = None + self._original_generate_content = Models.generate_content + self._original_generate_content_stream = Models.generate_content_stream + self._original_async_generate_content = AsyncModels.generate_content + self._original_async_generate_content_stream = ( + AsyncModels.generate_content_stream + ) + self._responses = [] + self._response_index = 0 + + @property + def mock_generate_content(self): + if self._generate_content_mock is None: + self._create_and_install_mocks() + return self._generate_content_mock + + @property + def mock_generate_content_stream(self): + if self._generate_content_stream_mock is None: + self._create_and_install_mocks() + return self._generate_content_stream_mock + + def configure_valid_response(self, **kwargs): + self._create_and_install_mocks() + response = create_response(**kwargs) + self._responses.append(response) + + def configure_exception(self, e, **kwargs): + self._create_and_install_mocks(e) + + def _create_and_install_mocks(self, e=None): + if self._generate_content_mock is not None: + return + self.reset_client() + self.reset_instrumentation() + self._generate_content_mock = self._create_nonstream_mock(e) + self._generate_content_stream_mock = self._create_stream_mock(e) + self._install_mocks() + + def _create_nonstream_mock(self, e=None): + mock = unittest.mock.MagicMock() + + def _default_impl(*args, **kwargs): + if not self._responses: + return create_response(text="Some response") + index = self._response_index % len(self._responses) + result = self._responses[index] + self._response_index += 1 + return result + + mock.side_effect = e or _default_impl + return mock + + def _create_stream_mock(self, e=None): + mock = unittest.mock.MagicMock() + + def _default_impl(*args, **kwargs): + yield from self._responses + + if not e: + mock.side_effect = _default_impl + else: + mock.side_effect = e + return mock + + def _install_mocks(self): + output_wrapped = _wrap_output(self._generate_content_mock) + output_wrapped_stream = _wrap_output_stream( + self._generate_content_stream_mock + ) + Models.generate_content = output_wrapped + Models.generate_content_stream = output_wrapped_stream + AsyncModels.generate_content = _async_wrapper(output_wrapped) + AsyncModels.generate_content_stream = _async_stream_wrapper( + output_wrapped_stream + ) + + def tearDown(self): + super().tearDown() + if self._generate_content_mock is None: + assert Models.generate_content == self._original_generate_content + assert ( + Models.generate_content_stream + == self._original_generate_content_stream + ) + assert ( + AsyncModels.generate_content + == self._original_async_generate_content + ) + assert ( + AsyncModels.generate_content_stream + == self._original_async_generate_content_stream + ) + Models.generate_content = self._original_generate_content + Models.generate_content_stream = self._original_generate_content_stream + AsyncModels.generate_content = self._original_async_generate_content + AsyncModels.generate_content_stream = ( + self._original_async_generate_content_stream + ) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/nonstreaming_base.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/nonstreaming_base.py new file mode 100644 index 000000000..2533f4160 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/nonstreaming_base.py @@ -0,0 +1,652 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import unittest +from unittest.mock import AsyncMock, create_autospec, patch + +import pytest +from google.genai.types import ( + FunctionDeclarationDict, + GenerateContentConfig, + GoogleMaps, + ToolDict, +) +from pydantic import BaseModel, Field + +from opentelemetry import context as context_api +from opentelemetry.instrumentation.google_genai import ( + GENERATE_CONTENT_EXTRA_ATTRIBUTES_CONTEXT_KEY, +) +from opentelemetry.semconv._incubating.attributes import gen_ai_attributes + +from .base import TestCase + +_is_mcp_imported = False +McpClientSession = McpTool = McpListToolsResult = None +try: + from mcp import ClientSession as McpClientSession + from mcp import ListToolsResult as McpListToolsResult + from mcp import Tool as McpTool + + _is_mcp_imported = True +except ImportError: + pass +# pylint: disable=too-many-public-methods + +GEN_AI_TOOL_DEFINITIONS = getattr( + gen_ai_attributes, "GEN_AI_TOOL_DEFINITIONS", "gen_ai.tool.definitions" +) + + +def _mock_callable_tool(): + """Description of some tool.""" + return "result" + + +def _mock_mcp_client_session() -> McpClientSession: + mock_session = create_autospec(spec=McpClientSession, instance=True) + + mock_tool_obj = McpTool( + name="mcp_tool", + description="Tool from session", + inputSchema={ + "type": "object", + "properties": {"id": {"type": "integer"}}, + }, + ) + mock_result = create_autospec(McpListToolsResult, instance=True) + mock_result.tools = [mock_tool_obj] + + mock_session.list_tools = AsyncMock(return_value=mock_result) + + return mock_session + + +def _mock_mcp_tool(): + return McpTool( + name="mcp_tool", + description="A standalone mcp tool", + inputSchema={ + "type": "object", + "properties": {"id": {"type": "integer"}}, + }, + ) + + +def _mock_tool_dict() -> ToolDict: + return ToolDict( + function_declarations=[ + FunctionDeclarationDict( + name="mock_tool", + description="Description of mock tool.", + ), + ], + google_maps=GoogleMaps(), + ) + + +class ExampleResponseSchema(BaseModel): + name: str = Field(description="A Destination's Name") + + +class NonStreamingTestCase(TestCase): + # The "setUp" function is defined by "unittest.TestCase" and thus + # this name must be used. Uncertain why pylint doesn't seem to + # recognize that this is a unit test class for which this is inherited. + def setUp(self): # pylint: disable=invalid-name + super().setUp() + if self.__class__ == NonStreamingTestCase: + raise unittest.SkipTest("Skipping testcase base.") + self.tools = [ + _mock_callable_tool, + _mock_tool_dict(), + ] + if _is_mcp_imported: + self.tools.append(_mock_mcp_client_session()) + self.tools.append(_mock_mcp_tool()) + self.base_tools_definition = ( + { + "name": "_mock_callable_tool", + "description": "Description of some tool.", + "parameters": None, + "type": "function", + }, + { + "name": "mock_tool", + "description": "Description of mock tool.", + "parameters": None, + "type": "function", + }, + { + "name": "google_maps", + "type": "google_maps", + }, + ) + if _is_mcp_imported: + self.mcp_tools_no_content = ( + ( + { + "name": "mcp_tool", + "description": "A standalone mcp tool", + "parameters": None, + "type": "function", + }, + ), + ( + { + "name": "mcp_tool", + "description": "Tool from session", + "parameters": None, + "type": "function", + }, + { + "name": "mcp_tool", + "description": "A standalone mcp tool", + "parameters": None, + "type": "function", + }, + ), + ) + self.mcp_tools_with_content = ( + ( + { + "name": "mcp_tool", + "description": "A standalone mcp tool", + "parameters": { + "type": "object", + "properties": {"id": {"type": "integer"}}, + }, + "type": "function", + }, + ), + ( + { + "name": "mcp_tool", + "description": "Tool from session", + "parameters": { + "type": "object", + "properties": {"id": {"type": "integer"}}, + }, + "type": "function", + }, + { + "name": "mcp_tool", + "description": "A standalone mcp tool", + "parameters": { + "type": "object", + "properties": {"id": {"type": "integer"}}, + }, + "type": "function", + }, + ), + ) + + def generate_content(self, *args, **kwargs): + raise NotImplementedError("Must implement 'generate_content'.") + + @property + def expected_function_name(self): + raise NotImplementedError("Must implement 'expected_function_name'.") + + def _generate_and_get_span(self, config): + self.generate_content( + model="gemini-2.0-flash", + contents="Some input prompt", + config=config, + ) + self.otel.assert_has_span_named("generate_content gemini-2.0-flash") + return self.otel.get_span_named("generate_content gemini-2.0-flash") + + def test_instrumentation_does_not_break_core_functionality(self): + self.configure_valid_response(text="Yep, it works!") + response = self.generate_content( + model="gemini-2.0-flash", contents="Does this work?" + ) + self.assertEqual(response.text, "Yep, it works!") + + def test_generates_span(self): + self.configure_valid_response(text="Yep, it works!") + response = self.generate_content( + model="gemini-2.0-flash", contents="Does this work?" + ) + self.assertEqual(response.text, "Yep, it works!") + self.otel.assert_has_span_named("generate_content gemini-2.0-flash") + + def test_model_reflected_into_span_name(self): + self.configure_valid_response(text="Yep, it works!") + response = self.generate_content( + model="gemini-1.5-flash", contents="Does this work?" + ) + self.assertEqual(response.text, "Yep, it works!") + self.otel.assert_has_span_named("generate_content gemini-1.5-flash") + + def test_generated_span_has_minimal_genai_attributes(self): + self.configure_valid_response(text="Yep, it works!") + self.generate_content( + model="gemini-2.0-flash", contents="Does this work?" + ) + self.otel.assert_has_span_named("generate_content gemini-2.0-flash") + span = self.otel.get_span_named("generate_content gemini-2.0-flash") + self.assertEqual(span.attributes["gen_ai.provider.name"], "gemini") + self.assertEqual( + span.attributes["gen_ai.operation.name"], "generate_content" + ) + self.assertEqual( + span.attributes["server.address"], + "generativelanguage.googleapis.com", + ) + + def test_generated_span_has_extra_genai_attributes(self): + self.configure_valid_response(text="Yep, it works!") + tok = context_api.attach( + context_api.set_value( + GENERATE_CONTENT_EXTRA_ATTRIBUTES_CONTEXT_KEY, + {"extra_attribute_key": "extra_attribute_value"}, + ) + ) + try: + self.generate_content( + model="gemini-2.0-flash", contents="Does this work?" + ) + self.otel.assert_has_span_named( + "generate_content gemini-2.0-flash" + ) + span = self.otel.get_span_named( + "generate_content gemini-2.0-flash" + ) + self.assertEqual( + span.attributes["extra_attribute_key"], "extra_attribute_value" + ) + finally: + context_api.detach(tok) + + def test_span_and_event_still_written_when_response_is_exception(self): + self.configure_exception(ValueError("Uh oh!")) + with pytest.raises(ValueError): + self.generate_content( + model="gemini-2.0-flash", contents="Does this work?" + ) + self.otel.assert_has_span_named("generate_content gemini-2.0-flash") + span = self.otel.get_span_named("generate_content gemini-2.0-flash") + self.otel.assert_has_event_named( + "gen_ai.client.inference.operation.details" + ) + event = self.otel.get_event_named( + "gen_ai.client.inference.operation.details" + ) + assert ( + span.attributes["error.type"] + == event.attributes["error.type"] + == "ValueError" + ) + + def test_generated_span_has_vertex_ai_system_when_configured(self): + self.set_use_vertex(True) + self.configure_valid_response(text="Yep, it works!") + self.generate_content( + model="gemini-2.0-flash", contents="Does this work?" + ) + self.otel.assert_has_span_named("generate_content gemini-2.0-flash") + span = self.otel.get_span_named("generate_content gemini-2.0-flash") + self.assertEqual(span.attributes["gen_ai.provider.name"], "vertex_ai") + self.assertEqual( + span.attributes["gen_ai.operation.name"], "generate_content" + ) + + def test_generated_span_counts_tokens(self): + self.configure_valid_response( + input_tokens=123, + output_tokens=456, + cached_tokens=50, + thinking_tokens=17, + ) + self.generate_content(model="gemini-2.0-flash", contents="Some input") + self.otel.assert_has_span_named("generate_content gemini-2.0-flash") + span = self.otel.get_span_named("generate_content gemini-2.0-flash") + self.assertEqual(span.attributes["gen_ai.usage.input_tokens"], 123) + self.assertEqual( + span.attributes["gen_ai.usage.output_tokens"], 456 + 17 + ) + self.assertEqual( + span.attributes["gen_ai.usage.cache_read.input_tokens"], 50 + ) + self.assertEqual( + span.attributes["gen_ai.usage.reasoning.output_tokens"], 17 + ) + + def test_generated_span_records_response_model(self): + self.configure_valid_response(model_version="gemini-2.0-flash-001") + self.generate_content(model="gemini-2.0-flash", contents="Some input") + span = self.otel.get_span_named("generate_content gemini-2.0-flash") + self.assertEqual( + span.attributes["gen_ai.response.model"], "gemini-2.0-flash-001" + ) + self.otel.assert_has_event_named( + "gen_ai.client.inference.operation.details" + ) + event = self.otel.get_event_named( + "gen_ai.client.inference.operation.details" + ) + self.assertEqual( + event.attributes["gen_ai.response.model"], "gemini-2.0-flash-001" + ) + + @patch.dict( + "os.environ", + { + "OTEL_GOOGLE_GENAI_GENERATE_CONTENT_CONFIG_INCLUDES": "gcp.gen_ai.operation.config.response_schema", + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "NO_CONTENT", + }, + ) + def test_log_event_no_content_capture(self): + self.configure_valid_response( + text="Some response content", + cached_tokens=50, + thinking_tokens=17, + ) + self.generate_content( + model="gemini-2.0-flash", + contents="Some input", + config=GenerateContentConfig( + system_instruction="System instruction", + response_schema=ExampleResponseSchema, + tools=self.tools, + ), + ) + self.otel.assert_has_event_named( + "gen_ai.client.inference.operation.details" + ) + event = self.otel.get_event_named( + "gen_ai.client.inference.operation.details" + ) + self.assertEqual( + event.attributes["gen_ai.usage.cache_read.input_tokens"], + 50, + ) + self.assertNotIn( + gen_ai_attributes.GEN_AI_RESPONSE_ID, event.attributes + ) + self.assertEqual( + event.attributes["gen_ai.usage.reasoning.output_tokens"], + 17, + ) + self.assertEqual( + event.attributes["gen_ai.usage.output_tokens"], + 17, + ) + assert ( + event.attributes["gcp.gen_ai.operation.config.response_schema"] + == "" + ) + + self.assertNotIn( + gen_ai_attributes.GEN_AI_INPUT_MESSAGES, + event.attributes, + ) + self.assertNotIn( + gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES, + event.attributes, + ) + self.assertNotIn( + gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS, + event.attributes, + ) + if _is_mcp_imported: + self.assertIn( + event.attributes[GEN_AI_TOOL_DEFINITIONS], + [ + self.base_tools_definition + mcp_var + for mcp_var in self.mcp_tools_no_content + ], + ) + else: + self.assertEqual( + event.attributes[GEN_AI_TOOL_DEFINITIONS], + self.base_tools_definition, + ) + + @patch.dict( + "os.environ", + { + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "EVENT_ONLY", + }, + ) + def test_log_event_with_content_capture(self): + self.configure_valid_response( + text="Some response content", + cached_tokens=50, + thinking_tokens=17, + response_id="qwerty17", + ) + self.generate_content( + model="gemini-2.0-flash", + contents="Some input", + config=GenerateContentConfig( + system_instruction="System instruction", + response_schema=ExampleResponseSchema, + tools=self.tools, + ), + ) + self.otel.assert_has_event_named( + "gen_ai.client.inference.operation.details" + ) + event = self.otel.get_event_named( + "gen_ai.client.inference.operation.details" + ) + self.assertEqual( + event.attributes[gen_ai_attributes.GEN_AI_RESPONSE_ID], "qwerty17" + ) + self.assertEqual( + event.attributes["gen_ai.usage.cache_read.input_tokens"], + 50, + ) + self.assertEqual( + event.attributes["gen_ai.usage.reasoning.output_tokens"], + 17, + ) + self.assertEqual( + event.attributes["gen_ai.usage.output_tokens"], + 17, + ) + self.assertEqual( + event.attributes[gen_ai_attributes.GEN_AI_INPUT_MESSAGES], + ( + { + "role": "user", + "parts": ({"content": "Some input", "type": "text"},), + }, + ), + ) + self.assertEqual( + event.attributes[gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES], + ( + { + "role": "assistant", + "parts": ( + {"content": "Some response content", "type": "text"}, + ), + "finish_reason": "", + }, + ), + ) + self.assertEqual( + event.attributes[gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS], + ({"content": "System instruction", "type": "text"},), + ) + if _is_mcp_imported: + self.assertIn( + event.attributes[GEN_AI_TOOL_DEFINITIONS], + [ + self.base_tools_definition + mcp_var + for mcp_var in self.mcp_tools_with_content + ], + ) + else: + self.assertEqual( + event.attributes[GEN_AI_TOOL_DEFINITIONS], + self.base_tools_definition, + ) + + @patch.dict( + "os.environ", + {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "NO_CONTENT"}, + ) + def test_span_attributes_no_content_capture(self): + self.configure_valid_response( + text="Some response content", + cached_tokens=50, + thinking_tokens=19, + response_id="qwerty17", + ) + self.generate_content( + model="gemini-2.0-flash", + contents="Some input", + config=GenerateContentConfig( + system_instruction="System instruction", + response_schema=ExampleResponseSchema, + tools=self.tools, + ), + ) + span = self.otel.get_span_named("generate_content gemini-2.0-flash") + self.assertEqual(span.attributes["gen_ai.provider.name"], "gemini") + self.assertEqual( + span.attributes["gen_ai.usage.cache_read.input_tokens"], + 50, + ) + self.assertEqual( + span.attributes[gen_ai_attributes.GEN_AI_RESPONSE_ID], "qwerty17" + ) + self.assertEqual( + span.attributes["gen_ai.usage.reasoning.output_tokens"], + 19, + ) + self.assertEqual( + span.attributes["gen_ai.usage.output_tokens"], + 19, + ) + for attribute in ( + gen_ai_attributes.GEN_AI_INPUT_MESSAGES, + gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES, + gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS, + ): + self.assertNotIn(attribute, span.attributes) + if _is_mcp_imported: + self.assertIn( + span.attributes[GEN_AI_TOOL_DEFINITIONS], + [ + '[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"},{"name":"mcp_tool","description":"Tool from session","parameters":null,"type":"function"},{"name":"mcp_tool","description":"A standalone mcp tool","parameters":null,"type":"function"}]', + '[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"},{"name":"mcp_tool","description":"A standalone mcp tool","parameters":null,"type":"function"}]', + ], + ) + else: + self.assertEqual( + span.attributes[GEN_AI_TOOL_DEFINITIONS], + '[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"}]', + ) + + @patch.dict( + "os.environ", + {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "SPAN_ONLY"}, + ) + def test_span_attributes_with_content_capture(self): + self.configure_valid_response( + text="Some response content", + cached_tokens=50, + thinking_tokens=19, + ) + self.generate_content( + model="gemini-2.0-flash", + contents="Some input", + config=GenerateContentConfig( + system_instruction="System instruction", + response_schema=ExampleResponseSchema, + tools=self.tools, + ), + ) + span = self.otel.get_span_named("generate_content gemini-2.0-flash") + self.assertEqual(span.attributes["gen_ai.provider.name"], "gemini") + self.assertEqual( + span.attributes["gen_ai.usage.cache_read.input_tokens"], + 50, + ) + self.assertEqual( + span.attributes["gen_ai.usage.reasoning.output_tokens"], + 19, + ) + self.assertEqual( + span.attributes["gen_ai.usage.output_tokens"], + 19, + ) + self.assertEqual( + span.attributes[gen_ai_attributes.GEN_AI_INPUT_MESSAGES], + '[{"role":"user","parts":[{"content":"Some input","type":"text"}]}]', + ) + self.assertEqual( + span.attributes[gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES], + '[{"role":"assistant","parts":[{"content":"Some response content","type":"text"}],"finish_reason":""}]', + ) + self.assertEqual( + span.attributes[gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS], + '[{"content":"System instruction","type":"text"}]', + ) + if _is_mcp_imported: + self.assertIn( + span.attributes[GEN_AI_TOOL_DEFINITIONS], + [ + '[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"},{"name":"mcp_tool","description":"Tool from session","parameters":{"type":"object","properties":{"id":{"type":"integer"}}},"type":"function"},{"name":"mcp_tool","description":"A standalone mcp tool","parameters":{"type":"object","properties":{"id":{"type":"integer"}}},"type":"function"}]', + '[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"},{"name":"mcp_tool","description":"A standalone mcp tool","parameters":{"type":"object","properties":{"id":{"type":"integer"}}},"type":"function"}]', + ], + ) + else: + self.assertEqual( + span.attributes[GEN_AI_TOOL_DEFINITIONS], + '[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"}]', + ) + + def test_log_has_extra_genai_attributes(self): + self.configure_valid_response(text="Yep, it works!") + tok = context_api.attach( + context_api.set_value( + GENERATE_CONTENT_EXTRA_ATTRIBUTES_CONTEXT_KEY, + {"extra_attribute_key": "extra_attribute_value"}, + ) + ) + try: + self.generate_content( + model="gemini-2.0-flash", + contents="Does this work?", + ) + self.otel.assert_has_event_named( + "gen_ai.client.inference.operation.details" + ) + event = self.otel.get_event_named( + "gen_ai.client.inference.operation.details" + ) + assert ( + event.attributes["extra_attribute_key"] + == "extra_attribute_value" + ) + finally: + context_api.detach(tok) + + def test_records_metrics_data(self): + self.configure_valid_response() + self.generate_content(model="gemini-2.0-flash", contents="Some input") + self.otel.assert_has_metrics_data_named("gen_ai.client.token.usage") + self.otel.assert_has_metrics_data_named( + "gen_ai.client.operation.duration" + ) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/streaming_base.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/streaming_base.py new file mode 100644 index 000000000..da5d03f02 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/streaming_base.py @@ -0,0 +1,195 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import json +import unittest + +from google.genai import types + +from opentelemetry import context as context_api +from opentelemetry.instrumentation.google_genai import ( + GENERATE_CONTENT_EXTRA_ATTRIBUTES_CONTEXT_KEY, +) + +from .base import TestCase + + +class StreamingTestCase(TestCase): + # The "setUp" function is defined by "unittest.TestCase" and thus + # this name must be used. Uncertain why pylint doesn't seem to + # recognize that this is a unit test class for which this is inherited. + def setUp(self): # pylint: disable=invalid-name + super().setUp() + if self.__class__ == StreamingTestCase: + raise unittest.SkipTest("Skipping testcase base.") + + def generate_content(self, *args, **kwargs): + raise NotImplementedError("Must implement 'generate_content'.") + + @property + def expected_function_name(self): + raise NotImplementedError("Must implement 'expected_function_name'.") + + def test_instrumentation_does_not_break_core_functionality(self): + self.configure_valid_response(text="Yep, it works!") + responses = self.generate_content( + model="gemini-2.0-flash", contents="Does this work?" + ) + self.assertEqual(len(responses), 1) + response = responses[0] + self.assertEqual(response.text, "Yep, it works!") + + def test_generated_span_has_extra_genai_attributes(self): + self.configure_valid_response(text="Yep, it works!") + tok = context_api.attach( + context_api.set_value( + GENERATE_CONTENT_EXTRA_ATTRIBUTES_CONTEXT_KEY, + {"extra_attribute_key": "extra_attribute_value"}, + ) + ) + try: + self.generate_content( + model="gemini-2.0-flash", contents="Does this work?" + ) + self.otel.assert_has_span_named( + "generate_content gemini-2.0-flash" + ) + span = self.otel.get_span_named( + "generate_content gemini-2.0-flash" + ) + self.assertEqual( + span.attributes["extra_attribute_key"], "extra_attribute_value" + ) + finally: + context_api.detach(tok) + + def test_handles_multiple_responses(self): + self.configure_valid_response(text="First response") + self.configure_valid_response(text="Second response") + responses = self.generate_content( + model="gemini-2.0-flash", contents="Does this work?" + ) + self.assertEqual(len(responses), 2) + self.assertEqual(responses[0].text, "First response") + self.assertEqual(responses[1].text, "Second response") + self.otel.assert_has_event_named( + "gen_ai.client.inference.operation.details" + ) + + def test_stream_records_time_to_first_token(self): + self.configure_valid_response(text="First response") + self.generate_content( + model="gemini-2.0-flash", contents="Does this work?" + ) + span = self.otel.get_span_named("generate_content gemini-2.0-flash") + self.assertGreater( + span.attributes["gen_ai.response.time_to_first_token"], 0 + ) + + def test_stream_aggregates_chunks_into_one_output_message(self): + self.configure_valid_response( + candidate=types.Candidate( + index=0, + content=types.Content( + role="model", + parts=[types.Part(text="stream")], + ), + ) + ) + self.configure_valid_response( + candidate=types.Candidate( + index=0, + content=types.Content( + role="model", + parts=[types.Part(text=" output")], + ), + ) + ) + self.configure_valid_response( + candidate=types.Candidate( + index=0, + finish_reason=types.FinishReason.STOP, + content=types.Content( + role="model", + parts=[types.Part(text="")], + ), + ) + ) + + self.generate_content( + model="gemini-2.0-flash", + contents="Some input", + ) + + span = self.otel.get_span_named("generate_content gemini-2.0-flash") + messages = json.loads(span.attributes["gen_ai.output.messages"]) + self.assertEqual( + messages, + [ + { + "role": "assistant", + "parts": [{"content": "stream output", "type": "text"}], + "finish_reason": "stop", + } + ], + ) + + def test_includes_token_counts_in_span_not_aggregated_from_responses(self): + # Tokens should not be aggregated in streaming. Cumulative counts are returned on each response. + self.configure_valid_response( + input_tokens=3, output_tokens=5, response_id="qwerty17" + ) + self.configure_valid_response( + input_tokens=3, output_tokens=5, response_id="qwerty18" + ) + self.configure_valid_response( + input_tokens=3, output_tokens=5, response_id="qwerty19" + ) + + self.generate_content(model="gemini-2.0-flash", contents="Some input") + + self.otel.assert_has_span_named("generate_content gemini-2.0-flash") + span = self.otel.get_span_named("generate_content gemini-2.0-flash") + self.assertEqual(span.attributes["gen_ai.response.id"], "qwerty19") + self.assertEqual(span.attributes["gen_ai.usage.input_tokens"], 3) + self.assertEqual(span.attributes["gen_ai.usage.output_tokens"], 5) + + def test_log_has_extra_genai_attributes(self): + self.configure_valid_response(text="Yep, it works!") + tok = context_api.attach( + context_api.set_value( + GENERATE_CONTENT_EXTRA_ATTRIBUTES_CONTEXT_KEY, + {"extra_attribute_key": "extra_attribute_value"}, + ) + ) + try: + self.generate_content( + model="gemini-2.0-flash", + contents="Does this work?", + ) + self.otel.assert_has_event_named( + "gen_ai.client.inference.operation.details" + ) + event = self.otel.get_event_named( + "gen_ai.client.inference.operation.details" + ) + assert ( + event.attributes["extra_attribute_key"] + == "extra_attribute_value" + ) + finally: + context_api.detach(tok) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_async_nonstreaming.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_async_nonstreaming.py new file mode 100644 index 000000000..8873825c2 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_async_nonstreaming.py @@ -0,0 +1,31 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import asyncio + +from .nonstreaming_base import NonStreamingTestCase + + +class TestGenerateContentAsyncNonstreaming(NonStreamingTestCase): + def generate_content(self, *args, **kwargs): + return asyncio.run( + self.client.aio.models.generate_content(*args, **kwargs) # pylint: disable=missing-kwoa + ) + + @property + def expected_function_name(self): + return "google.genai.AsyncModels.generate_content" diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_async_streaming.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_async_streaming.py new file mode 100644 index 000000000..31e8b7e74 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_async_streaming.py @@ -0,0 +1,58 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import asyncio + +from .nonstreaming_base import NonStreamingTestCase +from .streaming_base import StreamingTestCase + + +class AsyncStreamingMixin: + @property + def expected_function_name(self): + return "google.genai.AsyncModels.generate_content_stream" + + async def _generate_content_stream_helper(self, *args, **kwargs): + result = [] + async for ( + response + ) in await self.client.aio.models.generate_content_stream( # pylint: disable=missing-kwoa + *args, **kwargs + ): + result.append(response) + return result + + def generate_content_stream(self, *args, **kwargs): + return asyncio.run( + self._generate_content_stream_helper(*args, **kwargs) + ) + + +class TestGenerateContentAsyncStreamingWithSingleResult( + AsyncStreamingMixin, NonStreamingTestCase +): + def generate_content(self, *args, **kwargs): + responses = self.generate_content_stream(*args, **kwargs) + self.assertEqual(len(responses), 1) + return responses[0] + + +class TestGenerateContentAsyncStreamingWithStreamedResults( + AsyncStreamingMixin, StreamingTestCase +): + def generate_content(self, *args, **kwargs): + return self.generate_content_stream(*args, **kwargs) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_config_span_attributes.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_config_span_attributes.py new file mode 100644 index 000000000..e9590f990 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_config_span_attributes.py @@ -0,0 +1,169 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import os +from unittest import mock + +from google.genai.types import GenerateContentConfig + +from opentelemetry.instrumentation.google_genai.allowlist_util import AllowList + +from .base import TestCase + + +class ConfigSpanAttributesTestCase(TestCase): + def setUp(self): + super().setUp() + self.configure_valid_response(text="Some response") + + def generate_content(self, *args, **kwargs): + return self.client.models.generate_content(*args, **kwargs) + + def generate_and_get_span(self, config): + self.client.models.generate_content( + model="gemini-2.0-flash", + contents="Some input prompt", + config=config, + ) + self.otel.assert_has_span_named("generate_content gemini-2.0-flash") + return self.otel.get_span_named("generate_content gemini-2.0-flash") + + def test_option_reflected_to_span_attribute_choice_count_config_dict(self): + span = self.generate_and_get_span(config={"candidate_count": 2}) + self.assertEqual(span.attributes["gen_ai.request.choice.count"], 2) + + def test_option_reflected_to_span_attribute_choice_count_config_obj(self): + span = self.generate_and_get_span( + config=GenerateContentConfig(candidate_count=2) + ) + self.assertEqual(span.attributes["gen_ai.request.choice.count"], 2) + + def test_option_reflected_to_span_attribute_seed_config_dict(self): + span = self.generate_and_get_span(config={"seed": 12345}) + self.assertEqual(span.attributes["gen_ai.request.seed"], 12345) + + def test_option_reflected_to_span_attribute_seed_config_obj(self): + span = self.generate_and_get_span( + config=GenerateContentConfig(seed=12345) + ) + self.assertEqual(span.attributes["gen_ai.request.seed"], 12345) + + def test_option_reflected_to_span_attribute_frequency_penalty(self): + span = self.generate_and_get_span(config={"frequency_penalty": 1.0}) + self.assertEqual( + span.attributes["gen_ai.request.frequency_penalty"], 1.0 + ) + + def test_option_reflected_to_span_attribute_max_tokens(self): + span = self.generate_and_get_span( + config=GenerateContentConfig(max_output_tokens=5000) + ) + self.assertEqual(span.attributes["gen_ai.request.max_tokens"], 5000) + + def test_option_reflected_to_span_attribute_presence_penalty(self): + span = self.generate_and_get_span( + config=GenerateContentConfig(presence_penalty=0.5) + ) + self.assertEqual( + span.attributes["gen_ai.request.presence_penalty"], 0.5 + ) + + def test_option_reflected_to_span_attribute_stop_sequences(self): + span = self.generate_and_get_span( + config={"stop_sequences": ["foo", "bar"]} + ) + stop_sequences = span.attributes["gen_ai.request.stop_sequences"] + self.assertEqual(len(stop_sequences), 2) + self.assertEqual(stop_sequences[0], "foo") + self.assertEqual(stop_sequences[1], "bar") + + def test_option_reflected_to_span_attribute_top_k(self): + span = self.generate_and_get_span( + config=GenerateContentConfig(top_k=20) + ) + self.assertEqual(span.attributes["gen_ai.request.top_k"], 20) + + def test_option_reflected_to_span_attribute_top_p(self): + span = self.generate_and_get_span(config={"top_p": 10}) + self.assertEqual(span.attributes["gen_ai.request.top_p"], 10) + + @mock.patch.dict( + os.environ, + { + "OTEL_GOOGLE_GENAI_GENERATE_CONTENT_CONFIG_INCLUDES": "*", + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "NO_CONTENT", + }, + ) + def test_option_not_reflected_to_span_attribute_system_instruction(self): + span = self.generate_and_get_span( + config={"system_instruction": "Yadda yadda yadda"} + ) + self.assertNotIn( + "gcp.gen_ai.operation.config.system_instruction", span.attributes + ) + self.assertNotIn("gen_ai.request.system_instruction", span.attributes) + for key in span.attributes: + value = span.attributes[key] + if isinstance(value, str): + self.assertNotIn("Yadda yadda yadda", value) + + @mock.patch.dict( + os.environ, {"OTEL_GOOGLE_GENAI_GENERATE_CONTENT_CONFIG_INCLUDES": "*"} + ) + def test_option_reflected_to_span_attribute_automatic_func_calling(self): + span = self.generate_and_get_span( + config={ + "automatic_function_calling": { + "ignore_call_history": True, + } + } + ) + self.assertTrue( + span.attributes[ + "gcp.gen_ai.operation.config.automatic_function_calling.ignore_call_history" + ] + ) + + def test_dynamic_config_options_not_included_without_allow_list(self): + span = self.generate_and_get_span( + config={ + "automatic_function_calling": { + "ignore_call_history": True, + } + } + ) + self.assertNotIn( + "gcp.gen_ai.operation.config.automatic_function_calling.ignore_call_history", + span.attributes, + ) + + def test_can_supply_allow_list_via_instrumentor_constructor(self): + self.set_instrumentor_constructor_kwarg( + "generate_content_config_key_allowlist", AllowList(includes=["*"]) + ) + span = self.generate_and_get_span( + config={ + "automatic_function_calling": { + "ignore_call_history": True, + } + } + ) + self.assertTrue( + span.attributes[ + "gcp.gen_ai.operation.config.automatic_function_calling.ignore_call_history" + ] + ) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_finish_reasons.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_finish_reasons.py new file mode 100644 index 000000000..1508bddbb --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_finish_reasons.py @@ -0,0 +1,157 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + + +from google.genai import types as genai_types + +from .base import TestCase + + +class FinishReasonsTestCase(TestCase): + def generate_and_get_span_finish_reasons(self): + self.client.models.generate_content( + model="gemini-2.5-flash-001", contents="Some prompt" + ) + span = self.otel.get_span_named( + "generate_content gemini-2.5-flash-001" + ) + assert span is not None + if "gen_ai.response.finish_reasons" not in span.attributes: + return [] + return list(span.attributes["gen_ai.response.finish_reasons"]) + + def test_single_candidate_with_valid_reason(self): + self.configure_valid_response( + candidate=genai_types.Candidate( + finish_reason=genai_types.FinishReason.STOP + ) + ) + self.assertEqual(self.generate_and_get_span_finish_reasons(), ["stop"]) + + def test_single_candidate_with_safety_reason(self): + self.configure_valid_response( + candidate=genai_types.Candidate( + finish_reason=genai_types.FinishReason.SAFETY + ) + ) + self.assertEqual( + self.generate_and_get_span_finish_reasons(), ["safety"] + ) + + def test_single_candidate_with_max_tokens_reason(self): + self.configure_valid_response( + candidate=genai_types.Candidate( + finish_reason=genai_types.FinishReason.MAX_TOKENS + ) + ) + self.assertEqual( + self.generate_and_get_span_finish_reasons(), ["max_tokens"] + ) + + def test_single_candidate_with_no_reason(self): + self.configure_valid_response( + candidate=genai_types.Candidate(finish_reason=None) + ) + self.assertEqual(self.generate_and_get_span_finish_reasons(), []) + + def test_single_candidate_with_unspecified_reason(self): + self.configure_valid_response( + candidate=genai_types.Candidate( + finish_reason=genai_types.FinishReason.FINISH_REASON_UNSPECIFIED + ) + ) + self.assertEqual( + self.generate_and_get_span_finish_reasons(), + ["finish_reason_unspecified"], + ) + + def test_multiple_candidates_with_valid_reasons(self): + self.configure_valid_response( + candidates=[ + genai_types.Candidate( + finish_reason=genai_types.FinishReason.MAX_TOKENS + ), + genai_types.Candidate( + finish_reason=genai_types.FinishReason.STOP + ), + ] + ) + self.assertEqual( + self.generate_and_get_span_finish_reasons(), ["max_tokens", "stop"] + ) + + def test_doesnt_sort_finish_reasons(self): + self.configure_valid_response( + candidates=[ + genai_types.Candidate( + finish_reason=genai_types.FinishReason.STOP + ), + genai_types.Candidate( + finish_reason=genai_types.FinishReason.MAX_TOKENS + ), + genai_types.Candidate( + finish_reason=genai_types.FinishReason.SAFETY + ), + ] + ) + self.assertEqual( + self.generate_and_get_span_finish_reasons(), + ["stop", "max_tokens", "safety"], + ) + + def test_doesnt_deduplicate_finish_reasons(self): + self.configure_valid_response( + candidates=[ + genai_types.Candidate( + finish_reason=genai_types.FinishReason.STOP + ), + genai_types.Candidate( + finish_reason=genai_types.FinishReason.MAX_TOKENS + ), + genai_types.Candidate( + finish_reason=genai_types.FinishReason.STOP + ), + genai_types.Candidate( + finish_reason=genai_types.FinishReason.STOP + ), + genai_types.Candidate( + finish_reason=genai_types.FinishReason.SAFETY + ), + genai_types.Candidate( + finish_reason=genai_types.FinishReason.STOP + ), + genai_types.Candidate( + finish_reason=genai_types.FinishReason.STOP + ), + genai_types.Candidate( + finish_reason=genai_types.FinishReason.STOP + ), + ] + ) + self.assertEqual( + self.generate_and_get_span_finish_reasons(), + [ + "stop", + "max_tokens", + "stop", + "stop", + "safety", + "stop", + "stop", + "stop", + ], + ) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_stream_accumulator.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_stream_accumulator.py new file mode 100644 index 000000000..1cd0dbe33 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_stream_accumulator.py @@ -0,0 +1,137 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +from google.genai import types + +from opentelemetry.instrumentation.google_genai.message import ( + StreamOutputMessageAccumulator, +) +from opentelemetry.util.genai.types import Reasoning, Text, ToolCall + + +def _candidate( + index: int, + *parts: types.Part, + finish_reason: Optional[types.FinishReason] = None, +) -> types.Candidate: + return types.Candidate( + index=index, + finish_reason=finish_reason, + content=types.Content(role="model", parts=list(parts)), + ) + + +def test_groups_text_chunks_by_candidate_index() -> None: + accumulator = StreamOutputMessageAccumulator() + accumulator.add_candidates( + [ + _candidate(0, types.Part(text="first")), + _candidate(1, types.Part(text="second")), + ] + ) + accumulator.add_candidates( + [ + _candidate(0, types.Part(text=" answer")), + _candidate(1, types.Part(text=" choice")), + ] + ) + accumulator.add_candidates( + [ + _candidate( + 0, + types.Part(text=""), + finish_reason=types.FinishReason.STOP, + ), + _candidate( + 1, + types.Part(text=""), + finish_reason=types.FinishReason.MAX_TOKENS, + ), + ] + ) + + messages = accumulator.to_output_messages() + + assert len(messages) == 2 + assert messages[0].parts == [Text(content="first answer")] + assert messages[0].finish_reason == "stop" + assert messages[1].parts == [Text(content="second choice")] + assert messages[1].finish_reason == "length" + + +def test_preserves_reasoning_and_text_as_separate_parts() -> None: + accumulator = StreamOutputMessageAccumulator() + accumulator.add_candidates( + [_candidate(0, types.Part(text="think ", thought=True))] + ) + accumulator.add_candidates( + [_candidate(0, types.Part(text="carefully", thought=True))] + ) + accumulator.add_candidates([_candidate(0, types.Part(text="answer"))]) + accumulator.add_candidates( + [ + _candidate( + 0, + types.Part(text=""), + finish_reason=types.FinishReason.STOP, + ) + ] + ) + + message = accumulator.to_output_messages()[0] + + assert message.parts == [ + Reasoning(content="think carefully"), + Text(content="answer"), + ] + assert message.finish_reason == "stop" + + +def test_merges_function_call_fragments() -> None: + accumulator = StreamOutputMessageAccumulator() + accumulator.add_candidates( + [ + _candidate( + 0, + types.Part.from_function_call( + name="get_weather", + args={"city": "Hangzhou"}, + ), + ) + ] + ) + accumulator.add_candidates( + [ + _candidate( + 0, + types.Part.from_function_call( + name="get_weather", + args={"unit": "C"}, + ), + finish_reason=types.FinishReason.STOP, + ) + ] + ) + + message = accumulator.to_output_messages()[0] + + assert len(message.parts) == 1 + assert isinstance(message.parts[0], ToolCall) + assert message.parts[0].name == "get_weather" + assert message.parts[0].arguments == { + "city": "Hangzhou", + "unit": "C", + } diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_sync_nonstreaming.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_sync_nonstreaming.py new file mode 100644 index 000000000..3dc644ed2 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_sync_nonstreaming.py @@ -0,0 +1,28 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + + +from .nonstreaming_base import NonStreamingTestCase + + +class TestGenerateContentSyncNonstreaming(NonStreamingTestCase): + def generate_content(self, *args, **kwargs): + return self.client.models.generate_content(*args, **kwargs) + + @property + def expected_function_name(self): + return "google.genai.Models.generate_content" diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_sync_streaming.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_sync_streaming.py new file mode 100644 index 000000000..660aaa857 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/test_sync_streaming.py @@ -0,0 +1,50 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + + +from .nonstreaming_base import NonStreamingTestCase +from .streaming_base import StreamingTestCase + + +class StreamingMixin: + @property + def expected_function_name(self): + return "google.genai.Models.generate_content_stream" + + def generate_content_stream(self, *args, **kwargs): + result = [] + for response in self.client.models.generate_content_stream( # pylint: disable=missing-kwoa + *args, **kwargs + ): + result.append(response) + return result + + +class TestGenerateContentStreamingWithSingleResult( + StreamingMixin, NonStreamingTestCase +): + def generate_content(self, *args, **kwargs): + responses = self.generate_content_stream(*args, **kwargs) + self.assertEqual(len(responses), 1) + return responses[0] + + +class TestGenerateContentStreamingWithStreamedResults( + StreamingMixin, StreamingTestCase +): + def generate_content(self, *args, **kwargs): + return self.generate_content_stream(*args, **kwargs) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/util.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/util.py new file mode 100644 index 000000000..a5c552e49 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/generate_content/util.py @@ -0,0 +1,84 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from typing import List, Optional, Union + +import google.genai.types as genai_types + + +def create_response( + part: Optional[genai_types.Part] = None, + parts: Optional[List[genai_types.Part]] = None, + content: Optional[genai_types.Content] = None, + candidate: Optional[genai_types.Candidate] = None, + candidates: Optional[List[genai_types.Candidate]] = None, + text: Optional[str] = None, + input_tokens: Optional[int] = 0, + thinking_tokens: Optional[int] = None, + output_tokens: Optional[int] = 0, + cached_tokens: Optional[int] = None, + model_version: Optional[str] = None, + usage_metadata: Optional[ + genai_types.GenerateContentResponseUsageMetadata + ] = None, + **kwargs, +) -> genai_types.GenerateContentResponse: + # Build up the "candidates" subfield + if text is None: + text = "Some response text" + if part is None: + part = genai_types.Part(text=text) + if parts is None: + parts = [part] + if content is None: + content = genai_types.Content(parts=parts, role="model") + if candidate is None: + candidate = genai_types.Candidate(content=content) + if candidates is None: + candidates = [candidate] + + # Build up the "usage_metadata" subfield + if usage_metadata is None: + usage_metadata = genai_types.GenerateContentResponseUsageMetadata() + if input_tokens is not None: + usage_metadata.prompt_token_count = input_tokens + if output_tokens is not None: + usage_metadata.candidates_token_count = output_tokens + if cached_tokens is not None: + usage_metadata.cached_content_token_count = cached_tokens + if thinking_tokens is not None: + usage_metadata.thoughts_token_count = thinking_tokens + return genai_types.GenerateContentResponse( + candidates=candidates, + usage_metadata=usage_metadata, + model_version=model_version, + **kwargs, + ) + + +def convert_to_response( + arg: Union[str, genai_types.GenerateContentResponse, dict], +) -> genai_types.GenerateContentResponse: + if isinstance(arg, str): + return create_response(text=arg) + if isinstance(arg, genai_types.GenerateContentResponse): + return arg + if isinstance(arg, dict): + return create_response(**arg) + raise ValueError( + f"Unsure how to convert {arg} of type {arg.__class__.__name__} to response." + ) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/__init__.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/__init__.py new file mode 100644 index 000000000..cceb2a6af --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/__init__.py @@ -0,0 +1,16 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/base.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/base.py new file mode 100644 index 000000000..a788d91b4 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/base.py @@ -0,0 +1,315 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import unittest +import unittest.mock +from typing import Any, List, Optional +from unittest.mock import patch + +try: + try: + from google.genai._interactions.resources.interactions import ( + AsyncInteractionsResource, + InteractionsResource, + ) + except ImportError: + # In version 2.9 of google-genai these were moved. + from google.genai._gaos.interactions import ( + AsyncInteractions as AsyncInteractionsResource, + ) + from google.genai._gaos.interactions import ( + Interactions as InteractionsResource, + ) + _HAS_INTERACTIONS = True +except ImportError: + _HAS_INTERACTIONS = False + + # Placeholders to prevent compilation errors + class AsyncInteractionsResource: + create = None + + class InteractionsResource: + create = None + + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) + +from ..common.base import TestCase as CommonTestCaseBase +from .util import create_mock_completed_event, create_mock_interaction + + +class TestCase(CommonTestCaseBase): + def setUp(self) -> None: + super().setUp() + if not _HAS_INTERACTIONS: + raise unittest.SkipTest( + "Interactions are not supported in this version of google-genai" + ) + if self.__class__ == TestCase: + raise unittest.SkipTest("Skipping testcase base.") + self._create_mock = None + self._original_create = InteractionsResource.create + self._original_async_create = AsyncInteractionsResource.create + self._interactions: List[Any] = [] + self._interaction_index = 0 + self._completed_event_type = "interaction_completed" + + @property + def mock_create(self) -> unittest.mock.MagicMock: + if self._create_mock is None: + self._create_and_install_mocks() + assert self._create_mock is not None + return self._create_mock + + def configure_valid_interaction(self, **kwargs: Any) -> None: + self._create_and_install_mocks() + interaction = create_mock_interaction(**kwargs) + self._interactions.append(interaction) + + def configure_exception(self, e: Exception) -> None: + self._create_and_install_mocks(e) + + def _create_and_install_mocks(self, e: Optional[Exception] = None) -> None: + if self._create_mock is not None: + return + self.reset_client() + self.reset_instrumentation() + self._create_mock = self._create_mock_impl(e) + self._install_mocks() + + def _create_mock_impl( + self, e: Optional[Exception] = None + ) -> unittest.mock.MagicMock: + mock = unittest.mock.MagicMock() + + def _default_impl(*args: Any, **kwargs: Any) -> Any: + if not self._interactions: + result = create_mock_interaction() + else: + index = self._interaction_index % len(self._interactions) + result = self._interactions[index] + self._interaction_index += 1 + + if kwargs.get("stream"): + completed_event = create_mock_completed_event(result) + completed_event.event_type = self._completed_event_type + return [completed_event] + return result + + mock.side_effect = e or _default_impl + return mock + + def _install_mocks(self) -> None: + def _sync_create_wrapped(*args: Any, **kwargs: Any) -> Any: + assert self._create_mock is not None + return self._create_mock(*args, **kwargs) + + async def _async_create_wrapped(*args: Any, **kwargs: Any) -> Any: + assert self._create_mock is not None + res = self._create_mock(*args, **kwargs) + if kwargs.get("stream"): + + async def _async_generator() -> Any: + for item in res: + yield item + + return _async_generator() + return res + + InteractionsResource.create = _sync_create_wrapped + AsyncInteractionsResource.create = _async_create_wrapped + + def tearDown(self) -> None: + super().tearDown() + InteractionsResource.create = self._original_create + AsyncInteractionsResource.create = self._original_async_create + + # Abstract methods to be overridden by subclasses + def run_interaction(self, *args: Any, **kwargs: Any) -> Any: + raise NotImplementedError() + + def run_streaming_interaction( + self, *args: Any, **kwargs: Any + ) -> List[Any]: + raise NotImplementedError() + + # The actual collapsed test cases: + def test_instrumentation_does_not_break_core_functionality(self) -> None: + self.configure_valid_interaction( + interaction_id="test-id", + output_text="Yep, it works!", + ) + response = self.run_interaction( + model="gemini-2.5-flash", input="Does this work?" + ) + self.assertEqual(response.id, "test-id") + self.assertEqual(response.steps[1].content[0].text, "Yep, it works!") + + def test_generates_span(self) -> None: + self.configure_valid_interaction() + self.run_interaction(model="gemini-2.5-flash", input="Does this work?") + self.otel.assert_has_span_named("interactions.create gemini-2.5-flash") + + def test_model_reflected_into_span_name(self) -> None: + self.configure_valid_interaction() + self.run_interaction(model="gemini-1.5-flash", input="Does this work?") + self.otel.assert_has_span_named("interactions.create gemini-1.5-flash") + + def test_generated_span_has_minimal_genai_attributes(self) -> None: + self.configure_valid_interaction() + self.run_interaction(model="gemini-2.5-flash", input="Does this work?") + span = self.otel.get_span_named("interactions.create gemini-2.5-flash") + self.assertEqual(span.attributes["gen_ai.provider.name"], "gemini") + self.assertEqual( + span.attributes["gen_ai.operation.name"], "interactions.create" + ) + self.assertEqual( + span.attributes["server.address"], + "generativelanguage.googleapis.com", + ) + + def test_span_and_event_still_written_when_response_is_exception( + self, + ) -> None: + self.configure_exception(ValueError("Uh oh!")) + with self.assertRaises(ValueError): + self.run_interaction( + model="gemini-2.5-flash", input="Does this work?" + ) + self.otel.assert_has_span_named("interactions.create gemini-2.5-flash") + span = self.otel.get_span_named("interactions.create gemini-2.5-flash") + self.otel.assert_has_event_named( + "gen_ai.client.inference.operation.details" + ) + event = self.otel.get_event_named( + "gen_ai.client.inference.operation.details" + ) + self.assertEqual(span.attributes["error.type"], "ValueError") + self.assertEqual(event.attributes["error.type"], "ValueError") + + def test_generated_span_has_vertex_ai_system_when_configured(self) -> None: + self.set_use_vertex(True) + self.configure_valid_interaction() + self.run_interaction(model="gemini-2.5-flash", input="Does this work?") + span = self.otel.get_span_named("interactions.create gemini-2.5-flash") + self.assertEqual(span.attributes["gen_ai.provider.name"], "vertex_ai") + self.assertEqual( + span.attributes["gen_ai.operation.name"], "interactions.create" + ) + + def test_generated_span_counts_tokens(self) -> None: + self.configure_valid_interaction( + input_tokens=15, + output_tokens=25, + ) + self.run_interaction(model="gemini-2.5-flash", input="Some input") + span = self.otel.get_span_named("interactions.create gemini-2.5-flash") + self.assertEqual(span.attributes["gen_ai.usage.input_tokens"], 15) + self.assertEqual(span.attributes["gen_ai.usage.output_tokens"], 25) + + @patch.dict( + "os.environ", + {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "SPAN_ONLY"}, + ) + def test_span_attributes_with_content_capture(self) -> None: + self.configure_valid_interaction( + input_text="Hello interactions!", + output_text="Response from interactions!", + ) + self.run_interaction( + model="gemini-2.5-flash", + input="Hello interactions!", + ) + span = self.otel.get_span_named("interactions.create gemini-2.5-flash") + self.assertEqual( + span.attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES], + '[{"role":"user","parts":[{"content":"Hello interactions!","type":"text"}]}]', + ) + self.assertEqual( + span.attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES], + '[{"role":"assistant","parts":[{"content":"Response from interactions!","type":"text"}],"finish_reason":"stop"}]', + ) + + @patch.dict( + "os.environ", + {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "NO_CONTENT"}, + ) + def test_span_attributes_no_content_capture(self) -> None: + self.configure_valid_interaction( + input_text="Hello interactions!", + output_text="Response from interactions!", + ) + self.run_interaction( + model="gemini-2.5-flash", + input="Hello interactions!", + ) + span = self.otel.get_span_named("interactions.create gemini-2.5-flash") + for attribute in ( + GenAIAttributes.GEN_AI_INPUT_MESSAGES, + GenAIAttributes.GEN_AI_OUTPUT_MESSAGES, + ): + self.assertNotIn(attribute, span.attributes) + + def test_streaming_generates_span(self) -> None: + self.configure_valid_interaction( + interaction_id="stream-id-1", + model_name="gemini-2.5-flash", + output_text="Streaming response!", + input_tokens=5, + output_tokens=8, + ) + events = self.run_streaming_interaction( + model="gemini-2.5-flash", + input="Streaming test", + stream=True, + ) + self.assertEqual(len(events), 1) + self.assertEqual(events[0].interaction.id, "stream-id-1") + + self.otel.assert_has_span_named("interactions.create gemini-2.5-flash") + span = self.otel.get_span_named("interactions.create gemini-2.5-flash") + self.assertEqual(span.attributes["gen_ai.usage.input_tokens"], 5) + self.assertEqual(span.attributes["gen_ai.usage.output_tokens"], 8) + self.assertGreater( + span.attributes["gen_ai.response.time_to_first_token"], 0 + ) + + def test_streaming_accepts_real_sdk_completed_event_name(self) -> None: + self._completed_event_type = "interaction.completed" + self.configure_valid_interaction( + interaction_id="stream-id-real-sdk", + model_name="gemini-2.5-flash", + output_text="Real SDK streaming response!", + input_tokens=6, + output_tokens=9, + ) + events = self.run_streaming_interaction( + model="gemini-2.5-flash", + input="Real SDK streaming test", + stream=True, + ) + self.assertEqual(events[0].event_type, "interaction.completed") + span = self.otel.get_span_named("interactions.create gemini-2.5-flash") + self.assertEqual(span.attributes["gen_ai.usage.input_tokens"], 6) + self.assertEqual(span.attributes["gen_ai.usage.output_tokens"], 9) + self.assertEqual( + span.attributes["gen_ai.response.model"], "gemini-2.5-flash" + ) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/test_interactions_async.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/test_interactions_async.py new file mode 100644 index 000000000..35bdeaf10 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/test_interactions_async.py @@ -0,0 +1,42 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import asyncio +from typing import Any, List + +from .base import TestCase + + +class TestInteractionsAsync(TestCase): + def run_interaction(self, *args: Any, **kwargs: Any) -> Any: + return asyncio.run( + self.client.aio.interactions.create(*args, **kwargs) + ) + + def run_streaming_interaction( + self, *args: Any, **kwargs: Any + ) -> List[Any]: + async def _run() -> List[Any]: + stream = await self.client.aio.interactions.create(*args, **kwargs) + events = [] + async for event in stream: + events.append(event) + return events + + return asyncio.run(_run()) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/test_interactions_sync.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/test_interactions_sync.py new file mode 100644 index 000000000..18a9218f8 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/test_interactions_sync.py @@ -0,0 +1,33 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Any, List + +from .base import TestCase + + +class TestInteractionsSync(TestCase): + def run_interaction(self, *args: Any, **kwargs: Any) -> Any: + return self.client.interactions.create(*args, **kwargs) + + def run_streaming_interaction( + self, *args: Any, **kwargs: Any + ) -> List[Any]: + stream = self.client.interactions.create(*args, **kwargs) + return list(stream) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/test_parser.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/test_parser.py new file mode 100644 index 000000000..9a0490db0 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/test_parser.py @@ -0,0 +1,145 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import unittest +import unittest.mock + +from opentelemetry.instrumentation.google_genai._compat import GenericPart +from opentelemetry.instrumentation.google_genai.interactions import ( + _HAS_INTERACTIONS, + _interactions_input_to_messages, + _interactions_response_to_messages, +) +from opentelemetry.util.genai.types import ( + Text, + ToolCallResponse, + Uri, +) +from opentelemetry.util.genai.types import ( + ToolCall as ToolCallRequest, +) + + +@unittest.skipIf( + not _HAS_INTERACTIONS, + "Interactions are not supported in this version of google-genai", +) +class TestInteractionsParser(unittest.TestCase): + def test_input_to_messages_none(self) -> None: + self.assertEqual(_interactions_input_to_messages(None), []) + + def test_input_to_messages_str(self) -> None: + messages = _interactions_input_to_messages("Hello world") + self.assertEqual(messages[0].role, "user") + self.assertEqual(len(messages[0].parts), 1) + self.assertIsInstance(messages[0].parts[0], Text) + self.assertEqual(messages[0].parts[0].content, "Hello world") + + def test_input_to_messages_list_of_strings(self) -> None: + messages = _interactions_input_to_messages(["Hello", "world"]) + self.assertEqual(len(messages[0].parts), 2) + self.assertIsInstance(messages[0].parts[0], Text) + self.assertEqual(messages[0].parts[0].content, "Hello") + self.assertIsInstance(messages[0].parts[1], Text) + self.assertEqual(messages[0].parts[1].content, "world") + + def test_input_to_messages_text_step(self) -> None: + steps = [{"type": "text", "text": "Hello text step"}] + messages = _interactions_input_to_messages(steps) + self.assertEqual(len(messages[0].parts), 1) + self.assertIsInstance(messages[0].parts[0], Text) + self.assertEqual(messages[0].parts[0].content, "Hello text step") + + def test_input_to_messages_document_step(self) -> None: + steps = [ + { + "type": "document", + "mime_type": "application/pdf", + "uri": "https://example.com/doc.pdf", + } + ] + messages = _interactions_input_to_messages(steps) + self.assertEqual(len(messages[0].parts), 1) + self.assertIsInstance(messages[0].parts[0], Uri) + self.assertEqual(messages[0].parts[0].mime_type, "application/pdf") + self.assertEqual(messages[0].parts[0].modality, "document") + self.assertEqual( + messages[0].parts[0].uri, "https://example.com/doc.pdf" + ) + + def test_input_to_messages_tool_call_step(self) -> None: + steps = [ + { + "type": "function_call", + "id": "call-123", + "name": "calc", + "arguments": {"x": 5}, + } + ] + messages = _interactions_input_to_messages(steps) + self.assertEqual(len(messages[0].parts), 1) + self.assertIsInstance(messages[0].parts[0], ToolCallRequest) + self.assertEqual(messages[0].parts[0].id, "call-123") + self.assertEqual(messages[0].parts[0].name, "calc") + self.assertEqual(messages[0].parts[0].arguments, {"x": 5}) + + def test_input_to_messages_tool_result_step(self) -> None: + steps = [ + { + "type": "function_result", + "call_id": "call-123", + "result": {"val": 10}, + } + ] + messages = _interactions_input_to_messages(steps) + self.assertEqual(len(messages[0].parts), 1) + self.assertIsInstance(messages[0].parts[0], ToolCallResponse) + self.assertEqual(messages[0].parts[0].id, "call-123") + self.assertEqual(messages[0].parts[0].response, {"val": 10}) + + def test_input_to_messages_generic_fallback(self) -> None: + steps = [{"type": "some_unsupported_type"}] + messages = _interactions_input_to_messages(steps) + self.assertEqual(len(messages[0].parts), 1) + self.assertIsInstance(messages[0].parts[0], GenericPart) + self.assertEqual(messages[0].parts[0].value, "dict") + + def test_input_to_messages_single_non_sequence_step(self) -> None: + step = {"type": "text", "text": "Hello single step"} + messages = _interactions_input_to_messages(step) + self.assertEqual(len(messages[0].parts), 1) + self.assertIsInstance(messages[0].parts[0], Text) + self.assertEqual(messages[0].parts[0].content, "Hello single step") + + def test_input_to_messages_none_type_fall_through(self) -> None: + step = {"other_field": "no type specified"} + messages = _interactions_input_to_messages(step) + self.assertEqual(len(messages[0].parts), 0) + + def test_response_to_messages(self) -> None: + mock_interaction = unittest.mock.MagicMock() + mock_interaction.output_text = "Model response text" + + messages = _interactions_response_to_messages(mock_interaction) + + self.assertEqual(messages[0].role, "assistant") + self.assertEqual(messages[0].finish_reason, "stop") + self.assertEqual(len(messages[0].parts), 1) + self.assertIsInstance(messages[0].parts[0], Text) + self.assertEqual(messages[0].parts[0].content, "Model response text") diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/util.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/util.py new file mode 100644 index 000000000..29a66f8da --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/interactions/util.py @@ -0,0 +1,64 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import unittest.mock +from typing import Any + + +def create_mock_interaction( + interaction_id: str = "test-id", + model_name: str = "test-model", + input_text: str = "user input", + output_text: str = "model output", + input_tokens: int = 10, + output_tokens: int = 20, +) -> Any: + mock_usage = unittest.mock.MagicMock() + mock_usage.total_input_tokens = input_tokens + mock_usage.total_output_tokens = output_tokens + mock_usage.total_thought_tokens = 0 + mock_usage.total_cached_tokens = 0 + + mock_user_step = unittest.mock.MagicMock() + mock_user_step.type = "user_input" + mock_user_part = unittest.mock.MagicMock() + mock_user_part.text = input_text + mock_user_step.content = [mock_user_part] + + mock_model_step = unittest.mock.MagicMock() + mock_model_step.type = "model_output" + mock_model_part = unittest.mock.MagicMock() + mock_model_part.text = output_text + mock_model_step.content = [mock_model_part] + + mock_interaction = unittest.mock.MagicMock() + mock_interaction.id = interaction_id + mock_interaction.model = model_name + mock_interaction.usage = mock_usage + mock_interaction.steps = [mock_user_step, mock_model_step] + mock_interaction.output_text = output_text + + return mock_interaction + + +def create_mock_completed_event(interaction: Any) -> Any: + event = unittest.mock.MagicMock() + event.event_type = "interaction_completed" + event.interaction = interaction + return event diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/requirements.latest.txt b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/requirements.latest.txt new file mode 100644 index 000000000..df0e7edcc --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/requirements.latest.txt @@ -0,0 +1,9 @@ +google-genai==1.47.0; python_version < "3.10" +google-genai>=2.14.0,<3; python_version >= "3.10" +pytest +pytest-asyncio +pytest-vcr +vcrpy +-e opentelemetry-instrumentation +-e util/opentelemetry-util-genai +-e instrumentation-loongsuite/loongsuite-instrumentation-google-genai diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/requirements.oldest.txt b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/requirements.oldest.txt new file mode 100644 index 000000000..5c0eeb034 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/requirements.oldest.txt @@ -0,0 +1,11 @@ +google-genai==1.32.0 +opentelemetry-api==1.37.0 +opentelemetry-sdk==1.37.0 +opentelemetry-instrumentation==0.58b0 +opentelemetry-semantic-conventions==0.58b0 +pytest +pytest-asyncio +pytest-vcr +vcrpy +-e util/opentelemetry-util-genai +-e instrumentation-loongsuite/loongsuite-instrumentation-google-genai diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/test_client_info.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/test_client_info.py new file mode 100644 index 000000000..2c0974f13 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/test_client_info.py @@ -0,0 +1,89 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from types import SimpleNamespace + +import pytest + +from opentelemetry.instrumentation.google_genai.client_info import ( + get_client_info, +) + + +def _client(server_url: str) -> SimpleNamespace: + return SimpleNamespace( + sdk_configuration=SimpleNamespace(server_url=server_url) + ) + + +@pytest.mark.parametrize( + "server_url, expected_address", + [ + ( + "https://aiplatform.googleapis.com/v1", + "aiplatform.googleapis.com", + ), + ( + "https://us-central1-aiplatform.googleapis.com/v1", + "us-central1-aiplatform.googleapis.com", + ), + ( + "HTTPS://AIPLATFORM.GOOGLEAPIS.COM/v1", + "aiplatform.googleapis.com", + ), + ( + "us-central1-aiplatform.googleapis.com", + "us-central1-aiplatform.googleapis.com", + ), + ], +) +def test_vertex_hostname_is_recognized( + server_url: str, + expected_address: str, +) -> None: + is_vertex, server_address = get_client_info(_client(server_url)) + + assert is_vertex + assert server_address == expected_address + + +@pytest.mark.parametrize( + "server_url, expected_address", + [ + ( + "https://aiplatform.googleapis.com.evil.example/v1", + "aiplatform.googleapis.com.evil.example", + ), + ( + "https://evil.example/v1?next=aiplatform.googleapis.com", + "evil.example", + ), + ( + "https://aiplatform.googleapis.com@evil.example/v1", + "evil.example", + ), + ( + "aiplatform.googleapis.com.evil.example", + "aiplatform.googleapis.com.evil.example", + ), + ], +) +def test_vertex_hostname_rejects_substring_confusion( + server_url: str, + expected_address: str, +) -> None: + is_vertex, server_address = get_client_info(_client(server_url)) + + assert not is_vertex + assert server_address == expected_address diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/test_compat.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/test_compat.py new file mode 100644 index 000000000..ab202137b --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/test_compat.py @@ -0,0 +1,241 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import os +from typing import Tuple +from unittest.mock import patch + +from google.genai import types as genai_types + +from opentelemetry import context as context_api +from opentelemetry._logs import get_logger_provider +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY +from opentelemetry.instrumentation.google_genai._compat import TelemetryHandler +from opentelemetry.instrumentation.google_genai._stream import ( + AsyncStreamWrapper, +) +from opentelemetry.instrumentation.google_genai.allowlist_util import AllowList +from opentelemetry.instrumentation.google_genai.generate_content import ( + _create_instrumented_generate_content_stream, + _wrapped_config_with_tools, + instrument_generate_content, + uninstrument_generate_content, +) +from opentelemetry.instrumentation.google_genai.interactions import ( + _create_instrumented_interactions_create, +) +from opentelemetry.instrumentation.google_genai.message import _to_part +from opentelemetry.metrics import get_meter_provider +from opentelemetry.trace import SpanKind, get_tracer_provider +from opentelemetry.util.genai.types import Reasoning, Text + +from .common.otel_mocker import OTelMocker + + +def _handler() -> Tuple[TelemetryHandler, OTelMocker]: + otel = OTelMocker() + otel.install() + return ( + TelemetryHandler( + tracer_provider=get_tracer_provider(), + meter_provider=get_meter_provider(), + logger_provider=get_logger_provider(), + ), + otel, + ) + + +def test_standard_instrumentation_suppression_creates_no_span(): + handler, otel = _handler() + token = context_api.attach( + context_api.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True) + ) + try: + with handler.inference( + provider="gemini", + request_model="gemini-test", + operation_name="generate_content", + ): + pass + assert otel.get_finished_spans() == [] + finally: + context_api.detach(token) + otel.uninstall() + + +def test_automatic_function_call_uses_internal_span(): + handler, otel = _handler() + try: + with handler.tool("add"): + pass + span = otel.get_span_named("execute_tool add") + assert span.kind is SpanKind.INTERNAL + finally: + otel.uninstall() + + +@patch.dict( + "os.environ", + {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "SPAN_ONLY"}, + clear=False, +) +def test_content_capture_does_not_require_legacy_stability_flag(): + handler, otel = _handler() + try: + invocation = handler.inference( + provider="gemini", + request_model="gemini-test", + operation_name="generate_content", + ) + invocation.input_messages = [] + invocation.output_messages = [] + invocation.system_instruction = [Text(content="system")] + invocation.stop() + span = otel.get_span_named("generate_content gemini-test") + assert span.attributes["gen_ai.system_instructions"] == ( + '[{"content":"system","type":"text"}]' + ) + finally: + otel.uninstall() + + +def test_google_thought_part_maps_to_reasoning(): + part = _to_part(genai_types.Part(text="internal thought", thought=True), 0) + assert isinstance(part, Reasoning) + assert part.content == "internal thought" + + +def test_wrapping_tools_does_not_mutate_reusable_config(): + handler, otel = _handler() + + def get_temperature(city: str) -> str: + return city + + config = genai_types.GenerateContentConfig(tools=[get_temperature]) + try: + first, first_changed = _wrapped_config_with_tools(handler, config) + second, second_changed = _wrapped_config_with_tools(handler, config) + + assert first_changed and second_changed + assert config.tools == [get_temperature] + assert first.tools[0].__wrapped__ is get_temperature + assert second.tools[0].__wrapped__ is get_temperature + finally: + otel.uninstall() + + +def test_invalid_dict_config_is_passed_through_to_sdk(): + handler, otel = _handler() + invalid_config = {"temperature": object()} + try: + _, has_wrapped_tools = _wrapped_config_with_tools( + handler, invalid_config + ) + assert not has_wrapped_tools + finally: + otel.uninstall() + + +def test_sync_stream_creation_failure_finishes_span(): + handler, otel = _handler() + wrapped = _create_instrumented_generate_content_stream( + handler, AllowList() + ) + + def fail(*args, **kwargs): + raise ValueError("stream creation failed") + + try: + try: + wrapped( + fail, + object(), + (), + {"model": "gemini-test", "contents": "hello"}, + ) + except ValueError: + pass + span = otel.get_span_named("generate_content gemini-test") + assert span is not None + assert span.attributes["error.type"] == "ValueError" + finally: + otel.uninstall() + + +def test_interactions_stream_creation_failure_finishes_span(): + handler, otel = _handler() + wrapped = _create_instrumented_interactions_create(handler) + + def fail(*args, **kwargs): + raise ValueError("stream creation failed") + + try: + try: + wrapped( + fail, + object(), + (), + { + "model": "gemini-test", + "input": "hello", + "stream": True, + }, + ) + except ValueError: + pass + span = otel.get_span_named("interactions.create gemini-test") + assert span is not None + assert span.attributes["error.type"] == "ValueError" + finally: + otel.uninstall() + + +def test_async_stream_aclose_finalizes_google_style_stream(): + finalized = [] + + class Wrapper(AsyncStreamWrapper): + def _process_chunk(self, chunk): + pass + + def _on_stream_end(self): + finalized.append("success") + + def _on_stream_error(self, error): + finalized.append(error) + + async def exercise(): + async def stream(): + yield "chunk" + + underlying = stream() + wrapper = Wrapper(underlying) + await wrapper.aclose() + + asyncio.run(exercise()) + assert finalized == ["success"] + + +@patch.dict( + os.environ, + {"OTEL_INSTRUMENTATION_GENAI_EMIT_EVENT": "false"}, +) +def test_instrumentation_preserves_explicit_event_setting(): + handler, otel = _handler() + snapshot = instrument_generate_content(handler, AllowList()) + try: + assert os.environ["OTEL_INSTRUMENTATION_GENAI_EMIT_EVENT"] == "false" + finally: + uninstrument_generate_content(snapshot) + otel.uninstall() diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/test_fail_open.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/test_fail_open.py new file mode 100644 index 000000000..1e04137a1 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/test_fail_open.py @@ -0,0 +1,676 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from concurrent.futures import ThreadPoolExecutor +from typing import Tuple + +import pytest +from google.genai import types + +from opentelemetry import trace +from opentelemetry._logs import get_logger_provider +from opentelemetry.instrumentation.google_genai._compat import TelemetryHandler +from opentelemetry.instrumentation.google_genai._stream import ( + AsyncStreamWrapper, + SyncStreamWrapper, +) +from opentelemetry.instrumentation.google_genai.allowlist_util import AllowList +from opentelemetry.instrumentation.google_genai.embeddings import ( + _CAPTURE_RAW_RESPONSE, + _RAW_RESPONSE_BODY, + _complete_embedding_advice, + _EmbeddingAdviceState, +) +from opentelemetry.instrumentation.google_genai.generate_content import ( + _create_instrumented_generate_content, + _create_instrumented_generate_content_stream, +) +from opentelemetry.instrumentation.google_genai.interactions import ( + AsyncInteractionsStreamWrapper, + InteractionsStreamWrapper, + _apply_interaction_response_attributes, +) +from opentelemetry.instrumentation.google_genai.message import ( + StreamOutputMessageAccumulator, +) +from opentelemetry.instrumentation.google_genai.tool_call_wrapper import ( + _wrap_tool_function, +) +from opentelemetry.metrics import get_meter_provider +from opentelemetry.trace import get_tracer_provider + +from .common.otel_mocker import OTelMocker + + +def _handler() -> Tuple[TelemetryHandler, OTelMocker]: + otel = OTelMocker() + otel.install() + return ( + TelemetryHandler( + tracer_provider=get_tracer_provider(), + meter_provider=get_meter_provider(), + logger_provider=get_logger_provider(), + ), + otel, + ) + + +def test_prepare_failure_calls_sdk_once_and_returns_original( + monkeypatch: pytest.MonkeyPatch, +) -> None: + handler, otel = _handler() + wrapped = _create_instrumented_generate_content(handler, AllowList()) + response = object() + calls = 0 + + def fail_prepare(*args, **kwargs): + raise RuntimeError("probe prepare failed") + + def sdk_call(*args, **kwargs): + nonlocal calls + calls += 1 + return response + + monkeypatch.setattr( + "opentelemetry.instrumentation.google_genai.generate_content." + "_wrapped_config_with_tools", + fail_prepare, + ) + try: + actual = wrapped( + sdk_call, + object(), + (), + {"model": "gemini-test", "contents": "hello"}, + ) + assert actual is response + assert calls == 1 + assert otel.get_finished_spans() == [] + finally: + otel.uninstall() + + +def test_response_mapping_failure_returns_original_and_ends_span( + monkeypatch: pytest.MonkeyPatch, +) -> None: + handler, otel = _handler() + wrapped = _create_instrumented_generate_content(handler, AllowList()) + response = object() + calls = 0 + + def fail_mapping(*args, **kwargs): + raise RuntimeError("probe mapping failed") + + def sdk_call(*args, **kwargs): + nonlocal calls + calls += 1 + return response + + monkeypatch.setattr( + "opentelemetry.instrumentation.google_genai.generate_content." + "_apply_response_attributes", + fail_mapping, + ) + try: + actual = wrapped( + sdk_call, + object(), + (), + {"model": "gemini-test", "contents": "hello"}, + ) + assert actual is response + assert calls == 1 + assert len(otel.get_finished_spans()) == 1 + assert not trace.get_current_span().get_span_context().is_valid + finally: + otel.uninstall() + + +def test_interaction_usage_missing_optional_fields_is_tolerated() -> None: + class Usage: + total_input_tokens = 3 + total_output_tokens = 2 + + class Response: + model = "gemini-test" + usage = Usage() + + class Invocation: + response_model_name = None + input_tokens = None + output_tokens = None + thinking_tokens = None + cache_read_input_tokens = None + + class Handler: + @staticmethod + def should_capture_content() -> bool: + return False + + invocation = Invocation() + _apply_interaction_response_attributes(Response(), invocation, Handler()) + + assert invocation.input_tokens == 3 + assert invocation.output_tokens == 2 + assert invocation.thinking_tokens is None + assert invocation.cache_read_input_tokens is None + + +@pytest.mark.parametrize( + "wrapper_factory", + [ + lambda invocation, handler: InteractionsStreamWrapper( + iter(()), invocation, handler + ), + lambda invocation, handler: AsyncInteractionsStreamWrapper( + _empty_async_stream(), invocation, handler + ), + ], +) +def test_interaction_stream_mapping_failure_still_stops_invocation( + monkeypatch: pytest.MonkeyPatch, + wrapper_factory, +) -> None: + class Invocation: + stopped = False + + def stop(self) -> None: + self.stopped = True + + def fail_mapping(*args, **kwargs): + raise RuntimeError("probe mapping failed") + + invocation = Invocation() + wrapper = wrapper_factory(invocation, object()) + wrapper._self_last_interaction = object() + monkeypatch.setattr( + "opentelemetry.instrumentation.google_genai.interactions." + "_apply_interaction_response_attributes", + fail_mapping, + ) + + with pytest.raises(RuntimeError, match="probe mapping failed"): + wrapper._on_stream_end() + + assert invocation.stopped + + +def test_embedding_mapping_failure_still_stops_invocation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class Invocation: + stopped = False + + def stop(self) -> None: + self.stopped = True + + def fail_mapping(*args, **kwargs): + raise RuntimeError("probe mapping failed") + + invocation = Invocation() + raw_body_token = _RAW_RESPONSE_BODY.set(None) + capture_token = _CAPTURE_RAW_RESPONSE.set(True) + state = _EmbeddingAdviceState( + invocation=invocation, + raw_body_token=raw_body_token, + capture_token=capture_token, + ) + monkeypatch.setattr( + "opentelemetry.instrumentation.google_genai.embeddings." + "_apply_embedding_response_attributes", + fail_mapping, + ) + + _complete_embedding_advice(state, object()) + + assert invocation.stopped + assert _RAW_RESPONSE_BODY.get() is None + assert not _CAPTURE_RAW_RESPONSE.get() + + +async def _empty_async_stream(): + if False: + yield None + + +def test_failure_reporter_failure_preserves_original_exception( + monkeypatch: pytest.MonkeyPatch, +) -> None: + handler, otel = _handler() + wrapped = _create_instrumented_generate_content(handler, AllowList()) + original = ValueError("business failure") + calls = 0 + + def sdk_call(*args, **kwargs): + nonlocal calls + calls += 1 + raise original + + def fail_reporter(invocation, *args, **kwargs): + invocation._owner.abandon_inference(invocation) + raise RuntimeError("probe failure reporter failed") + + monkeypatch.setattr( + "opentelemetry.instrumentation.google_genai._compat." + "InferenceInvocation.fail", + fail_reporter, + ) + try: + with pytest.raises(ValueError) as raised: + wrapped( + sdk_call, + object(), + (), + {"model": "gemini-test", "contents": "hello"}, + ) + assert raised.value is original + assert calls == 1 + finally: + otel.uninstall() + + +class _FaultySyncWrapper(SyncStreamWrapper[object]): + def __init__( + self, + stream, + *, + chunk_failure: bool = False, + final_failure: bool = False, + ) -> None: + super().__init__(stream) + self._self_chunk_failure = chunk_failure + self._self_final_failure = final_failure + self._self_finalizations = 0 + + def _process_chunk(self, chunk: object) -> None: + if self._self_chunk_failure: + raise RuntimeError("probe chunk failed") + + def _on_stream_end(self) -> None: + self._self_finalizations += 1 + if self._self_final_failure: + raise RuntimeError("probe finalization failed") + + def _on_stream_error(self, error: BaseException) -> None: + self._self_finalizations += 1 + raise RuntimeError("probe error reporter failed") + + +def test_stream_probe_failures_preserve_chunks_and_finalize_once() -> None: + chunks = [object(), object()] + wrapper = _FaultySyncWrapper( + iter(chunks), + chunk_failure=True, + final_failure=True, + ) + actual = list(wrapper) + wrapper.close() + assert actual[0] is chunks[0] + assert actual[1] is chunks[1] + assert wrapper._self_finalizations == 1 + + +def test_stream_error_reporter_preserves_original_exception() -> None: + original = ValueError("business stream failed") + + def stream(): + yield object() + raise original + + wrapper = _FaultySyncWrapper(stream()) + iterator = iter(wrapper) + next(iterator) + with pytest.raises(ValueError) as raised: + next(iterator) + assert raised.value is original + assert wrapper._self_finalizations == 1 + + +def test_stream_close_failure_does_not_replace_business_outcome() -> None: + class CloseFailure: + def __iter__(self): + return iter(()) + + def close(self): + raise RuntimeError("provider close failed") + + wrapper = _FaultySyncWrapper(CloseFailure()) + wrapper.close() + assert wrapper._self_finalizations == 1 + + +def test_stream_creation_detaches_before_cross_thread_consumption( + monkeypatch: pytest.MonkeyPatch, +) -> None: + handler, otel = _handler() + wrapped = _create_instrumented_generate_content_stream( + handler, + AllowList(), + ) + chunks = [object(), object()] + + def sdk_call(*args, **kwargs): + return iter(chunks) + + monkeypatch.setattr( + "opentelemetry.instrumentation.google_genai.generate_content." + "_apply_response_attributes", + lambda *args, **kwargs: None, + ) + try: + stream = wrapped( + sdk_call, + object(), + (), + {"model": "gemini-test", "contents": "hello"}, + ) + assert not trace.get_current_span().get_span_context().is_valid + with ThreadPoolExecutor(max_workers=1) as executor: + actual = executor.submit(list, stream).result() + assert actual[0] is chunks[0] + assert actual[1] is chunks[1] + assert len(otel.get_finished_spans()) == 1 + finally: + otel.uninstall() + + +def test_stream_wrapper_construction_failure_returns_original_stream( + monkeypatch: pytest.MonkeyPatch, +) -> None: + handler, otel = _handler() + wrapped = _create_instrumented_generate_content_stream( + handler, + AllowList(), + ) + raw_stream = iter([object()]) + + def sdk_call(*args, **kwargs): + return raw_stream + + class ConstructionFailure: + def __init__(self, *args, **kwargs): + raise RuntimeError("probe wrapper construction failed") + + monkeypatch.setattr( + "opentelemetry.instrumentation.google_genai.generate_content." + "GenerateContentStreamWrapper", + ConstructionFailure, + ) + try: + actual = wrapped( + sdk_call, + object(), + (), + {"model": "gemini-test", "contents": "hello"}, + ) + assert actual is raw_stream + assert len(otel.get_finished_spans()) == 1 + assert not trace.get_current_span().get_span_context().is_valid + finally: + otel.uninstall() + + +def test_stream_accumulator_failure_preserves_chunks_and_ends_span( + monkeypatch: pytest.MonkeyPatch, +) -> None: + handler, otel = _handler() + wrapped = _create_instrumented_generate_content_stream( + handler, + AllowList(), + ) + chunks = [ + types.GenerateContentResponse( + candidates=[ + types.Candidate( + index=0, + content=types.Content( + role="model", + parts=[types.Part(text="business output")], + ), + ) + ] + ) + ] + + def sdk_call(*args, **kwargs): + return iter(chunks) + + def fail_accumulation(self, candidates): + raise RuntimeError("probe accumulation failed") + + monkeypatch.setattr( + StreamOutputMessageAccumulator, + "add_candidates", + fail_accumulation, + ) + try: + actual = list( + wrapped( + sdk_call, + object(), + (), + {"model": "gemini-test", "contents": "hello"}, + ) + ) + assert actual[0] is chunks[0] + assert len(otel.get_finished_spans()) == 1 + assert not trace.get_current_span().get_span_context().is_valid + finally: + otel.uninstall() + + +def test_stream_output_mapping_failure_still_ends_span( + monkeypatch: pytest.MonkeyPatch, +) -> None: + handler, otel = _handler() + wrapped = _create_instrumented_generate_content_stream( + handler, + AllowList(), + ) + chunk = types.GenerateContentResponse( + candidates=[ + types.Candidate( + index=0, + content=types.Content( + role="model", + parts=[types.Part(text="business output")], + ), + ) + ] + ) + + def sdk_call(*args, **kwargs): + return iter([chunk]) + + def fail_output_mapping(self): + raise RuntimeError("probe output mapping failed") + + monkeypatch.setattr( + StreamOutputMessageAccumulator, + "to_output_messages", + fail_output_mapping, + ) + try: + actual = list( + wrapped( + sdk_call, + object(), + (), + {"model": "gemini-test", "contents": "hello"}, + ) + ) + assert actual == [chunk] + assert len(otel.get_finished_spans()) == 1 + assert not trace.get_current_span().get_span_context().is_valid + finally: + otel.uninstall() + + +def test_generator_exit_identity_and_cleanup_are_preserved() -> None: + original = GeneratorExit("consumer closed") + + class GeneratorExitIterator: + def __iter__(self): + return self + + def __next__(self): + raise original + + def close(self): + return None + + wrapper = _FaultySyncWrapper(GeneratorExitIterator()) + with pytest.raises(GeneratorExit) as raised: + next(wrapper) + assert raised.value is original + assert wrapper._self_finalizations == 1 + + +def test_one_faulty_concurrent_stream_does_not_contaminate_sibling() -> None: + chunks = [[object(), object()], [object(), object()]] + wrappers = [ + _FaultySyncWrapper(iter(chunks[0]), chunk_failure=True), + _FaultySyncWrapper(iter(chunks[1])), + ] + with ThreadPoolExecutor(max_workers=2) as executor: + actual = list(executor.map(list, wrappers)) + assert actual[0][0] is chunks[0][0] + assert actual[1][0] is chunks[1][0] + assert wrappers[0]._self_finalizations == 1 + assert wrappers[1]._self_finalizations == 1 + + +class _FaultyAsyncWrapper(AsyncStreamWrapper[object]): + def __init__(self, stream) -> None: + super().__init__(stream) + self._self_finalizations = 0 + + def _process_chunk(self, chunk: object) -> None: + raise RuntimeError("probe async chunk failed") + + def _on_stream_end(self) -> None: + self._self_finalizations += 1 + raise RuntimeError("probe async finalization failed") + + def _on_stream_error(self, error: BaseException) -> None: + self._self_finalizations += 1 + raise RuntimeError("probe async error reporter failed") + + +def test_async_stream_probe_failures_and_aclose_are_fail_open() -> None: + chunks = [object(), object()] + + async def exercise(): + async def source(): + for chunk in chunks: + yield chunk + + wrapper = _FaultyAsyncWrapper(source()) + actual = [chunk async for chunk in wrapper] + await wrapper.aclose() + return wrapper, actual + + wrapper, actual = asyncio.run(exercise()) + assert actual[0] is chunks[0] + assert actual[1] is chunks[1] + assert wrapper._self_finalizations == 1 + + +def test_async_cancellation_preserves_control_flow_and_finalizes_once() -> ( + None +): + async def exercise(): + gate = asyncio.Event() + + async def source(): + await gate.wait() + yield object() + + wrapper = _FaultyAsyncWrapper(source()) + task = asyncio.create_task(wrapper.__anext__()) + await asyncio.sleep(0) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + return wrapper + + wrapper = asyncio.run(exercise()) + assert wrapper._self_finalizations == 1 + + +def test_tool_prepare_and_finalize_failures_do_not_change_business_result() -> ( + None +): + result = object() + calls = 0 + + class Invocation: + should_capture_content_on_span = True + + def stop(self): + raise RuntimeError("probe stop failed") + + def fail(self, error): + raise RuntimeError("probe fail failed") + + class Handler: + def __init__(self): + self.prepare_fails = True + + def tool(self, *args, **kwargs): + if self.prepare_fails: + raise RuntimeError("probe prepare failed") + return Invocation() + + def abandon_tool(self, invocation): + return None + + handler = Handler() + + def business_tool(value): + nonlocal calls + calls += 1 + return result + + wrapped = _wrap_tool_function(business_tool, handler) + assert wrapped("first") is result + handler.prepare_fails = False + assert wrapped("second") is result + assert calls == 2 + + +def test_tool_failure_reporter_does_not_replace_business_exception() -> None: + original = ValueError("tool business failure") + + class Invocation: + should_capture_content_on_span = False + + def fail(self, error): + raise RuntimeError("probe tool reporter failed") + + class Handler: + def tool(self, *args, **kwargs): + return Invocation() + + def abandon_tool(self, invocation): + return None + + def business_tool(): + raise original + + wrapped = _wrap_tool_function(business_tool, Handler()) + with pytest.raises(ValueError) as raised: + wrapped() + assert raised.value is original diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/test_live.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/test_live.py new file mode 100644 index 000000000..0c43abf79 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/test_live.py @@ -0,0 +1,252 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Opt-in smoke test against the real Gemini Developer API.""" + +import asyncio +import json +import os + +import pytest +from google import genai +from google.genai import types + +from opentelemetry.instrumentation.google_genai import ( + GoogleGenAiSdkInstrumentor, +) +from opentelemetry.sdk import metrics, trace +from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.trace import SpanKind + +_LIVE_TEST_ENV = "RUN_GOOGLE_GENAI_LIVE_TESTS" +_DEFAULT_MODEL = "gemini-3.5-flash-lite" +_EMBEDDING_MODEL = "gemini-embedding-001" +_INTERACTIONS_MODEL = _DEFAULT_MODEL + + +@pytest.mark.skipif( + os.getenv(_LIVE_TEST_ENV) != "1", + reason=f"set {_LIVE_TEST_ENV}=1 to call the real Gemini API", +) +def test_live_google_genai_all_instrumented_methods(): + api_key = os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY") + assert api_key, "GEMINI_API_KEY or GOOGLE_API_KEY is required" + model = os.getenv("GOOGLE_GENAI_LIVE_MODEL", _DEFAULT_MODEL) + + span_exporter = InMemorySpanExporter() + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + metric_reader = InMemoryMetricReader() + meter_provider = metrics.MeterProvider(metric_readers=[metric_reader]) + instrumentor = GoogleGenAiSdkInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + ) + client = genai.Client(api_key=api_key) + try: + sync_response = client.models.generate_content( + model=model, + contents="Reply with the single word OK.", + ) + sync_stream = list( + client.models.generate_content_stream( + model=model, + contents="Reply with exactly: stream OK", + ) + ) + sync_embedding = client.models.embed_content( + model=_EMBEDDING_MODEL, + contents="LoongSuite live sync embedding verification", + config=types.EmbedContentConfig(output_dimensionality=8), + ) + sync_interaction = client.interactions.create( + model=_INTERACTIONS_MODEL, + input="Reply with exactly: live interaction OK", + ) + sync_interaction_stream = list( + client.interactions.create( + model=_INTERACTIONS_MODEL, + input="Reply with exactly: live interaction stream OK", + stream=True, + ) + ) + + def get_temperature(city: str) -> str: + """Return the current temperature for a city.""" + + assert city + return "17 C" + + tool_response = client.models.generate_content( + model=model, + contents=( + "You must call get_temperature for Paris, then reply with " + "only the returned temperature." + ), + config=types.GenerateContentConfig(tools=[get_temperature]), + ) + + async def run_async_methods(): + try: + async_response = await client.aio.models.generate_content( + model=model, + contents="Reply with exactly: async OK", + ) + stream = await client.aio.models.generate_content_stream( + model=model, + contents="Reply with exactly: async stream OK", + ) + async_stream = [chunk async for chunk in stream] + async_embedding = await client.aio.models.embed_content( + model=_EMBEDDING_MODEL, + contents="LoongSuite live async embedding verification", + config=types.EmbedContentConfig(output_dimensionality=8), + ) + async_interaction = await client.aio.interactions.create( + model=_INTERACTIONS_MODEL, + input="Reply with exactly: live async interaction OK", + ) + interaction_stream = await client.aio.interactions.create( + model=_INTERACTIONS_MODEL, + input=( + "Reply with exactly: live async interaction stream OK" + ), + stream=True, + ) + async_interaction_stream = [ + event async for event in interaction_stream + ] + return ( + async_response, + async_stream, + async_embedding, + async_interaction, + async_interaction_stream, + ) + finally: + await client.aio.aclose() + + ( + async_response, + async_stream, + async_embedding, + async_interaction, + async_interaction_stream, + ) = asyncio.run(run_async_methods()) + finally: + client.close() + instrumentor.uninstrument() + + for response in (sync_response, async_response): + assert response.text + assert response.response_id + assert response.usage_metadata is not None + assert response.usage_metadata.prompt_token_count > 0 + assert response.usage_metadata.candidates_token_count > 0 + assert any(chunk.text for chunk in sync_stream) + assert any(chunk.text for chunk in async_stream) + assert len(sync_embedding.embeddings[0].values) == 8 + assert len(async_embedding.embeddings[0].values) == 8 + assert sync_interaction.id + assert async_interaction.id + assert sync_interaction_stream[-1].interaction.id + assert async_interaction_stream[-1].interaction.id + assert "17" in tool_response.text + + spans = span_exporter.get_finished_spans() + generation_spans = [ + span for span in spans if span.name == f"generate_content {model}" + ] + embedding_spans = [ + span for span in spans if span.name == f"embeddings {_EMBEDDING_MODEL}" + ] + interaction_spans = [ + span + for span in spans + if span.name == f"interactions.create {_INTERACTIONS_MODEL}" + ] + tool_spans = [ + span for span in spans if span.name == "execute_tool get_temperature" + ] + assert len(generation_spans) >= 5 + assert len(embedding_spans) == 2 + assert len(interaction_spans) == 4 + assert len(tool_spans) == 1 + assert tool_spans[0].kind is SpanKind.INTERNAL + assert all( + span.attributes["gen_ai.response.id"] for span in generation_spans + ) + assert all( + span.attributes["gen_ai.usage.input_tokens"] > 0 + for span in generation_spans + ) + assert all( + span.attributes["gen_ai.usage.output_tokens"] > 0 + for span in generation_spans + ) + assert all( + span.attributes["gen_ai.embeddings.dimension.count"] == 8 + for span in embedding_spans + ) + assert all( + span.attributes["gen_ai.response.model"] == _INTERACTIONS_MODEL + for span in interaction_spans + ) + streaming_spans = [ + span + for span in generation_spans + interaction_spans + if "gen_ai.response.time_to_first_token" in span.attributes + ] + assert len(streaming_spans) == 4 + assert all( + span.attributes["gen_ai.response.time_to_first_token"] > 0 + for span in streaming_spans + ) + generation_streaming_spans = [ + span + for span in generation_spans + if "gen_ai.response.time_to_first_token" in span.attributes + ] + assert len(generation_streaming_spans) == 2 + expected_stream_outputs = { + "".join(chunk.text or "" for chunk in sync_stream), + "".join(chunk.text or "" for chunk in async_stream), + } + observed_stream_outputs = set() + for span in generation_streaming_spans: + messages = json.loads(span.attributes["gen_ai.output.messages"]) + assert len(messages) == 1 + assert messages[0]["finish_reason"] == "stop" + observed_stream_outputs.add( + "".join( + part["content"] + for part in messages[0]["parts"] + if part["type"] == "text" + ) + ) + assert observed_stream_outputs == expected_stream_outputs + + metric_names = { + metric.name + for resource_metrics in metric_reader.get_metrics_data().resource_metrics + for scope_metrics in resource_metrics.scope_metrics + for metric in scope_metrics.metrics + } + assert "gen_ai.client.operation.duration" in metric_names + assert "gen_ai.client.token.usage" in metric_names diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/test_vcr.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/test_vcr.py new file mode 100644 index 000000000..c5ab75d11 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/test_vcr.py @@ -0,0 +1,287 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Recorded real-API coverage for every instrumented Google GenAI method.""" + +import json +import os +from importlib.metadata import version + +import pytest +from google import genai +from google.genai import types + +from opentelemetry.instrumentation.google_genai import ( + GoogleGenAiSdkInstrumentor, +) +from opentelemetry.sdk import metrics, trace +from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.trace import SpanKind + +_GENERATION_MODEL = "gemini-3.5-flash-lite" +_EMBEDDING_MODEL = "gemini-embedding-001" +_INTERACTIONS_MODEL = _GENERATION_MODEL + +pytestmark = pytest.mark.skipif( + int(version("google-genai").split(".", maxsplit=1)[0]) < 2, + reason="recorded SDK transport contract targets google-genai 2.x", +) + + +@pytest.fixture +def telemetry(): + span_exporter = InMemorySpanExporter() + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + metric_reader = InMemoryMetricReader() + meter_provider = metrics.MeterProvider(metric_readers=[metric_reader]) + instrumentor = GoogleGenAiSdkInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + ) + try: + yield span_exporter, metric_reader + finally: + instrumentor.uninstrument() + + +@pytest.fixture +def client(): + api_key = os.getenv("GEMINI_API_KEY") or os.getenv( + "GOOGLE_API_KEY", "test-google-key" + ) + value = genai.Client(api_key=api_key) + try: + yield value + finally: + value.close() + + +def _assert_generation_telemetry( + span_exporter, + metric_reader, + *, + expected_output=None, +): + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert span.name == f"generate_content {_GENERATION_MODEL}" + assert span.attributes["gen_ai.response.id"] + assert span.attributes["gen_ai.usage.input_tokens"] > 0 + assert span.attributes["gen_ai.usage.output_tokens"] > 0 + if expected_output is not None: + messages = json.loads(span.attributes["gen_ai.output.messages"]) + assert len(messages) == 1 + assert messages[0]["parts"] == [ + {"content": expected_output, "type": "text"} + ] + assert messages[0]["finish_reason"] == "stop" + _assert_standard_metrics(metric_reader) + + +def _assert_embedding_telemetry(span_exporter, metric_reader): + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert span.name == f"embeddings {_EMBEDDING_MODEL}" + assert span.attributes["gen_ai.embeddings.dimension.count"] == 8 + # OSS extended metrics intentionally cover LLM calls only. Embedding + # metrics remain an enterprise extension, while the OSS span is complete. + assert metric_reader.get_metrics_data() is None + + +def _assert_interactions_telemetry( + span_exporter, metric_reader, *, streaming=False +): + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert span.name == f"interactions.create {_INTERACTIONS_MODEL}" + assert span.attributes["gen_ai.operation.name"] == "interactions.create" + assert span.attributes["gen_ai.response.model"] == _INTERACTIONS_MODEL + assert span.attributes["gen_ai.usage.input_tokens"] > 0 + assert span.attributes["gen_ai.usage.output_tokens"] > 0 + if streaming: + assert span.attributes["gen_ai.response.time_to_first_token"] > 0 + _assert_standard_metrics(metric_reader) + + +def _assert_standard_metrics(metric_reader): + metric_names = { + metric.name + for resource_metrics in metric_reader.get_metrics_data().resource_metrics + for scope_metrics in resource_metrics.scope_metrics + for metric in scope_metrics.metrics + } + assert "gen_ai.client.operation.duration" in metric_names + + +@pytest.mark.vcr +def test_sync_generate_content(client, telemetry): + response = client.models.generate_content( + model=_GENERATION_MODEL, + contents="Reply with exactly: sync ok", + ) + assert response.text + _assert_generation_telemetry(*telemetry) + + +@pytest.mark.vcr +def test_sync_generate_content_stream(client, telemetry): + chunks = list( + client.models.generate_content_stream( + model=_GENERATION_MODEL, + contents="Reply with exactly: sync stream ok", + ) + ) + assert any(chunk.text for chunk in chunks) + _assert_generation_telemetry( + *telemetry, + expected_output="sync stream ok", + ) + + +@pytest.mark.vcr +def test_sync_embed_content(client, telemetry): + response = client.models.embed_content( + model=_EMBEDDING_MODEL, + contents="LoongSuite sync embedding verification", + config=types.EmbedContentConfig(output_dimensionality=8), + ) + assert len(response.embeddings[0].values) == 8 + _assert_embedding_telemetry(*telemetry) + + +@pytest.mark.asyncio +@pytest.mark.vcr +async def test_async_generate_content(client, telemetry): + response = await client.aio.models.generate_content( + model=_GENERATION_MODEL, + contents="Reply with exactly: async ok", + ) + assert response.text + _assert_generation_telemetry(*telemetry) + + +@pytest.mark.asyncio +@pytest.mark.vcr +async def test_async_generate_content_stream(client, telemetry): + stream = await client.aio.models.generate_content_stream( + model=_GENERATION_MODEL, + contents="Reply with exactly: async stream ok", + ) + chunks = [chunk async for chunk in stream] + assert any(chunk.text for chunk in chunks) + _assert_generation_telemetry( + *telemetry, + expected_output="async stream ok", + ) + + +@pytest.mark.asyncio +@pytest.mark.vcr +async def test_async_embed_content(client, telemetry): + response = await client.aio.models.embed_content( + model=_EMBEDDING_MODEL, + contents="LoongSuite async embedding verification", + config=types.EmbedContentConfig(output_dimensionality=8), + ) + assert len(response.embeddings[0].values) == 8 + _assert_embedding_telemetry(*telemetry) + + +@pytest.mark.vcr +def test_sync_interactions_create(client, telemetry): + response = client.interactions.create( + model=_INTERACTIONS_MODEL, + input="Reply with exactly: interaction ok", + ) + assert response.id + _assert_interactions_telemetry(*telemetry) + + +@pytest.mark.asyncio +@pytest.mark.vcr +async def test_async_interactions_create(client, telemetry): + response = await client.aio.interactions.create( + model=_INTERACTIONS_MODEL, + input="Reply with exactly: async interaction ok", + ) + assert response.id + _assert_interactions_telemetry(*telemetry) + + +@pytest.mark.vcr +def test_sync_interactions_create_stream(client, telemetry): + events = list( + client.interactions.create( + model=_INTERACTIONS_MODEL, + input="Reply with exactly: interaction stream ok", + stream=True, + ) + ) + assert events[-1].interaction.id + _assert_interactions_telemetry(*telemetry, streaming=True) + + +@pytest.mark.asyncio +@pytest.mark.vcr +async def test_async_interactions_create_stream(client, telemetry): + stream = await client.aio.interactions.create( + model=_INTERACTIONS_MODEL, + input="Reply with exactly: async interaction stream ok", + stream=True, + ) + events = [event async for event in stream] + assert events[-1].interaction.id + _assert_interactions_telemetry(*telemetry, streaming=True) + + +@pytest.mark.vcr +def test_automatic_function_call(client, telemetry): + def get_temperature(city: str) -> str: + """Return the current temperature for a city.""" + + assert city + return "17 C" + + response = client.models.generate_content( + model=_GENERATION_MODEL, + contents=( + "You must call get_temperature for Paris, then reply with only " + "the returned temperature." + ), + config=types.GenerateContentConfig(tools=[get_temperature]), + ) + assert "17" in response.text + + spans = telemetry[0].get_finished_spans() + tool_spans = [ + span for span in spans if span.name == "execute_tool get_temperature" + ] + generation_spans = [ + span + for span in spans + if span.name == f"generate_content {_GENERATION_MODEL}" + ] + assert len(tool_spans) == 1 + assert tool_spans[0].kind is SpanKind.INTERNAL + assert generation_spans + _assert_standard_metrics(telemetry[1]) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/utils/__init__.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/utils/__init__.py new file mode 100644 index 000000000..b0a6f4284 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/utils/test_allowlist_util.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/utils/test_allowlist_util.py new file mode 100644 index 000000000..d2a23d736 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/utils/test_allowlist_util.py @@ -0,0 +1,165 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import os +from unittest import mock + +from opentelemetry.instrumentation.google_genai.allowlist_util import AllowList + + +def test_empty_allowlist_allows_nothing(): + allow_list = AllowList() + assert not allow_list.allowed("") + assert not allow_list.allowed("foo") + assert not allow_list.allowed("bar") + assert not allow_list.allowed("baz") + assert not allow_list.allowed("anything at all") + + +def test_simple_include_allow_list(): + allow_list = AllowList(includes=["abc", "xyz"]) + assert allow_list.allowed("abc") + assert not allow_list.allowed("abc.xyz") + assert allow_list.allowed("xyz") + assert not allow_list.allowed("blah") + assert not allow_list.allowed("other value not in includes") + + +def test_allow_list_with_prefix_matching(): + allow_list = AllowList(includes=["abc.*", "xyz"]) + assert not allow_list.allowed("abc") + assert allow_list.allowed("abc.foo") + assert allow_list.allowed("abc.bar") + assert allow_list.allowed("xyz") + assert not allow_list.allowed("blah") + assert not allow_list.allowed("other value not in includes") + + +def test_allow_list_with_array_wildcard_matching(): + allow_list = AllowList(includes=["abc[*].foo", "xyz[*].*"]) + assert not allow_list.allowed("abc") + assert allow_list.allowed("abc[0].foo") + assert not allow_list.allowed("abc[0].bar") + assert allow_list.allowed("abc[1].foo") + assert allow_list.allowed("xyz[0].blah") + assert allow_list.allowed("xyz[1].yadayada") + assert not allow_list.allowed("blah") + assert not allow_list.allowed("other value not in includes") + + +def test_includes_and_excludes(): + allow_list = AllowList(includes=["abc", "xyz"], excludes=["xyz"]) + assert allow_list.allowed("abc") + assert not allow_list.allowed("xyz") + assert not allow_list.allowed("blah") + assert not allow_list.allowed("other value not in includes") + + +def test_includes_and_excludes_with_wildcards(): + allow_list = AllowList( + includes=["abc", "xyz", "xyz.*"], excludes=["xyz.foo", "xyz.foo.*"] + ) + assert allow_list.allowed("abc") + assert allow_list.allowed("xyz") + assert not allow_list.allowed("xyz.foo") + assert not allow_list.allowed("xyz.foo.bar") + assert not allow_list.allowed("xyz.foo.baz") + assert allow_list.allowed("xyz.not_foo") + assert allow_list.allowed("xyz.blah") + assert not allow_list.allowed("blah") + assert not allow_list.allowed("other value not in includes") + + +def test_default_include_with_excludes(): + allow_list = AllowList(includes=["*"], excludes=["foo", "bar"]) + assert not allow_list.allowed("foo") + assert not allow_list.allowed("bar") + assert allow_list.allowed("abc") + assert allow_list.allowed("xyz") + assert allow_list.allowed("blah") + assert allow_list.allowed("other value not in includes") + + +def test_default_exclude_with_includes(): + allow_list = AllowList(includes=["foo", "bar"], excludes=["*"]) + assert allow_list.allowed("foo") + assert allow_list.allowed("bar") + assert not allow_list.allowed("abc") + assert not allow_list.allowed("xyz") + assert not allow_list.allowed("blah") + assert not allow_list.allowed("other value not in includes") + + +@mock.patch.dict(os.environ, {"TEST_ALLOW_LIST_INCLUDE_KEYS": "abc,xyz"}) +def test_can_load_from_env_with_just_include_list(): + allow_list = AllowList.from_env("TEST_ALLOW_LIST_INCLUDE_KEYS") + assert allow_list.allowed("abc") + assert allow_list.allowed("xyz") + assert not allow_list.allowed("blah") + assert not allow_list.allowed("other value not in includes") + + +@mock.patch.dict( + os.environ, {"TEST_ALLOW_LIST_INCLUDE_KEYS": " abc , , xyz ,"} +) +def test_can_handle_spaces_and_empty_entries(): + allow_list = AllowList.from_env("TEST_ALLOW_LIST_INCLUDE_KEYS") + assert allow_list.allowed("abc") + assert allow_list.allowed("xyz") + assert not allow_list.allowed("") + assert not allow_list.allowed(",") + assert not allow_list.allowed("blah") + assert not allow_list.allowed("other value not in includes") + + +@mock.patch.dict( + os.environ, + { + "TEST_ALLOW_LIST_INCLUDE_KEYS": "abc,xyz", + "TEST_ALLOW_LIST_EXCLUDE_KEYS": "xyz, foo, bar", + }, +) +def test_can_load_from_env_with_includes_and_excludes(): + allow_list = AllowList.from_env( + "TEST_ALLOW_LIST_INCLUDE_KEYS", + excludes_env_var="TEST_ALLOW_LIST_EXCLUDE_KEYS", + ) + assert allow_list.allowed("abc") + assert not allow_list.allowed("xyz") + assert not allow_list.allowed("foo") + assert not allow_list.allowed("bar") + assert not allow_list.allowed("not in the list") + + +@mock.patch.dict( + os.environ, + { + "TEST_ALLOW_LIST_INCLUDE_KEYS": "*", + "TEST_ALLOW_LIST_EXCLUDE_KEYS": "xyz, foo, bar", + }, +) +def test_supports_wildcards_in_loading_from_env(): + allow_list = AllowList.from_env( + "TEST_ALLOW_LIST_INCLUDE_KEYS", + excludes_env_var="TEST_ALLOW_LIST_EXCLUDE_KEYS", + ) + assert allow_list.allowed("abc") + assert not allow_list.allowed("xyz") + assert not allow_list.allowed("foo") + assert not allow_list.allowed("bar") + assert allow_list.allowed("blah") + assert allow_list.allowed("not in the list") diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/utils/test_dict_util.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/utils/test_dict_util.py new file mode 100644 index 000000000..fc6e8e48c --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/utils/test_dict_util.py @@ -0,0 +1,281 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from pydantic import BaseModel + +from opentelemetry.instrumentation.google_genai import dict_util + + +class PydanticModel(BaseModel): + """Used to verify handling of pydantic models in the flattener.""" + + str_value: str = "" + int_value: int = 0 + + +class ModelDumpableNotPydantic: + """Used to verify general handling of 'model_dump'.""" + + def __init__(self, dump_output): + self._dump_output = dump_output + + def model_dump(self): + return self._dump_output + + +class NotJsonSerializable: + def __init__(self): + pass + + +def test_flatten_empty_dict(): + input_dict = {} + output_dict = dict_util.flatten_dict(input_dict, "", set()) + assert output_dict is not None + assert isinstance(output_dict, dict) + assert not output_dict + + +def test_flatten_simple_dict(): + input_dict = { + "int_key": 1, + "string_key": "somevalue", + "float_key": 3.14, + "bool_key": True, + } + assert dict_util.flatten_dict(input_dict, "gcp", set()) == { + "gcp.int_key": 1, + "gcp.string_key": "somevalue", + "gcp.float_key": 3.14, + "gcp.bool_key": True, + } + + +def test_flatten_nested_dict(): + input_dict = { + "int_key": 1, + "string_key": "somevalue", + "float_key": 3.14, + "bool_key": True, + "object_key": { + "nested": { + "foo": 1, + "bar": "baz", + }, + "qux": 54321, + }, + } + assert dict_util.flatten_dict(input_dict, "gcp", set()) == { + "gcp.int_key": 1, + "gcp.string_key": "somevalue", + "gcp.float_key": 3.14, + "gcp.bool_key": True, + "gcp.object_key.nested.foo": 1, + "gcp.object_key.nested.bar": "baz", + "gcp.object_key.qux": 54321, + } + + +def test_flatten_with_key_exclusion(): + input_dict = { + "int_key": 1, + "string_key": "somevalue", + "float_key": 3.14, + "bool_key": True, + } + output = dict_util.flatten_dict(input_dict, "gcp", {"gcp.int_key"}) + assert "gcp.int_key" not in output + assert output == { + "gcp.string_key": "somevalue", + "gcp.float_key": 3.14, + "gcp.bool_key": True, + } + + +def test_flatten_with_prefixing(): + input_dict = { + "int_key": 1, + "string_key": "somevalue", + "float_key": 3.14, + "bool_key": True, + } + output = dict_util.flatten_dict(input_dict, "gcp.someprefix", set()) + assert output == { + "gcp.someprefix.int_key": 1, + "gcp.someprefix.string_key": "somevalue", + "gcp.someprefix.float_key": 3.14, + "gcp.someprefix.bool_key": True, + } + + +def test_flatten_with_pydantic_model_value(): + input_dict = { + "foo": PydanticModel(str_value="bar", int_value=123), + } + + output = dict_util.flatten_dict(input_dict, "gcp", set()) + assert output == { + "gcp.foo.str_value": "bar", + "gcp.foo.int_value": 123, + } + assert dict_util.flatten_dict({"foo": PydanticModel}, "gcp", set()) == { + "gcp.foo": "" + } + + +def test_flatten_with_model_dumpable_value(): + input_dict = { + "foo": ModelDumpableNotPydantic( + { + "str_value": "bar", + "int_value": 123, + } + ), + } + + output = dict_util.flatten_dict(input_dict, "gcp", set()) + assert output == { + "gcp.foo.str_value": "bar", + "gcp.foo.int_value": 123, + } + + +def test_flatten_with_mixed_structures(): + input_dict = { + "foo": ModelDumpableNotPydantic( + { + "pydantic": PydanticModel(str_value="bar", int_value=123), + } + ), + } + + output = dict_util.flatten_dict(input_dict, "gcp", set()) + assert output == { + "gcp.foo.pydantic.str_value": "bar", + "gcp.foo.pydantic.int_value": 123, + } + + +def test_converts_tuple_with_json_fallback(): + input_dict = { + "foo": ("abc", 123), + } + output = dict_util.flatten_dict(input_dict, "gcp", set()) + assert output == { + "gcp.foo.length": 2, + "gcp.foo[0]": "abc", + "gcp.foo[1]": 123, + } + + +def test_json_conversion_handles_unicode(): + input_dict = { + "foo": ("❤️", 123), + } + output = dict_util.flatten_dict(input_dict, "gcp", set()) + assert output == { + "gcp.foo.length": 2, + "gcp.foo[0]": "❤️", + "gcp.foo[1]": 123, + } + + +def test_flatten_with_complex_object_not_json_serializable(): + result = dict_util.flatten_dict( + { + "cannot_serialize_directly": NotJsonSerializable(), + }, + "", + set(), + ) + assert result is not None + assert isinstance(result, dict) + assert len(result) == 0 + + +def test_flatten_good_with_non_serializable_complex_object(): + result = dict_util.flatten_dict( + { + "foo": { + "bar": "blah", + "baz": 5, + }, + "cannot_serialize_directly": NotJsonSerializable(), + }, + "gcp", + set(), + ) + assert result == { + "gcp.foo.bar": "blah", + "gcp.foo.baz": 5, + } + + +def test_flatten_simple_homogenous_primitive_string_list(): + input_dict = {"list_value": ["abc", "def"]} + assert dict_util.flatten_dict(input_dict, "gcp", set()) == { + "gcp.list_value": ["abc", "def"], + } + + +def test_flatten_simple_homogenous_primitive_int_list(): + input_dict = {"list_value": [123, 456]} + assert dict_util.flatten_dict(input_dict, "gcp", set()) == { + "gcp.list_value": [123, 456], + } + + +def test_flatten_simple_homogenous_primitive_bool_list(): + input_dict = {"list_value": [True, False]} + assert dict_util.flatten_dict(input_dict, "gcp", set()) == { + "gcp.list_value": [True, False], + } + + +def test_flatten_simple_heterogenous_primitive_list(): + input_dict = {"list_value": ["abc", 123]} + assert dict_util.flatten_dict(input_dict, "gcp", set()) == { + "gcp.list_value.length": 2, + "gcp.list_value[0]": "abc", + "gcp.list_value[1]": 123, + } + + +def test_flatten_list_of_compound_types(): + input_dict = { + "list_value": [ + {"a": 1, "b": 2}, + {"x": 100, "y": 123, "z": 321}, + "blah", + [ + "abc", + 123, + ], + ] + } + assert dict_util.flatten_dict(input_dict, "gcp", set()) == { + "gcp.list_value.length": 4, + "gcp.list_value[0].a": 1, + "gcp.list_value[0].b": 2, + "gcp.list_value[1].x": 100, + "gcp.list_value[1].y": 123, + "gcp.list_value[1].z": 321, + "gcp.list_value[2]": "blah", + "gcp.list_value[3].length": 2, + "gcp.list_value[3][0]": "abc", + "gcp.list_value[3][1]": 123, + } diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/utils/test_tool_call_wrapper.py b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/utils/test_tool_call_wrapper.py new file mode 100644 index 000000000..db0e5b372 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/utils/test_tool_call_wrapper.py @@ -0,0 +1,246 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +import json +import unittest +from unittest.mock import patch + +from google.genai import types as genai_types + +from opentelemetry._logs import get_logger_provider +from opentelemetry.instrumentation.google_genai import tool_call_wrapper +from opentelemetry.instrumentation.google_genai._compat import TelemetryHandler +from opentelemetry.metrics import get_meter_provider +from opentelemetry.trace import get_tracer_provider + +from ..common import otel_mocker + + +class TestCase(unittest.TestCase): + def setUp(self): + self.env_patcher = patch.dict( + "os.environ", + { + "OTEL_SEMCONV_STABILITY_OPT_IN": "gen_ai_latest_experimental", + }, + ) + self.env_patcher.start() + self._otel = otel_mocker.OTelMocker() + self._otel.install() + self._otel_wrapper = TelemetryHandler( + tracer_provider=get_tracer_provider(), + logger_provider=get_logger_provider(), + meter_provider=get_meter_provider(), + ) + + def tearDown(self): + self._otel.uninstall() + self.env_patcher.stop() + + @property + def otel(self): + return self._otel + + @property + def otel_wrapper(self): + return self._otel_wrapper + + def wrap(self, tool_or_tools): + return tool_call_wrapper.wrapped_tool(tool_or_tools, self.otel_wrapper) + + def test_wraps_none(self): + result = self.wrap(None) + self.assertIsNone(result) + + def test_wraps_multiple_tool_functions_as_list(self): + def somefunction(): + pass + + def otherfunction(): + pass + + wrapped_functions = self.wrap([somefunction, otherfunction]) + wrapped_somefunction = wrapped_functions[0] + wrapped_otherfunction = wrapped_functions[1] + self.otel.assert_does_not_have_span_named("execute_tool somefunction") + self.otel.assert_does_not_have_span_named("execute_tool otherfunction") + somefunction() + otherfunction() + self.otel.assert_does_not_have_span_named("execute_tool somefunction") + self.otel.assert_does_not_have_span_named("execute_tool otherfunction") + wrapped_somefunction() + self.otel.assert_has_span_named("execute_tool somefunction") + self.otel.assert_does_not_have_span_named("execute_tool otherfunction") + wrapped_otherfunction() + self.otel.assert_has_span_named("execute_tool otherfunction") + + def test_wraps_multiple_tool_functions_as_dict(self): + def somefunction(): + pass + + def otherfunction(): + pass + + wrapped_functions = self.wrap( + {"somefunction": somefunction, "otherfunction": otherfunction} + ) + wrapped_somefunction = wrapped_functions["somefunction"] + wrapped_otherfunction = wrapped_functions["otherfunction"] + self.otel.assert_does_not_have_span_named("execute_tool somefunction") + self.otel.assert_does_not_have_span_named("execute_tool otherfunction") + somefunction() + otherfunction() + self.otel.assert_does_not_have_span_named("execute_tool somefunction") + self.otel.assert_does_not_have_span_named("execute_tool otherfunction") + wrapped_somefunction() + self.otel.assert_has_span_named("execute_tool somefunction") + self.otel.assert_does_not_have_span_named("execute_tool otherfunction") + wrapped_otherfunction() + self.otel.assert_has_span_named("execute_tool otherfunction") + + def test_wraps_async_tool_function(self): + async def somefunction(): + pass + + wrapped_somefunction = self.wrap(somefunction) + self.otel.assert_does_not_have_span_named("execute_tool somefunction") + asyncio.run(somefunction()) + self.otel.assert_does_not_have_span_named("execute_tool somefunction") + asyncio.run(wrapped_somefunction()) + self.otel.assert_has_span_named("execute_tool somefunction") + + def test_preserves_tool_dict(self): + tool_dict = genai_types.ToolDict() + wrapped_tool_dict = self.wrap(tool_dict) + self.assertEqual(tool_dict, wrapped_tool_dict) + + def test_does_not_have_description_if_no_doc_string(self): + def somefunction(): + pass + + wrapped_somefunction = self.wrap(somefunction) + self.otel.assert_does_not_have_span_named("execute_tool somefunction") + somefunction() + self.otel.assert_does_not_have_span_named("execute_tool somefunction") + wrapped_somefunction() + self.otel.assert_has_span_named("execute_tool somefunction") + span = self.otel.get_span_named("execute_tool somefunction") + self.assertNotIn("gen_ai.tool.description", span.attributes) + + def test_has_description_if_doc_string_present(self): + def somefunction(): + """An example tool call function.""" + + wrapped_somefunction = self.wrap(somefunction) + self.otel.assert_does_not_have_span_named("execute_tool somefunction") + somefunction() + self.otel.assert_does_not_have_span_named("execute_tool somefunction") + wrapped_somefunction() + self.otel.assert_has_span_named("execute_tool somefunction") + span = self.otel.get_span_named("execute_tool somefunction") + self.assertEqual( + span.attributes["gen_ai.tool.description"], + "An example tool call function.", + ) + + # Capture content must be enabled to get arguments + @patch.dict( + "os.environ", + { + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "SPAN_AND_EVENT", + }, + ) + def test_handles_various_arg_types(self): + def somefunction( + primitive_int=None, + dict_arg=None, + list_arg=None, + heterogenous_list_arg=None, + ): + pass + + wrapped_somefunction = self.wrap(somefunction) + self.otel.assert_does_not_have_span_named("execute_tool somefunction") + somefunction(12345) + self.otel.assert_does_not_have_span_named("execute_tool somefunction") + wrapped_somefunction(12345, {"key": "value"}, [1, 2, 3], [123, "abc"]) + self.otel.assert_has_span_named("execute_tool somefunction") + span = self.otel.get_span_named("execute_tool somefunction") + arguments = json.loads(span.attributes["gen_ai.tool.call.arguments"]) + self.assertEqual( + arguments["code.function.parameters.primitive_int.type"], "int" + ) + self.assertEqual(span.attributes["gen_ai.tool.name"], "somefunction") + self.assertEqual( + arguments["code.function.parameters.primitive_int.value"], 12345 + ) + self.assertEqual( + arguments["code.function.parameters.dict_arg.type"], "dict" + ) + self.assertEqual( + arguments["code.function.parameters.dict_arg.value"], + {"key": "value"}, + ) + self.assertEqual( + arguments["code.function.parameters.list_arg.type"], "list" + ) + self.assertEqual( + arguments["code.function.parameters.list_arg.value"], [1, 2, 3] + ) + self.assertEqual( + arguments["code.function.parameters.heterogenous_list_arg.type"], + "list", + ) + self.assertEqual( + arguments["code.function.parameters.heterogenous_list_arg.value"], + [123, "abc"], + ) + + @patch.dict( + "os.environ", + { + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "NO_CONTENT", + }, + ) + def test_with_capture_content_disabled(self): + def somefunction(arg=None): + return arg + + wrapped_somefunction = self.wrap(somefunction) + wrapped_somefunction("a string value") + span = self.otel.get_span_named("execute_tool somefunction") + + self.assertNotIn( + "gen_ai.tool.call.arguments", + span.attributes, + ) + self.assertNotIn( + "gen_ai.tool.call.result", + span.attributes, + ) + + def test_function_that_throws_exception(self): + def somefunction(arg=None): + raise Exception("Something went wrong") + + wrapped_somefunction = self.wrap(somefunction) + try: + wrapped_somefunction(12345) + except Exception: + span = self.otel.get_span_named("execute_tool somefunction") + self.assertEqual(span.attributes["error.type"], "Exception") diff --git a/loongsuite-distro/src/loongsuite/distro/bootstrap_registry/loongsuite_instrumentation_google_genai.py b/loongsuite-distro/src/loongsuite/distro/bootstrap_registry/loongsuite_instrumentation_google_genai.py new file mode 100644 index 000000000..b6c11d6d3 --- /dev/null +++ b/loongsuite-distro/src/loongsuite/distro/bootstrap_registry/loongsuite_instrumentation_google_genai.py @@ -0,0 +1,30 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT. THIS FILE WAS AUTOGENERATED FROM INSTRUMENTATION PACKAGES. +# RUN `python scripts/loongsuite/generate_loongsuite_bootstrap.py` TO REGENERATE. +# +# Generated with options: +# --upstream-version: (from source) +# --loongsuite-version: (from source) + +REGISTRY = { + "source": "loongsuite", + "package": "loongsuite-instrumentation-google-genai", + "instrumentation": "loongsuite-instrumentation-google-genai==0.8.0.dev", + "libraries": [ + "google-genai >= 1.32.0, < 3", + ], + "default": False, +} diff --git a/tox-loongsuite.ini b/tox-loongsuite.ini index 75558b36d..3906e904d 100644 --- a/tox-loongsuite.ini +++ b/tox-loongsuite.ini @@ -29,6 +29,10 @@ envlist = py3{10,11,12,13}-test-loongsuite-instrumentation-google-adk-{oldest,latest} lint-loongsuite-instrumentation-google-adk + ; loongsuite-instrumentation-google-genai + py3{9,10,11,12,13}-test-loongsuite-instrumentation-google-genai{-oldest,-latest} + lint-loongsuite-instrumentation-google-genai + ; loongsuite-instrumentation-agno py3{9,10,11,12,13}-test-loongsuite-instrumentation-agno lint-loongsuite-instrumentation-agno @@ -181,6 +185,11 @@ deps = google-adk-latest: -r {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-google-adk/tests/requirements.latest.txt lint-loongsuite-instrumentation-google-adk: -r {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-google-adk/tests/requirements.oldest.txt + google-genai-oldest: -r {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/requirements.oldest.txt + google-genai-latest: {[testenv]test_deps} # loongsuite-instrumentation-google-genai + google-genai-latest: -r {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/requirements.latest.txt + lint-loongsuite-instrumentation-google-genai: -r {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests/requirements.oldest.txt + agno: {[testenv]test_deps} agno: -r {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-agno/test-requirements.txt @@ -315,6 +324,9 @@ commands = test-loongsuite-instrumentation-google-adk: pytest {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-google-adk/tests {posargs} lint-loongsuite-instrumentation-google-adk: python -m ruff check {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-google-adk + test-loongsuite-instrumentation-google-genai: pytest {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-google-genai/tests --vcr-record=none {posargs} + lint-loongsuite-instrumentation-google-genai: python -m ruff check {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-google-genai + test-loongsuite-instrumentation-agno: pytest {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-agno/tests {posargs} lint-loongsuite-instrumentation-agno: python -m ruff check {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-agno