Skip to content

fix(logger): auto-convert non-string metadata values to strings #488

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
calebe94 wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

fix(logger): auto-convert non-string metadata values to strings #488

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 38 additions & 8 deletions src/galileo/logger/logger.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -864,7 +864,7 @@ def add_single_llm_span_trace(
name: Optional[str] = None,
created_at: Optional[datetime] = None,
duration_ns: Optional[int] = None,
metadata: Optional[dict[str, str]] = None,
metadata: Optional[dict[str, MetadataValue]] = None,
tags: Optional[list[str]] = None,
num_input_tokens: Optional[int] = None,
num_output_tokens: Optional[int] = None,
Expand All @@ -874,7 +874,7 @@ def add_single_llm_span_trace(
time_to_first_token_ns: Optional[int] = None,
dataset_input: Optional[str] = None,
dataset_output: Optional[str] = None,
dataset_metadata: Optional[dict[str, str]] = None,
dataset_metadata: Optional[dict[str, MetadataValue]] = None,
span_step_number: Optional[int] = None,
) -> Trace:
"""
Expand Down Expand Up @@ -958,6 +958,12 @@ def add_single_llm_span_trace(
Trace
The created trace.
"""
# Auto-convert non-string metadata values to strings
if metadata:
metadata = {k: GalileoLogger._convert_metadata_value(v) for k, v in metadata.items()}
if dataset_metadata:
dataset_metadata = {k: GalileoLogger._convert_metadata_value(v) for k, v in dataset_metadata.items()}

trace = super().add_single_llm_span_trace(
input=input,
output=output,
Expand Down Expand Up @@ -1003,7 +1009,7 @@ def add_llm_span(
name: Optional[str] = None,
created_at: Optional[datetime] = None,
duration_ns: Optional[int] = None,
metadata: Optional[dict[str, str]] = None,
metadata: Optional[dict[str, MetadataValue]] = None,
tags: Optional[list[str]] = None,
num_input_tokens: Optional[int] = None,
num_output_tokens: Optional[int] = None,
Expand Down Expand Up @@ -1087,6 +1093,10 @@ def add_llm_span(
LlmSpan
The created span.
"""
# Auto-convert non-string metadata values to strings
if metadata:
metadata = {k: GalileoLogger._convert_metadata_value(v) for k, v in metadata.items()}

kwargs = {
"input": input,
"output": output,
Expand Down Expand Up @@ -1128,7 +1138,7 @@ def add_retriever_span(
name: Optional[str] = None,
duration_ns: Optional[int] = None,
created_at: Optional[datetime] = None,
metadata: Optional[dict[str, str]] = None,
metadata: Optional[dict[str, MetadataValue]] = None,
tags: Optional[list[str]] = None,
status_code: Optional[int] = None,
step_number: Optional[int] = None,
Expand Down Expand Up @@ -1167,6 +1177,10 @@ def add_retriever_span(
documents = convert_to_documents(output, "output")
redacted_documents = convert_to_documents(redacted_output, "redacted_output")

# Auto-convert non-string metadata values to strings
if metadata:
metadata = {k: GalileoLogger._convert_metadata_value(v) for k, v in metadata.items()}

kwargs = {
"input": input,
"documents": documents,
Expand Down Expand Up @@ -1199,7 +1213,7 @@ def add_tool_span(
name: Optional[str] = None,
duration_ns: Optional[int] = None,
created_at: Optional[datetime] = None,
metadata: Optional[dict[str, str]] = None,
metadata: Optional[dict[str, MetadataValue]] = None,
tags: Optional[list[str]] = None,
status_code: Optional[int] = None,
tool_call_id: Optional[str] = None,
Expand Down Expand Up @@ -1251,6 +1265,10 @@ def add_tool_span(
ToolSpan
The created span.
"""
# Auto-convert non-string metadata values to strings
if metadata:
metadata = {k: GalileoLogger._convert_metadata_value(v) for k, v in metadata.items()}

kwargs = {
"input": input,
"redacted_input": redacted_input,
Expand Down Expand Up @@ -1282,7 +1300,7 @@ def add_protect_span(
response: Optional[Response] = None,
redacted_response: Optional[Response] = None,
created_at: Optional[datetime] = None,
metadata: Optional[dict[str, str]] = None,
metadata: Optional[dict[str, MetadataValue]] = None,
tags: Optional[list[str]] = None,
status_code: Optional[int] = None,
step_number: Optional[int] = None,
Expand Down Expand Up @@ -1325,6 +1343,10 @@ def add_protect_span(
ToolSpan
The created Protect tool span.
"""
# Auto-convert non-string metadata values to strings
if metadata:
metadata = {k: GalileoLogger._convert_metadata_value(v) for k, v in metadata.items()}

kwargs = {
"input": json.dumps(payload.model_dump(mode="json")),
"redacted_input": json.dumps(redacted_payload.model_dump(mode="json")) if redacted_payload else None,
Expand Down Expand Up @@ -1359,7 +1381,7 @@ def add_workflow_span(
name: Optional[str] = None,
duration_ns: Optional[int] = None,
created_at: Optional[datetime] = None,
metadata: Optional[dict[str, str]] = None,
metadata: Optional[dict[str, MetadataValue]] = None,
tags: Optional[list[str]] = None,
step_number: Optional[int] = None,
status_code: Optional[int] = None,
Expand Down Expand Up @@ -1408,6 +1430,10 @@ def add_workflow_span(
WorkflowSpan
The created span.
"""
# Auto-convert non-string metadata values to strings
if metadata:
metadata = {k: GalileoLogger._convert_metadata_value(v) for k, v in metadata.items()}

kwargs = {
"input": input,
"redacted_input": redacted_input,
Expand Down Expand Up @@ -1442,7 +1468,7 @@ def add_agent_span(
name: Optional[str] = None,
duration_ns: Optional[int] = None,
created_at: Optional[datetime] = None,
metadata: Optional[dict[str, str]] = None,
metadata: Optional[dict[str, MetadataValue]] = None,
tags: Optional[list[str]] = None,
agent_type: Optional[AgentType] = None,
step_number: Optional[int] = None,
Expand Down Expand Up @@ -1494,6 +1520,10 @@ def add_agent_span(
AgentSpan
The created span.
"""
# Auto-convert non-string metadata values to strings
if metadata:
metadata = {k: GalileoLogger._convert_metadata_value(v) for k, v in metadata.items()}

kwargs = {
"input": input,
"redacted_input": redacted_input,
Expand Down
120 changes: 120 additions & 0 deletions tests/test_logger_batch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1754,6 +1754,126 @@ def test_start_trace_auto_conversion(
assert getattr(payload_trace, attr) == expected_value, f"payload.{attr} mismatch"


@pytest.mark.parametrize(
"span_method,span_kwargs,expected_metadata",
[
pytest.param(
"add_llm_span",
{"input": "prompt", "output": "response", "model": "gpt-4o"},
{"intMeta": "1", "boolMeta": "True", "ratio": "3.14", "name": "test"},
id="llm_span",
),
pytest.param(
"add_retriever_span",
{"input": "query", "output": [Document(content="doc1")]},
{"intMeta": "1", "boolMeta": "True", "ratio": "3.14", "name": "test"},
id="retriever_span",
),
pytest.param(
"add_tool_span",
{"input": "tool input"},
{"intMeta": "1", "boolMeta": "True", "ratio": "3.14", "name": "test"},
id="tool_span",
),
pytest.param(
"add_workflow_span",
{"input": "workflow input"},
{"intMeta": "1", "boolMeta": "True", "ratio": "3.14", "name": "test"},
id="workflow_span",
),
pytest.param(
"add_agent_span",
{"input": "agent input"},
{"intMeta": "1", "boolMeta": "True", "ratio": "3.14", "name": "test"},
id="agent_span",
),
],
)
def test_span_metadata_auto_conversion(
span_method: str,
span_kwargs: dict,
expected_metadata: dict,
) -> None:
"""Test that span methods auto-convert non-string metadata values to strings.

Regression test for Shortcut #54947: passing int/bool metadata to span methods
caused a silent Pydantic ValidationError, dropping the span entirely.
"""
# Given: a logger with an active trace and non-string metadata values
ingestion_hook = Mock()
logger = GalileoLogger(project="my_project", log_stream="my_log_stream", ingestion_hook=ingestion_hook)
logger.start_trace(input="test input")

non_string_metadata = {"intMeta": 1, "boolMeta": True, "ratio": 3.14, "name": "test"}

# When: adding a span with non-string metadata
method = getattr(logger, span_method)
span = method(**span_kwargs, metadata=non_string_metadata)

# Then: the span is created successfully with metadata converted to strings
assert span is not None, f"{span_method} returned None — metadata conversion failed"
assert span.user_metadata == expected_metadata, f"{span_method} metadata mismatch"


@pytest.mark.parametrize(
"span_method,span_kwargs",
[
pytest.param("add_llm_span", {"input": "prompt", "output": "response", "model": "gpt-4o"}, id="llm_span"),
pytest.param("add_tool_span", {"input": "tool input"}, id="tool_span"),
pytest.param("add_workflow_span", {"input": "workflow input"}, id="workflow_span"),
pytest.param("add_agent_span", {"input": "agent input"}, id="agent_span"),
],
)
def test_span_metadata_none_values_converted(
span_method: str,
span_kwargs: dict,
) -> None:
"""Test that None metadata values are converted to the string 'None'."""
# Given: a logger with an active trace and metadata containing None values
ingestion_hook = Mock()
logger = GalileoLogger(project="my_project", log_stream="my_log_stream", ingestion_hook=ingestion_hook)
logger.start_trace(input="test input")

metadata_with_none = {"key1": "value1", "key2": None, "key3": 42}

# When: adding a span with metadata containing None
method = getattr(logger, span_method)
span = method(**span_kwargs, metadata=metadata_with_none)

# Then: the span is created with None converted to string "None"
assert span is not None, f"{span_method} returned None"
assert span.user_metadata["key1"] == "value1"
assert span.user_metadata["key2"] == "None"
assert span.user_metadata["key3"] == "42"


def test_add_single_llm_span_trace_metadata_auto_conversion() -> None:
"""Test that add_single_llm_span_trace auto-converts non-string metadata and dataset_metadata."""
# Given: non-string metadata and dataset_metadata values
ingestion_hook = Mock()
logger = GalileoLogger(project="my_project", log_stream="my_log_stream", ingestion_hook=ingestion_hook)

non_string_metadata = {"intMeta": 1, "boolMeta": True, "strMeta": "physics"}
non_string_dataset_metadata = {"enabled": True, "count": 42}

# When: creating a single LLM span trace with non-string metadata
trace = logger.add_single_llm_span_trace(
input="prompt",
output="response",
model="gpt-4o",
metadata=non_string_metadata,
dataset_metadata=non_string_dataset_metadata,
)

# Then: both metadata dicts are converted to strings
assert trace is not None, "add_single_llm_span_trace returned None"
assert trace.user_metadata == {"intMeta": "1", "boolMeta": "True", "strMeta": "physics"}
assert trace.dataset_metadata == {"enabled": "True", "count": "42"}

# Then: the child span also has converted metadata
llm_span = trace.spans[0]
assert llm_span.user_metadata == {"intMeta": "1", "boolMeta": "True", "strMeta": "physics"}

class TestMultipleLoggerInstanceIsolation:
"""Test that multiple GalileoLogger instances have fully isolated state.

Expand Down
Loading