diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index d376d4a24..e4e73848b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -33,10 +33,7 @@ services: OPAMP_SERVER_URL: 'http://host.docker.internal:${HYPERDX_OPAMP_PORT}' CUSTOM_OTELCOL_CONFIG_FILE: '/etc/otelcol-contrib/custom.config.yaml' # Uncomment to enable stdout logging for the OTel collector - # OTEL_SUPERVISOR_LOGS: 'true' - # Uncomment to enable JSON schema in ClickHouse - # Be sure to also set BETA_CH_OTEL_JSON_SCHEMA_ENABLED to 'true' in ch-server - # OTEL_AGENT_FEATURE_GATE_ARG: '--feature-gates=clickhouse.json' + OTEL_SUPERVISOR_LOGS: 'true' volumes: - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml - ./docker/otel-collector/supervisor_docker.yaml.tmpl:/etc/otel/supervisor.yaml.tmpl @@ -54,6 +51,37 @@ services: depends_on: ch-server: condition: service_healthy + otel-collector-json: + # image: otel/opentelemetry-collector-contrib:0.120.0 + build: + context: ./docker/otel-collector + target: dev + environment: + CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s' + CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363' + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: 'otel_json' + HYPERDX_API_KEY: ${HYPERDX_API_KEY} + HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} + OPAMP_SERVER_URL: 'http://host.docker.internal:${HYPERDX_OPAMP_PORT}' + CUSTOM_OTELCOL_CONFIG_FILE: '/etc/otelcol-contrib/custom.config.yaml' + # Uncomment to enable stdout logging for the OTel collector + OTEL_SUPERVISOR_LOGS: 'true' + # Uncomment to enable JSON schema in ClickHouse + # Be sure to also set BETA_CH_OTEL_JSON_SCHEMA_ENABLED to 'true' in ch-server + OTEL_AGENT_FEATURE_GATE_ARG: '--feature-gates=clickhouse.json' + volumes: + - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml + - ./docker/otel-collector/supervisor_docker.yaml.tmpl:/etc/otel/supervisor.yaml.tmpl + # Add a custom config file + - ./docker/otel-collector/dev.json.config.yaml:/etc/otelcol-contrib/custom.config.yaml + ports: + - '14318:4318' # OTLP http receiver + restart: always + networks: + - internal + depends_on: + ch-server: + condition: service_healthy ch-server: image: clickhouse/clickhouse-server:25.7-alpine ports: diff --git a/docker/otel-collector/custom.config.yaml b/docker/otel-collector/custom.config.yaml index 2f5714379..97f4e4765 100644 --- a/docker/otel-collector/custom.config.yaml +++ b/docker/otel-collector/custom.config.yaml @@ -15,11 +15,45 @@ receivers: # timeout: 10s # memory_limiter: # limit_mib: 2000 +exporters: + # fork data to otel-collector-json service (DEV ONLY) + otlphttp/json: + endpoint: http://otel-collector-json:4318 + headers: + authorization: "super-secure-ingestion-api-key" + compression: gzip service: pipelines: metrics/hostmetrics: - receivers: [hostmetrics] - # attach existing processors - processors: [memory_limiter, batch] - # attach existing exporters - exporters: [clickhouse] + exporters: + - clickhouse + processors: + - memory_limiter + - batch + receivers: + - hostmetrics + traces/json: + exporters: + - otlphttp/json + processors: + - memory_limiter + - batch + receivers: + - otlp/hyperdx + # Metrics hasn't used JSON attributes yet + # metrics/json: + # exporters: + # - otlphttp/json + # processors: + # - memory_limiter + # - batch + # receivers: + # - otlp/hyperdx + logs/json: + exporters: + - otlphttp/json + processors: + - memory_limiter + - batch + receivers: + - otlp/hyperdx diff --git a/docker/otel-collector/dev.json.config.yaml b/docker/otel-collector/dev.json.config.yaml new file mode 100644 index 000000000..6fd1563b2 --- /dev/null +++ b/docker/otel-collector/dev.json.config.yaml @@ -0,0 +1,11 @@ +service: + pipelines: + # ignore rrweb events + logs/out-rrweb: + exporters: + - nop + processors: + - memory_limiter + - batch + receivers: + - routing/logs diff --git a/packages/api/.env.development b/packages/api/.env.development index 95c18e480..ac5ef1aa3 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -22,7 +22,7 @@ USAGE_STATS_ENABLED=false NODE_OPTIONS="--max-http-header-size=131072" ENABLE_SWAGGER=true DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:8123","username":"default","password":""}] -DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","bodyExpression":"SpanName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"}] +DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","bodyExpression":"SpanName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","bodyExpression":"SpanName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"}] INGESTION_API_KEY="super-secure-ingestion-api-key" HYPERDX_API_KEY=$INGESTION_API_KEY ANTHROPIC_API_KEY="your-anthropic-api-key-here" diff --git a/packages/api/src/opamp/controllers/opampController.ts b/packages/api/src/opamp/controllers/opampController.ts index b838bd39d..d6f1cb33b 100644 --- a/packages/api/src/opamp/controllers/opampController.ts +++ b/packages/api/src/opamp/controllers/opampController.ts @@ -67,6 +67,7 @@ type CollectorConfig = { }; }; exporters?: { + nop?: null; debug?: { verbosity: string; sampling_initial: number; @@ -184,6 +185,7 @@ export const buildOtelCollectorConfig = (teams: ITeam[]): CollectorConfig => { }, }, exporters: { + nop: null, debug: { verbosity: 'detailed', sampling_initial: 5,