From c1d098be2d594618f6a407ae8139373d51b52294 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Thu, 23 Oct 2025 18:58:42 +0000 Subject: [PATCH 1/4] Add docker_otel_input package following OpenTelemetry input pattern This package implements a Docker Stats input using the OpenTelemetry Collector's dockerstats receiver, following the pattern established in PR #14315. Key features: - Type: integration with otelcol input (not content package) - Configurable collection interval, endpoint, and filtering - Comprehensive field definitions for container metrics - Full documentation and test policy Resolves: elastic/integrations#15731 Co-authored-by: William Easton --- packages/docker_otel_input/LICENSE.txt | 93 ++++++++++ .../_dev/test/policy/test-docker-stats.yml | 3 + .../agent/input/input.yml.hbs | 37 ++++ packages/docker_otel_input/changelog.yml | 5 + packages/docker_otel_input/docs/README.md | 162 ++++++++++++++++++ .../docker_otel_input/fields/base-fields.yml | 84 +++++++++ .../docker_otel_input/img/docker_logo.svg | 1 + packages/docker_otel_input/manifest.yml | 75 ++++++++ 8 files changed, 460 insertions(+) create mode 100644 packages/docker_otel_input/LICENSE.txt create mode 100644 packages/docker_otel_input/_dev/test/policy/test-docker-stats.yml create mode 100644 packages/docker_otel_input/agent/input/input.yml.hbs create mode 100644 packages/docker_otel_input/changelog.yml create mode 100644 packages/docker_otel_input/docs/README.md create mode 100644 packages/docker_otel_input/fields/base-fields.yml create mode 100644 packages/docker_otel_input/img/docker_logo.svg create mode 100644 packages/docker_otel_input/manifest.yml diff --git a/packages/docker_otel_input/LICENSE.txt b/packages/docker_otel_input/LICENSE.txt new file mode 100644 index 00000000000..d317b57b294 --- /dev/null +++ b/packages/docker_otel_input/LICENSE.txt @@ -0,0 +1,93 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license + +## Acceptance + +By using the software, you agree to all of the terms and conditions below. + +## Copyright License + +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. + +## Limitations + +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor's trademarks is subject +to applicable law. + +## Patents + +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. + +## Notices + +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. + +## No Other Rights + +These terms do not imply any licenses other than those expressly granted in +these terms. + +## Termination + +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. + +## No Liability + +*As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim.* + +## Definitions + +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. + +**you** refers to the individual or entity agreeing to these terms. + +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. **control** means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. + +**your licenses** are all the licenses granted to you for the software under +these terms. + +**use** means anything you do with the software requiring one of your licenses. + +**trademark** means trademarks, service marks, and similar rights. diff --git a/packages/docker_otel_input/_dev/test/policy/test-docker-stats.yml b/packages/docker_otel_input/_dev/test/policy/test-docker-stats.yml new file mode 100644 index 00000000000..574e375e564 --- /dev/null +++ b/packages/docker_otel_input/_dev/test/policy/test-docker-stats.yml @@ -0,0 +1,3 @@ +vars: + period: "10s" + endpoint: "unix:///var/run/docker.sock" diff --git a/packages/docker_otel_input/agent/input/input.yml.hbs b/packages/docker_otel_input/agent/input/input.yml.hbs new file mode 100644 index 00000000000..a0b14173f54 --- /dev/null +++ b/packages/docker_otel_input/agent/input/input.yml.hbs @@ -0,0 +1,37 @@ +receivers: + dockerstats: +{{#if period}} + collection_interval: {{period}} +{{/if}} +{{#if endpoint}} + endpoint: {{endpoint}} +{{/if}} +{{#if excluded_images}} + excluded_images: +{{#each excluded_images as |image|}} + - {{image}} +{{/each}} +{{/if}} +{{#if timeout}} + timeout: {{timeout}} +{{/if}} +{{#if api_version}} + api_version: {{api_version}} +{{/if}} +{{#if provide_per_core_cpu_metrics}} + provide_per_core_cpu_metrics: {{provide_per_core_cpu_metrics}} +{{/if}} +{{#if container_labels_to_metric_labels}} + container_labels_to_metric_labels: +{{container_labels_to_metric_labels}} +{{/if}} +{{#if env_vars_to_metric_labels}} + env_vars_to_metric_labels: +{{env_vars_to_metric_labels}} +{{/if}} + +service: + pipelines: + metrics: + receivers: + - dockerstats diff --git a/packages/docker_otel_input/changelog.yml b/packages/docker_otel_input/changelog.yml new file mode 100644 index 00000000000..d0ecbd2299f --- /dev/null +++ b/packages/docker_otel_input/changelog.yml @@ -0,0 +1,5 @@ +- version: "0.0.1" + changes: + - description: Initial draft of Docker Stats OpenTelemetry input package + type: enhancement + link: https://github.com/elastic/integrations/issues/15731 diff --git a/packages/docker_otel_input/docs/README.md b/packages/docker_otel_input/docs/README.md new file mode 100644 index 00000000000..e02838e0824 --- /dev/null +++ b/packages/docker_otel_input/docs/README.md @@ -0,0 +1,162 @@ +# Docker Stats (OpenTelemetry) + +This integration collects Docker container metrics using the OpenTelemetry Collector's [dockerstats receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/dockerstatsreceiver). + +## Overview + +The Docker Stats receiver collects container metrics from the Docker daemon using the Docker Stats API. It provides comprehensive metrics about container resource usage including CPU, memory, network, and block I/O statistics. + +## Requirements + +- Docker API version 1.22 or higher +- Access to the Docker daemon socket (default: `unix:///var/run/docker.sock`) +- Not supported on Darwin/Windows platforms +- OpenTelemetry Collector integration enabled in your Elastic Agent + +## Compatibility + +This integration uses the OpenTelemetry dockerstats receiver which is currently in **alpha** stability. The configuration and metrics may change between versions. + +## Configuration + +### Collection Interval + +How often to collect metrics from Docker containers. Default is `10s`. + +### Docker Endpoint + +The endpoint of the Docker daemon. Default is `unix:///var/run/docker.sock`. + +For remote Docker hosts, you can use TCP endpoints like `tcp://docker-host:2375`. + +### Excluded Images + +A list of container image names to exclude from metrics collection. Supports wildcards. + +Example: +``` +- "nginx:*" +- "redis:latest" +``` + +### Timeout + +Timeout for Docker API requests. Default is `5s`. + +### API Version + +The Docker API version to use. Default is `1.22` (the minimum supported version). + +### Per-Core CPU Metrics + +When enabled, provides CPU metrics broken down by individual CPU cores. Default is `false`. + +**Note:** Enabling this option can significantly increase the number of metrics collected. + +### Container Labels to Metric Labels + +Map container labels to metric resource attributes. This allows you to add custom dimensions to your metrics based on container labels. + +Example YAML format: +```yaml +my.container.label: my_metric_label +app.version: version +``` + +### Environment Variables to Metric Labels + +Map container environment variables to metric resource attributes. + +Example YAML format: +```yaml +MY_ENV_VAR: my_metric_label +APP_VERSION: version +``` + +## Metrics + +This integration collects the following types of metrics: + +### CPU Metrics + +- Total CPU usage (nanoseconds) +- Kernel mode CPU usage (nanoseconds) +- User mode CPU usage (nanoseconds) +- CPU limit + +### Memory Metrics + +- Memory usage (bytes) +- Memory limit (bytes) +- Memory cache (bytes) +- Memory RSS (bytes) + +### Network Metrics + +- Bytes received/transmitted +- Packets received/transmitted +- Receive/transmit errors +- Receive/transmit dropped packets + +### Block I/O Metrics + +- Bytes read/written +- Read/write operations + +### Metadata + +- Container ID +- Container name +- Image name +- Container runtime + +## Example Configuration + +```yaml +receivers: + dockerstats: + collection_interval: 10s + endpoint: unix:///var/run/docker.sock + excluded_images: + - "nginx:*" + timeout: 5s + api_version: "1.22" + provide_per_core_cpu_metrics: false + +service: + pipelines: + metrics: + receivers: + - dockerstats +``` + +## Troubleshooting + +### Permission Denied + +If you see "permission denied" errors, ensure the Elastic Agent has access to the Docker socket. You may need to add the agent user to the `docker` group: + +```bash +sudo usermod -aG docker elastic-agent +``` + +### No Metrics Collected + +- Verify Docker is running: `docker ps` +- Check the Docker endpoint configuration +- Ensure the API version is compatible with your Docker installation +- Review Elastic Agent logs for errors + +### High Cardinality + +If you're experiencing high cardinality issues: + +- Disable per-core CPU metrics +- Use the `excluded_images` option to filter out noisy containers +- Be selective with container labels and environment variables mapped to metric labels + +## Additional Resources + +- [OpenTelemetry dockerstats receiver documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/dockerstatsreceiver) +- [Docker Stats API documentation](https://docs.docker.com/engine/api/v1.43/#tag/Container/operation/ContainerStats) +- [Elastic Observability documentation](https://www.elastic.co/guide/en/observability/current/index.html) diff --git a/packages/docker_otel_input/fields/base-fields.yml b/packages/docker_otel_input/fields/base-fields.yml new file mode 100644 index 00000000000..2a7de921ade --- /dev/null +++ b/packages/docker_otel_input/fields/base-fields.yml @@ -0,0 +1,84 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: container.id + type: keyword + description: Container ID. +- name: container.name + type: keyword + description: Container name. +- name: container.image.name + type: keyword + description: Container image name. +- name: container.runtime + type: keyword + description: Container runtime. +- name: container.cpu.usage + type: long + description: Total CPU usage in nanoseconds. +- name: container.cpu.kernel.usage + type: long + description: Kernel CPU usage in nanoseconds. +- name: container.cpu.user.usage + type: long + description: User mode CPU usage in nanoseconds. +- name: container.cpu.limit + type: long + description: CPU limit. +- name: container.memory.usage + type: long + description: Memory usage in bytes. +- name: container.memory.limit + type: long + description: Memory limit in bytes. +- name: container.memory.cache + type: long + description: Memory cache in bytes. +- name: container.memory.rss + type: long + description: Memory RSS in bytes. +- name: container.network.io.usage.rx_bytes + type: long + description: Network bytes received. +- name: container.network.io.usage.tx_bytes + type: long + description: Network bytes transmitted. +- name: container.network.io.usage.rx_packets + type: long + description: Network packets received. +- name: container.network.io.usage.tx_packets + type: long + description: Network packets transmitted. +- name: container.network.io.usage.rx_errors + type: long + description: Network receive errors. +- name: container.network.io.usage.tx_errors + type: long + description: Network transmit errors. +- name: container.network.io.usage.rx_dropped + type: long + description: Network packets dropped on receive. +- name: container.network.io.usage.tx_dropped + type: long + description: Network packets dropped on transmit. +- name: container.blockio.io_service_bytes_recursive.read + type: long + description: Block I/O bytes read. +- name: container.blockio.io_service_bytes_recursive.write + type: long + description: Block I/O bytes written. +- name: container.blockio.io_serviced_recursive.read + type: long + description: Block I/O read operations. +- name: container.blockio.io_serviced_recursive.write + type: long + description: Block I/O write operations. diff --git a/packages/docker_otel_input/img/docker_logo.svg b/packages/docker_otel_input/img/docker_logo.svg new file mode 100644 index 00000000000..b61583f95d7 --- /dev/null +++ b/packages/docker_otel_input/img/docker_logo.svg @@ -0,0 +1 @@ + diff --git a/packages/docker_otel_input/manifest.yml b/packages/docker_otel_input/manifest.yml new file mode 100644 index 00000000000..3bb67885e05 --- /dev/null +++ b/packages/docker_otel_input/manifest.yml @@ -0,0 +1,75 @@ +format_version: 3.4.0 +name: docker_otel_input +title: "Docker Stats (OpenTelemetry)" +version: 0.0.1 +description: "Collect Docker container metrics using OpenTelemetry Collector" +type: integration +categories: + - containers + - monitoring + - observability + - opentelemetry +conditions: + kibana: + version: "^9.0.0" + elastic: + subscription: "basic" +policy_templates: + - name: docker_stats + title: Docker Stats + description: Collect Docker container metrics using OpenTelemetry dockerstats receiver + inputs: + - type: otelcol + title: Docker Stats Metrics + description: Collect Docker container performance metrics including CPU, memory, network, and block I/O + template_path: input.yml.hbs + vars: + - name: period + type: text + title: Collection Interval + description: How often to collect metrics (e.g., 10s, 1m) + default: 10s + required: true + - name: endpoint + type: text + title: Docker Endpoint + description: Docker daemon endpoint + default: "unix:///var/run/docker.sock" + required: false + - name: excluded_images + type: text + title: Excluded Images + description: List of image names to exclude from metrics collection + multi: true + required: false + - name: timeout + type: text + title: Timeout + description: Timeout for Docker API requests + default: 5s + required: false + - name: api_version + type: text + title: API Version + description: Docker API version to use + default: "1.22" + required: false + - name: provide_per_core_cpu_metrics + type: bool + title: Provide Per-Core CPU Metrics + description: Enable per-core CPU metrics collection + default: false + required: false + - name: container_labels_to_metric_labels + type: yaml + title: Container Labels to Metric Labels + description: Mapping of container labels to metric labels (YAML format) + required: false + - name: env_vars_to_metric_labels + type: yaml + title: Environment Variables to Metric Labels + description: Mapping of environment variables to metric labels (YAML format) + required: false +owner: + github: elastic/obs-infraobs-integrations + type: elastic From 10587985c1e34af171a25c996dac86b9ba59dd5c Mon Sep 17 00:00:00 2001 From: William Easton Date: Thu, 23 Oct 2025 15:09:27 -0500 Subject: [PATCH 2/4] updates to match httpcheck --- packages/docker_otel_input/manifest.yml | 106 ++++++++++++------------ 1 file changed, 52 insertions(+), 54 deletions(-) diff --git a/packages/docker_otel_input/manifest.yml b/packages/docker_otel_input/manifest.yml index 3bb67885e05..f4bbc4408ad 100644 --- a/packages/docker_otel_input/manifest.yml +++ b/packages/docker_otel_input/manifest.yml @@ -3,7 +3,7 @@ name: docker_otel_input title: "Docker Stats (OpenTelemetry)" version: 0.0.1 description: "Collect Docker container metrics using OpenTelemetry Collector" -type: integration +type: input categories: - containers - monitoring @@ -17,59 +17,57 @@ conditions: policy_templates: - name: docker_stats title: Docker Stats - description: Collect Docker container metrics using OpenTelemetry dockerstats receiver - inputs: - - type: otelcol - title: Docker Stats Metrics - description: Collect Docker container performance metrics including CPU, memory, network, and block I/O - template_path: input.yml.hbs - vars: - - name: period - type: text - title: Collection Interval - description: How often to collect metrics (e.g., 10s, 1m) - default: 10s - required: true - - name: endpoint - type: text - title: Docker Endpoint - description: Docker daemon endpoint - default: "unix:///var/run/docker.sock" - required: false - - name: excluded_images - type: text - title: Excluded Images - description: List of image names to exclude from metrics collection - multi: true - required: false - - name: timeout - type: text - title: Timeout - description: Timeout for Docker API requests - default: 5s - required: false - - name: api_version - type: text - title: API Version - description: Docker API version to use - default: "1.22" - required: false - - name: provide_per_core_cpu_metrics - type: bool - title: Provide Per-Core CPU Metrics - description: Enable per-core CPU metrics collection - default: false - required: false - - name: container_labels_to_metric_labels - type: yaml - title: Container Labels to Metric Labels - description: Mapping of container labels to metric labels (YAML format) - required: false - - name: env_vars_to_metric_labels - type: yaml - title: Environment Variables to Metric Labels - description: Mapping of environment variables to metric labels (YAML format) - required: false + description: Collect Docker container performance metrics including CPU, memory, network, and block I/O + input: otelcol + type: logs + template_path: input.yml.hbs + vars: + - name: period + type: text + title: Collection Interval + description: How often to collect metrics (e.g., 10s, 1m) + default: 10s + required: true + - name: endpoint + type: text + title: Docker Endpoint + description: Docker daemon endpoint + default: "unix:///var/run/docker.sock" + required: false + - name: excluded_images + type: text + title: Excluded Images + description: List of image names to exclude from metrics collection + multi: true + required: false + - name: timeout + type: text + title: Timeout + description: Timeout for Docker API requests + default: 5s + required: false + - name: api_version + type: text + title: API Version + description: Docker API version to use + default: "1.22" + required: false + - name: provide_per_core_cpu_metrics + type: bool + title: Provide Per-Core CPU Metrics + description: Enable per-core CPU metrics collection + default: false + required: false + - name: container_labels_to_metric_labels + type: yaml + title: Container Labels to Metric Labels + description: Mapping of container labels to metric labels (YAML format) + required: false + - name: env_vars_to_metric_labels + type: yaml + title: Environment Variables to Metric Labels + description: Mapping of environment variables to metric labels (YAML format) + required: false owner: github: elastic/obs-infraobs-integrations type: elastic From f3b50cff51138387ff9900bd8ef1a22ace7cf44b Mon Sep 17 00:00:00 2001 From: William Easton Date: Thu, 23 Oct 2025 15:19:53 -0500 Subject: [PATCH 3/4] Update manifest --- .../docker_otel_input/fields/base-fields.yml | 74 +------------------ packages/docker_otel_input/manifest.yml | 6 +- 2 files changed, 4 insertions(+), 76 deletions(-) diff --git a/packages/docker_otel_input/fields/base-fields.yml b/packages/docker_otel_input/fields/base-fields.yml index 2a7de921ade..4da5428ca93 100644 --- a/packages/docker_otel_input/fields/base-fields.yml +++ b/packages/docker_otel_input/fields/base-fields.yml @@ -9,76 +9,4 @@ description: Data stream namespace. - name: '@timestamp' type: date - description: Event timestamp. -- name: container.id - type: keyword - description: Container ID. -- name: container.name - type: keyword - description: Container name. -- name: container.image.name - type: keyword - description: Container image name. -- name: container.runtime - type: keyword - description: Container runtime. -- name: container.cpu.usage - type: long - description: Total CPU usage in nanoseconds. -- name: container.cpu.kernel.usage - type: long - description: Kernel CPU usage in nanoseconds. -- name: container.cpu.user.usage - type: long - description: User mode CPU usage in nanoseconds. -- name: container.cpu.limit - type: long - description: CPU limit. -- name: container.memory.usage - type: long - description: Memory usage in bytes. -- name: container.memory.limit - type: long - description: Memory limit in bytes. -- name: container.memory.cache - type: long - description: Memory cache in bytes. -- name: container.memory.rss - type: long - description: Memory RSS in bytes. -- name: container.network.io.usage.rx_bytes - type: long - description: Network bytes received. -- name: container.network.io.usage.tx_bytes - type: long - description: Network bytes transmitted. -- name: container.network.io.usage.rx_packets - type: long - description: Network packets received. -- name: container.network.io.usage.tx_packets - type: long - description: Network packets transmitted. -- name: container.network.io.usage.rx_errors - type: long - description: Network receive errors. -- name: container.network.io.usage.tx_errors - type: long - description: Network transmit errors. -- name: container.network.io.usage.rx_dropped - type: long - description: Network packets dropped on receive. -- name: container.network.io.usage.tx_dropped - type: long - description: Network packets dropped on transmit. -- name: container.blockio.io_service_bytes_recursive.read - type: long - description: Block I/O bytes read. -- name: container.blockio.io_service_bytes_recursive.write - type: long - description: Block I/O bytes written. -- name: container.blockio.io_serviced_recursive.read - type: long - description: Block I/O read operations. -- name: container.blockio.io_serviced_recursive.write - type: long - description: Block I/O write operations. + description: Event timestamp. \ No newline at end of file diff --git a/packages/docker_otel_input/manifest.yml b/packages/docker_otel_input/manifest.yml index f4bbc4408ad..38b18b29ff5 100644 --- a/packages/docker_otel_input/manifest.yml +++ b/packages/docker_otel_input/manifest.yml @@ -1,4 +1,4 @@ -format_version: 3.4.0 +format_version: 3.5.0 name: docker_otel_input title: "Docker Stats (OpenTelemetry)" version: 0.0.1 @@ -11,7 +11,7 @@ categories: - opentelemetry conditions: kibana: - version: "^9.0.0" + version: "^9.2.0" elastic: subscription: "basic" policy_templates: @@ -19,7 +19,7 @@ policy_templates: title: Docker Stats description: Collect Docker container performance metrics including CPU, memory, network, and block I/O input: otelcol - type: logs + type: metrics template_path: input.yml.hbs vars: - name: period From ec643479676c4878556cc369f9728260f63d8f09 Mon Sep 17 00:00:00 2001 From: William Easton Date: Thu, 23 Oct 2025 16:54:00 -0500 Subject: [PATCH 4/4] Add docker container stats input package --- .../agent/input/input.yml.hbs | 17 ++- packages/docker_otel_input/changelog.yml | 4 +- packages/docker_otel_input/docs/README.md | 135 ++++++------------ packages/docker_otel_input/manifest.yml | 15 +- 4 files changed, 64 insertions(+), 107 deletions(-) diff --git a/packages/docker_otel_input/agent/input/input.yml.hbs b/packages/docker_otel_input/agent/input/input.yml.hbs index a0b14173f54..59ba4daa15d 100644 --- a/packages/docker_otel_input/agent/input/input.yml.hbs +++ b/packages/docker_otel_input/agent/input/input.yml.hbs @@ -1,7 +1,9 @@ receivers: - dockerstats: + docker_stats: {{#if period}} collection_interval: {{period}} +{{else}} + collection_interval: 60s {{/if}} {{#if endpoint}} endpoint: {{endpoint}} @@ -16,7 +18,7 @@ receivers: timeout: {{timeout}} {{/if}} {{#if api_version}} - api_version: {{api_version}} + api_version: "{{api_version}}" {{/if}} {{#if provide_per_core_cpu_metrics}} provide_per_core_cpu_metrics: {{provide_per_core_cpu_metrics}} @@ -30,8 +32,17 @@ receivers: {{env_vars_to_metric_labels}} {{/if}} +processors: + resourcedetection: + # Modify the list of detectors to match the cloud environment + detectors: [env, system, gcp, ec2, azure] + timeout: 2s + override: false + service: pipelines: metrics: receivers: - - dockerstats + - docker_stats + processors: + - resourcedetection \ No newline at end of file diff --git a/packages/docker_otel_input/changelog.yml b/packages/docker_otel_input/changelog.yml index d0ecbd2299f..3fff2e21967 100644 --- a/packages/docker_otel_input/changelog.yml +++ b/packages/docker_otel_input/changelog.yml @@ -1,5 +1,5 @@ -- version: "0.0.1" +- version: "0.2.0" changes: - description: Initial draft of Docker Stats OpenTelemetry input package type: enhancement - link: https://github.com/elastic/integrations/issues/15731 + link: https://github.com/elastic/integrations/pull/15739 diff --git a/packages/docker_otel_input/docs/README.md b/packages/docker_otel_input/docs/README.md index e02838e0824..5ecade2bfc0 100644 --- a/packages/docker_otel_input/docs/README.md +++ b/packages/docker_otel_input/docs/README.md @@ -1,78 +1,20 @@ -# Docker Stats (OpenTelemetry) +# Docker Container Stats (OpenTelemetry) This integration collects Docker container metrics using the OpenTelemetry Collector's [dockerstats receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/dockerstatsreceiver). ## Overview -The Docker Stats receiver collects container metrics from the Docker daemon using the Docker Stats API. It provides comprehensive metrics about container resource usage including CPU, memory, network, and block I/O statistics. +Docker Container Stats are collected via the `docker_stats` receiver. It collects container metrics from the Docker daemon using the Docker Stats API. It provides comprehensive metrics about container resource usage including CPU, memory, network, and block I/O statistics. ## Requirements -- Docker API version 1.22 or higher -- Access to the Docker daemon socket (default: `unix:///var/run/docker.sock`) -- Not supported on Darwin/Windows platforms -- OpenTelemetry Collector integration enabled in your Elastic Agent +- Docker API version 1.25 or higher +- Access to the Docker daemon socket (default: `unix:///var/run/docker.sock` on linux and `npipe:////./pipe/docker_engine` on Windows) ## Compatibility This integration uses the OpenTelemetry dockerstats receiver which is currently in **alpha** stability. The configuration and metrics may change between versions. -## Configuration - -### Collection Interval - -How often to collect metrics from Docker containers. Default is `10s`. - -### Docker Endpoint - -The endpoint of the Docker daemon. Default is `unix:///var/run/docker.sock`. - -For remote Docker hosts, you can use TCP endpoints like `tcp://docker-host:2375`. - -### Excluded Images - -A list of container image names to exclude from metrics collection. Supports wildcards. - -Example: -``` -- "nginx:*" -- "redis:latest" -``` - -### Timeout - -Timeout for Docker API requests. Default is `5s`. - -### API Version - -The Docker API version to use. Default is `1.22` (the minimum supported version). - -### Per-Core CPU Metrics - -When enabled, provides CPU metrics broken down by individual CPU cores. Default is `false`. - -**Note:** Enabling this option can significantly increase the number of metrics collected. - -### Container Labels to Metric Labels - -Map container labels to metric resource attributes. This allows you to add custom dimensions to your metrics based on container labels. - -Example YAML format: -```yaml -my.container.label: my_metric_label -app.version: version -``` - -### Environment Variables to Metric Labels - -Map container environment variables to metric resource attributes. - -Example YAML format: -```yaml -MY_ENV_VAR: my_metric_label -APP_VERSION: version -``` - ## Metrics This integration collects the following types of metrics: @@ -110,50 +52,53 @@ This integration collects the following types of metrics: - Image name - Container runtime -## Example Configuration +## Advanced Configuration -```yaml -receivers: - dockerstats: - collection_interval: 10s - endpoint: unix:///var/run/docker.sock - excluded_images: - - "nginx:*" - timeout: 5s - api_version: "1.22" - provide_per_core_cpu_metrics: false - -service: - pipelines: - metrics: - receivers: - - dockerstats -``` +### Docker Endpoint + +The endpoint of the Docker daemon. Default is `unix:///var/run/docker.sock`. -## Troubleshooting +For remote Docker hosts, you can use TCP endpoints like `tcp://docker-host:2375`. -### Permission Denied +### Excluded Images -If you see "permission denied" errors, ensure the Elastic Agent has access to the Docker socket. You may need to add the agent user to the `docker` group: +A list of container image names to exclude from metrics collection. Supports wildcards. -```bash -sudo usermod -aG docker elastic-agent +Example: +``` +- "nginx:*" +- "redis:latest" ``` -### No Metrics Collected +### API Version -- Verify Docker is running: `docker ps` -- Check the Docker endpoint configuration -- Ensure the API version is compatible with your Docker installation -- Review Elastic Agent logs for errors +The Docker API version to use. Default is `1.25` (the minimum supported version). -### High Cardinality +### Per-Core CPU Metrics -If you're experiencing high cardinality issues: +When enabled, provides CPU metrics broken down by individual CPU cores. Default is `false`. -- Disable per-core CPU metrics -- Use the `excluded_images` option to filter out noisy containers -- Be selective with container labels and environment variables mapped to metric labels +**Note:** Enabling this option can significantly increase the number of metrics collected. + +### Container Labels to Metric Labels + +Map container labels to metric resource attributes. This allows you to add custom dimensions to your metrics based on container labels. + +Example YAML format: +```yaml +my.container.label: my_metric_label +app.version: version +``` + +### Environment Variables to Metric Labels + +Map container environment variables to metric resource attributes. + +Example YAML format: +```yaml +MY_ENV_VAR: my_metric_label +APP_VERSION: version +``` ## Additional Resources diff --git a/packages/docker_otel_input/manifest.yml b/packages/docker_otel_input/manifest.yml index 38b18b29ff5..4032b894349 100644 --- a/packages/docker_otel_input/manifest.yml +++ b/packages/docker_otel_input/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.5.0 name: docker_otel_input -title: "Docker Stats (OpenTelemetry)" -version: 0.0.1 +title: "Docker (OpenTelemetry)" +version: 0.2.0 description: "Collect Docker container metrics using OpenTelemetry Collector" type: input categories: @@ -14,6 +14,11 @@ conditions: version: "^9.2.0" elastic: subscription: "basic" +icons: + - src: /img/docker_logo.svg + title: Sample logo + size: 32x32 + type: image/svg+xml policy_templates: - name: docker_stats title: Docker Stats @@ -25,14 +30,12 @@ policy_templates: - name: period type: text title: Collection Interval - description: How often to collect metrics (e.g., 10s, 1m) - default: 10s + description: How often to collect metrics (e.g., 10s, 1m). Default is 60s. required: true - name: endpoint type: text title: Docker Endpoint description: Docker daemon endpoint - default: "unix:///var/run/docker.sock" required: false - name: excluded_images type: text @@ -44,13 +47,11 @@ policy_templates: type: text title: Timeout description: Timeout for Docker API requests - default: 5s required: false - name: api_version type: text title: API Version description: Docker API version to use - default: "1.22" required: false - name: provide_per_core_cpu_metrics type: bool