Dash0 acquires Polar Signals
Capture Prometheus metrics with OpenTelemetry collector. Gain real-time visibility into your systems and applications.
The OpenTelemetry Collector can be configured to scrape Prometheus metrics, effectively acting as a drop-in replacement for the Prometheus server. This integration allows for a seamless transition from Prometheus to OpenTelemetry, enabling users to leverage both systems' strengths.
Many Kubernetes services and pods expose metrics through a Prometheus endpoint that you can scrape. We recommend to scrape and forward these metrics via an OpenTelemetry Collector.
To make your services scrapeable, add these annotations to your pod or service definitions
prometheus.io/scrape: 'true'
prometheus.io/path: '/metrics'
prometheus.io/port: '8888'The following code snippet shows an OpenTelemetry Collector configuration file.
receivers:
prometheus:
config:
scrape_configs:
# This config is used to find all Kubernetes pods that have the prometheus.io/scrape annotation
- job_name: "kubernetes-pods"
kubernetes_sd_configs:
- role: pod
scrape_interval: 30s
scrape_timeout: 5s
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: false
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
regex: true
action: keep
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
regex: false
action: drop
- source_labels: [__meta_kubernetes_pod_ip, __meta_kubernetes_pod_annotation_prometheus_io_port]
regex: (.+);(.+)
replacement: $1:$2
target_label: __address__
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
action: replace
regex: ()
replacement: /metrics
target_label: __meta_kubernetes_pod_annotation_prometheus_io_path
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: pod_name
- source_labels: [__meta_kubernetes_pod_container_name]
action: replace
target_label: container_name
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- action: labeldrop
regex: "container_id|image_id|controller_revision_hash|pod_template_generation|pod_template_hash"
metric_relabel_configs:
- source_labels: [__name__]
regex: "process_virtual.*|process_max_fds|process_start_time.*|http_.*|go_gc.*"
action: drop
# This config is used to find all Kubernetes service endpoints that have the prometheus.io/scrape annotation
- job_name: kubernetes-service-endpoints
# This specifies that Prometheus should discover targets based on Kubernetes endpoints.
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
# Keep targets with prometheus.io/scrape annotation set to true
- action: keep
regex: true
source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_scrape
# Drop targets with prometheus.io/scrape_slow annotation set to true
- action: drop
regex: true
source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_scrape_slow
# Set the scheme (http/https) based on prometheus.io/scheme annotation
- action: replace
regex: (https?)
source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_scheme
target_label: __scheme__
# Set the metrics path based on prometheus.io/path annotation
- action: replace
regex: (.+)
source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_path
target_label: __metrics_path__
# Set the address based on the prometheus.io/port annotation
- action: replace
regex: (.+?)(?::\d+)?;(\d+)
replacement: $$1:$$2
source_labels:
- __address__
- __meta_kubernetes_service_annotation_prometheus_io_port
target_label: __address__
# Map annotations prefixed with prometheus.io/param_ to Prometheus parameters
- action: labelmap
regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+)
replacement: __param_$$1
# Map service labels to Prometheus labels
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
# Add namespace label based on the Kubernetes namespace
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: namespace
# Add service label based on the Kubernetes service name
- action: replace
source_labels:
- __meta_kubernetes_service_name
target_label: service
# Add node label based on the Kubernetes pod node name
- action: replace
source_labels:
- __meta_kubernetes_pod_node_name
target_label: node
processors:
batch:
exporters:
otlp/dash0:
endpoint: "{{endpoint_otlp_grpc}}"
headers:
Authorization: "Bearer {{token}}"
"Dash0-Dataset": "{{dataset}}"
service:
pipelines:
metrics:
receivers: [prometheus]
processors: [batch]
exporters: [otlp/dash0]Learn how to deploy the collector within our OpenTelemetry collector integration documentation.