    tag_emails_logs:
      type: remap
      inputs:
        - normalize_logs

role: Aggregator
replicas: ${replicas}
podAnnotations:
  reloader.stakater.com/auto: "true"
service:
  enabled: true
  type: ClusterIP
  ports:
    - name: otlp-grpc
      port: 4317
      protocol: TCP
    - name: otlp-http
      port: 4318
      protocol: TCP
    - name: dd-agent
      port: 8282
      protocol: TCP
env:
  - name: DATADOG_API_KEY
    valueFrom:
      secretKeyRef:
        name: vector-datadog
        key: api_key
resources:
  requests:
    cpu: ${requests_cpu}
    memory: ${requests_memory}
  limits:
    cpu: "${limits_cpu}"
    memory: ${limits_memory}
podDisruptionBudget:
  enabled: true
  minAvailable: 2

topologySpreadConstraints:
  - maxSkew: 1
    topologyKey: topology.kubernetes.io/zone
    whenUnsatisfiable: ScheduleAnyway
    labelSelector:
      matchLabels:
        app.kubernetes.io/name: vector

affinity:
  podAntiAffinity:
    preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 100
        podAffinityTerm:
          topologyKey: kubernetes.io/hostname
          labelSelector:
            matchLabels:
              app.kubernetes.io/name: vector

%{ if service_monitor_enabled ~}
serviceMonitor:
  enabled: true
%{ endif ~}
customConfig:
  data_dir: /vector-data-dir
  api:
    enabled: true
    address: 127.0.0.1:8686
  sources:
    otlp_in:
      type: opentelemetry
      grpc:
        address: 0.0.0.0:4317
      http:
        address: 0.0.0.0:4318
    dd_agent_in:
      type: datadog_agent
      address: 0.0.0.0:8282
      multiple_outputs: true
      disable_metrics: true
      disable_traces: true
      store_api_key: false
      acknowledgements:
        enabled: true
  transforms:
    drop_audit:
      type: filter
      inputs:
        - otlp_in.logs
        - dd_agent_in.logs
      condition: |
        msg = if is_string(.message) { string!(.message) } else { "" }
        ${audit_drop_condition}
    normalize_logs:
      type: remap
      inputs:
        - drop_audit
      source: |
        if exists(.severity) {
          .status = downcase(to_string!(.severity))
          del(.severity)
        }
%{ if email_redaction_enabled ~}
    redact_emails_logs:
      type: remap
      inputs:
        - normalize_logs
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_string(.message) {
          .message = replace(string!(.message), email_re, "${email_redaction_replacement}")
        }
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
    redact_emails_traces:
      type: remap
      inputs:
        - otlp_in.traces
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
    redact_emails_metrics:
      type: remap
      inputs:
        - otlp_in.metrics
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
%{ endif ~}
%{ if !email_redaction_enabled && email_redaction_dry_run ~}
    tag_emails_logs:
      type: remap
      inputs:
        - drop_audit
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        detected = false
        if is_string(.message) {
          detected = match(string!(.message), email_re)
        }
        if !detected && is_object(.attributes) {
          detected = match(encode_json(.attributes), email_re)
        }
        if detected {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_redaction:true" } else { "vector_redaction:true" }
        }
%{ endif ~}
  sinks:
    datadog_logs:
      type: datadog_logs
      inputs:
        - ${ email_redaction_enabled ? "redact_emails_logs" : email_redaction_dry_run ? "tag_emails_logs" : "drop_audit" }
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_traces:
      type: datadog_traces
      inputs:
        - ${email_redaction_enabled ? "redact_emails_traces" : "otlp_in.traces"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_metrics:
      type: datadog_metrics
      inputs:
        - ${email_redaction_enabled ? "redact_emails_metrics" : "otlp_in.metrics"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}


Review moved Terraform monitoring modules


Edit

- ${ email_redaction_enabled ? "redact_emails_logs" : email_redaction_dry_run ? "tag_emails_logs" : "drop_audit" }

- ${ email_redaction_enabled ? "redact_emails_logs" : email_redaction_dry_run ? "tag_emails_logs" : "normalize_logs" }


The file /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/templates/values.yaml.tpl has been updated successfully. (file state is current in your context — no need to Read it back)

role: Aggregator
replicas: ${replicas}
podAnnotations:
  reloader.stakater.com/auto: "true"
service:
  enabled: true
  type: ClusterIP
  ports:
    - name: otlp-grpc
      port: 4317
      protocol: TCP
    - name: otlp-http
      port: 4318
      protocol: TCP
    - name: dd-agent
      port: 8282
      protocol: TCP
env:
  - name: DATADOG_API_KEY
    valueFrom:
      secretKeyRef:
        name: vector-datadog
        key: api_key
resources:
  requests:
    cpu: ${requests_cpu}
    memory: ${requests_memory}
  limits:
    cpu: "${limits_cpu}"
    memory: ${limits_memory}
podDisruptionBudget:
  enabled: true
  minAvailable: 2

topologySpreadConstraints:
  - maxSkew: 1
    topologyKey: topology.kubernetes.io/zone
    whenUnsatisfiable: ScheduleAnyway
    labelSelector:
      matchLabels:
        app.kubernetes.io/name: vector

affinity:
  podAntiAffinity:
    preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 100
        podAffinityTerm:
          topologyKey: kubernetes.io/hostname
          labelSelector:
            matchLabels:
              app.kubernetes.io/name: vector

%{ if service_monitor_enabled ~}
serviceMonitor:
  enabled: true
%{ endif ~}
customConfig:
  data_dir: /vector-data-dir
  api:
    enabled: true
    address: 127.0.0.1:8686
  sources:
    otlp_in:
      type: opentelemetry
      grpc:
        address: 0.0.0.0:4317
      http:
        address: 0.0.0.0:4318
    dd_agent_in:
      type: datadog_agent
      address: 0.0.0.0:8282
      multiple_outputs: true
      disable_metrics: true
      disable_traces: true
      store_api_key: false
      acknowledgements:
        enabled: true
  transforms:
    drop_audit:
      type: filter
      inputs:
        - otlp_in.logs
        - dd_agent_in.logs
      condition: |
        msg = if is_string(.message) { string!(.message) } else { "" }
        ${audit_drop_condition}
    normalize_logs:
      type: remap
      inputs:
        - drop_audit
      source: |
        if exists(.severity) {
          .status = downcase(to_string!(.severity))
          del(.severity)
        }
%{ if email_redaction_enabled ~}
    redact_emails_logs:
      type: remap
      inputs:
        - normalize_logs
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_string(.message) {
          .message = replace(string!(.message), email_re, "${email_redaction_replacement}")
        }
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
    redact_emails_traces:
      type: remap
      inputs:
        - otlp_in.traces
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
    redact_emails_metrics:
      type: remap
      inputs:
        - otlp_in.metrics
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
%{ endif ~}
%{ if !email_redaction_enabled && email_redaction_dry_run ~}
    tag_emails_logs:
      type: remap
      inputs:
        - normalize_logs
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        detected = false
        if is_string(.message) {
          detected = match(string!(.message), email_re)
        }
        if !detected && is_object(.attributes) {
          detected = match(encode_json(.attributes), email_re)
        }
        if detected {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_redaction:true" } else { "vector_redaction:true" }
        }
%{ endif ~}
  sinks:
    datadog_logs:
      type: datadog_logs
      inputs:
        - ${ email_redaction_enabled ? "redact_emails_logs" : email_redaction_dry_run ? "tag_emails_logs" : "drop_audit" }
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_traces:
      type: datadog_traces
      inputs:
        - ${email_redaction_enabled ? "redact_emails_traces" : "otlp_in.traces"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_metrics:
      type: datadog_metrics
      inputs:
        - ${email_redaction_enabled ? "redact_emails_metrics" : "otlp_in.metrics"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}

-        - ${ email_redaction_enabled ? "redact_emails_logs" : email_redaction_dry_run ? "tag_emails_logs" : "drop_audit" }

+        - ${ email_redaction_enabled ? "redact_emails_logs" : email_redaction_dry_run ? "tag_emails_logs" : "normalize_logs" }


Done. The pipeline is now:

```
otlp_in.logs ──┐
               ├─→ drop_audit → normalize_logs → [redact/tag/passthrough] → datadog_logs
dd_agent_in ───┘
```

`normalize_logs` maps `.severity` → `.status` and deletes `.severity` on every log, so the Datadog sink no longer sees the conflicting field and the warning goes away. No other transforms needed changes — traces and metrics don't go through this path and weren't affected by the severity collision.


afplay ~/.claude/sounds/mario/stop/stomp.wav 2>/dev/null || true


Adding email redaction to a Vector log pipeline. Current task is fixing a severity field collision warning. Next action is testing the changes in a non-prod cluster.


still seeing these; 2026-05-06T20:21:01.014648Z  WARN vector::sinks::datadog::logs::sink: Semantic meaning is defined, but the event path already exists. Renaming to not overwrite. meaning="severity" renamed="_RESERVED_severity"


Our VRL `del(.severity)` removes both the field and the semantic meaning annotation — so if the warning persists, we're deleting the wrong field. The `severity` semantic meaning is probably registered on a different path. The `datadog_agent` source typically assigns it to `.status`, and the OTel source may assign it to `.severity_text` rather than `.severity`.

Diagnose it by tapping the `normalize_logs` output from inside the pod:

```bash
kubectl exec -n vector -it deploy/vector -- \
  curl -s -N -X POST http://localhost:8686/tap \
  -H 'Content-Type: application/json' \
  -d '{"component_id_patterns": ["normalize_logs"], "limit": 3}' \
  | jq .
```

That will show you the actual event shape after `normalize_logs` runs. Look for any field that could be carrying severity info — likely `.status`, `.severity_text`, or `.severity_number`.

Once you know the field, the fix in `normalize_logs` is straightforward — we just need to target the right path. My best guesses based on the two sources:

- **OTel logs**: semantic meaning is on `.severity_text` → we should `del(.severity_text)`
- **DD Agent logs**: semantic meaning is on `.status` → can't delete it (Datadog needs it), need a source-level fix instead

Run the tap and share what the event fields look like — then we can target the right one.


still seeing these; 2026-05-06T20:21:01.014648Z  WARN vector::sinks::datadog::logs::sink: Semantic meaning is defined, but the event path already exists. Renaming to not overwrite. meaning="severity"…


i don't have a port 8686 in vector?


We removed 8686 from the Kubernetes Service and netpol, but the Vector API is still listening on `127.0.0.1:8686` inside the pod — it's just not reachable from outside. `kubectl exec` runs inside the pod so it can still hit localhost:

```bash
kubectl exec -n vector deploy/vector -- \
  curl -s -X POST http://localhost:8686/tap \
  -H 'Content-Type: application/json' \
  -d '{"component_id_patterns":["normalize_logs"],"limit":3}' \
  | jq .
```

If the API isn't enabled in your deployed instance, `vector top` is another option:

```bash
kubectl exec -n vector deploy/vector -- vector top
```

That uses the same localhost API and shows per-component event rates — useful for seeing which component is producing events and confirming the pipeline topology is wired as expected.


Adding a tag to logs Vector would have redacted, to aid non-prod visibility. Currently diagnosing a semantic meaning warning in the Datadog logs sink — next step is running the tap command inside the Vector pod to identify which field carries the severity meaning.