  bearer_stage_active = local.bearer_do_redact || local.bearer_do_tag
  bearer_stage_output = local.bearer_do_redact ? "redact_bearer_tokens_logs" : "tag_bearer_tokens_logs"

  bearer_token_logs_input = local.email_stage_output
  logs_sink_input         = local.bearer_stage_active ? local.bearer_stage_output : local.email_stage_output

  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
    "${path.module}/templates/values.yaml.tpl",
    {
      replicas                           = var.replicas
      requests_cpu                       = var.resources_requests.cpu
      requests_memory                    = var.resources_requests.memory
      limits_cpu                         = var.resources_limits.cpu
      limits_memory                      = var.resources_limits.memory
      datadog_site                       = var.datadog_site
      audit_drop_condition               = local.audit_drop_condition
      email_do_redact                    = local.email_do_redact
      email_do_tag                       = local.email_do_tag
      email_redaction_replacement        = var.email_redaction_replacement
      service_monitor_enabled            = var.service_monitor_enabled
      bearer_do_redact                   = local.bearer_do_redact
      bearer_do_tag                      = local.bearer_do_tag
      bearer_token_redaction_replacement = var.bearer_token_redaction_replacement
      bearer_token_services_arr          = local.bearer_token_services_arr
      bearer_token_exclusion_condition   = local.bearer_token_exclusion_condition
      bearer_token_logs_input            = local.bearer_token_logs_input
      logs_sink_input                    = local.logs_sink_input
    },
  )
}

  bearer_stage_active = local.bearer_do_redact || local.bearer_do_tag
  bearer_stage_output = local.bearer_do_redact ? "redact_bearer_tokens_logs" : "tag_bearer_tokens_logs"

  bearer_token_logs_input = local.email_stage_output

  # AWS Access Key ID scanner
  aws_key_do_redact = var.aws_access_key_redaction_enabled && !var.aws_access_key_redaction_dry_run
  aws_key_do_tag    = var.aws_access_key_redaction_dry_run

  aws_key_services_arr = "[\"${join("\", \"", var.aws_access_key_redaction_services)}\"]"

  aws_key_string_exclusion_condition = length(var.aws_access_key_redaction_exclusion_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.aws_access_key_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
  )

  # Each entry generates a VRL boolean: true when the attribute matches and should be excluded.
  # Short-circuits on is_object(.attributes) so get() is never called on a non-object.
  aws_key_attr_exclusion_condition = length(var.aws_access_key_redaction_exclusion_attributes) == 0 ? "false" : join(
    " || ",
    [for k, v in var.aws_access_key_redaction_exclusion_attributes :
      "(is_object(.attributes) && (to_string(get(.attributes, [\"${k}\"]) ?? \"\") == \"${v}\"))"
    ],
  )

  aws_key_stage_active = local.aws_key_do_redact || local.aws_key_do_tag
  aws_key_stage_output = local.aws_key_do_redact ? "redact_aws_access_keys_logs" : "tag_aws_access_keys_logs"

  aws_key_logs_input = local.bearer_stage_active ? local.bearer_stage_output : local.email_stage_output

  logs_sink_input = (
    local.aws_key_stage_active ? local.aws_key_stage_output :
    local.bearer_stage_active  ? local.bearer_stage_output :
    local.email_stage_output
  )

  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
    "${path.module}/templates/values.yaml.tpl",
    {
      replicas                              = var.replicas
      requests_cpu                          = var.resources_requests.cpu
      requests_memory                       = var.resources_requests.memory
      limits_cpu                            = var.resources_limits.cpu
      limits_memory                         = var.resources_limits.memory
      datadog_site                          = var.datadog_site
      audit_drop_condition                  = local.audit_drop_condition
      email_do_redact                       = local.email_do_redact
      email_do_tag                          = local.email_do_tag
      email_redaction_replacement           = var.email_redaction_replacement
      service_monitor_enabled               = var.service_monitor_enabled
      bearer_do_redact                      = local.bearer_do_redact
      bearer_do_tag                         = local.bearer_do_tag
      bearer_token_redaction_replacement    = var.bearer_token_redaction_replacement
      bearer_token_services_arr             = local.bearer_token_services_arr
      bearer_token_exclusion_condition      = local.bearer_token_exclusion_condition
      bearer_token_logs_input               = local.bearer_token_logs_input
      aws_key_do_redact                     = local.aws_key_do_redact
      aws_key_do_tag                        = local.aws_key_do_tag
      aws_access_key_redaction_replacement  = var.aws_access_key_redaction_replacement
      aws_key_services_arr                  = local.aws_key_services_arr
      aws_key_string_exclusion_condition    = local.aws_key_string_exclusion_condition
      aws_key_attr_exclusion_condition      = local.aws_key_attr_exclusion_condition
      aws_key_logs_input                    = local.aws_key_logs_input
      logs_sink_input                       = local.logs_sink_input
    },
  )
}


The file /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/main.tf has been updated successfully. (file state is current in your context — no need to Read it back)

  bearer_stage_active = local.bearer_do_redact || local.bearer_do_tag
  bearer_stage_output = local.bearer_do_redact ? "redact_bearer_tokens_logs" : "tag_bearer_tokens_logs"

  bearer_token_logs_input = local.email_stage_output

  # AWS Access Key ID scanner
  aws_key_do_redact = var.aws_access_key_redaction_enabled && !var.aws_access_key_redaction_dry_run
  aws_key_do_tag    = var.aws_access_key_redaction_dry_run

  aws_key_services_arr = "[\"${join("\", \"", var.aws_access_key_redaction_services)}\"]"

  aws_key_string_exclusion_condition = length(var.aws_access_key_redaction_exclusion_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.aws_access_key_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
  )

  # Each entry generates a VRL boolean: true when the attribute matches and should be excluded.
  # Short-circuits on is_object(.attributes) so get() is never called on a non-object.
  aws_key_attr_exclusion_condition = length(var.aws_access_key_redaction_exclusion_attributes) == 0 ? "false" : join(
    " || ",
    [for k, v in var.aws_access_key_redaction_exclusion_attributes :
      "(is_object(.attributes) && (to_string(get(.attributes, [\"${k}\"]) ?? \"\") == \"${v}\"))"
    ],
  )

  aws_key_stage_active = local.aws_key_do_redact || local.aws_key_do_tag
  aws_key_stage_output = local.aws_key_do_redact ? "redact_aws_access_keys_logs" : "tag_aws_access_keys_logs"

  aws_key_logs_input = local.bearer_stage_active ? local.bearer_stage_output : local.email_stage_output

  logs_sink_input = (
    local.aws_key_stage_active ? local.aws_key_stage_output :
    local.bearer_stage_active  ? local.bearer_stage_output :
    local.email_stage_output
  )

  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
    "${path.module}/templates/values.yaml.tpl",
    {
      replicas                              = var.replicas
      requests_cpu                          = var.resources_requests.cpu
      requests_memory                       = var.resources_requests.memory
      limits_cpu                            = var.resources_limits.cpu
      limits_memory                         = var.resources_limits.memory
      datadog_site                          = var.datadog_site
      audit_drop_condition                  = local.audit_drop_condition
      email_do_redact                       = local.email_do_redact
      email_do_tag                          = local.email_do_tag
      email_redaction_replacement           = var.email_redaction_replacement
      service_monitor_enabled               = var.service_monitor_enabled
      bearer_do_redact                      = local.bearer_do_redact
      bearer_do_tag                         = local.bearer_do_tag
      bearer_token_redaction_replacement    = var.bearer_token_redaction_replacement
      bearer_token_services_arr             = local.bearer_token_services_arr
      bearer_token_exclusion_condition      = local.bearer_token_exclusion_condition
      bearer_token_logs_input               = local.bearer_token_logs_input
      aws_key_do_redact                     = local.aws_key_do_redact
      aws_key_do_tag                        = local.aws_key_do_tag
      aws_access_key_redaction_replacement  = var.aws_access_key_redaction_replacement
      aws_key_services_arr                  = local.aws_key_services_arr
      aws_key_string_exclusion_condition    = local.aws_key_string_exclusion_condition
      aws_key_attr_exclusion_condition      = local.aws_key_attr_exclusion_condition
      aws_key_logs_input                    = local.aws_key_logs_input
      logs_sink_input                       = local.logs_sink_input
    },
  )
}

terraform {
  required_version = ">= 1.9.0"

  required_providers {
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = ">= 2.20"
    }
    helm = {
      source  = "hashicorp/helm"
      version = ">= 2.10"
    }
  }
}

locals {
  audit_drop_condition = length(var.audit_drop_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.audit_drop_patterns : "!match(msg, r'${p}')"],
  )

  bearer_token_exclusion_condition = length(var.bearer_token_redaction_exclusion_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.bearer_token_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
  )

  bearer_token_services_arr = "[\"${join("\", \"", var.bearer_token_redaction_services)}\"]"

  # dry_run always wins over enabled for both scanner types
  email_do_redact = var.email_redaction_enabled && !var.email_redaction_dry_run
  email_do_tag    = var.email_redaction_dry_run

  bearer_do_redact = var.bearer_token_redaction_enabled && !var.bearer_token_redaction_dry_run
  bearer_do_tag    = var.bearer_token_redaction_dry_run

  # named output of each stage used to wire transform inputs and the final sink
  email_stage_output = (
    local.email_do_redact ? "redact_emails_logs" :
    local.email_do_tag    ? "tag_emails_logs" :
    "normalize_logs"
  )

  bearer_stage_active = local.bearer_do_redact || local.bearer_do_tag
  bearer_stage_output = local.bearer_do_redact ? "redact_bearer_tokens_logs" : "tag_bearer_tokens_logs"

  bearer_token_logs_input = local.email_stage_output
  logs_sink_input         = local.bearer_stage_active ? local.bearer_stage_output : local.email_stage_output

  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
    "${path.module}/templates/values.yaml.tpl",
    {
      replicas                           = var.replicas
      requests_cpu                       = var.resources_requests.cpu
      requests_memory                    = var.resources_requests.memory
      limits_cpu                         = var.resources_limits.cpu
      limits_memory                      = var.resources_limits.memory
      datadog_site                       = var.datadog_site
      audit_drop_condition               = local.audit_drop_condition
      email_do_redact                    = local.email_do_redact
      email_do_tag                       = local.email_do_tag
      email_redaction_replacement        = var.email_redaction_replacement
      service_monitor_enabled            = var.service_monitor_enabled
      bearer_do_redact                   = local.bearer_do_redact
      bearer_do_tag                      = local.bearer_do_tag
      bearer_token_redaction_replacement = var.bearer_token_redaction_replacement
      bearer_token_services_arr          = local.bearer_token_services_arr
      bearer_token_exclusion_condition   = local.bearer_token_exclusion_condition
      bearer_token_logs_input            = local.bearer_token_logs_input
      logs_sink_input                    = local.logs_sink_input
    },
  )
}

resource "kubernetes_namespace_v1" "vector" {
  metadata {
    name = var.namespace
    labels = {
      "app.kubernetes.io/name"      = "vector"
      "app.kubernetes.io/component" = "telemetry-aggregator"
    }
  }
}

resource "kubernetes_manifest" "datadog_external_secret" {
  manifest = {
    apiVersion = "external-secrets.io/v1"
    kind       = "ExternalSecret"
    metadata = {
      name      = "vector-datadog"
      namespace = kubernetes_namespace_v1.vector.metadata[0].name
    }
    spec = {
      refreshInterval = "1h"
      secretStoreRef = {
        kind = "ClusterSecretStore"
        name = var.external_secrets_cluster_store_name
      }
      target = {
        name           = "vector-datadog"
        creationPolicy = "Owner"
      }
      data = [{
        secretKey = "api_key"
        remoteRef = {
          key      = var.datadog_api_key_vault_path
          property = "apiKey"
        }
      }]
    }
  }

  wait {
    fields = {
      "status.conditions[0].type"   = "Ready"
      "status.conditions[0].status" = "True"
    }
  }

  timeouts {
    create = "5m"
    update = "5m"
  }
}

resource "helm_release" "vector" {
  name       = "vector"
  namespace  = kubernetes_namespace_v1.vector.metadata[0].name
  repository = "https://helm.vector.dev"
  chart      = "vector"
  version    = var.chart_version

  values = [local.rendered_values]

  set {
    name  = "image.repository"
    value = var.image
  }
  set {
    name  = "image.tag"
    value = var.image_tag_version
  }

  depends_on = [
    kubernetes_manifest.datadog_external_secret,
  ]
}

module "netpol" {
  source = "crow.infrastructure.immuta.io/internal/security-network-policy/kubernetes"
  version = "1.1.0"

  name             = "vector-netpol"
  namespace        = var.namespace
  network_policies = local.netpols

  depends_on = [
    kubernetes_namespace_v1.vector,
  ]
}

-  logs_sink_input         = local.bearer_stage_active ? local.bearer_stage_output : local.email_stage_output

+  aws_key_do_redact = var.aws_access_key_redaction_enabled && !var.aws_access_key_redaction_dry_run

+  aws_key_do_tag    = var.aws_access_key_redaction_dry_run

+  aws_key_services_arr = "[\"${join("\", \"", var.aws_access_key_redaction_services)}\"]"

+  aws_key_string_exclusion_condition = length(var.aws_access_key_redaction_exclusion_patterns) == 0 ? "true" : join(

+    [for p in var.aws_access_key_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],

+  # Each entry generates a VRL boolean: true when the attribute matches and should be excluded.

+  # Short-circuits on is_object(.attributes) so get() is never called on a non-object.

+  aws_key_attr_exclusion_condition = length(var.aws_access_key_redaction_exclusion_attributes) == 0 ? "false" : join(

+    [for k, v in var.aws_access_key_redaction_exclusion_attributes :

+  aws_key_stage_active = local.aws_key_do_redact || local.aws_key_do_tag

+  aws_key_stage_output = local.aws_key_do_redact ? "redact_aws_access_keys_logs" : "tag_aws_access_keys_logs"

+  aws_key_logs_input = local.bearer_stage_active ? local.bearer_stage_output : local.email_stage_output

+    local.aws_key_stage_active ? local.aws_key_stage_output :

+    local.bearer_stage_active  ? local.bearer_stage_output :

-      replicas                           = var.replicas

-      requests_cpu                       = var.resources_requests.cpu

-      requests_memory                    = var.resources_requests.memory

-      limits_cpu                         = var.resources_limits.cpu

-      limits_memory                      = var.resources_limits.memory

-      datadog_site                       = var.datadog_site

-      audit_drop_condition               = local.audit_drop_condition

-      email_do_redact                    = local.email_do_redact

-      email_do_tag                       = local.email_do_tag

-      email_redaction_replacement        = var.email_redaction_replacement

-      service_monitor_enabled            = var.service_monitor_enabled

-      bearer_do_redact                   = local.bearer_do_redact

-      bearer_do_tag                      = local.bearer_do_tag

-      bearer_token_redaction_replacement = var.bearer_token_redaction_replacement

-      bearer_token_services_arr          = local.bearer_token_services_arr

-      bearer_token_exclusion_condition   = local.bearer_token_exclusion_condition

-      bearer_token_logs_input            = local.bearer_token_logs_input

-      logs_sink_input                    = local.logs_sink_input

+      replicas                              = var.replicas

+      requests_cpu                          = var.resources_requests.cpu

+      requests_memory                       = var.resources_requests.memory

+      limits_cpu                            = var.resources_limits.cpu

+      limits_memory                         = var.resources_limits.memory

+      datadog_site                          = var.datadog_site

+      audit_drop_condition                  = local.audit_drop_condition

+      email_do_redact                       = local.email_do_redact

+      email_do_tag                          = local.email_do_tag

+      email_redaction_replacement           = var.email_redaction_replacement

+      service_monitor_enabled               = var.service_monitor_enabled

+      bearer_do_redact                      = local.bearer_do_redact

+      bearer_do_tag                         = local.bearer_do_tag

+      bearer_token_redaction_replacement    = var.bearer_token_redaction_replacement

+      bearer_token_services_arr             = local.bearer_token_services_arr

+      bearer_token_exclusion_condition      = local.bearer_token_exclusion_condition

+      bearer_token_logs_input               = local.bearer_token_logs_input

+      aws_key_do_redact                     = local.aws_key_do_redact

+      aws_key_do_tag                        = local.aws_key_do_tag

+      aws_access_key_redaction_replacement  = var.aws_access_key_redaction_replacement

+      aws_key_services_arr                  = local.aws_key_services_arr

+      aws_key_string_exclusion_condition    = local.aws_key_string_exclusion_condition

+      aws_key_attr_exclusion_condition      = local.aws_key_attr_exclusion_condition

+      aws_key_logs_input                    = local.aws_key_logs_input

+      logs_sink_input                       = local.logs_sink_input


I also need to add another rule for AWS Access Key ID Scanner, the datadog sensitive data scanner query is `env:prod AND service:marketplace-api AND NOT @req.headers.x-synthetic-test:true`


Add Bearer Token Scanner redaction filter


Now add the two new transform blocks to the template:


Edit

%{ if bearer_do_tag ~}
    tag_bearer_tokens_logs:
      type: remap
      inputs:
        - ${bearer_token_logs_input}
      source: |
        bearer_re = r'(?i)Bearer\s+[A-Za-z0-9+/._~-]+=*'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${bearer_token_services_arr}
        would_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${bearer_token_exclusion_condition}
        if would_redact && is_string(.message) && match(string!(.message), bearer_re) {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_bearer_token_detected:true" } else { "vector_bearer_token_detected:true" }
        }
%{ endif ~}
  sinks:

%{ if bearer_do_tag ~}
    tag_bearer_tokens_logs:
      type: remap
      inputs:
        - ${bearer_token_logs_input}
      source: |
        bearer_re = r'(?i)Bearer\s+[A-Za-z0-9+/._~-]+=*'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${bearer_token_services_arr}
        would_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${bearer_token_exclusion_condition}
        if would_redact && is_string(.message) && match(string!(.message), bearer_re) {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_bearer_token_detected:true" } else { "vector_bearer_token_detected:true" }
        }
%{ endif ~}
%{ if aws_key_do_redact ~}
    redact_aws_access_keys_logs:
      type: remap
      inputs:
        - ${aws_key_logs_input}
      source: |
        aws_key_re = r'(AKIA|ASIA|AROA|AIPA|ANPA|ANVA|APKA)[A-Z0-9]{16}'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${aws_key_services_arr}
        attr_excluded = ${aws_key_attr_exclusion_condition}
        should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${aws_key_string_exclusion_condition} && !attr_excluded
        if should_redact && is_string(.message) {
          .message = replace(string!(.message), aws_key_re, "${aws_access_key_redaction_replacement}")
        }
%{ endif ~}
%{ if aws_key_do_tag ~}
    tag_aws_access_keys_logs:
      type: remap
      inputs:
        - ${aws_key_logs_input}
      source: |
        aws_key_re = r'(AKIA|ASIA|AROA|AIPA|ANPA|ANVA|APKA)[A-Z0-9]{16}'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${aws_key_services_arr}
        attr_excluded = ${aws_key_attr_exclusion_condition}
        would_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${aws_key_string_exclusion_condition} && !attr_excluded
        if would_redact && is_string(.message) && match(string!(.message), aws_key_re) {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_aws_key_detected:true" } else { "vector_aws_key_detected:true" }
        }
%{ endif ~}
  sinks:


The file /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/templates/values.yaml.tpl has been updated successfully. (file state is current in your context — no need to Read it back)

%{ if bearer_do_tag ~}
    tag_bearer_tokens_logs:
      type: remap
      inputs:
        - ${bearer_token_logs_input}
      source: |
        bearer_re = r'(?i)Bearer\s+[A-Za-z0-9+/._~-]+=*'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${bearer_token_services_arr}
        would_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${bearer_token_exclusion_condition}
        if would_redact && is_string(.message) && match(string!(.message), bearer_re) {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_bearer_token_detected:true" } else { "vector_bearer_token_detected:true" }
        }
%{ endif ~}
%{ if aws_key_do_redact ~}
    redact_aws_access_keys_logs:
      type: remap
      inputs:
        - ${aws_key_logs_input}
      source: |
        aws_key_re = r'(AKIA|ASIA|AROA|AIPA|ANPA|ANVA|APKA)[A-Z0-9]{16}'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${aws_key_services_arr}
        attr_excluded = ${aws_key_attr_exclusion_condition}
        should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${aws_key_string_exclusion_condition} && !attr_excluded
        if should_redact && is_string(.message) {
          .message = replace(string!(.message), aws_key_re, "${aws_access_key_redaction_replacement}")
        }
%{ endif ~}
%{ if aws_key_do_tag ~}
    tag_aws_access_keys_logs:
      type: remap
      inputs:
        - ${aws_key_logs_input}
      source: |
        aws_key_re = r'(AKIA|ASIA|AROA|AIPA|ANPA|ANVA|APKA)[A-Z0-9]{16}'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${aws_key_services_arr}
        attr_excluded = ${aws_key_attr_exclusion_condition}
        would_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${aws_key_string_exclusion_condition} && !attr_excluded
        if would_redact && is_string(.message) && match(string!(.message), aws_key_re) {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_aws_key_detected:true" } else { "vector_aws_key_detected:true" }
        }
%{ endif ~}
  sinks:

role: Aggregator
replicas: ${replicas}
podAnnotations:
  reloader.stakater.com/auto: "true"
service:
  enabled: true
  type: ClusterIP
  ports:
    - name: otlp-grpc
      port: 4317
      protocol: TCP
    - name: otlp-http
      port: 4318
      protocol: TCP
    - name: dd-agent
      port: 8282
      protocol: TCP
env:
  - name: DATADOG_API_KEY
    valueFrom:
      secretKeyRef:
        name: vector-datadog
        key: api_key
resources:
  requests:
    cpu: ${requests_cpu}
    memory: ${requests_memory}
  limits:
    cpu: "${limits_cpu}"
    memory: ${limits_memory}
podDisruptionBudget:
  enabled: true
  minAvailable: 2

topologySpreadConstraints:
  - maxSkew: 1
    topologyKey: topology.kubernetes.io/zone
    whenUnsatisfiable: ScheduleAnyway
    labelSelector:
      matchLabels:
        app.kubernetes.io/name: vector

affinity:
  podAntiAffinity:
    preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 100
        podAffinityTerm:
          topologyKey: kubernetes.io/hostname
          labelSelector:
            matchLabels:
              app.kubernetes.io/name: vector

%{ if service_monitor_enabled ~}
serviceMonitor:
  enabled: true
%{ endif ~}
customConfig:
  data_dir: /vector-data-dir
  api:
    enabled: true
    address: 0.0.0.0:8686
  sources:
    otlp_in:
      type: opentelemetry
      grpc:
        address: 0.0.0.0:4317
      http:
        address: 0.0.0.0:4318
    dd_agent_in:
      type: datadog_agent
      address: 0.0.0.0:8282
      multiple_outputs: true
      disable_metrics: true
      disable_traces: true
      store_api_key: false
      acknowledgements:
        enabled: true
  transforms:
    drop_audit:
      type: filter
      inputs:
        - otlp_in.logs
        - dd_agent_in.logs
      condition: |
        msg = if is_string(.message) { string!(.message) } else { "" }
        ${audit_drop_condition}
    normalize_logs:
      type: remap
      inputs:
        - drop_audit
      source: |
        if exists(.severity) {
          .status = downcase(to_string!(.severity))
          del(.severity)
        }
%{ if email_do_redact ~}
    redact_emails_logs:
      type: remap
      inputs:
        - normalize_logs
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_string(.message) {
          .message = replace(string!(.message), email_re, "${email_redaction_replacement}")
        }
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
    redact_emails_traces:
      type: remap
      inputs:
        - otlp_in.traces
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
    redact_emails_metrics:
      type: remap
      inputs:
        - otlp_in.metrics
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
%{ endif ~}
%{ if email_do_tag ~}
    tag_emails_logs:
      type: remap
      inputs:
        - normalize_logs
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        detected = false
        if is_string(.message) {
          detected = match(string!(.message), email_re)
        }
        if !detected && is_object(.attributes) {
          detected = match(encode_json(.attributes), email_re)
        }
        if detected {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_redaction:true" } else { "vector_redaction:true" }
        }
%{ endif ~}
%{ if bearer_do_redact ~}
    redact_bearer_tokens_logs:
      type: remap
      inputs:
        - ${bearer_token_logs_input}
      source: |
        bearer_re = r'(?i)Bearer\s+[A-Za-z0-9+/._~-]+=*'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${bearer_token_services_arr}
        should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${bearer_token_exclusion_condition}
        if should_redact && is_string(.message) {
          .message = replace(string!(.message), bearer_re, "${bearer_token_redaction_replacement}")
        }
%{ endif ~}
%{ if bearer_do_tag ~}
    tag_bearer_tokens_logs:
      type: remap
      inputs:
        - ${bearer_token_logs_input}
      source: |
        bearer_re = r'(?i)Bearer\s+[A-Za-z0-9+/._~-]+=*'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${bearer_token_services_arr}
        would_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${bearer_token_exclusion_condition}
        if would_redact && is_string(.message) && match(string!(.message), bearer_re) {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_bearer_token_detected:true" } else { "vector_bearer_token_detected:true" }
        }
%{ endif ~}
  sinks:
    datadog_logs:
      type: datadog_logs
      inputs:
        - ${logs_sink_input}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_traces:
      type: datadog_traces
      inputs:
        - ${email_do_redact ? "redact_emails_traces" : "otlp_in.traces"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_metrics:
      type: datadog_metrics
      inputs:
        - ${email_do_redact ? "redact_emails_metrics" : "otlp_in.metrics"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}

+        aws_key_re = r'(AKIA|ASIA|AROA|AIPA|ANPA|ANVA|APKA)[A-Z0-9]{16}'

+        svc = to_string(.service) ?? ""

+        ddtags = to_string(.ddtags) ?? ""

+        msg = if is_string(.message) { string!(.message) } else { "" }

+        target_services = ${aws_key_services_arr}

+        attr_excluded = ${aws_key_attr_exclusion_condition}

+        should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${aws_key_string_exclusion_condition} && !attr_excluded

+        if should_redact && is_string(.message) {

+          .message = replace(string!(.message), aws_key_re, "${aws_access_key_redaction_replacement}")

+        would_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${aws_key_string_exclusion_condition} && !attr_excluded

+        if would_redact && is_string(.message) && match(string!(.message), aws_key_re) {

+          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_aws_key_detected:true" } else { "vector_aws_key_detected:true" }


Now write and verify the VRL tests. Let me start with the redact program to validate the attribute accessor syntax:


Write

aws_key_re = r'(AKIA|ASIA|AROA|AIPA|ANPA|ANVA|APKA)[A-Z0-9]{16}'
svc = to_string(.service) ?? ""
ddtags = to_string(.ddtags) ?? ""
msg = if is_string(.message) { string!(.message) } else { "" }
combined = msg + " " + ddtags
target_services = ["marketplace-api"]
attr_excluded = (is_object(.attributes) && (to_string(get(.attributes, ["req.headers.x-synthetic-test"]) ?? "") == "true"))
should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && true && !attr_excluded
if should_redact && is_string(.message) {
  .message = replace(string!(.message), aws_key_re, "[REDACTED-AWS-ACCESS-KEY]")
}
.


File created successfully at: /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/redact_aws_access_keys_logs/program.vrl (file state is current in your context — no need to Read it back)

aws_key_re = r'(AKIA|ASIA|AROA|AIPA|ANPA|ANVA|APKA)[A-Z0-9]{16}'
svc = to_string(.service) ?? ""
ddtags = to_string(.ddtags) ?? ""
msg = if is_string(.message) { string!(.message) } else { "" }
combined = msg + " " + ddtags
target_services = ["marketplace-api"]
attr_excluded = (is_object(.attributes) && (to_string(get(.attributes, ["req.headers.x-synthetic-test"]) ?? "") == "true"))
should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && true && !attr_excluded
if should_redact && is_string(.message) {
  .message = replace(string!(.message), aws_key_re, "[REDACTED-AWS-ACCESS-KEY]")
}
.


{"message": "Leaked key AKIAIOSFODNN7EXAMPLE in request", "service": "marketplace-api", "ddtags": "env:prod,service:marketplace-api"}
{"message": "Leaked key AKIAIOSFODNN7EXAMPLE in request", "service": "marketplace-api", "ddtags": "env:prod,service:marketplace-api", "attributes": {"req.headers.x-synthetic-test": "true"}}
{"message": "Leaked key AKIAIOSFODNN7EXAMPLE in request", "service": "marketplace-api", "ddtags": "env:prod,service:marketplace-api", "attributes": {"req.headers.x-synthetic-test": "false"}}
{"message": "Leaked key AKIAIOSFODNN7EXAMPLE in request", "service": "other-service", "ddtags": "env:prod,service:other-service"}
{"message": "Leaked key AKIAIOSFODNN7EXAMPLE in request", "service": "marketplace-api", "ddtags": "env:staging,service:marketplace-api"}
{"message": "Temporary key ASIAIOSFODNN7EXAMPLE12 found", "service": "marketplace-api", "ddtags": "env:prod,service:marketplace-api"}
{"message": "No key here", "service": "marketplace-api", "ddtags": "env:prod,service:marketplace-api"}


File created successfully at: /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/redact_aws_access_keys_logs/inputs.ndjson (file state is current in your context — no need to Read it back)

{"message": "Leaked key AKIAIOSFODNN7EXAMPLE in request", "service": "marketplace-api", "ddtags": "env:prod,service:marketplace-api"}
{"message": "Leaked key AKIAIOSFODNN7EXAMPLE in request", "service": "marketplace-api", "ddtags": "env:prod,service:marketplace-api", "attributes": {"req.headers.x-synthetic-test": "true"}}
{"message": "Leaked key AKIAIOSFODNN7EXAMPLE in request", "service": "marketplace-api", "ddtags": "env:prod,service:marketplace-api", "attributes": {"req.headers.x-synthetic-test": "false"}}
{"message": "Leaked key AKIAIOSFODNN7EXAMPLE in request", "service": "other-service", "ddtags": "env:prod,service:other-service"}
{"message": "Leaked key AKIAIOSFODNN7EXAMPLE in request", "service": "marketplace-api", "ddtags": "env:staging,service:marketplace-api"}
{"message": "Temporary key ASIAIOSFODNN7EXAMPLE12 found", "service": "marketplace-api", "ddtags": "env:prod,service:marketplace-api"}
{"message": "No key here", "service": "marketplace-api", "ddtags": "env:prod,service:marketplace-api"}


Bash

cd /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/redact_aws_access_keys_logs && vector vrl -p program.vrl -i inputs.ndjson -q -o 2>&1

Validate VRL syntax and check output


Error: Exit code 70
2026-05-08T17:03:03.115629Z  INFO vector::app: Log level is enabled. level="info"

error[E103]: unhandled fallible assignment
  ┌─ :7:17
  │
7 │ attr_excluded = (is_object(.attributes) && (to_string(get(.attributes, ["req.headers.x-synthetic-test"]) ?? "") == "true"))
  │ --------------- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ this expression is fallible because at least one argument's type cannot be verified to be valid
  │ │
  │ or change this to an infallible assignment:
  │ attr_excluded, err = (is_object(.attributes) && (to_string(get(.attributes, ["req.headers.x-synthetic-test"]) ?? "") == "true"))
  │
  = see documentation about error handling at https://errors.vrl.dev/#handling
  = see functions characteristics documentation at https://vrl.dev/expressions/#function-call-characteristics
  = learn more about error code 103 at https://errors.vrl.dev/103
  = see language documentation at https://vrl.dev
  = try your code in the VRL REPL, learn more at https://vrl.dev/examples

error[E701]: call to undefined variable
  ┌─ :8:92
  │
8 │ should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && true && !attr_excluded
  │                                                                                ^^^^^^^^^^^^^
  │                                                                                │
  │                                                                                undefined variable
  │                                                                                did you mean "true"?
  │
  = see language documentation at https://vrl.dev
  = try your code in the VRL REPL, learn more at https://vrl.dev/examples


`get()` is fallible in VRL — I need to use quoted path access instead. Let me fix the VRL and the terraform local that generates it:


cd /tmp && cat > test_attr.vrl << 'EOF'
attr_excluded = (to_string(.attributes."req.headers.x-synthetic-test") ?? "") == "true"
.
EOF
echo '{"attributes": {"req.headers.x-synthetic-test": "true"}}' | vector vrl -p test_attr.vrl -i - -q -o 2>&1
echo '{"attributes": {"req.headers.x-synthetic-test": "false"}}' | vector vrl -p test_attr.vrl -i - -q -o 2>&1
echo '{"attributes": {}}' | vector vrl -p test_attr.vrl -i - -q -o 2>&1
echo '{}' | vector vrl -p test_attr.vrl -i - -q -o 2>&1

Test quoted path access as infallible VRL alternative to get()


Error: Exit code 70
2026-05-08T17:04:50.487825Z  INFO vector::app: Log level is enabled. level="info"
io error: No such file or directory (os error 2)
2026-05-08T17:04:50.640604Z  INFO vector::app: Log level is enabled. level="info"
io error: No such file or directory (os error 2)
2026-05-08T17:04:50.793811Z  INFO vector::app: Log level is enabled. level="info"
io error: No such file or directory (os error 2)
2026-05-08T17:04:50.939759Z  INFO vector::app: Log level is enabled. level="info"
io error: No such file or directory (os error 2)


cd /tmp && cat > test_attr.vrl << 'EOF'
attr_excluded = (to_string(.attributes."req.headers.x-synthetic-test") ?? "") == "true"
.
EOF
cat > test_inputs.ndjson << 'EOF'
{"attributes": {"req.headers.x-synthetic-test": "true"}}
{"attributes": {"req.headers.x-synthetic-test": "false"}}
{"attributes": {}}
{}
EOF
vector vrl -p test_attr.vrl -i test_inputs.ndjson -q -o 2>&1

Test quoted path access for dotted attribute key in VRL


2026-05-08T17:05:10.002333Z  INFO vector::app: Log level is enabled. level="info"
{ "attributes": { "req.headers.x-synthetic-test": "true" } }
{ "attributes": { "req.headers.x-synthetic-test": "false" } }
{ "attributes": {  } }
{  }
Shell cwd was reset to /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector


Good — no error, the syntax is accepted. Let me verify the values are computed correctly:


cd /tmp && cat > test_attr.vrl << 'EOF'
attr_excluded = (to_string(.attributes."req.headers.x-synthetic-test") ?? "") == "true"
.attr_excluded = attr_excluded
.
EOF
vector vrl -p test_attr.vrl -i test_inputs.ndjson -q -o 2>&1 | jq -c '{attr_excluded, attributes}'

Verify attr_excluded values for each test case


Error: Exit code 5
jq: parse error: Invalid numeric literal at line 1, column 14


cd /tmp && vector vrl -p test_attr.vrl -i test_inputs.ndjson -q -o 2>&1 | jq -c '{attr_excluded: .attr_excluded, has_attrs: (.attributes != null)}'

Verify attr_excluded values


cd /tmp && vector vrl -p test_attr.vrl -i test_inputs.ndjson -q -o 2>/dev/null | jq '.'

Show VRL output with attr_excluded field


{
  "attr_excluded": true,
  "attributes": {
    "req.headers.x-synthetic-test": "true"
  }
}
{
  "attr_excluded": false,
  "attributes": {
    "req.headers.x-synthetic-test": "false"
  }
}
{
  "attr_excluded": false,
  "attributes": {}
}
{
  "attr_excluded": false
}
Shell cwd was reset to /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector


`attr_excluded` is computed correctly across all cases. Now update the terraform local to use path access instead of `get()`, and fix the VRL program:


Edit

  # Each entry generates a VRL boolean: true when the attribute matches and should be excluded.
  # Short-circuits on is_object(.attributes) so get() is never called on a non-object.
  aws_key_attr_exclusion_condition = length(var.aws_access_key_redaction_exclusion_attributes) == 0 ? "false" : join(
    " || ",
    [for k, v in var.aws_access_key_redaction_exclusion_attributes :
      "(is_object(.attributes) && (to_string(get(.attributes, [\"${k}\"]) ?? \"\") == \"${v}\"))"
    ],
  )

  # Each entry generates a VRL boolean: true when the attribute matches and should be excluded.
  # Uses quoted path access (.attributes."dotted.key") which is infallible; returns null for missing paths.
  aws_key_attr_exclusion_condition = length(var.aws_access_key_redaction_exclusion_attributes) == 0 ? "false" : join(
    " || ",
    [for k, v in var.aws_access_key_redaction_exclusion_attributes :
      "(to_string(.attributes.\"${k}\") ?? \"\") == \"${v}\""
    ],
  )


  # Each entry generates a VRL boolean: true when the attribute matches and should be excluded.
  # Uses quoted path access (.attributes."dotted.key") which is infallible; returns null for missing paths.
  aws_key_attr_exclusion_condition = length(var.aws_access_key_redaction_exclusion_attributes) == 0 ? "false" : join(
    " || ",
    [for k, v in var.aws_access_key_redaction_exclusion_attributes :
      "(to_string(.attributes.\"${k}\") ?? \"\") == \"${v}\""
    ],
  )

terraform {
  required_version = ">= 1.9.0"

  required_providers {
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = ">= 2.20"
    }
    helm = {
      source  = "hashicorp/helm"
      version = ">= 2.10"
    }
  }
}

locals {
  audit_drop_condition = length(var.audit_drop_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.audit_drop_patterns : "!match(msg, r'${p}')"],
  )

  bearer_token_exclusion_condition = length(var.bearer_token_redaction_exclusion_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.bearer_token_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
  )

  bearer_token_services_arr = "[\"${join("\", \"", var.bearer_token_redaction_services)}\"]"

  # dry_run always wins over enabled for both scanner types
  email_do_redact = var.email_redaction_enabled && !var.email_redaction_dry_run
  email_do_tag    = var.email_redaction_dry_run

  bearer_do_redact = var.bearer_token_redaction_enabled && !var.bearer_token_redaction_dry_run
  bearer_do_tag    = var.bearer_token_redaction_dry_run

  # named output of each stage used to wire transform inputs and the final sink
  email_stage_output = (
    local.email_do_redact ? "redact_emails_logs" :
    local.email_do_tag    ? "tag_emails_logs" :
    "normalize_logs"
  )

  bearer_stage_active = local.bearer_do_redact || local.bearer_do_tag
  bearer_stage_output = local.bearer_do_redact ? "redact_bearer_tokens_logs" : "tag_bearer_tokens_logs"

  bearer_token_logs_input = local.email_stage_output

  # AWS Access Key ID scanner
  aws_key_do_redact = var.aws_access_key_redaction_enabled && !var.aws_access_key_redaction_dry_run
  aws_key_do_tag    = var.aws_access_key_redaction_dry_run

  aws_key_services_arr = "[\"${join("\", \"", var.aws_access_key_redaction_services)}\"]"

  aws_key_string_exclusion_condition = length(var.aws_access_key_redaction_exclusion_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.aws_access_key_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
  )

  # Each entry generates a VRL boolean: true when the attribute matches and should be excluded.
  # Short-circuits on is_object(.attributes) so get() is never called on a non-object.
  aws_key_attr_exclusion_condition = length(var.aws_access_key_redaction_exclusion_attributes) == 0 ? "false" : join(
    " || ",
    [for k, v in var.aws_access_key_redaction_exclusion_attributes :
      "(is_object(.attributes) && (to_string(get(.attributes, [\"${k}\"]) ?? \"\") == \"${v}\"))"
    ],
  )

  aws_key_stage_active = local.aws_key_do_redact || local.aws_key_do_tag
  aws_key_stage_output = local.aws_key_do_redact ? "redact_aws_access_keys_logs" : "tag_aws_access_keys_logs"

  aws_key_logs_input = local.bearer_stage_active ? local.bearer_stage_output : local.email_stage_output

  logs_sink_input = (
    local.aws_key_stage_active ? local.aws_key_stage_output :
    local.bearer_stage_active  ? local.bearer_stage_output :
    local.email_stage_output
  )

  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
    "${path.module}/templates/values.yaml.tpl",
    {
      replicas                              = var.replicas
      requests_cpu                          = var.resources_requests.cpu
      requests_memory                       = var.resources_requests.memory
      limits_cpu                            = var.resources_limits.cpu
      limits_memory                         = var.resources_limits.memory
      datadog_site                          = var.datadog_site
      audit_drop_condition                  = local.audit_drop_condition
      email_do_redact                       = local.email_do_redact
      email_do_tag                          = local.email_do_tag
      email_redaction_replacement           = var.email_redaction_replacement
      service_monitor_enabled               = var.service_monitor_enabled
      bearer_do_redact                      = local.bearer_do_redact
      bearer_do_tag                         = local.bearer_do_tag
      bearer_token_redaction_replacement    = var.bearer_token_redaction_replacement
      bearer_token_services_arr             = local.bearer_token_services_arr
      bearer_token_exclusion_condition      = local.bearer_token_exclusion_condition
      bearer_token_logs_input               = local.bearer_token_logs_input
      aws_key_do_redact                     = local.aws_key_do_redact
      aws_key_do_tag                        = local.aws_key_do_tag
      aws_access_key_redaction_replacement  = var.aws_access_key_redaction_replacement
      aws_key_services_arr                  = local.aws_key_services_arr
      aws_key_string_exclusion_condition    = local.aws_key_string_exclusion_condition
      aws_key_attr_exclusion_condition      = local.aws_key_attr_exclusion_condition
      aws_key_logs_input                    = local.aws_key_logs_input
      logs_sink_input                       = local.logs_sink_input
    },
  )
}

resource "kubernetes_namespace_v1" "vector" {
  metadata {
    name = var.namespace
    labels = {
      "app.kubernetes.io/name"      = "vector"
      "app.kubernetes.io/component" = "telemetry-aggregator"
    }
  }
}

resource "kubernetes_manifest" "datadog_external_secret" {
  manifest = {
    apiVersion = "external-secrets.io/v1"
    kind       = "ExternalSecret"
    metadata = {
      name      = "vector-datadog"
      namespace = kubernetes_namespace_v1.vector.metadata[0].name
    }
    spec = {
      refreshInterval = "1h"
      secretStoreRef = {
        kind = "ClusterSecretStore"
        name = var.external_secrets_cluster_store_name
      }
      target = {
        name           = "vector-datadog"
        creationPolicy = "Owner"
      }
      data = [{
        secretKey = "api_key"
        remoteRef = {
          key      = var.datadog_api_key_vault_path
          property = "apiKey"
        }
      }]
    }
  }

  wait {
    fields = {
      "status.conditions[0].type"   = "Ready"
      "status.conditions[0].status" = "True"
    }
  }

  timeouts {
    create = "5m"
    update = "5m"
  }
}

resource "helm_release" "vector" {
  name       = "vector"
  namespace  = kubernetes_namespace_v1.vector.metadata[0].name
  repository = "https://helm.vector.dev"
  chart      = "vector"
  version    = var.chart_version

  values = [local.rendered_values]

  set {
    name  = "image.repository"
    value = var.image
  }
  set {
    name  = "image.tag"
    value = var.image_tag_version
  }

  depends_on = [
    kubernetes_manifest.datadog_external_secret,
  ]
}

module "netpol" {
  source = "crow.infrastructure.immuta.io/internal/security-network-policy/kubernetes"
  version = "1.1.0"

  name             = "vector-netpol"
  namespace        = var.namespace
  network_policies = local.netpols

  depends_on = [
    kubernetes_namespace_v1.vector,
  ]
}

-  # Short-circuits on is_object(.attributes) so get() is never called on a non-object.

+  # Uses quoted path access (.attributes."dotted.key") which is infallible; returns null for missing paths.


Now update the VRL program to use the same syntax:


Write

aws_key_re = r'(AKIA|ASIA|AROA|AIPA|ANPA|ANVA|APKA)[A-Z0-9]{16}'
svc = to_string(.service) ?? ""
ddtags = to_string(.ddtags) ?? ""
msg = if is_string(.message) { string!(.message) } else { "" }
combined = msg + " " + ddtags
target_services = ["marketplace-api"]
attr_excluded = (to_string(.attributes."req.headers.x-synthetic-test") ?? "") == "true"
should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && !attr_excluded
if should_redact && is_string(.message) {
  .message = replace(string!(.message), aws_key_re, "[REDACTED-AWS-ACCESS-KEY]")
}
.


The file /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/redact_aws_access_keys_logs/program.vrl has been updated successfully. (file state is current in your context — no need to Read it back)

aws_key_re = r'(AKIA|ASIA|AROA|AIPA|ANPA|ANVA|APKA)[A-Z0-9]{16}'
svc = to_string(.service) ?? ""
ddtags = to_string(.ddtags) ?? ""
msg = if is_string(.message) { string!(.message) } else { "" }
combined = msg + " " + ddtags
target_services = ["marketplace-api"]
attr_excluded = (to_string(.attributes."req.headers.x-synthetic-test") ?? "") == "true"
should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && !attr_excluded
if should_redact && is_string(.message) {
  .message = replace(string!(.message), aws_key_re, "[REDACTED-AWS-ACCESS-KEY]")
}
.

-attr_excluded = (is_object(.attributes) && (to_string(get(.attributes, ["req.headers.x-synthetic-test"]) ?? "") == "true"))

-should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && true && !attr_excluded

+attr_excluded = (to_string(.attributes."req.headers.x-synthetic-test") ?? "") == "true"

+should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && !attr_excluded

aws_key_re = r'(AKIA|ASIA|AROA|AIPA|ANPA|ANVA|APKA)[A-Z0-9]{16}'
svc = to_string(.service) ?? ""
ddtags = to_string(.ddtags) ?? ""
msg = if is_string(.message) { string!(.message) } else { "" }
combined = msg + " " + ddtags
target_services = ["marketplace-api"]
attr_excluded = (is_object(.attributes) && (to_string(get(.attributes, ["req.headers.x-synthetic-test"]) ?? "") == "true"))
should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && true && !attr_excluded
if should_redact && is_string(.message) {
  .message = replace(string!(.message), aws_key_re, "[REDACTED-AWS-ACCESS-KEY]")
}
.


Bash

cd /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/redact_aws_access_keys_logs && vector vrl -p program.vrl -i inputs.ndjson -q -o 2>/dev/null | jq -c . > expected.ndjson && cat expected.ndjson

Generate expected output for redact_aws_access_keys_logs


{"ddtags":"env:prod,service:marketplace-api","message":"Leaked key [REDACTED-AWS-ACCESS-KEY] in request","service":"marketplace-api"}
{"attributes":{"req.headers.x-synthetic-test":"true"},"ddtags":"env:prod,service:marketplace-api","message":"Leaked key AKIAIOSFODNN7EXAMPLE in request","service":"marketplace-api"}
{"attributes":{"req.headers.x-synthetic-test":"false"},"ddtags":"env:prod,service:marketplace-api","message":"Leaked key [REDACTED-AWS-ACCESS-KEY] in request","service":"marketplace-api"}
{"ddtags":"env:prod,service:other-service","message":"Leaked key AKIAIOSFODNN7EXAMPLE in request","service":"other-service"}
{"ddtags":"env:staging,service:marketplace-api","message":"Leaked key AKIAIOSFODNN7EXAMPLE in request","service":"marketplace-api"}
{"ddtags":"env:prod,service:marketplace-api","message":"Temporary key [REDACTED-AWS-ACCESS-KEY]12 found","service":"marketplace-api"}
{"ddtags":"env:prod,service:marketplace-api","message":"No key here","service":"marketplace-api"}


Results are correct — redacts for matching service+env, passes through for synthetic-test header `true`, wrong service, and wrong env. Now add the run script and build the tag variant:


Write

#!/usr/bin/env bash
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ACTUAL="$DIR/.actual.ndjson"
vector vrl -p "$DIR/program.vrl" -i "$DIR/inputs.ndjson" -q -o > "$ACTUAL"
diff -u <(jq -c . "$DIR/expected.ndjson") <(jq -c . "$ACTUAL")
echo "redact_aws_access_keys_logs VRL OK"


File created successfully at: /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/redact_aws_access_keys_logs/run.sh (file state is current in your context — no need to Read it back)

#!/usr/bin/env bash
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ACTUAL="$DIR/.actual.ndjson"
vector vrl -p "$DIR/program.vrl" -i "$DIR/inputs.ndjson" -q -o > "$ACTUAL"
diff -u <(jq -c . "$DIR/expected.ndjson") <(jq -c . "$ACTUAL")
echo "redact_aws_access_keys_logs VRL OK"


aws_key_re = r'(AKIA|ASIA|AROA|AIPA|ANPA|ANVA|APKA)[A-Z0-9]{16}'
svc = to_string(.service) ?? ""
ddtags = to_string(.ddtags) ?? ""
msg = if is_string(.message) { string!(.message) } else { "" }
combined = msg + " " + ddtags
target_services = ["marketplace-api"]
attr_excluded = (to_string(.attributes."req.headers.x-synthetic-test") ?? "") == "true"
would_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && !attr_excluded
if would_redact && is_string(.message) && match(string!(.message), aws_key_re) {
  .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_aws_key_detected:true" } else { "vector_aws_key_detected:true" }
}
.


File created successfully at: /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/tag_aws_access_keys_logs/program.vrl (file state is current in your context — no need to Read it back)

aws_key_re = r'(AKIA|ASIA|AROA|AIPA|ANPA|ANVA|APKA)[A-Z0-9]{16}'
svc = to_string(.service) ?? ""
ddtags = to_string(.ddtags) ?? ""
msg = if is_string(.message) { string!(.message) } else { "" }
combined = msg + " " + ddtags
target_services = ["marketplace-api"]
attr_excluded = (to_string(.attributes."req.headers.x-synthetic-test") ?? "") == "true"
would_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && !attr_excluded
if would_redact && is_string(.message) && match(string!(.message), aws_key_re) {
  .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_aws_key_detected:true" } else { "vector_aws_key_detected:true" }
}
.


{"message": "Leaked key AKIAIOSFODNN7EXAMPLE in request", "service": "marketplace-api", "ddtags": "env:prod,service:marketplace-api"}
{"message": "Leaked key AKIAIOSFODNN7EXAMPLE in request", "service": "marketplace-api", "ddtags": "env:prod,service:marketplace-api", "attributes": {"req.headers.x-synthetic-test": "true"}}
{"message": "Leaked key AKIAIOSFODNN7EXAMPLE in request", "service": "other-service", "ddtags": "env:prod,service:other-service"}
{"message": "No key here", "service": "marketplace-api", "ddtags": "env:prod,service:marketplace-api"}


File created successfully at: /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/tag_aws_access_keys_logs/inputs.ndjson (file state is current in your context — no need to Read it back)

{"message": "Leaked key AKIAIOSFODNN7EXAMPLE in request", "service": "marketplace-api", "ddtags": "env:prod,service:marketplace-api"}
{"message": "Leaked key AKIAIOSFODNN7EXAMPLE in request", "service": "marketplace-api", "ddtags": "env:prod,service:marketplace-api", "attributes": {"req.headers.x-synthetic-test": "true"}}
{"message": "Leaked key AKIAIOSFODNN7EXAMPLE in request", "service": "other-service", "ddtags": "env:prod,service:other-service"}
{"message": "No key here", "service": "marketplace-api", "ddtags": "env:prod,service:marketplace-api"}


Bash

cd /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/tag_aws_access_keys_logs && vector vrl -p program.vrl -i inputs.ndjson -q -o 2>/dev/null | jq -c . > expected.ndjson && cat expected.ndjson

Generate expected output for tag_aws_access_keys_logs


{"ddtags":"env:prod,service:marketplace-api,vector_aws_key_detected:true","message":"Leaked key AKIAIOSFODNN7EXAMPLE in request","service":"marketplace-api"}
{"attributes":{"req.headers.x-synthetic-test":"true"},"ddtags":"env:prod,service:marketplace-api","message":"Leaked key AKIAIOSFODNN7EXAMPLE in request","service":"marketplace-api"}
{"ddtags":"env:prod,service:other-service","message":"Leaked key AKIAIOSFODNN7EXAMPLE in request","service":"other-service"}
{"ddtags":"env:prod,service:marketplace-api","message":"No key here","service":"marketplace-api"}


Write

#!/usr/bin/env bash
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ACTUAL="$DIR/.actual.ndjson"
vector vrl -p "$DIR/program.vrl" -i "$DIR/inputs.ndjson" -q -o > "$ACTUAL"
diff -u <(jq -c . "$DIR/expected.ndjson") <(jq -c . "$ACTUAL")
echo "tag_aws_access_keys_logs VRL OK"


File created successfully at: /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/tag_aws_access_keys_logs/run.sh (file state is current in your context — no need to Read it back)

#!/usr/bin/env bash
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ACTUAL="$DIR/.actual.ndjson"
vector vrl -p "$DIR/program.vrl" -i "$DIR/inputs.ndjson" -q -o > "$ACTUAL"
diff -u <(jq -c . "$DIR/expected.ndjson") <(jq -c . "$ACTUAL")
echo "tag_aws_access_keys_logs VRL OK"


Bash

chmod +x /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/redact_aws_access_keys_logs/run.sh /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/tag_aws_access_keys_logs/run.sh && for f in /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/*/run.sh; do bash "$f" 2>/dev/null || echo "FAIL: $f"; done

Run full VRL test suite


drop_audit VRL OK
redact_aws_access_keys_logs VRL OK
redact_bearer_tokens_logs VRL OK
redact_emails_dd_agent_logs VRL OK
redact_emails_logs VRL OK
redact_emails_metrics VRL OK
redact_emails_traces VRL OK
tag_aws_access_keys_logs VRL OK
tag_bearer_tokens_logs VRL OK
tag_emails_logs VRL OK


All 10 tests pass. Here's a summary of what was added, Sir Charles of St Cheese:

**New variables** (`variables.tf`):
- `aws_access_key_redaction_enabled` / `_dry_run` — same on/off/dry_run semantics as the other scanners
- `aws_access_key_redaction_services` — default `["marketplace-api"]`
- `aws_access_key_redaction_exclusion_patterns` — string contains check against message+ddtags (empty by default)
- `aws_access_key_redaction_exclusion_attributes` — new type: `map(string)`, default `{"req.headers.x-synthetic-test" = "true"}`

**New mechanism** — attribute-based exclusion. The key insight is that `NOT @req.headers.x-synthetic-test:true` in the SDS query is a structured attribute check, not a string search. The terraform local generates infallible VRL using quoted path access (`.attributes."dotted.key"`) rather than `get()` which is fallible:

```vrl
attr_excluded = (to_string(.attributes."req.headers.x-synthetic-test") ?? "") == "true"
```

**Transform chain** now has three stages: email → bearer token → AWS key. The `logs_sink_input` local cascades through all three.

**Regex** matches all AWS key prefixes: `AKIA` (long-term), `ASIA` (STS temporary), `AROA`, `AIPA`, `ANPA`, `ANVA`, `APKA` followed by exactly 16 uppercase alphanumeric characters.


afplay ~/.claude/sounds/mario/stop/stomp.wav 2>/dev/null || true


can you also add a JSON Web Token rule? The datadog SDS query is `env:prod AND (service:service OR service:secure-metadata-database) AND NOT "internal-webhook" AND NOT "starting crawl" AND NOT "Databricks automatic subscription" AND NOT "audit/enrich" AND NOT "Background Job" AND NOT community-organization-id:immuta AND NOT resource_name:"pg - pool.connect"`


The filter query is identical to the bearer token scanner — same services, same exclusions — so the JWT scanner is essentially a new regex sitting at the end of the existing chain. I'll follow the same pattern: new variables, extend the chain locals, two new template blocks, two new VRL tests.


Edit

variable "aws_access_key_redaction_exclusion_attributes" {
  description = "Skip AWS Access Key ID redaction for any log where the named attribute equals the given value (keys are attribute paths, values are string values to match)"
  type        = map(string)
  default = {
    "req.headers.x-synthetic-test" = "true"
  }
}

variable "aws_access_key_redaction_exclusion_attributes" {
  description = "Skip AWS Access Key ID redaction for any log where the named attribute equals the given value (keys are attribute paths, values are string values to match)"
  type        = map(string)
  default = {
    "req.headers.x-synthetic-test" = "true"
  }
}

variable "jwt_redaction_enabled" {
  description = "Toggle JSON Web Token redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "jwt_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_jwt_detected:true instead of replacing. Takes priority over jwt_redaction_enabled."
  type        = bool
  default     = false
}

variable "jwt_redaction_replacement" {
  description = "String that replaces matched JSON Web Tokens"
  type        = string
  default     = "[REDACTED-JWT]"
}

variable "jwt_redaction_services" {
  description = "Service names (.service field on DD agent logs) to apply JWT redaction to"
  type        = list(string)
  default     = ["service", "secure-metadata-database"]
}

variable "jwt_redaction_exclusion_patterns" {
  description = "Skip JWT redaction for any log whose message or ddtags contains one of these strings"
  type        = list(string)
  default = [
    "internal-webhook",
    "starting crawl",
    "Databricks automatic subscription",
    "audit/enrich",
    "Background Job",
    "community-organization-id:immuta",
    "pg - pool.connect",
  ]
}


can you also add a JSON Web Token rule? The datadog SDS query is `env:prod AND (service:service OR service:secure-metadata-database) AND NOT "internal-webhook" AND NOT "starting crawl" AND NOT "Databr…


The file /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/variables.tf has been updated successfully. (file state is current in your context — no need to Read it back)

variable "aws_access_key_redaction_exclusion_attributes" {
  description = "Skip AWS Access Key ID redaction for any log where the named attribute equals the given value (keys are attribute paths, values are string values to match)"
  type        = map(string)
  default = {
    "req.headers.x-synthetic-test" = "true"
  }
}

variable "jwt_redaction_enabled" {
  description = "Toggle JSON Web Token redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "jwt_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_jwt_detected:true instead of replacing. Takes priority over jwt_redaction_enabled."
  type        = bool
  default     = false
}

variable "jwt_redaction_replacement" {
  description = "String that replaces matched JSON Web Tokens"
  type        = string
  default     = "[REDACTED-JWT]"
}

variable "jwt_redaction_services" {
  description = "Service names (.service field on DD agent logs) to apply JWT redaction to"
  type        = list(string)
  default     = ["service", "secure-metadata-database"]
}

variable "jwt_redaction_exclusion_patterns" {
  description = "Skip JWT redaction for any log whose message or ddtags contains one of these strings"
  type        = list(string)
  default = [
    "internal-webhook",
    "starting crawl",
    "Databricks automatic subscription",
    "audit/enrich",
    "Background Job",
    "community-organization-id:immuta",
    "pg - pool.connect",
  ]
}

# ---------------------------------------------------------------------------------------------------------------------
# REQUIRED PARAMETERS
# These variables must be set.
# ---------------------------------------------------------------------------------------------------------------------

variable "chart_version" {
  description = "Vector helm chart version pin"
  type        = string
}

variable "image_tag_version" {
  description = "Vector image tag"
  type        = string
}

# ---------------------------------------------------------------------------------------------------------------------
# OPTIONAL PARAMETERS
# These variables are configurable, but have reasonable defaults set.
# ---------------------------------------------------------------------------------------------------------------------
variable "datadog_site" {
  description = "Datadog site (e.g. datadoghq.com, us3.datadoghq.com)"
  type        = string
  default     = "datadoghq.com"
}

variable "image" {
  description = "Vector image source"
  type        = string
  default     = "registry.ic-ops.net/cache-docker-hub/timberio/vector"
}

variable "replicas" {
  description = "Number of Vector aggregator replicas"
  type        = number
  default     = 3
}

variable "audit_drop_patterns" {
  description = "Regex patterns; matching log message bodies are dropped"
  type        = list(string)
  default = [
    "Audit\\s-\\s.+",
    "\"level\":\"audit\"",
  ]
}

variable "datadog_api_key_vault_path" {
  description = "Vault path for the Datadog API key, consumed by ExternalSecret"
  type        = string
  default     = "secret/data/saas/infrastructure/datadog-api-key"
}

variable "external_secrets_cluster_store_name" {
  description = "Name of the ClusterSecretStore created by external-secrets module"
  type        = string
  default     = "vault"
}

variable "email_redaction_enabled" {
  description = "Toggle the email-redact VRL transform across all signals"
  type        = bool
  default     = true
}

variable "email_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_redaction:true instead of replacing. Takes priority over email_redaction_enabled."
  type        = bool
  default     = false
}

variable "email_redaction_replacement" {
  description = "String that replaces matched email addresses"
  type        = string
  default     = "[REDACTED-EMAIL]"
}

variable "additional_config" {
  description = "Escape hatch — overwrites helm customConfig wholesale (mirrors otel_collector_additional_config semantics)"
  type        = any
  default     = null
}

variable "resources_requests" {
  description = "Pod resource requests"
  type        = map(string)
  default = {
    cpu    = "200m"
    memory = "256Mi"
  }
}

variable "resources_limits" {
  description = "Pod resource limits"
  type        = map(string)
  default = {
    cpu    = "2"
    memory = "1Gi"
  }
}

variable "namespace" {
  description = "Kubernetes namespace for Vector"
  type        = string
  default     = "vector"
}

variable "service_monitor_enabled" {
  description = "Create a ServiceMonitor for Vector self-metrics. Requires kube-prometheus-stack CRDs in cluster."
  type        = bool
  default     = false
}

variable "bearer_token_redaction_enabled" {
  description = "Toggle bearer token redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "bearer_token_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_bearer_token_detected:true instead of replacing. Takes priority over bearer_token_redaction_enabled."
  type        = bool
  default     = false
}

variable "aws_access_key_redaction_enabled" {
  description = "Toggle AWS Access Key ID redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "aws_access_key_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_aws_key_detected:true instead of replacing. Takes priority over aws_access_key_redaction_enabled."
  type        = bool
  default     = false
}

variable "aws_access_key_redaction_replacement" {
  description = "String that replaces matched AWS Access Key IDs"
  type        = string
  default     = "[REDACTED-AWS-ACCESS-KEY]"
}

variable "aws_access_key_redaction_services" {
  description = "Service names (.service field on DD agent logs) to apply AWS Access Key ID redaction to"
  type        = list(string)
  default     = ["marketplace-api"]
}

variable "aws_access_key_redaction_exclusion_patterns" {
  description = "Skip AWS Access Key ID redaction for any log whose message or ddtags contains one of these strings"
  type        = list(string)
  default     = []
}

variable "aws_access_key_redaction_exclusion_attributes" {
  description = "Skip AWS Access Key ID redaction for any log where the named attribute equals the given value (keys are attribute paths, values are string values to match)"
  type        = map(string)
  default = {
    "req.headers.x-synthetic-test" = "true"
  }
}

variable "bearer_token_redaction_replacement" {
  description = "String that replaces matched bearer tokens (replaces the full 'Bearer <token>' match)"
  type        = string
  default     = "Bearer [REDACTED-BEARER-TOKEN]"
}

variable "bearer_token_redaction_services" {
  description = "Service names (.service field on DD agent logs) to apply bearer token redaction to"
  type        = list(string)
  default     = ["service", "secure-metadata-database"]
}

variable "bearer_token_redaction_exclusion_patterns" {
  description = "Skip bearer token redaction for any log whose message or ddtags contains one of these strings"
  type        = list(string)
  default = [
    "internal-webhook",
    "starting crawl",
    "Databricks automatic subscription",
    "audit/enrich",
    "Background Job",
    "community-organization-id:immuta",
    "pg - pool.connect",
  ]
}

+  description = "Toggle JSON Web Token redact VRL transform on logs"

+  description = "When true, tag logs that would have been redacted with vector_jwt_detected:true instead of replacing. Takes priority over jwt_redaction_enabled."

+  description = "String that replaces matched JSON Web Tokens"

+  description = "Service names (.service field on DD agent logs) to apply JWT redaction to"

+  default     = ["service", "secure-metadata-database"]

+  description = "Skip JWT redaction for any log whose message or ddtags contains one of these strings"

+    "Databricks automatic subscription",

+    "community-organization-id:immuta",


Now extend `main.tf` — add JWT locals and update `logs_sink_input` to include the fourth stage:


  aws_key_stage_active = local.aws_key_do_redact || local.aws_key_do_tag
  aws_key_stage_output = local.aws_key_do_redact ? "redact_aws_access_keys_logs" : "tag_aws_access_keys_logs"

  aws_key_logs_input = local.bearer_stage_active ? local.bearer_stage_output : local.email_stage_output

  logs_sink_input = (
    local.aws_key_stage_active ? local.aws_key_stage_output :
    local.bearer_stage_active  ? local.bearer_stage_output :
    local.email_stage_output
  )

  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
    "${path.module}/templates/values.yaml.tpl",
    {
      replicas                              = var.replicas
      requests_cpu                          = var.resources_requests.cpu
      requests_memory                       = var.resources_requests.memory
      limits_cpu                            = var.resources_limits.cpu
      limits_memory                         = var.resources_limits.memory
      datadog_site                          = var.datadog_site
      audit_drop_condition                  = local.audit_drop_condition
      email_do_redact                       = local.email_do_redact
      email_do_tag                          = local.email_do_tag
      email_redaction_replacement           = var.email_redaction_replacement
      service_monitor_enabled               = var.service_monitor_enabled
      bearer_do_redact                      = local.bearer_do_redact
      bearer_do_tag                         = local.bearer_do_tag
      bearer_token_redaction_replacement    = var.bearer_token_redaction_replacement
      bearer_token_services_arr             = local.bearer_token_services_arr
      bearer_token_exclusion_condition      = local.bearer_token_exclusion_condition
      bearer_token_logs_input               = local.bearer_token_logs_input
      aws_key_do_redact                     = local.aws_key_do_redact
      aws_key_do_tag                        = local.aws_key_do_tag
      aws_access_key_redaction_replacement  = var.aws_access_key_redaction_replacement
      aws_key_services_arr                  = local.aws_key_services_arr
      aws_key_string_exclusion_condition    = local.aws_key_string_exclusion_condition
      aws_key_attr_exclusion_condition      = local.aws_key_attr_exclusion_condition
      aws_key_logs_input                    = local.aws_key_logs_input
      logs_sink_input                       = local.logs_sink_input
    },
  )
}

  aws_key_stage_active = local.aws_key_do_redact || local.aws_key_do_tag
  aws_key_stage_output = local.aws_key_do_redact ? "redact_aws_access_keys_logs" : "tag_aws_access_keys_logs"

  aws_key_logs_input = local.bearer_stage_active ? local.bearer_stage_output : local.email_stage_output

  # JWT scanner
  jwt_do_redact = var.jwt_redaction_enabled && !var.jwt_redaction_dry_run
  jwt_do_tag    = var.jwt_redaction_dry_run

  jwt_services_arr = "[\"${join("\", \"", var.jwt_redaction_services)}\"]"

  jwt_exclusion_condition = length(var.jwt_redaction_exclusion_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.jwt_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
  )

  jwt_stage_active = local.jwt_do_redact || local.jwt_do_tag
  jwt_stage_output = local.jwt_do_redact ? "redact_jwts_logs" : "tag_jwts_logs"

  jwt_logs_input = (
    local.aws_key_stage_active ? local.aws_key_stage_output :
    local.bearer_stage_active  ? local.bearer_stage_output :
    local.email_stage_output
  )

  logs_sink_input = (
    local.jwt_stage_active     ? local.jwt_stage_output :
    local.aws_key_stage_active ? local.aws_key_stage_output :
    local.bearer_stage_active  ? local.bearer_stage_output :
    local.email_stage_output
  )

  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
    "${path.module}/templates/values.yaml.tpl",
    {
      replicas                              = var.replicas
      requests_cpu                          = var.resources_requests.cpu
      requests_memory                       = var.resources_requests.memory
      limits_cpu                            = var.resources_limits.cpu
      limits_memory                         = var.resources_limits.memory
      datadog_site                          = var.datadog_site
      audit_drop_condition                  = local.audit_drop_condition
      email_do_redact                       = local.email_do_redact
      email_do_tag                          = local.email_do_tag
      email_redaction_replacement           = var.email_redaction_replacement
      service_monitor_enabled               = var.service_monitor_enabled
      bearer_do_redact                      = local.bearer_do_redact
      bearer_do_tag                         = local.bearer_do_tag
      bearer_token_redaction_replacement    = var.bearer_token_redaction_replacement
      bearer_token_services_arr             = local.bearer_token_services_arr
      bearer_token_exclusion_condition      = local.bearer_token_exclusion_condition
      bearer_token_logs_input               = local.bearer_token_logs_input
      aws_key_do_redact                     = local.aws_key_do_redact
      aws_key_do_tag                        = local.aws_key_do_tag
      aws_access_key_redaction_replacement  = var.aws_access_key_redaction_replacement
      aws_key_services_arr                  = local.aws_key_services_arr
      aws_key_string_exclusion_condition    = local.aws_key_string_exclusion_condition
      aws_key_attr_exclusion_condition      = local.aws_key_attr_exclusion_condition
      aws_key_logs_input                    = local.aws_key_logs_input
      jwt_do_redact                         = local.jwt_do_redact
      jwt_do_tag                            = local.jwt_do_tag
      jwt_redaction_replacement             = var.jwt_redaction_replacement
      jwt_services_arr                      = local.jwt_services_arr
      jwt_exclusion_condition               = local.jwt_exclusion_condition
      jwt_logs_input                        = local.jwt_logs_input
      logs_sink_input                       = local.logs_sink_input
    },
  )
}


The file /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/main.tf has been updated successfully. (file state is current in your context — no need to Read it back)

  aws_key_stage_active = local.aws_key_do_redact || local.aws_key_do_tag
  aws_key_stage_output = local.aws_key_do_redact ? "redact_aws_access_keys_logs" : "tag_aws_access_keys_logs"

  aws_key_logs_input = local.bearer_stage_active ? local.bearer_stage_output : local.email_stage_output

  # JWT scanner
  jwt_do_redact = var.jwt_redaction_enabled && !var.jwt_redaction_dry_run
  jwt_do_tag    = var.jwt_redaction_dry_run

  jwt_services_arr = "[\"${join("\", \"", var.jwt_redaction_services)}\"]"

  jwt_exclusion_condition = length(var.jwt_redaction_exclusion_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.jwt_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
  )

  jwt_stage_active = local.jwt_do_redact || local.jwt_do_tag
  jwt_stage_output = local.jwt_do_redact ? "redact_jwts_logs" : "tag_jwts_logs"

  jwt_logs_input = (
    local.aws_key_stage_active ? local.aws_key_stage_output :
    local.bearer_stage_active  ? local.bearer_stage_output :
    local.email_stage_output
  )

  logs_sink_input = (
    local.jwt_stage_active     ? local.jwt_stage_output :
    local.aws_key_stage_active ? local.aws_key_stage_output :
    local.bearer_stage_active  ? local.bearer_stage_output :
    local.email_stage_output
  )

  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
    "${path.module}/templates/values.yaml.tpl",
    {
      replicas                              = var.replicas
      requests_cpu                          = var.resources_requests.cpu
      requests_memory                       = var.resources_requests.memory
      limits_cpu                            = var.resources_limits.cpu
      limits_memory                         = var.resources_limits.memory
      datadog_site                          = var.datadog_site
      audit_drop_condition                  = local.audit_drop_condition
      email_do_redact                       = local.email_do_redact
      email_do_tag                          = local.email_do_tag
      email_redaction_replacement           = var.email_redaction_replacement
      service_monitor_enabled               = var.service_monitor_enabled
      bearer_do_redact                      = local.bearer_do_redact
      bearer_do_tag                         = local.bearer_do_tag
      bearer_token_redaction_replacement    = var.bearer_token_redaction_replacement
      bearer_token_services_arr             = local.bearer_token_services_arr
      bearer_token_exclusion_condition      = local.bearer_token_exclusion_condition
      bearer_token_logs_input               = local.bearer_token_logs_input
      aws_key_do_redact                     = local.aws_key_do_redact
      aws_key_do_tag                        = local.aws_key_do_tag
      aws_access_key_redaction_replacement  = var.aws_access_key_redaction_replacement
      aws_key_services_arr                  = local.aws_key_services_arr
      aws_key_string_exclusion_condition    = local.aws_key_string_exclusion_condition
      aws_key_attr_exclusion_condition      = local.aws_key_attr_exclusion_condition
      aws_key_logs_input                    = local.aws_key_logs_input
      jwt_do_redact                         = local.jwt_do_redact
      jwt_do_tag                            = local.jwt_do_tag
      jwt_redaction_replacement             = var.jwt_redaction_replacement
      jwt_services_arr                      = local.jwt_services_arr
      jwt_exclusion_condition               = local.jwt_exclusion_condition
      jwt_logs_input                        = local.jwt_logs_input
      logs_sink_input                       = local.logs_sink_input
    },
  )
}

terraform {
  required_version = ">= 1.9.0"

  required_providers {
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = ">= 2.20"
    }
    helm = {
      source  = "hashicorp/helm"
      version = ">= 2.10"
    }
  }
}

locals {
  audit_drop_condition = length(var.audit_drop_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.audit_drop_patterns : "!match(msg, r'${p}')"],
  )

  bearer_token_exclusion_condition = length(var.bearer_token_redaction_exclusion_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.bearer_token_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
  )

  bearer_token_services_arr = "[\"${join("\", \"", var.bearer_token_redaction_services)}\"]"

  # dry_run always wins over enabled for both scanner types
  email_do_redact = var.email_redaction_enabled && !var.email_redaction_dry_run
  email_do_tag    = var.email_redaction_dry_run

  bearer_do_redact = var.bearer_token_redaction_enabled && !var.bearer_token_redaction_dry_run
  bearer_do_tag    = var.bearer_token_redaction_dry_run

  # named output of each stage used to wire transform inputs and the final sink
  email_stage_output = (
    local.email_do_redact ? "redact_emails_logs" :
    local.email_do_tag    ? "tag_emails_logs" :
    "normalize_logs"
  )

  bearer_stage_active = local.bearer_do_redact || local.bearer_do_tag
  bearer_stage_output = local.bearer_do_redact ? "redact_bearer_tokens_logs" : "tag_bearer_tokens_logs"

  bearer_token_logs_input = local.email_stage_output

  # AWS Access Key ID scanner
  aws_key_do_redact = var.aws_access_key_redaction_enabled && !var.aws_access_key_redaction_dry_run
  aws_key_do_tag    = var.aws_access_key_redaction_dry_run

  aws_key_services_arr = "[\"${join("\", \"", var.aws_access_key_redaction_services)}\"]"

  aws_key_string_exclusion_condition = length(var.aws_access_key_redaction_exclusion_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.aws_access_key_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
  )

  # Each entry generates a VRL boolean: true when the attribute matches and should be excluded.
  # Uses quoted path access (.attributes."dotted.key") which is infallible; returns null for missing paths.
  aws_key_attr_exclusion_condition = length(var.aws_access_key_redaction_exclusion_attributes) == 0 ? "false" : join(
    " || ",
    [for k, v in var.aws_access_key_redaction_exclusion_attributes :
      "(to_string(.attributes.\"${k}\") ?? \"\") == \"${v}\""
    ],
  )

  aws_key_stage_active = local.aws_key_do_redact || local.aws_key_do_tag
  aws_key_stage_output = local.aws_key_do_redact ? "redact_aws_access_keys_logs" : "tag_aws_access_keys_logs"

  aws_key_logs_input = local.bearer_stage_active ? local.bearer_stage_output : local.email_stage_output

  logs_sink_input = (
    local.aws_key_stage_active ? local.aws_key_stage_output :
    local.bearer_stage_active  ? local.bearer_stage_output :
    local.email_stage_output
  )

  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
    "${path.module}/templates/values.yaml.tpl",
    {
      replicas                              = var.replicas
      requests_cpu                          = var.resources_requests.cpu
      requests_memory                       = var.resources_requests.memory
      limits_cpu                            = var.resources_limits.cpu
      limits_memory                         = var.resources_limits.memory
      datadog_site                          = var.datadog_site
      audit_drop_condition                  = local.audit_drop_condition
      email_do_redact                       = local.email_do_redact
      email_do_tag                          = local.email_do_tag
      email_redaction_replacement           = var.email_redaction_replacement
      service_monitor_enabled               = var.service_monitor_enabled
      bearer_do_redact                      = local.bearer_do_redact
      bearer_do_tag                         = local.bearer_do_tag
      bearer_token_redaction_replacement    = var.bearer_token_redaction_replacement
      bearer_token_services_arr             = local.bearer_token_services_arr
      bearer_token_exclusion_condition      = local.bearer_token_exclusion_condition
      bearer_token_logs_input               = local.bearer_token_logs_input
      aws_key_do_redact                     = local.aws_key_do_redact
      aws_key_do_tag                        = local.aws_key_do_tag
      aws_access_key_redaction_replacement  = var.aws_access_key_redaction_replacement
      aws_key_services_arr                  = local.aws_key_services_arr
      aws_key_string_exclusion_condition    = local.aws_key_string_exclusion_condition
      aws_key_attr_exclusion_condition      = local.aws_key_attr_exclusion_condition
      aws_key_logs_input                    = local.aws_key_logs_input
      logs_sink_input                       = local.logs_sink_input
    },
  )
}

resource "kubernetes_namespace_v1" "vector" {
  metadata {
    name = var.namespace
    labels = {
      "app.kubernetes.io/name"      = "vector"
      "app.kubernetes.io/component" = "telemetry-aggregator"
    }
  }
}

resource "kubernetes_manifest" "datadog_external_secret" {
  manifest = {
    apiVersion = "external-secrets.io/v1"
    kind       = "ExternalSecret"
    metadata = {
      name      = "vector-datadog"
      namespace = kubernetes_namespace_v1.vector.metadata[0].name
    }
    spec = {
      refreshInterval = "1h"
      secretStoreRef = {
        kind = "ClusterSecretStore"
        name = var.external_secrets_cluster_store_name
      }
      target = {
        name           = "vector-datadog"
        creationPolicy = "Owner"
      }
      data = [{
        secretKey = "api_key"
        remoteRef = {
          key      = var.datadog_api_key_vault_path
          property = "apiKey"
        }
      }]
    }
  }

  wait {
    fields = {
      "status.conditions[0].type"   = "Ready"
      "status.conditions[0].status" = "True"
    }
  }

  timeouts {
    create = "5m"
    update = "5m"
  }
}

resource "helm_release" "vector" {
  name       = "vector"
  namespace  = kubernetes_namespace_v1.vector.metadata[0].name
  repository = "https://helm.vector.dev"
  chart      = "vector"
  version    = var.chart_version

  values = [local.rendered_values]

  set {
    name  = "image.repository"
    value = var.image
  }
  set {
    name  = "image.tag"
    value = var.image_tag_version
  }

  depends_on = [
    kubernetes_manifest.datadog_external_secret,
  ]
}

module "netpol" {
  source = "crow.infrastructure.immuta.io/internal/security-network-policy/kubernetes"
  version = "1.1.0"

  name             = "vector-netpol"
  namespace        = var.namespace
  network_policies = local.netpols

  depends_on = [
    kubernetes_namespace_v1.vector,
  ]
}

+  jwt_do_redact = var.jwt_redaction_enabled && !var.jwt_redaction_dry_run

+  jwt_do_tag    = var.jwt_redaction_dry_run

+  jwt_services_arr = "[\"${join("\", \"", var.jwt_redaction_services)}\"]"

+  jwt_exclusion_condition = length(var.jwt_redaction_exclusion_patterns) == 0 ? "true" : join(

+    [for p in var.jwt_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],

+  jwt_stage_active = local.jwt_do_redact || local.jwt_do_tag

+  jwt_stage_output = local.jwt_do_redact ? "redact_jwts_logs" : "tag_jwts_logs"

+    local.aws_key_stage_active ? local.aws_key_stage_output :

+    local.bearer_stage_active  ? local.bearer_stage_output :

+    local.jwt_stage_active     ? local.jwt_stage_output :

+      jwt_do_redact                         = local.jwt_do_redact

+      jwt_do_tag                            = local.jwt_do_tag

+      jwt_redaction_replacement             = var.jwt_redaction_replacement

+      jwt_services_arr                      = local.jwt_services_arr

+      jwt_exclusion_condition               = local.jwt_exclusion_condition

+      jwt_logs_input                        = local.jwt_logs_input

logs_sink_input                       = local.logs_sink_input


Add the two JWT transform blocks to the template, before the sinks section:


%{ if aws_key_do_tag ~}
    tag_aws_access_keys_logs:
      type: remap
      inputs:
        - ${aws_key_logs_input}
      source: |
        aws_key_re = r'(AKIA|ASIA|AROA|AIPA|ANPA|ANVA|APKA)[A-Z0-9]{16}'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${aws_key_services_arr}
        attr_excluded = ${aws_key_attr_exclusion_condition}
        would_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${aws_key_string_exclusion_condition} && !attr_excluded
        if would_redact && is_string(.message) && match(string!(.message), aws_key_re) {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_aws_key_detected:true" } else { "vector_aws_key_detected:true" }
        }
%{ endif ~}
  sinks:

%{ if aws_key_do_tag ~}
    tag_aws_access_keys_logs:
      type: remap
      inputs:
        - ${aws_key_logs_input}
      source: |
        aws_key_re = r'(AKIA|ASIA|AROA|AIPA|ANPA|ANVA|APKA)[A-Z0-9]{16}'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${aws_key_services_arr}
        attr_excluded = ${aws_key_attr_exclusion_condition}
        would_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${aws_key_string_exclusion_condition} && !attr_excluded
        if would_redact && is_string(.message) && match(string!(.message), aws_key_re) {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_aws_key_detected:true" } else { "vector_aws_key_detected:true" }
        }
%{ endif ~}
%{ if jwt_do_redact ~}
    redact_jwts_logs:
      type: remap
      inputs:
        - ${jwt_logs_input}
      source: |
        jwt_re = r'eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${jwt_services_arr}
        should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${jwt_exclusion_condition}
        if should_redact && is_string(.message) {
          .message = replace(string!(.message), jwt_re, "${jwt_redaction_replacement}")
        }
%{ endif ~}
%{ if jwt_do_tag ~}
    tag_jwts_logs:
      type: remap
      inputs:
        - ${jwt_logs_input}
      source: |
        jwt_re = r'eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${jwt_services_arr}
        would_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${jwt_exclusion_condition}
        if would_redact && is_string(.message) && match(string!(.message), jwt_re) {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_jwt_detected:true" } else { "vector_jwt_detected:true" }
        }
%{ endif ~}
  sinks: