164	%{ endif ~}
165	%{ if !email_redaction_enabled && email_redaction_dry_run ~}
166	    tag_emails_logs:
167	      type: remap
168	      inputs:
169	        - normalize_logs
170	      source: |
171	        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
172	        detected = false
173	        if is_string(.message) {
174	          detected = match(string!(.message), email_re)
175	        }
176	        if !detected && is_object(.attributes) {
177	          detected = match(encode_json(.attributes), email_re)
178	        }
179	        if detected {
180	          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_redaction:true" } else { "vector_redaction:true" }
181	        }
182	%{ endif ~}
183	%{ if bearer_token_redaction_enabled ~}
184	    redact_bearer_tokens_logs:
185	      type: remap
186	      inputs:
187	        - ${bearer_token_logs_input}
188	      source: |
189	        bearer_re = r'(?i)Bearer\s+[A-Za-z0-9+/._~-]+=*'
190	        svc = to_string(.service) ?? ""
191	        ddtags = to_string(.ddtags) ?? ""
192	        msg = if is_string(.message) { string!(.message) } else { "" }
193	        combined = msg + " " + ddtags
194	        target_services = ${bearer_token_services_arr}
195	        should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${bearer_token_exclusion_condition}
196	        if should_redact && is_string(.message) {
197	          .message = replace(string!(.message), bearer_re, "${bearer_token_redaction_replacement}")
198	        }
199	%{ endif ~}
200	  sinks:
201	    datadog_logs:
202	      type: datadog_logs
203	      inputs:
204	        - ${logs_sink_input}
205	      default_api_key: $${DATADOG_API_KEY}
206	      site: ${datadog_site}
207	    datadog_traces:
208	      type: datadog_traces
209	      inputs:
210	        - ${email_redaction_enabled ? "redact_emails_traces" : "otlp_in.traces"}
211	      default_api_key: $${DATADOG_API_KEY}
212	      site: ${datadog_site}
213	    datadog_metrics:

%{ endif ~}
%{ if !email_redaction_enabled && email_redaction_dry_run ~}
    tag_emails_logs:
      type: remap
      inputs:
        - normalize_logs
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        detected = false
        if is_string(.message) {
          detected = match(string!(.message), email_re)
        }
        if !detected && is_object(.attributes) {
          detected = match(encode_json(.attributes), email_re)
        }
        if detected {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_redaction:true" } else { "vector_redaction:true" }
        }
%{ endif ~}
%{ if bearer_token_redaction_enabled ~}
    redact_bearer_tokens_logs:
      type: remap
      inputs:
        - ${bearer_token_logs_input}
      source: |
        bearer_re = r'(?i)Bearer\s+[A-Za-z0-9+/._~-]+=*'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${bearer_token_services_arr}
        should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${bearer_token_exclusion_condition}
        if should_redact && is_string(.message) {
          .message = replace(string!(.message), bearer_re, "${bearer_token_redaction_replacement}")
        }
%{ endif ~}
  sinks:
    datadog_logs:
      type: datadog_logs
      inputs:
        - ${logs_sink_input}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_traces:
      type: datadog_traces
      inputs:
        - ${email_redaction_enabled ? "redact_emails_traces" : "otlp_in.traces"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_metrics:


Everything looks good. Here's a summary of what was added, Sir Charles of St Cheese:

**`variables.tf`** — 4 new variables:
- `bearer_token_redaction_enabled` (bool, default `false`)
- `bearer_token_redaction_replacement` (string, default `"Bearer [REDACTED-BEARER-TOKEN]"`)
- `bearer_token_redaction_services` (list, default `["service", "secure-metadata-database"]`)
- `bearer_token_redaction_exclusion_patterns` (list, all 7 NOT-conditions from the SDS query)

**`main.tf`** — 4 new locals that pre-compute the VRL expression fragments and chain the transform inputs correctly. The `logs_sink_input` local handles all ordering permutations (bearer token sits last in the chain, after email redaction).

**`templates/values.yaml.tpl`** — New `redact_bearer_tokens_logs` transform that gates on `env:prod`, the configured services list, and all exclusion patterns, then applies a bearer token regex. The `datadog_logs` sink now uses `${logs_sink_input}` instead of the inlined ternary.

**`tests/vrl/redact_bearer_tokens_logs/`** — Full test suite covering: redact when service+env match, pass-through for wrong service, wrong env, and each of the 7 exclusion patterns.

To enable it, just add `bearer_token_redaction_enabled = true` to your module invocation.


afplay ~/.claude/sounds/mario/stop/stomp.wav 2>/dev/null || true


currently if i enable email_redaction AND email_redaction_dry_run the emails still get redacted. Can we make this better and also maybe more generic? I.e. dry_run disables redaction and adds the tag?


Let me look at the current logic more carefully before touching anything.


78	  transforms:
79	    drop_audit:
80	      type: filter
81	      inputs:
82	        - otlp_in.logs
83	        - dd_agent_in.logs
84	      condition: |
85	        msg = if is_string(.message) { string!(.message) } else { "" }
86	        ${audit_drop_condition}
87	    normalize_logs:
88	      type: remap
89	      inputs:
90	        - drop_audit
91	      source: |
92	        if exists(.severity) {
93	          .status = downcase(to_string!(.severity))
94	          del(.severity)
95	        }
96	%{ if email_redaction_enabled ~}
97	    redact_emails_logs:
98	      type: remap
99	      inputs:
100	        - normalize_logs
101	      source: |
102	        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
103	        if is_string(.message) {
104	          .message = replace(string!(.message), email_re, "${email_redaction_replacement}")
105	        }
106	        if is_object(.attributes) {
107	          attrs, _err = object(.attributes)
108	          if is_object(attrs) {
109	            .attributes = map_values(attrs) -> |v| {
110	              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
111	            }
112	          }
113	        }
114	    redact_emails_traces:
115	      type: remap
116	      inputs:
117	        - otlp_in.traces
118	      source: |
119	        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
120	        if is_object(.attributes) {
121	          attrs, _err = object(.attributes)
122	          if is_object(attrs) {
123	            .attributes = map_values(attrs) -> |v| {
124	              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
125	            }
126	          }
127	        }
128	        if is_object(.resource) {
129	          res, _err = object(.resource)
130	          if is_object(res) && is_object(res.attributes) {
131	            rattrs, _err2 = object(res.attributes)
132	            if is_object(rattrs) {
133	              .resource.attributes = map_values(rattrs) -> |v| {
134	                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
135	              }
136	            }
137	          }
138	        }
139	    redact_emails_metrics:
140	      type: remap
141	      inputs:
142	        - otlp_in.metrics
143	      source: |
144	        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
145	        if is_object(.attributes) {
146	          attrs, _err = object(.attributes)
147	          if is_object(attrs) {
148	            .attributes = map_values(attrs) -> |v| {
149	              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
150	            }
151	          }
152	        }
153	        if is_object(.resource) {
154	          res, _err = object(.resource)
155	          if is_object(res) && is_object(res.attributes) {
156	            rattrs, _err2 = object(res.attributes)
157	            if is_object(rattrs) {
158	              .resource.attributes = map_values(rattrs) -> |v| {
159	                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
160	              }
161	            }
162	          }
163	        }
164	%{ endif ~}
165	%{ if !email_redaction_enabled && email_redaction_dry_run ~}
166	    tag_emails_logs:
167	      type: remap
168	      inputs:
169	        - normalize_logs
170	      source: |
171	        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
172	        detected = false
173	        if is_string(.message) {
174	          detected = match(string!(.message), email_re)
175	        }
176	        if !detected && is_object(.attributes) {
177	          detected = match(encode_json(.attributes), email_re)
178	        }
179	        if detected {
180	          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_redaction:true" } else { "vector_redaction:true" }
181	        }
182	%{ endif ~}
183	%{ if bearer_token_redaction_enabled ~}
184	    redact_bearer_tokens_logs:
185	      type: remap
186	      inputs:
187	        - ${bearer_token_logs_input}
188	      source: |
189	        bearer_re = r'(?i)Bearer\s+[A-Za-z0-9+/._~-]+=*'
190	        svc = to_string(.service) ?? ""
191	        ddtags = to_string(.ddtags) ?? ""
192	        msg = if is_string(.message) { string!(.message) } else { "" }
193	        combined = msg + " " + ddtags
194	        target_services = ${bearer_token_services_arr}
195	        should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${bearer_token_exclusion_condition}
196	        if should_redact && is_string(.message) {
197	          .message = replace(string!(.message), bearer_re, "${bearer_token_redaction_replacement}")

  transforms:
    drop_audit:
      type: filter
      inputs:
        - otlp_in.logs
        - dd_agent_in.logs
      condition: |
        msg = if is_string(.message) { string!(.message) } else { "" }
        ${audit_drop_condition}
    normalize_logs:
      type: remap
      inputs:
        - drop_audit
      source: |
        if exists(.severity) {
          .status = downcase(to_string!(.severity))
          del(.severity)
        }
%{ if email_redaction_enabled ~}
    redact_emails_logs:
      type: remap
      inputs:
        - normalize_logs
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_string(.message) {
          .message = replace(string!(.message), email_re, "${email_redaction_replacement}")
        }
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
    redact_emails_traces:
      type: remap
      inputs:
        - otlp_in.traces
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
    redact_emails_metrics:
      type: remap
      inputs:
        - otlp_in.metrics
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
%{ endif ~}
%{ if !email_redaction_enabled && email_redaction_dry_run ~}
    tag_emails_logs:
      type: remap
      inputs:
        - normalize_logs
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        detected = false
        if is_string(.message) {
          detected = match(string!(.message), email_re)
        }
        if !detected && is_object(.attributes) {
          detected = match(encode_json(.attributes), email_re)
        }
        if detected {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_redaction:true" } else { "vector_redaction:true" }
        }
%{ endif ~}
%{ if bearer_token_redaction_enabled ~}
    redact_bearer_tokens_logs:
      type: remap
      inputs:
        - ${bearer_token_logs_input}
      source: |
        bearer_re = r'(?i)Bearer\s+[A-Za-z0-9+/._~-]+=*'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${bearer_token_services_arr}
        should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${bearer_token_exclusion_condition}
        if should_redact && is_string(.message) {
          .message = replace(string!(.message), bearer_re, "${bearer_token_redaction_replacement}")


Add Bearer Token Scanner redaction filter


I see the full picture. The bug: `%{ if !email_redaction_enabled && email_redaction_dry_run ~}` means dry_run only fires when `enabled = false`, so enabling both → redaction wins silently. The fix is to make `dry_run` always override `enabled`, and extend that pattern to bearer tokens.

Plan:
1. Pre-compute `email_do_redact`/`email_do_tag`/`bearer_do_redact`/`bearer_do_tag` booleans in `main.tf` — dry_run always wins
2. Pass those to the template instead of the raw vars
3. Add `tag_bearer_tokens_logs` transform parallel to `tag_emails_logs`
4. Add `bearer_token_redaction_dry_run` variable
5. New VRL test for the bearer tag mode


Edit

variable "email_redaction_dry_run" {
  description = "When true (and email_redaction_enabled is false), tag logs that would have been redacted with vector_email_detected=true instead of replacing. Useful for non-prod visibility without masking."
  type        = bool
  default     = false
}

variable "email_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_redaction:true instead of replacing. Takes priority over email_redaction_enabled."
  type        = bool
  default     = false
}


The file /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/variables.tf has been updated successfully. (file state is current in your context — no need to Read it back)

variable "email_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_redaction:true instead of replacing. Takes priority over email_redaction_enabled."
  type        = bool
  default     = false
}

# ---------------------------------------------------------------------------------------------------------------------
# REQUIRED PARAMETERS
# These variables must be set.
# ---------------------------------------------------------------------------------------------------------------------

variable "chart_version" {
  description = "Vector helm chart version pin"
  type        = string
}

variable "image_tag_version" {
  description = "Vector image tag"
  type        = string
}

# ---------------------------------------------------------------------------------------------------------------------
# OPTIONAL PARAMETERS
# These variables are configurable, but have reasonable defaults set.
# ---------------------------------------------------------------------------------------------------------------------
variable "datadog_site" {
  description = "Datadog site (e.g. datadoghq.com, us3.datadoghq.com)"
  type        = string
  default     = "datadoghq.com"
}

variable "image" {
  description = "Vector image source"
  type        = string
  default     = "registry.ic-ops.net/cache-docker-hub/timberio/vector"
}

variable "replicas" {
  description = "Number of Vector aggregator replicas"
  type        = number
  default     = 3
}

variable "audit_drop_patterns" {
  description = "Regex patterns; matching log message bodies are dropped"
  type        = list(string)
  default = [
    "Audit\\s-\\s.+",
    "\"level\":\"audit\"",
  ]
}

variable "datadog_api_key_vault_path" {
  description = "Vault path for the Datadog API key, consumed by ExternalSecret"
  type        = string
  default     = "secret/data/saas/infrastructure/datadog-api-key"
}

variable "external_secrets_cluster_store_name" {
  description = "Name of the ClusterSecretStore created by external-secrets module"
  type        = string
  default     = "vault"
}

variable "email_redaction_enabled" {
  description = "Toggle the email-redact VRL transform across all signals"
  type        = bool
  default     = true
}

variable "email_redaction_dry_run" {
  description = "When true (and email_redaction_enabled is false), tag logs that would have been redacted with vector_email_detected=true instead of replacing. Useful for non-prod visibility without masking."
  type        = bool
  default     = false
}

variable "email_redaction_replacement" {
  description = "String that replaces matched email addresses"
  type        = string
  default     = "[REDACTED-EMAIL]"
}

variable "additional_config" {
  description = "Escape hatch — overwrites helm customConfig wholesale (mirrors otel_collector_additional_config semantics)"
  type        = any
  default     = null
}

variable "resources_requests" {
  description = "Pod resource requests"
  type        = map(string)
  default = {
    cpu    = "200m"
    memory = "256Mi"
  }
}

variable "resources_limits" {
  description = "Pod resource limits"
  type        = map(string)
  default = {
    cpu    = "2"
    memory = "1Gi"
  }
}

variable "namespace" {
  description = "Kubernetes namespace for Vector"
  type        = string
  default     = "vector"
}

variable "service_monitor_enabled" {
  description = "Create a ServiceMonitor for Vector self-metrics. Requires kube-prometheus-stack CRDs in cluster."
  type        = bool
  default     = false
}

variable "bearer_token_redaction_enabled" {
  description = "Toggle bearer token redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "bearer_token_redaction_replacement" {
  description = "String that replaces matched bearer tokens (replaces the full 'Bearer <token>' match)"
  type        = string
  default     = "Bearer [REDACTED-BEARER-TOKEN]"
}

variable "bearer_token_redaction_services" {
  description = "Service names (.service field on DD agent logs) to apply bearer token redaction to"
  type        = list(string)
  default     = ["service", "secure-metadata-database"]
}

variable "bearer_token_redaction_exclusion_patterns" {
  description = "Skip bearer token redaction for any log whose message or ddtags contains one of these strings"
  type        = list(string)
  default = [
    "internal-webhook",
    "starting crawl",
    "Databricks automatic subscription",
    "audit/enrich",
    "Background Job",
    "community-organization-id:immuta",
    "pg - pool.connect",
  ]
}

-  description = "When true (and email_redaction_enabled is false), tag logs that would have been redacted with vector_email_detected=true instead of replacing. Useful for non-prod visibility without masking."

+  description = "When true, tag logs that would have been redacted with vector_redaction:true instead of replacing. Takes priority over email_redaction_enabled."


variable "bearer_token_redaction_enabled" {
  description = "Toggle bearer token redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "bearer_token_redaction_enabled" {
  description = "Toggle bearer token redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "bearer_token_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_bearer_token_detected:true instead of replacing. Takes priority over bearer_token_redaction_enabled."
  type        = bool
  default     = false
}


variable "bearer_token_redaction_enabled" {
  description = "Toggle bearer token redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "bearer_token_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_bearer_token_detected:true instead of replacing. Takes priority over bearer_token_redaction_enabled."
  type        = bool
  default     = false
}

# ---------------------------------------------------------------------------------------------------------------------
# REQUIRED PARAMETERS
# These variables must be set.
# ---------------------------------------------------------------------------------------------------------------------

variable "chart_version" {
  description = "Vector helm chart version pin"
  type        = string
}

variable "image_tag_version" {
  description = "Vector image tag"
  type        = string
}

# ---------------------------------------------------------------------------------------------------------------------
# OPTIONAL PARAMETERS
# These variables are configurable, but have reasonable defaults set.
# ---------------------------------------------------------------------------------------------------------------------
variable "datadog_site" {
  description = "Datadog site (e.g. datadoghq.com, us3.datadoghq.com)"
  type        = string
  default     = "datadoghq.com"
}

variable "image" {
  description = "Vector image source"
  type        = string
  default     = "registry.ic-ops.net/cache-docker-hub/timberio/vector"
}

variable "replicas" {
  description = "Number of Vector aggregator replicas"
  type        = number
  default     = 3
}

variable "audit_drop_patterns" {
  description = "Regex patterns; matching log message bodies are dropped"
  type        = list(string)
  default = [
    "Audit\\s-\\s.+",
    "\"level\":\"audit\"",
  ]
}

variable "datadog_api_key_vault_path" {
  description = "Vault path for the Datadog API key, consumed by ExternalSecret"
  type        = string
  default     = "secret/data/saas/infrastructure/datadog-api-key"
}

variable "external_secrets_cluster_store_name" {
  description = "Name of the ClusterSecretStore created by external-secrets module"
  type        = string
  default     = "vault"
}

variable "email_redaction_enabled" {
  description = "Toggle the email-redact VRL transform across all signals"
  type        = bool
  default     = true
}

variable "email_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_redaction:true instead of replacing. Takes priority over email_redaction_enabled."
  type        = bool
  default     = false
}

variable "email_redaction_replacement" {
  description = "String that replaces matched email addresses"
  type        = string
  default     = "[REDACTED-EMAIL]"
}

variable "additional_config" {
  description = "Escape hatch — overwrites helm customConfig wholesale (mirrors otel_collector_additional_config semantics)"
  type        = any
  default     = null
}

variable "resources_requests" {
  description = "Pod resource requests"
  type        = map(string)
  default = {
    cpu    = "200m"
    memory = "256Mi"
  }
}

variable "resources_limits" {
  description = "Pod resource limits"
  type        = map(string)
  default = {
    cpu    = "2"
    memory = "1Gi"
  }
}

variable "namespace" {
  description = "Kubernetes namespace for Vector"
  type        = string
  default     = "vector"
}

variable "service_monitor_enabled" {
  description = "Create a ServiceMonitor for Vector self-metrics. Requires kube-prometheus-stack CRDs in cluster."
  type        = bool
  default     = false
}

variable "bearer_token_redaction_enabled" {
  description = "Toggle bearer token redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "bearer_token_redaction_replacement" {
  description = "String that replaces matched bearer tokens (replaces the full 'Bearer <token>' match)"
  type        = string
  default     = "Bearer [REDACTED-BEARER-TOKEN]"
}

variable "bearer_token_redaction_services" {
  description = "Service names (.service field on DD agent logs) to apply bearer token redaction to"
  type        = list(string)
  default     = ["service", "secure-metadata-database"]
}

variable "bearer_token_redaction_exclusion_patterns" {
  description = "Skip bearer token redaction for any log whose message or ddtags contains one of these strings"
  type        = list(string)
  default = [
    "internal-webhook",
    "starting crawl",
    "Databricks automatic subscription",
    "audit/enrich",
    "Background Job",
    "community-organization-id:immuta",
    "pg - pool.connect",
  ]
}

+  description = "When true, tag logs that would have been redacted with vector_bearer_token_detected:true instead of replacing. Takes priority over bearer_token_redaction_enabled."


Now restructure the locals in `main.tf`:


  bearer_token_exclusion_condition = length(var.bearer_token_redaction_exclusion_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.bearer_token_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
  )

  bearer_token_services_arr = "[\"${join("\", \"", var.bearer_token_redaction_services)}\"]"

  # The input to the bearer token transform is whatever the last logs transform in the chain is
  bearer_token_logs_input = (
    var.email_redaction_enabled ? "redact_emails_logs" :
    var.email_redaction_dry_run ? "tag_emails_logs" :
    "normalize_logs"
  )

  # The datadog_logs sink takes from the last transform in the full chain
  logs_sink_input = (
    var.bearer_token_redaction_enabled ? "redact_bearer_tokens_logs" :
    var.email_redaction_enabled ? "redact_emails_logs" :
    var.email_redaction_dry_run ? "tag_emails_logs" :
    "normalize_logs"
  )

  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
    "${path.module}/templates/values.yaml.tpl",
    {
      replicas                             = var.replicas
      requests_cpu                         = var.resources_requests.cpu
      requests_memory                      = var.resources_requests.memory
      limits_cpu                           = var.resources_limits.cpu
      limits_memory                        = var.resources_limits.memory
      datadog_site                         = var.datadog_site
      audit_drop_condition                 = local.audit_drop_condition
      email_redaction_enabled              = var.email_redaction_enabled
      email_redaction_dry_run              = var.email_redaction_dry_run
      email_redaction_replacement          = var.email_redaction_replacement
      service_monitor_enabled              = var.service_monitor_enabled
      bearer_token_redaction_enabled       = var.bearer_token_redaction_enabled
      bearer_token_redaction_replacement   = var.bearer_token_redaction_replacement
      bearer_token_services_arr            = local.bearer_token_services_arr
      bearer_token_exclusion_condition     = local.bearer_token_exclusion_condition
      bearer_token_logs_input              = local.bearer_token_logs_input
      logs_sink_input                      = local.logs_sink_input
    },
  )
}

  bearer_token_exclusion_condition = length(var.bearer_token_redaction_exclusion_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.bearer_token_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
  )

  bearer_token_services_arr = "[\"${join("\", \"", var.bearer_token_redaction_services)}\"]"

  # dry_run always wins over enabled for both scanner types
  email_do_redact = var.email_redaction_enabled && !var.email_redaction_dry_run
  email_do_tag    = var.email_redaction_dry_run

  bearer_do_redact = var.bearer_token_redaction_enabled && !var.bearer_token_redaction_dry_run
  bearer_do_tag    = var.bearer_token_redaction_dry_run

  # named output of each stage used to wire transform inputs and the final sink
  email_stage_output = (
    local.email_do_redact ? "redact_emails_logs" :
    local.email_do_tag    ? "tag_emails_logs" :
    "normalize_logs"
  )

  bearer_stage_active = local.bearer_do_redact || local.bearer_do_tag
  bearer_stage_output = local.bearer_do_redact ? "redact_bearer_tokens_logs" : "tag_bearer_tokens_logs"

  bearer_token_logs_input = local.email_stage_output
  logs_sink_input         = local.bearer_stage_active ? local.bearer_stage_output : local.email_stage_output

  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
    "${path.module}/templates/values.yaml.tpl",
    {
      replicas                           = var.replicas
      requests_cpu                       = var.resources_requests.cpu
      requests_memory                    = var.resources_requests.memory
      limits_cpu                         = var.resources_limits.cpu
      limits_memory                      = var.resources_limits.memory
      datadog_site                       = var.datadog_site
      audit_drop_condition               = local.audit_drop_condition
      email_do_redact                    = local.email_do_redact
      email_do_tag                       = local.email_do_tag
      email_redaction_replacement        = var.email_redaction_replacement
      service_monitor_enabled            = var.service_monitor_enabled
      bearer_do_redact                   = local.bearer_do_redact
      bearer_do_tag                      = local.bearer_do_tag
      bearer_token_redaction_replacement = var.bearer_token_redaction_replacement
      bearer_token_services_arr          = local.bearer_token_services_arr
      bearer_token_exclusion_condition   = local.bearer_token_exclusion_condition
      bearer_token_logs_input            = local.bearer_token_logs_input
      logs_sink_input                    = local.logs_sink_input
    },
  )
}


The file /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/main.tf has been updated successfully. (file state is current in your context — no need to Read it back)

  bearer_token_exclusion_condition = length(var.bearer_token_redaction_exclusion_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.bearer_token_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
  )

  bearer_token_services_arr = "[\"${join("\", \"", var.bearer_token_redaction_services)}\"]"

  # dry_run always wins over enabled for both scanner types
  email_do_redact = var.email_redaction_enabled && !var.email_redaction_dry_run
  email_do_tag    = var.email_redaction_dry_run

  bearer_do_redact = var.bearer_token_redaction_enabled && !var.bearer_token_redaction_dry_run
  bearer_do_tag    = var.bearer_token_redaction_dry_run

  # named output of each stage used to wire transform inputs and the final sink
  email_stage_output = (
    local.email_do_redact ? "redact_emails_logs" :
    local.email_do_tag    ? "tag_emails_logs" :
    "normalize_logs"
  )

  bearer_stage_active = local.bearer_do_redact || local.bearer_do_tag
  bearer_stage_output = local.bearer_do_redact ? "redact_bearer_tokens_logs" : "tag_bearer_tokens_logs"

  bearer_token_logs_input = local.email_stage_output
  logs_sink_input         = local.bearer_stage_active ? local.bearer_stage_output : local.email_stage_output

  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
    "${path.module}/templates/values.yaml.tpl",
    {
      replicas                           = var.replicas
      requests_cpu                       = var.resources_requests.cpu
      requests_memory                    = var.resources_requests.memory
      limits_cpu                         = var.resources_limits.cpu
      limits_memory                      = var.resources_limits.memory
      datadog_site                       = var.datadog_site
      audit_drop_condition               = local.audit_drop_condition
      email_do_redact                    = local.email_do_redact
      email_do_tag                       = local.email_do_tag
      email_redaction_replacement        = var.email_redaction_replacement
      service_monitor_enabled            = var.service_monitor_enabled
      bearer_do_redact                   = local.bearer_do_redact
      bearer_do_tag                      = local.bearer_do_tag
      bearer_token_redaction_replacement = var.bearer_token_redaction_replacement
      bearer_token_services_arr          = local.bearer_token_services_arr
      bearer_token_exclusion_condition   = local.bearer_token_exclusion_condition
      bearer_token_logs_input            = local.bearer_token_logs_input
      logs_sink_input                    = local.logs_sink_input
    },
  )
}

terraform {
  required_version = ">= 1.9.0"

  required_providers {
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = ">= 2.20"
    }
    helm = {
      source  = "hashicorp/helm"
      version = ">= 2.10"
    }
  }
}

locals {
  audit_drop_condition = length(var.audit_drop_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.audit_drop_patterns : "!match(msg, r'${p}')"],
  )

  bearer_token_exclusion_condition = length(var.bearer_token_redaction_exclusion_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.bearer_token_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
  )

  bearer_token_services_arr = "[\"${join("\", \"", var.bearer_token_redaction_services)}\"]"

  # The input to the bearer token transform is whatever the last logs transform in the chain is
  bearer_token_logs_input = (
    var.email_redaction_enabled ? "redact_emails_logs" :
    var.email_redaction_dry_run ? "tag_emails_logs" :
    "normalize_logs"
  )

  # The datadog_logs sink takes from the last transform in the full chain
  logs_sink_input = (
    var.bearer_token_redaction_enabled ? "redact_bearer_tokens_logs" :
    var.email_redaction_enabled ? "redact_emails_logs" :
    var.email_redaction_dry_run ? "tag_emails_logs" :
    "normalize_logs"
  )

  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
    "${path.module}/templates/values.yaml.tpl",
    {
      replicas                             = var.replicas
      requests_cpu                         = var.resources_requests.cpu
      requests_memory                      = var.resources_requests.memory
      limits_cpu                           = var.resources_limits.cpu
      limits_memory                        = var.resources_limits.memory
      datadog_site                         = var.datadog_site
      audit_drop_condition                 = local.audit_drop_condition
      email_redaction_enabled              = var.email_redaction_enabled
      email_redaction_dry_run              = var.email_redaction_dry_run
      email_redaction_replacement          = var.email_redaction_replacement
      service_monitor_enabled              = var.service_monitor_enabled
      bearer_token_redaction_enabled       = var.bearer_token_redaction_enabled
      bearer_token_redaction_replacement   = var.bearer_token_redaction_replacement
      bearer_token_services_arr            = local.bearer_token_services_arr
      bearer_token_exclusion_condition     = local.bearer_token_exclusion_condition
      bearer_token_logs_input              = local.bearer_token_logs_input
      logs_sink_input                      = local.logs_sink_input
    },
  )
}

resource "kubernetes_namespace_v1" "vector" {
  metadata {
    name = var.namespace
    labels = {
      "app.kubernetes.io/name"      = "vector"
      "app.kubernetes.io/component" = "telemetry-aggregator"
    }
  }
}

resource "kubernetes_manifest" "datadog_external_secret" {
  manifest = {
    apiVersion = "external-secrets.io/v1"
    kind       = "ExternalSecret"
    metadata = {
      name      = "vector-datadog"
      namespace = kubernetes_namespace_v1.vector.metadata[0].name
    }
    spec = {
      refreshInterval = "1h"
      secretStoreRef = {
        kind = "ClusterSecretStore"
        name = var.external_secrets_cluster_store_name
      }
      target = {
        name           = "vector-datadog"
        creationPolicy = "Owner"
      }
      data = [{
        secretKey = "api_key"
        remoteRef = {
          key      = var.datadog_api_key_vault_path
          property = "apiKey"
        }
      }]
    }
  }

  wait {
    fields = {
      "status.conditions[0].type"   = "Ready"
      "status.conditions[0].status" = "True"
    }
  }

  timeouts {
    create = "5m"
    update = "5m"
  }
}

resource "helm_release" "vector" {
  name       = "vector"
  namespace  = kubernetes_namespace_v1.vector.metadata[0].name
  repository = "https://helm.vector.dev"
  chart      = "vector"
  version    = var.chart_version

  values = [local.rendered_values]

  set {
    name  = "image.repository"
    value = var.image
  }
  set {
    name  = "image.tag"
    value = var.image_tag_version
  }

  depends_on = [
    kubernetes_manifest.datadog_external_secret,
  ]
}

module "netpol" {
  source = "crow.infrastructure.immuta.io/internal/security-network-policy/kubernetes"
  version = "1.1.0"

  name             = "vector-netpol"
  namespace        = var.namespace
  network_policies = local.netpols

  depends_on = [
    kubernetes_namespace_v1.vector,
  ]
}

-  # The input to the bearer token transform is whatever the last logs transform in the chain is

-    var.email_redaction_enabled ? "redact_emails_logs" :

-    var.email_redaction_dry_run ? "tag_emails_logs" :

+  # dry_run always wins over enabled for both scanner types

+  email_do_redact = var.email_redaction_enabled && !var.email_redaction_dry_run

+  email_do_tag    = var.email_redaction_dry_run

-  # The datadog_logs sink takes from the last transform in the full chain

-    var.bearer_token_redaction_enabled ? "redact_bearer_tokens_logs" :

+  bearer_do_redact = var.bearer_token_redaction_enabled && !var.bearer_token_redaction_dry_run

+  bearer_do_tag    = var.bearer_token_redaction_dry_run

+  # named output of each stage used to wire transform inputs and the final sink

+    local.email_do_redact ? "redact_emails_logs" :

+    local.email_do_tag    ? "tag_emails_logs" :

+  bearer_stage_active = local.bearer_do_redact || local.bearer_do_tag

+  bearer_stage_output = local.bearer_do_redact ? "redact_bearer_tokens_logs" : "tag_bearer_tokens_logs"

+  bearer_token_logs_input = local.email_stage_output

+  logs_sink_input         = local.bearer_stage_active ? local.bearer_stage_output : local.email_stage_output

-      replicas                             = var.replicas

-      requests_cpu                         = var.resources_requests.cpu

-      requests_memory                      = var.resources_requests.memory

-      limits_cpu                           = var.resources_limits.cpu

-      limits_memory                        = var.resources_limits.memory

-      datadog_site                         = var.datadog_site

-      audit_drop_condition                 = local.audit_drop_condition

-      email_redaction_enabled              = var.email_redaction_enabled

-      email_redaction_dry_run              = var.email_redaction_dry_run

-      email_redaction_replacement          = var.email_redaction_replacement

-      service_monitor_enabled              = var.service_monitor_enabled

-      bearer_token_redaction_enabled       = var.bearer_token_redaction_enabled

-      bearer_token_redaction_replacement   = var.bearer_token_redaction_replacement

-      bearer_token_services_arr            = local.bearer_token_services_arr

-      bearer_token_exclusion_condition     = local.bearer_token_exclusion_condition

-      bearer_token_logs_input              = local.bearer_token_logs_input

-      logs_sink_input                      = local.logs_sink_input

+      replicas                           = var.replicas

+      requests_cpu                       = var.resources_requests.cpu

+      requests_memory                    = var.resources_requests.memory

+      limits_cpu                         = var.resources_limits.cpu

+      limits_memory                      = var.resources_limits.memory

+      datadog_site                       = var.datadog_site

+      audit_drop_condition               = local.audit_drop_condition

+      email_do_redact                    = local.email_do_redact

+      email_do_tag                       = local.email_do_tag

+      email_redaction_replacement        = var.email_redaction_replacement

+      service_monitor_enabled            = var.service_monitor_enabled

+      bearer_do_redact                   = local.bearer_do_redact

+      bearer_do_tag                      = local.bearer_do_tag

+      bearer_token_redaction_replacement = var.bearer_token_redaction_replacement

+      bearer_token_services_arr          = local.bearer_token_services_arr

+      bearer_token_exclusion_condition   = local.bearer_token_exclusion_condition

+      bearer_token_logs_input            = local.bearer_token_logs_input

+      logs_sink_input                    = local.logs_sink_input


Now rewrite the template to use the new booleans and add the `tag_bearer_tokens_logs` block:


%{ if email_redaction_enabled ~}
    redact_emails_logs:
      type: remap
      inputs:
        - normalize_logs
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_string(.message) {
          .message = replace(string!(.message), email_re, "${email_redaction_replacement}")
        }
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
    redact_emails_traces:
      type: remap
      inputs:
        - otlp_in.traces
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
    redact_emails_metrics:
      type: remap
      inputs:
        - otlp_in.metrics
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
%{ endif ~}
%{ if !email_redaction_enabled && email_redaction_dry_run ~}
    tag_emails_logs:
      type: remap
      inputs:
        - normalize_logs
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        detected = false
        if is_string(.message) {
          detected = match(string!(.message), email_re)
        }
        if !detected && is_object(.attributes) {
          detected = match(encode_json(.attributes), email_re)
        }
        if detected {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_redaction:true" } else { "vector_redaction:true" }
        }
%{ endif ~}
%{ if bearer_token_redaction_enabled ~}
    redact_bearer_tokens_logs:
      type: remap
      inputs:
        - ${bearer_token_logs_input}
      source: |
        bearer_re = r'(?i)Bearer\s+[A-Za-z0-9+/._~-]+=*'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${bearer_token_services_arr}
        should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${bearer_token_exclusion_condition}
        if should_redact && is_string(.message) {
          .message = replace(string!(.message), bearer_re, "${bearer_token_redaction_replacement}")
        }
%{ endif ~}
  sinks:
    datadog_logs:
      type: datadog_logs
      inputs:
        - ${logs_sink_input}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_traces:
      type: datadog_traces
      inputs:
        - ${email_redaction_enabled ? "redact_emails_traces" : "otlp_in.traces"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_metrics:
      type: datadog_metrics
      inputs:
        - ${email_redaction_enabled ? "redact_emails_metrics" : "otlp_in.metrics"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}

%{ if email_do_redact ~}
    redact_emails_logs:
      type: remap
      inputs:
        - normalize_logs
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_string(.message) {
          .message = replace(string!(.message), email_re, "${email_redaction_replacement}")
        }
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
    redact_emails_traces:
      type: remap
      inputs:
        - otlp_in.traces
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
    redact_emails_metrics:
      type: remap
      inputs:
        - otlp_in.metrics
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
%{ endif ~}
%{ if email_do_tag ~}
    tag_emails_logs:
      type: remap
      inputs:
        - normalize_logs
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        detected = false
        if is_string(.message) {
          detected = match(string!(.message), email_re)
        }
        if !detected && is_object(.attributes) {
          detected = match(encode_json(.attributes), email_re)
        }
        if detected {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_redaction:true" } else { "vector_redaction:true" }
        }
%{ endif ~}
%{ if bearer_do_redact ~}
    redact_bearer_tokens_logs:
      type: remap
      inputs:
        - ${bearer_token_logs_input}
      source: |
        bearer_re = r'(?i)Bearer\s+[A-Za-z0-9+/._~-]+=*'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${bearer_token_services_arr}
        should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${bearer_token_exclusion_condition}
        if should_redact && is_string(.message) {
          .message = replace(string!(.message), bearer_re, "${bearer_token_redaction_replacement}")
        }
%{ endif ~}
%{ if bearer_do_tag ~}
    tag_bearer_tokens_logs:
      type: remap
      inputs:
        - ${bearer_token_logs_input}
      source: |
        bearer_re = r'(?i)Bearer\s+[A-Za-z0-9+/._~-]+=*'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${bearer_token_services_arr}
        would_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${bearer_token_exclusion_condition}
        if would_redact && is_string(.message) && match(string!(.message), bearer_re) {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_bearer_token_detected:true" } else { "vector_bearer_token_detected:true" }
        }
%{ endif ~}
  sinks:
    datadog_logs:
      type: datadog_logs
      inputs:
        - ${logs_sink_input}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_traces:
      type: datadog_traces
      inputs:
        - ${email_do_redact ? "redact_emails_traces" : "otlp_in.traces"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_metrics:
      type: datadog_metrics
      inputs:
        - ${email_do_redact ? "redact_emails_metrics" : "otlp_in.metrics"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}


The file /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/templates/values.yaml.tpl has been updated successfully. (file state is current in your context — no need to Read it back)

%{ if email_do_redact ~}
    redact_emails_logs:
      type: remap
      inputs:
        - normalize_logs
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_string(.message) {
          .message = replace(string!(.message), email_re, "${email_redaction_replacement}")
        }
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
    redact_emails_traces:
      type: remap
      inputs:
        - otlp_in.traces
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
    redact_emails_metrics:
      type: remap
      inputs:
        - otlp_in.metrics
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
%{ endif ~}
%{ if email_do_tag ~}
    tag_emails_logs:
      type: remap
      inputs:
        - normalize_logs
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        detected = false
        if is_string(.message) {
          detected = match(string!(.message), email_re)
        }
        if !detected && is_object(.attributes) {
          detected = match(encode_json(.attributes), email_re)
        }
        if detected {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_redaction:true" } else { "vector_redaction:true" }
        }
%{ endif ~}
%{ if bearer_do_redact ~}
    redact_bearer_tokens_logs:
      type: remap
      inputs:
        - ${bearer_token_logs_input}
      source: |
        bearer_re = r'(?i)Bearer\s+[A-Za-z0-9+/._~-]+=*'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${bearer_token_services_arr}
        should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${bearer_token_exclusion_condition}
        if should_redact && is_string(.message) {
          .message = replace(string!(.message), bearer_re, "${bearer_token_redaction_replacement}")
        }
%{ endif ~}
%{ if bearer_do_tag ~}
    tag_bearer_tokens_logs:
      type: remap
      inputs:
        - ${bearer_token_logs_input}
      source: |
        bearer_re = r'(?i)Bearer\s+[A-Za-z0-9+/._~-]+=*'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${bearer_token_services_arr}
        would_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${bearer_token_exclusion_condition}
        if would_redact && is_string(.message) && match(string!(.message), bearer_re) {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_bearer_token_detected:true" } else { "vector_bearer_token_detected:true" }
        }
%{ endif ~}
  sinks:
    datadog_logs:
      type: datadog_logs
      inputs:
        - ${logs_sink_input}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_traces:
      type: datadog_traces
      inputs:
        - ${email_do_redact ? "redact_emails_traces" : "otlp_in.traces"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_metrics:
      type: datadog_metrics
      inputs:
        - ${email_do_redact ? "redact_emails_metrics" : "otlp_in.metrics"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}

role: Aggregator
replicas: ${replicas}
podAnnotations:
  reloader.stakater.com/auto: "true"
service:
  enabled: true
  type: ClusterIP
  ports:
    - name: otlp-grpc
      port: 4317
      protocol: TCP
    - name: otlp-http
      port: 4318
      protocol: TCP
    - name: dd-agent
      port: 8282
      protocol: TCP
env:
  - name: DATADOG_API_KEY
    valueFrom:
      secretKeyRef:
        name: vector-datadog
        key: api_key
resources:
  requests:
    cpu: ${requests_cpu}
    memory: ${requests_memory}
  limits:
    cpu: "${limits_cpu}"
    memory: ${limits_memory}
podDisruptionBudget:
  enabled: true
  minAvailable: 2

topologySpreadConstraints:
  - maxSkew: 1
    topologyKey: topology.kubernetes.io/zone
    whenUnsatisfiable: ScheduleAnyway
    labelSelector:
      matchLabels:
        app.kubernetes.io/name: vector

affinity:
  podAntiAffinity:
    preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 100
        podAffinityTerm:
          topologyKey: kubernetes.io/hostname
          labelSelector:
            matchLabels:
              app.kubernetes.io/name: vector

%{ if service_monitor_enabled ~}
serviceMonitor:
  enabled: true
%{ endif ~}
customConfig:
  data_dir: /vector-data-dir
  api:
    enabled: true
    address: 0.0.0.0:8686
  sources:
    otlp_in:
      type: opentelemetry
      grpc:
        address: 0.0.0.0:4317
      http:
        address: 0.0.0.0:4318
    dd_agent_in:
      type: datadog_agent
      address: 0.0.0.0:8282
      multiple_outputs: true
      disable_metrics: true
      disable_traces: true
      store_api_key: false
      acknowledgements:
        enabled: true
  transforms:
    drop_audit:
      type: filter
      inputs:
        - otlp_in.logs
        - dd_agent_in.logs
      condition: |
        msg = if is_string(.message) { string!(.message) } else { "" }
        ${audit_drop_condition}
    normalize_logs:
      type: remap
      inputs:
        - drop_audit
      source: |
        if exists(.severity) {
          .status = downcase(to_string!(.severity))
          del(.severity)
        }
%{ if email_redaction_enabled ~}
    redact_emails_logs:
      type: remap
      inputs:
        - normalize_logs
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_string(.message) {
          .message = replace(string!(.message), email_re, "${email_redaction_replacement}")
        }
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
    redact_emails_traces:
      type: remap
      inputs:
        - otlp_in.traces
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
    redact_emails_metrics:
      type: remap
      inputs:
        - otlp_in.metrics
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
%{ endif ~}
%{ if !email_redaction_enabled && email_redaction_dry_run ~}
    tag_emails_logs:
      type: remap
      inputs:
        - normalize_logs
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        detected = false
        if is_string(.message) {
          detected = match(string!(.message), email_re)
        }
        if !detected && is_object(.attributes) {
          detected = match(encode_json(.attributes), email_re)
        }
        if detected {
          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_redaction:true" } else { "vector_redaction:true" }
        }
%{ endif ~}
%{ if bearer_token_redaction_enabled ~}
    redact_bearer_tokens_logs:
      type: remap
      inputs:
        - ${bearer_token_logs_input}
      source: |
        bearer_re = r'(?i)Bearer\s+[A-Za-z0-9+/._~-]+=*'
        svc = to_string(.service) ?? ""
        ddtags = to_string(.ddtags) ?? ""
        msg = if is_string(.message) { string!(.message) } else { "" }
        combined = msg + " " + ddtags
        target_services = ${bearer_token_services_arr}
        should_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${bearer_token_exclusion_condition}
        if should_redact && is_string(.message) {
          .message = replace(string!(.message), bearer_re, "${bearer_token_redaction_replacement}")
        }
%{ endif ~}
  sinks:
    datadog_logs:
      type: datadog_logs
      inputs:
        - ${logs_sink_input}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_traces:
      type: datadog_traces
      inputs:
        - ${email_redaction_enabled ? "redact_emails_traces" : "otlp_in.traces"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_metrics:
      type: datadog_metrics
      inputs:
        - ${email_redaction_enabled ? "redact_emails_metrics" : "otlp_in.metrics"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}

-%{ if !email_redaction_enabled && email_redaction_dry_run ~}

-%{ if bearer_token_redaction_enabled ~}

+        svc = to_string(.service) ?? ""

+        ddtags = to_string(.ddtags) ?? ""

+        msg = if is_string(.message) { string!(.message) } else { "" }

+        target_services = ${bearer_token_services_arr}

+        would_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && ${bearer_token_exclusion_condition}

+        if would_redact && is_string(.message) && match(string!(.message), bearer_re) {

+          .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_bearer_token_detected:true" } else { "vector_bearer_token_detected:true" }

-        - ${email_redaction_enabled ? "redact_emails_traces" : "otlp_in.traces"}

+        - ${email_do_redact ? "redact_emails_traces" : "otlp_in.traces"}

-        - ${email_redaction_enabled ? "redact_emails_metrics" : "otlp_in.metrics"}

+        - ${email_do_redact ? "redact_emails_metrics" : "otlp_in.metrics"}


Now add the VRL test for `tag_bearer_tokens_logs`:


Write

bearer_re = r'(?i)Bearer\s+[A-Za-z0-9+/._~-]+=*'
svc = to_string(.service) ?? ""
ddtags = to_string(.ddtags) ?? ""
msg = if is_string(.message) { string!(.message) } else { "" }
combined = msg + " " + ddtags
target_services = ["service", "secure-metadata-database"]
would_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && !contains(combined, "internal-webhook") && !contains(combined, "starting crawl") && !contains(combined, "Databricks automatic subscription") && !contains(combined, "audit/enrich") && !contains(combined, "Background Job") && !contains(combined, "community-organization-id:immuta") && !contains(combined, "pg - pool.connect")
if would_redact && is_string(.message) && match(string!(.message), bearer_re) {
  .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_bearer_token_detected:true" } else { "vector_bearer_token_detected:true" }
}
.


File created successfully at: /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/tag_bearer_tokens_logs/program.vrl (file state is current in your context — no need to Read it back)

bearer_re = r'(?i)Bearer\s+[A-Za-z0-9+/._~-]+=*'
svc = to_string(.service) ?? ""
ddtags = to_string(.ddtags) ?? ""
msg = if is_string(.message) { string!(.message) } else { "" }
combined = msg + " " + ddtags
target_services = ["service", "secure-metadata-database"]
would_redact = contains(ddtags, "env:prod") && includes(target_services, svc) && !contains(combined, "internal-webhook") && !contains(combined, "starting crawl") && !contains(combined, "Databricks automatic subscription") && !contains(combined, "audit/enrich") && !contains(combined, "Background Job") && !contains(combined, "community-organization-id:immuta") && !contains(combined, "pg - pool.connect")
if would_redact && is_string(.message) && match(string!(.message), bearer_re) {
  .ddtags = if is_string(.ddtags) { string!(.ddtags) + ",vector_bearer_token_detected:true" } else { "vector_bearer_token_detected:true" }
}
.


{"message": "Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig", "service": "service", "ddtags": "env:prod,service:service"}
{"message": "Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig", "service": "secure-metadata-database", "ddtags": "env:prod,service:secure-metadata-database"}
{"message": "Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig", "service": "other-service", "ddtags": "env:prod,service:other-service"}
{"message": "Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig", "service": "service", "ddtags": "env:staging,service:service"}
{"message": "internal-webhook Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig", "service": "service", "ddtags": "env:prod,service:service"}
{"message": "No token here", "service": "service", "ddtags": "env:prod,service:service"}


File created successfully at: /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/tag_bearer_tokens_logs/inputs.ndjson (file state is current in your context — no need to Read it back)

{"message": "Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig", "service": "service", "ddtags": "env:prod,service:service"}
{"message": "Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig", "service": "secure-metadata-database", "ddtags": "env:prod,service:secure-metadata-database"}
{"message": "Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig", "service": "other-service", "ddtags": "env:prod,service:other-service"}
{"message": "Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig", "service": "service", "ddtags": "env:staging,service:service"}
{"message": "internal-webhook Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig", "service": "service", "ddtags": "env:prod,service:service"}
{"message": "No token here", "service": "service", "ddtags": "env:prod,service:service"}


Bash

cd /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/tag_bearer_tokens_logs && vector vrl -p program.vrl -i inputs.ndjson -q -o | jq -c . > expected.ndjson && cat expected.ndjson

Generate expected output for tag_bearer_tokens_logs


2026-05-07T22:02:07.639551Z  INFO vector::app: Log level is enabled. level="info"
{"ddtags":"env:prod,service:service,vector_bearer_token_detected:true","message":"Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig","service":"service"}
{"ddtags":"env:prod,service:secure-metadata-database,vector_bearer_token_detected:true","message":"Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig","service":"secure-metadata-database"}
{"ddtags":"env:prod,service:other-service","message":"Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig","service":"other-service"}
{"ddtags":"env:staging,service:service","message":"Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig","service":"service"}
{"ddtags":"env:prod,service:service","message":"internal-webhook Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig","service":"service"}
{"ddtags":"env:prod,service:service","message":"No token here","service":"service"}


Write

#!/usr/bin/env bash
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ACTUAL="$DIR/.actual.ndjson"
vector vrl -p "$DIR/program.vrl" -i "$DIR/inputs.ndjson" -q -o > "$ACTUAL"
diff -u <(jq -c . "$DIR/expected.ndjson") <(jq -c . "$ACTUAL")
echo "tag_bearer_tokens_logs VRL OK"


File created successfully at: /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/tag_bearer_tokens_logs/run.sh (file state is current in your context — no need to Read it back)

#!/usr/bin/env bash
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ACTUAL="$DIR/.actual.ndjson"
vector vrl -p "$DIR/program.vrl" -i "$DIR/inputs.ndjson" -q -o > "$ACTUAL"
diff -u <(jq -c . "$DIR/expected.ndjson") <(jq -c . "$ACTUAL")
echo "tag_bearer_tokens_logs VRL OK"


Bash

chmod +x /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/tag_bearer_tokens_logs/run.sh && for f in /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/*/run.sh; do bash "$f" 2>/dev/null || echo "FAIL: $f"; done

Run all VRL tests


drop_audit VRL OK
redact_bearer_tokens_logs VRL OK
redact_emails_dd_agent_logs VRL OK
redact_emails_logs VRL OK
redact_emails_metrics VRL OK
redact_emails_traces VRL OK
tag_bearer_tokens_logs VRL OK
--- /dev/fd/63	2026-05-07 18:02:33
+++ /dev/fd/62	2026-05-07 18:02:33
@@ -1,6 +1,6 @@
-{"message":"user alice@example.com signed in","ddtags":"vector_redaction:true"}
+{"ddtags":"vector_redaction:true","message":"user alice@example.com signed in"}
 {"message":"no email here"}
-{"message":"multiple a@b.com and c.d+tag@sub.example.co.uk","ddtags":"vector_redaction:true"}
-{"message":"ok","attributes":{"user.email":"bob@x.io","request.id":"abc-123"},"ddtags":"vector_redaction:true"}
-{"message":"clean message","attributes":{"request.id":"abc-123"}}
-{"message":"user alice@example.com signed in","ddtags":"env:nonprod,service:myapp,vector_redaction:true"}
+{"ddtags":"vector_redaction:true","message":"multiple a@b.com and c.d+tag@sub.example.co.uk"}
+{"attributes":{"request.id":"abc-123","user.email":"bob@x.io"},"ddtags":"vector_redaction:true","message":"ok"}
+{"attributes":{"request.id":"abc-123"},"message":"clean message"}
+{"ddtags":"env:nonprod,service:myapp,vector_redaction:true","message":"user alice@example.com signed in"}
FAIL: /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/tag_emails_logs/run.sh


All my tests pass. The `tag_emails_logs` failure is the pre-existing key-ordering issue in its expected file — let me fix that while I'm here:


cd /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/tests/vrl/tag_emails_logs && vector vrl -p program.vrl -i inputs.ndjson -q -o | jq -c . > expected.ndjson && bash run.sh 2>/dev/null

Regenerate tag_emails_logs expected.ndjson and verify


2026-05-07T22:02:42.176132Z  INFO vector::app: Log level is enabled. level="info"
tag_emails_logs VRL OK


All tests green. Here's what changed, Sir Charles of St Cheese:

**The bug fix:** The old condition `%{ if !email_redaction_enabled && email_redaction_dry_run ~}` required `enabled = false`, so setting both to `true` silently fell through to redaction. Now `dry_run` is computed first in `main.tf` and **wins unconditionally**:

```hcl
email_do_redact = var.email_redaction_enabled && !var.email_redaction_dry_run
email_do_tag    = var.email_redaction_dry_run  # no enabled check
```

**The generalization:** Bearer tokens now have the same pattern — `bearer_token_redaction_dry_run` (new var) sets `bearer_do_tag`, and the template has a `tag_bearer_tokens_logs` transform that appends `vector_bearer_token_detected:true` to `ddtags` instead of mutating the message.

**Behavior matrix** (same for both scanner types):

| `enabled` | `dry_run` | Result |
|-----------|-----------|--------|
| false | false | passthrough |
| true | false | redact |
| false | **true** | tag |
| true | **true** | tag ← was broken, now fixed |


I also need to add another rule for AWS Access Key ID Scanner, the datadog sensitive data scanner query is `env:prod AND service:marketplace-api AND NOT @req.headers.x-synthetic-test:true`


Let me review the current state of the locals and variable files before adding the new scanner.


1	terraform {
2	  required_version = ">= 1.9.0"
3
4	  required_providers {
5	    kubernetes = {
6	      source  = "hashicorp/kubernetes"
7	      version = ">= 2.20"
8	    }
9	    helm = {
10	      source  = "hashicorp/helm"
11	      version = ">= 2.10"
12	    }
13	  }
14	}
15
16	locals {
17	  audit_drop_condition = length(var.audit_drop_patterns) == 0 ? "true" : join(
18	    " && ",
19	    [for p in var.audit_drop_patterns : "!match(msg, r'${p}')"],
20	  )
21
22	  bearer_token_exclusion_condition = length(var.bearer_token_redaction_exclusion_patterns) == 0 ? "true" : join(
23	    " && ",
24	    [for p in var.bearer_token_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
25	  )
26
27	  bearer_token_services_arr = "[\"${join("\", \"", var.bearer_token_redaction_services)}\"]"
28
29	  # dry_run always wins over enabled for both scanner types
30	  email_do_redact = var.email_redaction_enabled && !var.email_redaction_dry_run
31	  email_do_tag    = var.email_redaction_dry_run
32
33	  bearer_do_redact = var.bearer_token_redaction_enabled && !var.bearer_token_redaction_dry_run
34	  bearer_do_tag    = var.bearer_token_redaction_dry_run
35
36	  # named output of each stage used to wire transform inputs and the final sink
37	  email_stage_output = (
38	    local.email_do_redact ? "redact_emails_logs" :
39	    local.email_do_tag    ? "tag_emails_logs" :
40	    "normalize_logs"
41	  )
42
43	  bearer_stage_active = local.bearer_do_redact || local.bearer_do_tag
44	  bearer_stage_output = local.bearer_do_redact ? "redact_bearer_tokens_logs" : "tag_bearer_tokens_logs"
45
46	  bearer_token_logs_input = local.email_stage_output
47	  logs_sink_input         = local.bearer_stage_active ? local.bearer_stage_output : local.email_stage_output
48
49	  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
50	    "${path.module}/templates/values.yaml.tpl",
51	    {
52	      replicas                           = var.replicas
53	      requests_cpu                       = var.resources_requests.cpu
54	      requests_memory                    = var.resources_requests.memory
55	      limits_cpu                         = var.resources_limits.cpu
56	      limits_memory                      = var.resources_limits.memory
57	      datadog_site                       = var.datadog_site
58	      audit_drop_condition               = local.audit_drop_condition
59	      email_do_redact                    = local.email_do_redact
60	      email_do_tag                       = local.email_do_tag
61	      email_redaction_replacement        = var.email_redaction_replacement
62	      service_monitor_enabled            = var.service_monitor_enabled
63	      bearer_do_redact                   = local.bearer_do_redact
64	      bearer_do_tag                      = local.bearer_do_tag
65	      bearer_token_redaction_replacement = var.bearer_token_redaction_replacement
66	      bearer_token_services_arr          = local.bearer_token_services_arr
67	      bearer_token_exclusion_condition   = local.bearer_token_exclusion_condition
68	      bearer_token_logs_input            = local.bearer_token_logs_input
69	      logs_sink_input                    = local.logs_sink_input
70	    },
71	  )
72	}
73
74	resource "kubernetes_namespace_v1" "vector" {
75	  metadata {
76	    name = var.namespace
77	    labels = {
78	      "app.kubernetes.io/name"      = "vector"
79	      "app.kubernetes.io/component" = "telemetry-aggregator"
80	    }
81	  }
82	}
83
84	resource "kubernetes_manifest" "datadog_external_secret" {
85	  manifest = {
86	    apiVersion = "external-secrets.io/v1"
87	    kind       = "ExternalSecret"
88	    metadata = {
89	      name      = "vector-datadog"
90	      namespace = kubernetes_namespace_v1.vector.metadata[0].name
91	    }
92	    spec = {
93	      refreshInterval = "1h"
94	      secretStoreRef = {
95	        kind = "ClusterSecretStore"
96	        name = var.external_secrets_cluster_store_name
97	      }
98	      target = {
99	        name           = "vector-datadog"
100	        creationPolicy = "Owner"
101	      }
102	      data = [{
103	        secretKey = "api_key"
104	        remoteRef = {
105	          key      = var.datadog_api_key_vault_path
106	          property = "apiKey"
107	        }
108	      }]
109	    }
110	  }
111
112	  wait {
113	    fields = {
114	      "status.conditions[0].type"   = "Ready"
115	      "status.conditions[0].status" = "True"
116	    }
117	  }
118
119	  timeouts {
120	    create = "5m"
121	    update = "5m"
122	  }
123	}
124
125	resource "helm_release" "vector" {
126	  name       = "vector"
127	  namespace  = kubernetes_namespace_v1.vector.metadata[0].name
128	  repository = "https://helm.vector.dev"
129	  chart      = "vector"
130	  version    = var.chart_version
131
132	  values = [local.rendered_values]
133
134	  set {
135	    name  = "image.repository"
136	    value = var.image
137	  }
138	  set {
139	    name  = "image.tag"
140	    value = var.image_tag_version
141	  }
142
143	  depends_on = [
144	    kubernetes_manifest.datadog_external_secret,
145	  ]
146	}
147
148	module "netpol" {
149	  source = "crow.infrastructure.immuta.io/internal/security-network-policy/kubernetes"
150	  version = "1.1.0"
151
152	  name             = "vector-netpol"
153	  namespace        = var.namespace
154	  network_policies = local.netpols
155
156	  depends_on = [
157	    kubernetes_namespace_v1.vector,
158	  ]
159	}
160

terraform {
  required_version = ">= 1.9.0"

  required_providers {
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = ">= 2.20"
    }
    helm = {
      source  = "hashicorp/helm"
      version = ">= 2.10"
    }
  }
}

locals {
  audit_drop_condition = length(var.audit_drop_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.audit_drop_patterns : "!match(msg, r'${p}')"],
  )

  bearer_token_exclusion_condition = length(var.bearer_token_redaction_exclusion_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.bearer_token_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
  )

  bearer_token_services_arr = "[\"${join("\", \"", var.bearer_token_redaction_services)}\"]"

  # dry_run always wins over enabled for both scanner types
  email_do_redact = var.email_redaction_enabled && !var.email_redaction_dry_run
  email_do_tag    = var.email_redaction_dry_run

  bearer_do_redact = var.bearer_token_redaction_enabled && !var.bearer_token_redaction_dry_run
  bearer_do_tag    = var.bearer_token_redaction_dry_run

  # named output of each stage used to wire transform inputs and the final sink
  email_stage_output = (
    local.email_do_redact ? "redact_emails_logs" :
    local.email_do_tag    ? "tag_emails_logs" :
    "normalize_logs"
  )

  bearer_stage_active = local.bearer_do_redact || local.bearer_do_tag
  bearer_stage_output = local.bearer_do_redact ? "redact_bearer_tokens_logs" : "tag_bearer_tokens_logs"

  bearer_token_logs_input = local.email_stage_output
  logs_sink_input         = local.bearer_stage_active ? local.bearer_stage_output : local.email_stage_output

  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
    "${path.module}/templates/values.yaml.tpl",
    {
      replicas                           = var.replicas
      requests_cpu                       = var.resources_requests.cpu
      requests_memory                    = var.resources_requests.memory
      limits_cpu                         = var.resources_limits.cpu
      limits_memory                      = var.resources_limits.memory
      datadog_site                       = var.datadog_site
      audit_drop_condition               = local.audit_drop_condition
      email_do_redact                    = local.email_do_redact
      email_do_tag                       = local.email_do_tag
      email_redaction_replacement        = var.email_redaction_replacement
      service_monitor_enabled            = var.service_monitor_enabled
      bearer_do_redact                   = local.bearer_do_redact
      bearer_do_tag                      = local.bearer_do_tag
      bearer_token_redaction_replacement = var.bearer_token_redaction_replacement
      bearer_token_services_arr          = local.bearer_token_services_arr
      bearer_token_exclusion_condition   = local.bearer_token_exclusion_condition
      bearer_token_logs_input            = local.bearer_token_logs_input
      logs_sink_input                    = local.logs_sink_input
    },
  )
}

resource "kubernetes_namespace_v1" "vector" {
  metadata {
    name = var.namespace
    labels = {
      "app.kubernetes.io/name"      = "vector"
      "app.kubernetes.io/component" = "telemetry-aggregator"
    }
  }
}

resource "kubernetes_manifest" "datadog_external_secret" {
  manifest = {
    apiVersion = "external-secrets.io/v1"
    kind       = "ExternalSecret"
    metadata = {
      name      = "vector-datadog"
      namespace = kubernetes_namespace_v1.vector.metadata[0].name
    }
    spec = {
      refreshInterval = "1h"
      secretStoreRef = {
        kind = "ClusterSecretStore"
        name = var.external_secrets_cluster_store_name
      }
      target = {
        name           = "vector-datadog"
        creationPolicy = "Owner"
      }
      data = [{
        secretKey = "api_key"
        remoteRef = {
          key      = var.datadog_api_key_vault_path
          property = "apiKey"
        }
      }]
    }
  }

  wait {
    fields = {
      "status.conditions[0].type"   = "Ready"
      "status.conditions[0].status" = "True"
    }
  }

  timeouts {
    create = "5m"
    update = "5m"
  }
}

resource "helm_release" "vector" {
  name       = "vector"
  namespace  = kubernetes_namespace_v1.vector.metadata[0].name
  repository = "https://helm.vector.dev"
  chart      = "vector"
  version    = var.chart_version

  values = [local.rendered_values]

  set {
    name  = "image.repository"
    value = var.image
  }
  set {
    name  = "image.tag"
    value = var.image_tag_version
  }

  depends_on = [
    kubernetes_manifest.datadog_external_secret,
  ]
}

module "netpol" {
  source = "crow.infrastructure.immuta.io/internal/security-network-policy/kubernetes"
  version = "1.1.0"

  name             = "vector-netpol"
  namespace        = var.namespace
  network_policies = local.netpols

  depends_on = [
    kubernetes_namespace_v1.vector,
  ]
}


The key difference here is `NOT @req.headers.x-synthetic-test:true` — that's a log **attribute** check, not a message/tag string match. I'll introduce a new `exclusion_attributes` variable type (`map(string)`) that generates the correct VRL attribute accessor, alongside the existing `exclusion_patterns` for string-based NOT conditions. The chain extends to a third stage: email → bearer → aws_key.


Edit

variable "bearer_token_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_bearer_token_detected:true instead of replacing. Takes priority over bearer_token_redaction_enabled."
  type        = bool
  default     = false
}

variable "bearer_token_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_bearer_token_detected:true instead of replacing. Takes priority over bearer_token_redaction_enabled."
  type        = bool
  default     = false
}

variable "aws_access_key_redaction_enabled" {
  description = "Toggle AWS Access Key ID redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "aws_access_key_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_aws_key_detected:true instead of replacing. Takes priority over aws_access_key_redaction_enabled."
  type        = bool
  default     = false
}

variable "aws_access_key_redaction_replacement" {
  description = "String that replaces matched AWS Access Key IDs"
  type        = string
  default     = "[REDACTED-AWS-ACCESS-KEY]"
}

variable "aws_access_key_redaction_services" {
  description = "Service names (.service field on DD agent logs) to apply AWS Access Key ID redaction to"
  type        = list(string)
  default     = ["marketplace-api"]
}

variable "aws_access_key_redaction_exclusion_patterns" {
  description = "Skip AWS Access Key ID redaction for any log whose message or ddtags contains one of these strings"
  type        = list(string)
  default     = []
}

variable "aws_access_key_redaction_exclusion_attributes" {
  description = "Skip AWS Access Key ID redaction for any log where the named attribute equals the given value (keys are attribute paths, values are string values to match)"
  type        = map(string)
  default = {
    "req.headers.x-synthetic-test" = "true"
  }
}


variable "bearer_token_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_bearer_token_detected:true instead of replacing. Takes priority over bearer_token_redaction_enabled."
  type        = bool
  default     = false
}

variable "aws_access_key_redaction_enabled" {
  description = "Toggle AWS Access Key ID redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "aws_access_key_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_aws_key_detected:true instead of replacing. Takes priority over aws_access_key_redaction_enabled."
  type        = bool
  default     = false
}

variable "aws_access_key_redaction_replacement" {
  description = "String that replaces matched AWS Access Key IDs"
  type        = string
  default     = "[REDACTED-AWS-ACCESS-KEY]"
}

variable "aws_access_key_redaction_services" {
  description = "Service names (.service field on DD agent logs) to apply AWS Access Key ID redaction to"
  type        = list(string)
  default     = ["marketplace-api"]
}

variable "aws_access_key_redaction_exclusion_patterns" {
  description = "Skip AWS Access Key ID redaction for any log whose message or ddtags contains one of these strings"
  type        = list(string)
  default     = []
}

variable "aws_access_key_redaction_exclusion_attributes" {
  description = "Skip AWS Access Key ID redaction for any log where the named attribute equals the given value (keys are attribute paths, values are string values to match)"
  type        = map(string)
  default = {
    "req.headers.x-synthetic-test" = "true"
  }
}

# ---------------------------------------------------------------------------------------------------------------------
# REQUIRED PARAMETERS
# These variables must be set.
# ---------------------------------------------------------------------------------------------------------------------

variable "chart_version" {
  description = "Vector helm chart version pin"
  type        = string
}

variable "image_tag_version" {
  description = "Vector image tag"
  type        = string
}

# ---------------------------------------------------------------------------------------------------------------------
# OPTIONAL PARAMETERS
# These variables are configurable, but have reasonable defaults set.
# ---------------------------------------------------------------------------------------------------------------------
variable "datadog_site" {
  description = "Datadog site (e.g. datadoghq.com, us3.datadoghq.com)"
  type        = string
  default     = "datadoghq.com"
}

variable "image" {
  description = "Vector image source"
  type        = string
  default     = "registry.ic-ops.net/cache-docker-hub/timberio/vector"
}

variable "replicas" {
  description = "Number of Vector aggregator replicas"
  type        = number
  default     = 3
}

variable "audit_drop_patterns" {
  description = "Regex patterns; matching log message bodies are dropped"
  type        = list(string)
  default = [
    "Audit\\s-\\s.+",
    "\"level\":\"audit\"",
  ]
}

variable "datadog_api_key_vault_path" {
  description = "Vault path for the Datadog API key, consumed by ExternalSecret"
  type        = string
  default     = "secret/data/saas/infrastructure/datadog-api-key"
}

variable "external_secrets_cluster_store_name" {
  description = "Name of the ClusterSecretStore created by external-secrets module"
  type        = string
  default     = "vault"
}

variable "email_redaction_enabled" {
  description = "Toggle the email-redact VRL transform across all signals"
  type        = bool
  default     = true
}

variable "email_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_redaction:true instead of replacing. Takes priority over email_redaction_enabled."
  type        = bool
  default     = false
}

variable "email_redaction_replacement" {
  description = "String that replaces matched email addresses"
  type        = string
  default     = "[REDACTED-EMAIL]"
}

variable "additional_config" {
  description = "Escape hatch — overwrites helm customConfig wholesale (mirrors otel_collector_additional_config semantics)"
  type        = any
  default     = null
}

variable "resources_requests" {
  description = "Pod resource requests"
  type        = map(string)
  default = {
    cpu    = "200m"
    memory = "256Mi"
  }
}

variable "resources_limits" {
  description = "Pod resource limits"
  type        = map(string)
  default = {
    cpu    = "2"
    memory = "1Gi"
  }
}

variable "namespace" {
  description = "Kubernetes namespace for Vector"
  type        = string
  default     = "vector"
}

variable "service_monitor_enabled" {
  description = "Create a ServiceMonitor for Vector self-metrics. Requires kube-prometheus-stack CRDs in cluster."
  type        = bool
  default     = false
}

variable "bearer_token_redaction_enabled" {
  description = "Toggle bearer token redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "bearer_token_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_bearer_token_detected:true instead of replacing. Takes priority over bearer_token_redaction_enabled."
  type        = bool
  default     = false
}

variable "bearer_token_redaction_replacement" {
  description = "String that replaces matched bearer tokens (replaces the full 'Bearer <token>' match)"
  type        = string
  default     = "Bearer [REDACTED-BEARER-TOKEN]"
}

variable "bearer_token_redaction_services" {
  description = "Service names (.service field on DD agent logs) to apply bearer token redaction to"
  type        = list(string)
  default     = ["service", "secure-metadata-database"]
}

variable "bearer_token_redaction_exclusion_patterns" {
  description = "Skip bearer token redaction for any log whose message or ddtags contains one of these strings"
  type        = list(string)
  default = [
    "internal-webhook",
    "starting crawl",
    "Databricks automatic subscription",
    "audit/enrich",
    "Background Job",
    "community-organization-id:immuta",
    "pg - pool.connect",
  ]
}

+  description = "Toggle AWS Access Key ID redact VRL transform on logs"

+  description = "When true, tag logs that would have been redacted with vector_aws_key_detected:true instead of replacing. Takes priority over aws_access_key_redaction_enabled."

+  description = "String that replaces matched AWS Access Key IDs"

+  default     = "[REDACTED-AWS-ACCESS-KEY]"

+  description = "Service names (.service field on DD agent logs) to apply AWS Access Key ID redaction to"

+  description = "Skip AWS Access Key ID redaction for any log whose message or ddtags contains one of these strings"

+  description = "Skip AWS Access Key ID redaction for any log where the named attribute equals the given value (keys are attribute paths, values are string values to match)"

+    "req.headers.x-synthetic-test" = "true"


Now update `main.tf` to add the AWS key locals and extend the chain: