1	locals {
2	  audit_drop_condition = length(var.audit_drop_patterns) == 0 ? "true" : join(
3	    " && ",
4	    [for p in var.audit_drop_patterns : "!match(msg, r'${p}')"],
5	  )
6
7	  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
8	    "${path.module}/templates/values.yaml.tpl",
9	    {
10	      replicas                    = var.replicas
11	      requests_cpu                = var.resources_requests.cpu
12	      requests_memory             = var.resources_requests.memory
13	      limits_cpu                  = var.resources_limits.cpu
14	      limits_memory               = var.resources_limits.memory
15	      datadog_site                = var.datadog_site
16	      audit_drop_condition        = local.audit_drop_condition
17	      email_redaction_enabled     = var.email_redaction_enabled
18	      email_redaction_replacement = var.email_redaction_replacement
19	      service_monitor_enabled     = var.service_monitor_enabled
20	    },
21	  )
22	}
23
24	resource "kubernetes_namespace_v1" "vector" {
25	  metadata {
26	    name = var.namespace
27	    labels = {
28	      "app.kubernetes.io/name"      = "vector"
29	      "app.kubernetes.io/component" = "telemetry-aggregator"
30	    }
31	  }
32	}
33
34	resource "kubernetes_manifest" "datadog_external_secret" {
35	  manifest = {
36	    apiVersion = "external-secrets.io/v1"
37	    kind       = "ExternalSecret"
38	    metadata = {
39	      name      = "vector-datadog"
40	      namespace = kubernetes_namespace_v1.vector.metadata[0].name
41	    }
42	    spec = {
43	      refreshInterval = "1h"
44	      secretStoreRef = {
45	        kind = "ClusterSecretStore"
46	        name = var.external_secrets_cluster_store_name
47	      }
48	      target = {
49	        name           = "vector-datadog"
50	        creationPolicy = "Owner"
51	      }
52	      data = [{
53	        secretKey = "api_key"
54	        remoteRef = {
55	          key      = var.datadog_api_key_vault_path
56	          property = "apiKey"
57	        }
58	      }]
59	    }
60	  }
61
62	  wait {
63	    fields = {
64	      "status.conditions[0].type"   = "Ready"
65	      "status.conditions[0].status" = "True"
66	    }
67	  }
68
69	  timeouts {
70	    create = "5m"
71	    update = "5m"
72	  }
73	}
74
75	resource "helm_release" "vector" {
76	  name       = "vector"
77	  namespace  = kubernetes_namespace_v1.vector.metadata[0].name
78	  repository = "https://helm.vector.dev"
79	  chart      = "vector"
80	  version    = var.chart_version
81
82	  values = [local.rendered_values]
83
84	  set {
85	    name  = "image.repository"
86	    value = var.image
87	  }
88	  set {
89	    name  = "image.tag"
90	    value = var.image_tag_version
91	  }
92
93	  depends_on = [
94	    kubernetes_manifest.datadog_external_secret,
95	  ]
96	}
97
98	module "netpol" {
99	  source = "git@gitlab.infrastructure.immuta.io:infrastructure/terraform/internal-modules/kubernetes/security/network-policy.git?ref=v1.1.0"
100
101	  name             = "vector-netpol"
102	  namespace        = var.namespace
103	  network_policies = local.netpols
104
105	  depends_on = [
106	    kubernetes_namespace_v1.vector,
107	  ]
108	}
109

locals {
  audit_drop_condition = length(var.audit_drop_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.audit_drop_patterns : "!match(msg, r'${p}')"],
  )

  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
    "${path.module}/templates/values.yaml.tpl",
    {
      replicas                    = var.replicas
      requests_cpu                = var.resources_requests.cpu
      requests_memory             = var.resources_requests.memory
      limits_cpu                  = var.resources_limits.cpu
      limits_memory               = var.resources_limits.memory
      datadog_site                = var.datadog_site
      audit_drop_condition        = local.audit_drop_condition
      email_redaction_enabled     = var.email_redaction_enabled
      email_redaction_replacement = var.email_redaction_replacement
      service_monitor_enabled     = var.service_monitor_enabled
    },
  )
}

resource "kubernetes_namespace_v1" "vector" {
  metadata {
    name = var.namespace
    labels = {
      "app.kubernetes.io/name"      = "vector"
      "app.kubernetes.io/component" = "telemetry-aggregator"
    }
  }
}

resource "kubernetes_manifest" "datadog_external_secret" {
  manifest = {
    apiVersion = "external-secrets.io/v1"
    kind       = "ExternalSecret"
    metadata = {
      name      = "vector-datadog"
      namespace = kubernetes_namespace_v1.vector.metadata[0].name
    }
    spec = {
      refreshInterval = "1h"
      secretStoreRef = {
        kind = "ClusterSecretStore"
        name = var.external_secrets_cluster_store_name
      }
      target = {
        name           = "vector-datadog"
        creationPolicy = "Owner"
      }
      data = [{
        secretKey = "api_key"
        remoteRef = {
          key      = var.datadog_api_key_vault_path
          property = "apiKey"
        }
      }]
    }
  }

  wait {
    fields = {
      "status.conditions[0].type"   = "Ready"
      "status.conditions[0].status" = "True"
    }
  }

  timeouts {
    create = "5m"
    update = "5m"
  }
}

resource "helm_release" "vector" {
  name       = "vector"
  namespace  = kubernetes_namespace_v1.vector.metadata[0].name
  repository = "https://helm.vector.dev"
  chart      = "vector"
  version    = var.chart_version

  values = [local.rendered_values]

  set {
    name  = "image.repository"
    value = var.image
  }
  set {
    name  = "image.tag"
    value = var.image_tag_version
  }

  depends_on = [
    kubernetes_manifest.datadog_external_secret,
  ]
}

module "netpol" {
  source = "git@gitlab.infrastructure.immuta.io:infrastructure/terraform/internal-modules/kubernetes/security/network-policy.git?ref=v1.1.0"

  name             = "vector-netpol"
  namespace        = var.namespace
  network_policies = local.netpols

  depends_on = [
    kubernetes_namespace_v1.vector,
  ]
}


1	# ---------------------------------------------------------------------------------------------------------------------
2	# REQUIRED PARAMETERS
3	# These variables must be set.
4	# ---------------------------------------------------------------------------------------------------------------------
5
6	variable "chart_version" {
7	  description = "Vector helm chart version pin"
8	  type        = string
9	}
10
11	variable "image_tag_version" {
12	  description = "Vector image tag"
13	  type        = string
14	}
15
16	# ---------------------------------------------------------------------------------------------------------------------
17	# OPTIONAL PARAMETERS
18	# These variables are configurable, but have reasonable defaults set.
19	# ---------------------------------------------------------------------------------------------------------------------
20	variable "datadog_site" {
21	  description = "Datadog site (e.g. datadoghq.com, us3.datadoghq.com)"
22	  type        = string
23	  default     = "datadoghq.com"
24	}
25
26	variable "image" {
27	  description = "Vector image source"
28	  type        = string
29	  default     = "registry.ic-ops.net/cache-docker-hub/timberio/vector"
30	}
31
32	variable "replicas" {
33	  description = "Number of Vector aggregator replicas"
34	  type        = number
35	  default     = 3
36	}
37
38	variable "audit_drop_patterns" {
39	  description = "Regex patterns; matching log message bodies are dropped"
40	  type        = list(string)
41	  default = [
42	    "Audit\\s-\\s.+",
43	    "\"level\":\"audit\"",
44	  ]
45	}
46
47	variable "datadog_api_key_vault_path" {
48	  description = "Vault path for the Datadog API key, consumed by ExternalSecret"
49	  type        = string
50	  default     = "secret/data/saas/infrastructure/datadog-api-key"
51	}
52
53	variable "external_secrets_cluster_store_name" {
54	  description = "Name of the ClusterSecretStore created by external-secrets module"
55	  type        = string
56	  default     = "vault"
57	}
58
59	variable "email_redaction_enabled" {
60	  description = "Toggle the email-redact VRL transform across all signals"
61	  type        = bool
62	  default     = true
63	}
64
65	variable "email_redaction_replacement" {
66	  description = "String that replaces matched email addresses"
67	  type        = string
68	  default     = "[REDACTED-EMAIL]"
69	}
70
71	variable "additional_config" {
72	  description = "Escape hatch — overwrites helm customConfig wholesale (mirrors otel_collector_additional_config semantics)"
73	  type        = any
74	  default     = null
75	}
76
77	variable "resources_requests" {
78	  description = "Pod resource requests"
79	  type        = map(string)
80	  default = {
81	    cpu    = "200m"
82	    memory = "256Mi"
83	  }
84	}
85
86	variable "resources_limits" {
87	  description = "Pod resource limits"
88	  type        = map(string)
89	  default = {
90	    cpu    = "2"
91	    memory = "1Gi"
92	  }
93	}
94
95	variable "namespace" {
96	  description = "Kubernetes namespace for Vector"
97	  type        = string
98	  default     = "vector"
99	}
100
101	variable "service_monitor_enabled" {
102	  description = "Create a ServiceMonitor for Vector self-metrics. Requires kube-prometheus-stack CRDs in cluster."
103	  type        = bool
104	  default     = false
105	}
106

# ---------------------------------------------------------------------------------------------------------------------
# REQUIRED PARAMETERS
# These variables must be set.
# ---------------------------------------------------------------------------------------------------------------------

variable "chart_version" {
  description = "Vector helm chart version pin"
  type        = string
}

variable "image_tag_version" {
  description = "Vector image tag"
  type        = string
}

# ---------------------------------------------------------------------------------------------------------------------
# OPTIONAL PARAMETERS
# These variables are configurable, but have reasonable defaults set.
# ---------------------------------------------------------------------------------------------------------------------
variable "datadog_site" {
  description = "Datadog site (e.g. datadoghq.com, us3.datadoghq.com)"
  type        = string
  default     = "datadoghq.com"
}

variable "image" {
  description = "Vector image source"
  type        = string
  default     = "registry.ic-ops.net/cache-docker-hub/timberio/vector"
}

variable "replicas" {
  description = "Number of Vector aggregator replicas"
  type        = number
  default     = 3
}

variable "audit_drop_patterns" {
  description = "Regex patterns; matching log message bodies are dropped"
  type        = list(string)
  default = [
    "Audit\\s-\\s.+",
    "\"level\":\"audit\"",
  ]
}

variable "datadog_api_key_vault_path" {
  description = "Vault path for the Datadog API key, consumed by ExternalSecret"
  type        = string
  default     = "secret/data/saas/infrastructure/datadog-api-key"
}

variable "external_secrets_cluster_store_name" {
  description = "Name of the ClusterSecretStore created by external-secrets module"
  type        = string
  default     = "vault"
}

variable "email_redaction_enabled" {
  description = "Toggle the email-redact VRL transform across all signals"
  type        = bool
  default     = true
}

variable "email_redaction_replacement" {
  description = "String that replaces matched email addresses"
  type        = string
  default     = "[REDACTED-EMAIL]"
}

variable "additional_config" {
  description = "Escape hatch — overwrites helm customConfig wholesale (mirrors otel_collector_additional_config semantics)"
  type        = any
  default     = null
}

variable "resources_requests" {
  description = "Pod resource requests"
  type        = map(string)
  default = {
    cpu    = "200m"
    memory = "256Mi"
  }
}

variable "resources_limits" {
  description = "Pod resource limits"
  type        = map(string)
  default = {
    cpu    = "2"
    memory = "1Gi"
  }
}

variable "namespace" {
  description = "Kubernetes namespace for Vector"
  type        = string
  default     = "vector"
}

variable "service_monitor_enabled" {
  description = "Create a ServiceMonitor for Vector self-metrics. Requires kube-prometheus-stack CRDs in cluster."
  type        = bool
  default     = false
}


Creating brainstorming checklist tasks and reading the Vector template + netpols to understand the current sources surface.


TaskCreate

Explore project context

Read datadog-agent + vector module sources, netpols, values templates to understand current state before designing log-routing change.


Task #1 created successfully: Explore project context

Explore project context


Ask clarifying questions

One at a time, narrow down purpose/constraints/success criteria for routing Datadog Agent logs through Vector.


Task #2 created successfully: Ask clarifying questions

Ask clarifying questions


Propose 2-3 approaches

Compare protocol options (Datadog Agent native intake vs OTLP vs other) and where the change lives (datadog-agent module vs vector module vs consumer).


Task #3 created successfully: Propose 2-3 approaches

Propose 2-3 approaches


Present design sections

Architecture, components, data flow, error handling, testing — get user approval section by section.


Task #4 created successfully: Present design sections

Present design sections


Write design doc to docs/superpowers/specs

Save validated spec to docs/superpowers/specs/2026-05-06-datadog-agent-logs-via-vector-design.md and commit.


Task #5 created successfully: Write design doc to docs/superpowers/specs


Spec self-review

Inline scan for placeholders, contradictions, ambiguity, scope. Fix and move on.


Task #6 created successfully: Spec self-review

Spec self-review


User reviews written spec

Ask user to review committed spec before invoking writing-plans.


Task #7 created successfully: User reviews written spec

User reviews written spec


Invoke writing-plans skill

Terminal step — only after spec approval, hand off to writing-plans for implementation plan.


Task #8 created successfully: Invoke writing-plans skill

Invoke writing-plans skill


1	# ---------------------------------------------------------------------------------------------------------------------
2	# VECTOR
3	# ---------------------------------------------------------------------------------------------------------------------
4	locals {
5	  vector_enabled = (
6	    var.vector_enabled != null
7	    ? var.vector_enabled
8	    : lookup(var.vector_enabled_cluster_types, local.cluster_type, true)
9	  )
10	  vector_endpoint = "http://vector.vector.svc.cluster.local:4317"
11	}
12
13	module "vector" {
14	  #source = "git@gitlab.infrastructure.immuta.io:infrastructure/terraform/internal-modules/kubernetes/monitoring/vector.git?ref=v1.0.0"
15	  source = "/Users/steve.briggs/projects/IAC/terraform/internal-modules/kubernetes/monitoring/vector"
16	  count  = local.vector_enabled ? 1 : 0
17
18	  providers = {
19	    kubernetes = kubernetes.local-cluster
20	    helm       = helm.local-cluster
21	  }
22
23	  chart_version               = var.vector_chart_version
24	  image                       = var.vector_image
25	  image_tag_version           = var.vector_image_tag
26	  replicas                    = var.vector_replicas
27	  datadog_site                = var.vector_datadog_site
28	  datadog_api_key_vault_path  = var.vector_datadog_api_key_vault_path
29	  audit_drop_patterns         = var.vector_audit_drop_patterns
30	  email_redaction_enabled     = var.vector_email_redaction_enabled
31	  email_redaction_replacement = var.vector_email_redaction_replacement
32	  additional_config           = var.vector_additional_config
33
34	  # NOTE: The external-secrets internal-module (v1.2.0) has an empty outputs.tf
35	  # and does not expose cluster_secret_store_name. Until that output is added
36	  # to the external-secrets module, the ClusterSecretStore name is passed
37	  # explicitly via var.vector_external_secrets_cluster_store_name.
38	  # TODO: replace with module.external_secrets[0].cluster_secret_store_name
39	  # once the external-secrets internal-module exposes that output.
40	  external_secrets_cluster_store_name = var.vector_external_secrets_cluster_store_name
41
42	  depends_on = [
43	    module.cilium,
44	    module.external_secrets,
45	  ]
46	}
47
48	data "external" "vector_validate" {
49	  count = local.vector_enabled && !local.external_secrets_enabled ? 1 : 0
50
51	  program = ["Error: vector can only be set if external-secrets is enabled as well"]
52	}
53
54	# ---------------------------------------------------------------------------------------------------------------------
55	# REQUIRED PARAMETERS
56	# These variables must be set.
57	# ---------------------------------------------------------------------------------------------------------------------
58	variable "vector_chart_version" {
59	  description = "Version of vector helm chart"
60	  type        = string
61	  default     = ""
62	  validation {
63	    condition     = var.vector_chart_version == "" && local.vector_enabled ? false : true
64	    error_message = "vector_chart_version must be set when vector is enabled"
65	  }
66	}
67
68	variable "vector_image_tag" {
69	  description = "Vector image tag"
70	  type        = string
71	  default     = ""
72	  validation {
73	    condition     = var.vector_image_tag == "" && local.vector_enabled ? false : true
74	    error_message = "vector_image_tag must be set when vector is enabled"
75	  }
76	}
77
78	variable "vector_datadog_site" {
79	  description = "Datadog site for vector sinks (e.g. datadoghq.com)"
80	  type        = string
81	  default     = "datadoghq.com"
82	  validation {
83	    condition     = var.vector_datadog_site == "" && local.vector_enabled ? false : true
84	    error_message = "vector_datadog_site must be set when vector is enabled"
85	  }
86	}
87
88	variable "vector_datadog_api_key_vault_path" {
89	  description = "Vault path for the Datadog API key (consumed by ExternalSecret in vector ns)"
90	  type        = string
91	  default     = "secret/data/saas/infrastructure/datadog-api-key"
92	  validation {
93	    condition     = var.vector_datadog_api_key_vault_path == "" && local.vector_enabled ? false : true
94	    error_message = "vector_datadog_api_key_vault_path must be set when vector is enabled"
95	  }
96	}
97
98	variable "vector_external_secrets_cluster_store_name" {
99	  description = "Name of the ClusterSecretStore to use for vector ExternalSecret resources. Pass the name created by the external-secrets module (typically 'vault-cluster-store'). Required when vector is enabled."
100	  type        = string
101	  default     = "vault"
102	  validation {
103	    condition     = var.vector_external_secrets_cluster_store_name == "" && local.vector_enabled ? false : true
104	    error_message = "vector_external_secrets_cluster_store_name must be set when vector is enabled"
105	  }
106	}
107
108	# ---------------------------------------------------------------------------------------------------------------------
109	# OPTIONAL PARAMETERS
110	# These variables are configurable, but have reasonable defaults set.
111	# ---------------------------------------------------------------------------------------------------------------------
112	variable "vector_enabled_cluster_types" {
113	  description = "Types of clusters Vector should be enabled on by default"
114	  type        = map(string)
115	  default = {
116	    "mgmt"  = true
117	    "cps"   = true
118	    "saas"  = true
119	    "infra" = false
120	    "dev"   = false
121	  }
122	}
123
124	variable "vector_enabled" {
125	  description = "Override for explicitly enabling or disabling Vector"
126	  type        = bool
127	  default     = null
128	}
129
130	variable "vector_image" {
131	  description = "Image for Vector"
132	  type        = string
133	  default     = "registry.ic-ops.net/cache-docker-hub/timberio/vector"
134	}
135
136	variable "vector_replicas" {
137	  description = "Number of Vector aggregator replicas"
138	  type        = number
139	  default     = 3
140	}
141
142	variable "vector_audit_drop_patterns" {
143	  description = "Regex patterns; matching log message bodies are dropped"
144	  type        = list(string)
145	  default = [
146	    "Audit\\s-\\s.+",
147	    "\"level\":\"audit\"",
148	  ]
149	}
150
151	variable "vector_email_redaction_enabled" {
152	  description = "Toggle the email-redact VRL transform across all signals"
153	  type        = bool
154	  default     = true
155	}
156
157	variable "vector_email_redaction_replacement" {
158	  description = "String that replaces matched email addresses"
159	  type        = string
160	  default     = "[REDACTED-EMAIL]"
161	}
162
163	variable "vector_additional_config" {
164	  description = "Alternate helm chart customConfig (overwrites entire helm config)"
165	  type        = any
166	  default     = null
167	}
168

# ---------------------------------------------------------------------------------------------------------------------
# VECTOR
# ---------------------------------------------------------------------------------------------------------------------
locals {
  vector_enabled = (
    var.vector_enabled != null
    ? var.vector_enabled
    : lookup(var.vector_enabled_cluster_types, local.cluster_type, true)
  )
  vector_endpoint = "http://vector.vector.svc.cluster.local:4317"
}

module "vector" {
  #source = "git@gitlab.infrastructure.immuta.io:infrastructure/terraform/internal-modules/kubernetes/monitoring/vector.git?ref=v1.0.0"
  source = "/Users/steve.briggs/projects/IAC/terraform/internal-modules/kubernetes/monitoring/vector"
  count  = local.vector_enabled ? 1 : 0

  providers = {
    kubernetes = kubernetes.local-cluster
    helm       = helm.local-cluster
  }

  chart_version               = var.vector_chart_version
  image                       = var.vector_image
  image_tag_version           = var.vector_image_tag
  replicas                    = var.vector_replicas
  datadog_site                = var.vector_datadog_site
  datadog_api_key_vault_path  = var.vector_datadog_api_key_vault_path
  audit_drop_patterns         = var.vector_audit_drop_patterns
  email_redaction_enabled     = var.vector_email_redaction_enabled
  email_redaction_replacement = var.vector_email_redaction_replacement
  additional_config           = var.vector_additional_config

  # NOTE: The external-secrets internal-module (v1.2.0) has an empty outputs.tf
  # and does not expose cluster_secret_store_name. Until that output is added
  # to the external-secrets module, the ClusterSecretStore name is passed
  # explicitly via var.vector_external_secrets_cluster_store_name.
  # TODO: replace with module.external_secrets[0].cluster_secret_store_name
  # once the external-secrets internal-module exposes that output.
  external_secrets_cluster_store_name = var.vector_external_secrets_cluster_store_name

  depends_on = [
    module.cilium,
    module.external_secrets,
  ]
}

data "external" "vector_validate" {
  count = local.vector_enabled && !local.external_secrets_enabled ? 1 : 0

  program = ["Error: vector can only be set if external-secrets is enabled as well"]
}

# ---------------------------------------------------------------------------------------------------------------------
# REQUIRED PARAMETERS
# These variables must be set.
# ---------------------------------------------------------------------------------------------------------------------
variable "vector_chart_version" {
  description = "Version of vector helm chart"
  type        = string
  default     = ""
  validation {
    condition     = var.vector_chart_version == "" && local.vector_enabled ? false : true
    error_message = "vector_chart_version must be set when vector is enabled"
  }
}

variable "vector_image_tag" {
  description = "Vector image tag"
  type        = string
  default     = ""
  validation {
    condition     = var.vector_image_tag == "" && local.vector_enabled ? false : true
    error_message = "vector_image_tag must be set when vector is enabled"
  }
}

variable "vector_datadog_site" {
  description = "Datadog site for vector sinks (e.g. datadoghq.com)"
  type        = string
  default     = "datadoghq.com"
  validation {
    condition     = var.vector_datadog_site == "" && local.vector_enabled ? false : true
    error_message = "vector_datadog_site must be set when vector is enabled"
  }
}

variable "vector_datadog_api_key_vault_path" {
  description = "Vault path for the Datadog API key (consumed by ExternalSecret in vector ns)"
  type        = string
  default     = "secret/data/saas/infrastructure/datadog-api-key"
  validation {
    condition     = var.vector_datadog_api_key_vault_path == "" && local.vector_enabled ? false : true
    error_message = "vector_datadog_api_key_vault_path must be set when vector is enabled"
  }
}

variable "vector_external_secrets_cluster_store_name" {
  description = "Name of the ClusterSecretStore to use for vector ExternalSecret resources. Pass the name created by the external-secrets module (typically 'vault-cluster-store'). Required when vector is enabled."
  type        = string
  default     = "vault"
  validation {
    condition     = var.vector_external_secrets_cluster_store_name == "" && local.vector_enabled ? false : true
    error_message = "vector_external_secrets_cluster_store_name must be set when vector is enabled"
  }
}

# ---------------------------------------------------------------------------------------------------------------------
# OPTIONAL PARAMETERS
# These variables are configurable, but have reasonable defaults set.
# ---------------------------------------------------------------------------------------------------------------------
variable "vector_enabled_cluster_types" {
  description = "Types of clusters Vector should be enabled on by default"
  type        = map(string)
  default = {
    "mgmt"  = true
    "cps"   = true
    "saas"  = true
    "infra" = false
    "dev"   = false
  }
}

variable "vector_enabled" {
  description = "Override for explicitly enabling or disabling Vector"
  type        = bool
  default     = null
}

variable "vector_image" {
  description = "Image for Vector"
  type        = string
  default     = "registry.ic-ops.net/cache-docker-hub/timberio/vector"
}

variable "vector_replicas" {
  description = "Number of Vector aggregator replicas"
  type        = number
  default     = 3
}

variable "vector_audit_drop_patterns" {
  description = "Regex patterns; matching log message bodies are dropped"
  type        = list(string)
  default = [
    "Audit\\s-\\s.+",
    "\"level\":\"audit\"",
  ]
}

variable "vector_email_redaction_enabled" {
  description = "Toggle the email-redact VRL transform across all signals"
  type        = bool
  default     = true
}

variable "vector_email_redaction_replacement" {
  description = "String that replaces matched email addresses"
  type        = string
  default     = "[REDACTED-EMAIL]"
}

variable "vector_additional_config" {
  description = "Alternate helm chart customConfig (overwrites entire helm config)"
  type        = any
  default     = null
}


Yesterday we worked on the vector terraform module and I was able to deploy that, currently only traces are going from datadog to the open telemetry collector which is fine, what I need to do next is…


Configure Datadog to ship logs to Vector


1	role: Aggregator
2	replicas: ${replicas}
3	podAnnotations:
4	  reloader.stakater.com/auto: "true"
5	service:
6	  enabled: true
7	  type: ClusterIP
8	  ports:
9	    - name: otlp-grpc
10	      port: 4317
11	      protocol: TCP
12	    - name: otlp-http
13	      port: 4318
14	      protocol: TCP
15	env:
16	  - name: DATADOG_API_KEY
17	    valueFrom:
18	      secretKeyRef:
19	        name: vector-datadog
20	        key: api_key
21	resources:
22	  requests:
23	    cpu: ${requests_cpu}
24	    memory: ${requests_memory}
25	  limits:
26	    cpu: "${limits_cpu}"
27	    memory: ${limits_memory}
28	podDisruptionBudget:
29	  enabled: true
30	  minAvailable: 2
31
32	topologySpreadConstraints:
33	  - maxSkew: 1
34	    topologyKey: topology.kubernetes.io/zone
35	    whenUnsatisfiable: ScheduleAnyway
36	    labelSelector:
37	      matchLabels:
38	        app.kubernetes.io/name: vector
39
40	affinity:
41	  podAntiAffinity:
42	    preferredDuringSchedulingIgnoredDuringExecution:
43	      - weight: 100
44	        podAffinityTerm:
45	          topologyKey: kubernetes.io/hostname
46	          labelSelector:
47	            matchLabels:
48	              app.kubernetes.io/name: vector
49
50	%{ if service_monitor_enabled ~}
51	serviceMonitor:
52	  enabled: true
53	%{ endif ~}
54	customConfig:
55	  data_dir: /vector-data-dir
56	  api:
57	    enabled: true
58	    address: 127.0.0.1:8686
59	  sources:
60	    otlp_in:
61	      type: opentelemetry
62	      grpc:
63	        address: 0.0.0.0:4317
64	      http:
65	        address: 0.0.0.0:4318
66	  transforms:
67	    drop_audit:
68	      type: filter
69	      inputs:
70	        - otlp_in.logs
71	      condition: |
72	        msg = if is_string(.message) { string!(.message) } else { "" }
73	        ${audit_drop_condition}
74	%{ if email_redaction_enabled ~}
75	    redact_emails_logs:
76	      type: remap
77	      inputs:
78	        - drop_audit
79	      source: |
80	        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
81	        if is_string(.message) {
82	          .message = replace(string!(.message), email_re, "${email_redaction_replacement}")
83	        }
84	        if is_object(.attributes) {
85	          attrs, _err = object(.attributes)
86	          if is_object(attrs) {
87	            .attributes = map_values(attrs) -> |v| {
88	              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
89	            }
90	          }
91	        }
92	    redact_emails_traces:
93	      type: remap
94	      inputs:
95	        - otlp_in.traces
96	      source: |
97	        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
98	        if is_object(.attributes) {
99	          attrs, _err = object(.attributes)
100	          if is_object(attrs) {
101	            .attributes = map_values(attrs) -> |v| {
102	              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
103	            }
104	          }
105	        }
106	        if is_object(.resource) {
107	          res, _err = object(.resource)
108	          if is_object(res) && is_object(res.attributes) {
109	            rattrs, _err2 = object(res.attributes)
110	            if is_object(rattrs) {
111	              .resource.attributes = map_values(rattrs) -> |v| {
112	                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
113	              }
114	            }
115	          }
116	        }
117	    redact_emails_metrics:
118	      type: remap
119	      inputs:
120	        - otlp_in.metrics
121	      source: |
122	        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
123	        if is_object(.attributes) {
124	          attrs, _err = object(.attributes)
125	          if is_object(attrs) {
126	            .attributes = map_values(attrs) -> |v| {
127	              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
128	            }
129	          }
130	        }
131	        if is_object(.resource) {
132	          res, _err = object(.resource)
133	          if is_object(res) && is_object(res.attributes) {
134	            rattrs, _err2 = object(res.attributes)
135	            if is_object(rattrs) {
136	              .resource.attributes = map_values(rattrs) -> |v| {
137	                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
138	              }
139	            }
140	          }
141	        }
142	%{ endif ~}
143	  sinks:
144	    datadog_logs:
145	      type: datadog_logs
146	      inputs:
147	        - ${email_redaction_enabled ? "redact_emails_logs" : "drop_audit"}
148	      default_api_key: $${DATADOG_API_KEY}
149	      site: ${datadog_site}
150	    datadog_traces:
151	      type: datadog_traces
152	      inputs:
153	        - ${email_redaction_enabled ? "redact_emails_traces" : "otlp_in.traces"}
154	      default_api_key: $${DATADOG_API_KEY}
155	      site: ${datadog_site}
156	    datadog_metrics:
157	      type: datadog_metrics
158	      inputs:
159	        - ${email_redaction_enabled ? "redact_emails_metrics" : "otlp_in.metrics"}
160	      default_api_key: $${DATADOG_API_KEY}
161	      site: ${datadog_site}
162

role: Aggregator
replicas: ${replicas}
podAnnotations:
  reloader.stakater.com/auto: "true"
service:
  enabled: true
  type: ClusterIP
  ports:
    - name: otlp-grpc
      port: 4317
      protocol: TCP
    - name: otlp-http
      port: 4318
      protocol: TCP
env:
  - name: DATADOG_API_KEY
    valueFrom:
      secretKeyRef:
        name: vector-datadog
        key: api_key
resources:
  requests:
    cpu: ${requests_cpu}
    memory: ${requests_memory}
  limits:
    cpu: "${limits_cpu}"
    memory: ${limits_memory}
podDisruptionBudget:
  enabled: true
  minAvailable: 2

topologySpreadConstraints:
  - maxSkew: 1
    topologyKey: topology.kubernetes.io/zone
    whenUnsatisfiable: ScheduleAnyway
    labelSelector:
      matchLabels:
        app.kubernetes.io/name: vector

affinity:
  podAntiAffinity:
    preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 100
        podAffinityTerm:
          topologyKey: kubernetes.io/hostname
          labelSelector:
            matchLabels:
              app.kubernetes.io/name: vector

%{ if service_monitor_enabled ~}
serviceMonitor:
  enabled: true
%{ endif ~}
customConfig:
  data_dir: /vector-data-dir
  api:
    enabled: true
    address: 127.0.0.1:8686
  sources:
    otlp_in:
      type: opentelemetry
      grpc:
        address: 0.0.0.0:4317
      http:
        address: 0.0.0.0:4318
  transforms:
    drop_audit:
      type: filter
      inputs:
        - otlp_in.logs
      condition: |
        msg = if is_string(.message) { string!(.message) } else { "" }
        ${audit_drop_condition}
%{ if email_redaction_enabled ~}
    redact_emails_logs:
      type: remap
      inputs:
        - drop_audit
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_string(.message) {
          .message = replace(string!(.message), email_re, "${email_redaction_replacement}")
        }
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
    redact_emails_traces:
      type: remap
      inputs:
        - otlp_in.traces
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
    redact_emails_metrics:
      type: remap
      inputs:
        - otlp_in.metrics
      source: |
        email_re = r'[\w._%+\-]+@[\w.-]+\.[A-Za-z]{2,}'
        if is_object(.attributes) {
          attrs, _err = object(.attributes)
          if is_object(attrs) {
            .attributes = map_values(attrs) -> |v| {
              if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
            }
          }
        }
        if is_object(.resource) {
          res, _err = object(.resource)
          if is_object(res) && is_object(res.attributes) {
            rattrs, _err2 = object(res.attributes)
            if is_object(rattrs) {
              .resource.attributes = map_values(rattrs) -> |v| {
                if is_string(v) { replace(string!(v), email_re, "${email_redaction_replacement}") } else { v }
              }
            }
          }
        }
%{ endif ~}
  sinks:
    datadog_logs:
      type: datadog_logs
      inputs:
        - ${email_redaction_enabled ? "redact_emails_logs" : "drop_audit"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_traces:
      type: datadog_traces
      inputs:
        - ${email_redaction_enabled ? "redact_emails_traces" : "otlp_in.traces"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}
    datadog_metrics:
      type: datadog_metrics
      inputs:
        - ${email_redaction_enabled ? "redact_emails_metrics" : "otlp_in.metrics"}
      default_api_key: $${DATADOG_API_KEY}
      site: ${datadog_site}


1	locals {
2	  netpols = {
3	    "vector" = local.vector_netpol
4	  }
5	  vector_netpol = {
6	    endpointSelector = {
7	      matchLabels = {
8	        "app.kubernetes.io/name" = "vector"
9	      }
10	    }
11	    ingress = [
12	      {
13	        # OTLP from any cluster-internal sender (primarily otel-collector).
14	        fromEntities = ["cluster"]
15	        toPorts = [{
16	          ports = [
17	            { port = "4317" },
18	            { port = "4318" },
19	          ]
20	        }]
21	      },
22	      {
23	        # Vector self-metrics scrape (Prometheus operator, Datadog Agent, etc.).
24	        fromEntities = ["cluster"]
25	        toPorts = [{
26	          ports = [{ port = "8686" }]
27	        }]
28	      },
29	      {
30	        # kubelet health probes from the node.
31	        fromEntities = ["host"]
32	        toPorts = [{
33	          ports = [
34	            { port = "4317" },
35	            { port = "4318" },
36	            { port = "8686" },
37	          ]
38	        }]
39	      },
40	      {
41	        fromEntities = ["kube-apiserver"]
42	      }
43	    ]
44	    egress = [
45	      {
46	        # Datadog SaaS HTTPS for logs/traces/metrics sinks.
47	        toEntities = ["world"]
48	        toPorts = [{
49	          ports = [{ port = "443" }]
50	        }]
51	      },
52	      {
53	        toEntities = ["kube-apiserver"]
54	      },
55	      {
56	        # DNS resolution + service discovery (CoreDNS lives here).
57	        toEntities = ["cluster"]
58	      }
59	    ]
60	  }
61	}
62

locals {
  netpols = {
    "vector" = local.vector_netpol
  }
  vector_netpol = {
    endpointSelector = {
      matchLabels = {
        "app.kubernetes.io/name" = "vector"
      }
    }
    ingress = [
      {
        # OTLP from any cluster-internal sender (primarily otel-collector).
        fromEntities = ["cluster"]
        toPorts = [{
          ports = [
            { port = "4317" },
            { port = "4318" },
          ]
        }]
      },
      {
        # Vector self-metrics scrape (Prometheus operator, Datadog Agent, etc.).
        fromEntities = ["cluster"]
        toPorts = [{
          ports = [{ port = "8686" }]
        }]
      },
      {
        # kubelet health probes from the node.
        fromEntities = ["host"]
        toPorts = [{
          ports = [
            { port = "4317" },
            { port = "4318" },
            { port = "8686" },
          ]
        }]
      },
      {
        fromEntities = ["kube-apiserver"]
      }
    ]
    egress = [
      {
        # Datadog SaaS HTTPS for logs/traces/metrics sinks.
        toEntities = ["world"]
        toPorts = [{
          ports = [{ port = "443" }]
        }]
      },
      {
        toEntities = ["kube-apiserver"]
      },
      {
        # DNS resolution + service discovery (CoreDNS lives here).
        toEntities = ["cluster"]
      }
    ]
  }
}


1	output "namespace" {
2	  description = "Namespace Vector is deployed into"
3	  value       = var.namespace
4	}
5
6	output "service_endpoint_grpc" {
7	  description = "Cluster-internal OTLP/gRPC endpoint to point upstream collectors at"
8	  value       = "http://vector.${var.namespace}.svc.cluster.local:4317"
9	}
10
11	output "service_endpoint_http" {
12	  description = "Cluster-internal OTLP/HTTP endpoint"
13	  value       = "http://vector.${var.namespace}.svc.cluster.local:4318"
14	}
15

output "namespace" {
  description = "Namespace Vector is deployed into"
  value       = var.namespace
}

output "service_endpoint_grpc" {
  description = "Cluster-internal OTLP/gRPC endpoint to point upstream collectors at"
  value       = "http://vector.${var.namespace}.svc.cluster.local:4317"
}

output "service_endpoint_http" {
  description = "Cluster-internal OTLP/HTTP endpoint"
  value       = "http://vector.${var.namespace}.svc.cluster.local:4318"
}


1	locals {
2	  netpols = {
3	    "datadog"       = local.datadog_netpol
4	    "allow-dns"     = local.allow_dns_netpol
5	  }
6
7	  datadog_netpol = {
8	    endpointSelector = {}
9	    ingress = [
10	      {
11	        fromEntities = ["kube-apiserver"]
12	      },
13	      {
14	        fromEntities = ["host"]
15	        toPorts = [{
16	          ports = [
17	            { port = "5000" },
18	            { port = "5005" },
19	            { port = "5556" }
20	          ]
21	        }]
22	      },
23	      {
24	        fromEntities = ["remote-node"]
25	        toPorts = [{
26	          ports = [
27	            { port = "5000" },
28	            { port = "5005" },
29	            { port = "5556" }
30	          ]
31	        }]
32	      },
33	      {
34	        fromEndpoints = [{}]
35	      }
36	    ]
37	    egress = [
38	      {
39	        toFQDNs = [
40	          { matchPattern = "*.*.*.*.*.elastic-cloud.com" },
41	          { matchPattern = "*.*.*.aws.found.io" },
42	          { matchPattern = "*.*.datadoghq.com" },
43	          { matchPattern = "*.datadoghq.com" },
44	          { matchPattern = "*.*.rds.amazonaws.com" },
45	          { matchPattern = "*.*.*.rds.amazonaws.com" }
46	        ]
47	        toPorts = [{
48	          ports = [{ port = "443" }]
49	        }]
50	      },
51	      {
52	        toCIDRSet = [{ cidr = "169.254.169.254/32" }]
53	        toPorts = [{
54	          ports = [{ port = "80" }]
55	        }]
56	      },
57	      {
58	        toCIDRSet = [{ cidr = "169.254.169.123/32" }]
59	        toPorts = [{
60	          ports = [{ port = "123" }]
61	        }]
62	      },
63	      {
64	        toCIDRSet = [{ cidr = "10.0.0.0/8" }]
65	      },
66	      {
67	        toEntities = ["cluster"]
68	      }
69	    ]
70	  }
71
72	  allow_dns_netpol = {
73	    endpointSelector = {}
74	    egress = [
75	      {
76	        toEndpoints = [{
77	          matchLabels = {
78	            "io.kubernetes.pod.namespace" = "kube-system"
79	            "k8s-app"                     = "kube-dns"
80	          }
81	        }]
82	        toPorts = [{
83	          ports = [{
84	            port     = "53"
85	            protocol = "UDP"
86	          }]
87	          rules = {
88	            dns = [{ matchPattern = "*" }]
89	          }
90	        }]
91	      }
92	    ]
93	  }
94
95	}
96

locals {
  netpols = {
    "datadog"       = local.datadog_netpol
    "allow-dns"     = local.allow_dns_netpol
  }

  datadog_netpol = {
    endpointSelector = {}
    ingress = [
      {
        fromEntities = ["kube-apiserver"]
      },
      {
        fromEntities = ["host"]
        toPorts = [{
          ports = [
            { port = "5000" },
            { port = "5005" },
            { port = "5556" }
          ]
        }]
      },
      {
        fromEntities = ["remote-node"]
        toPorts = [{
          ports = [
            { port = "5000" },
            { port = "5005" },
            { port = "5556" }
          ]
        }]
      },
      {
        fromEndpoints = [{}]
      }
    ]
    egress = [
      {
        toFQDNs = [
          { matchPattern = "*.*.*.*.*.elastic-cloud.com" },
          { matchPattern = "*.*.*.aws.found.io" },
          { matchPattern = "*.*.datadoghq.com" },
          { matchPattern = "*.datadoghq.com" },
          { matchPattern = "*.*.rds.amazonaws.com" },
          { matchPattern = "*.*.*.rds.amazonaws.com" }
        ]
        toPorts = [{
          ports = [{ port = "443" }]
        }]
      },
      {
        toCIDRSet = [{ cidr = "169.254.169.254/32" }]
        toPorts = [{
          ports = [{ port = "80" }]
        }]
      },
      {
        toCIDRSet = [{ cidr = "169.254.169.123/32" }]
        toPorts = [{
          ports = [{ port = "123" }]
        }]
      },
      {
        toCIDRSet = [{ cidr = "10.0.0.0/8" }]
      },
      {
        toEntities = ["cluster"]
      }
    ]
  }

  allow_dns_netpol = {
    endpointSelector = {}
    egress = [
      {
        toEndpoints = [{
          matchLabels = {
            "io.kubernetes.pod.namespace" = "kube-system"
            "k8s-app"                     = "kube-dns"
          }
        }]
        toPorts = [{
          ports = [{
            port     = "53"
            protocol = "UDP"
          }]
          rules = {
            dns = [{ matchPattern = "*" }]
          }
        }]
      }
    ]
  }

}


Marking exploration done. I have a clear picture of both modules now.


Updated task #1 status


Updated task #2 status


Quick state-of-the-board, Sir Charles of St Cheese, since framing matters before I ask anything:

- Today, DD Agent ships container logs **direct to `*.datadoghq.com:443`**, with audit-line dropping handled agent-side via `DD_LOGS_CONFIG_PROCESSING_RULES` (the `processing-rules/{env}/rules.json` config map).
- Vector currently only has an `otlp_in` source — its `drop_audit` and `redact_emails_logs` transforms only see OTLP-formatted logs (so anything DD Agent collects today bypasses redaction entirely).
- Vector already has a `datadog_logs` sink wired up. The missing piece is the *intake* side: a `datadog_agent` source plus a way to flip DD Agent's log destination to Vector's service.

Before I sketch approaches, one foundational question to make sure I'm solving the right problem.

**Q1 — What's the primary motivation for this routing change?**

a) **Apply email/PII redaction** to logs DD Agent collects (i.e., extend Vector's redaction surface to cover container stdout, not just OTLP).
b) **Centralize audit-drop logic** in Vector and retire the agent-side `processing-rules/{env}/rules.json` config map.
c) **Both** — full migration: DD Agent becomes a dumb collector, Vector owns all log filtering + redaction before anything reaches Datadog SaaS.
d) Something else (please describe).

Which of these matches the goal?


afplay ~/.claude/sounds/mario/stop/stomp.wav 2>/dev/null || true


a


Got it — Vector becomes an inline redaction stop on the existing DD-Agent log path; the agent-side `processing-rules/{env}/rules.json` audit-drop stays where it is (it'll run before logs ever leave the agent, reducing Vector's workload, and we keep defense-in-depth).

**Q2 — Rollout granularity / gating.**

Today `vector_enabled` is computed per cluster type (`mgmt`/`cps`/`saas` on; `infra`/`dev` off). The new DD-Agent → Vector log routing has a few shapes:

a) **Coupled** — if `vector_enabled` is true for the cluster, DD Agent ships logs to Vector automatically. No new flag. Simplest, fewest knobs, follows the existing pattern.

b) **Independent flag** — new `vector_dd_agent_logs_enabled` (or similar) flag, defaulted to follow `vector_enabled` but overridable. Lets you turn DD Agent's log redirection on/off without disabling the rest of Vector (useful if redaction misbehaves and you need a quick fallback to direct DD shipping while Vector keeps handling traces from OTel).

c) **Phased** — start hard-coupled, add a flag later if needed.

Which shape do you want? My recommendation is **(b)** — the cost is one extra variable but the safety hatch (flip logs back to direct DD shipping while Vector keeps doing OTel traces) is real, and matches how you already gate `vector_enabled` itself.


b


Good, that gives us the safety hatch. Marking the gating decision down.

**Q3 — Vector graph topology on the new DD-Agent log path.**

Today, OTLP logs flow through `otlp_in.logs → drop_audit → redact_emails_logs → datadog_logs sink`. DD-Agent logs would arrive on a new source (e.g. `dd_agent_in.logs`). Question is what transforms they pass through:

a) **Skip `drop_audit`, redact only.** DD Agent source → `redact_emails_logs` → `datadog_logs` sink. Cleanest graph; relies on the existing agent-side `processing-rules/{env}/rules.json` to do the audit-drop *before* logs leave the agent (which it already does). Tightest match for what you said in Q1 (redaction only).

b) **Defense-in-depth, single shared filter.** Make `drop_audit` accept both inputs (`otlp_in.logs` + `dd_agent_in.logs`). Same regex logic runs again in Vector. Catches anything the agent rules would miss / misconfigure.

c) **Defense-in-depth, separate filters.** A second `drop_audit_dd_agent` mirroring the OTLP one. Clearest per-source telemetry on `vector_component_discarded_events_total`, more YAML.

My recommendation is **(a)** — matches your stated intent (just add redaction), keeps the graph readable, and the agent-side rules are already authoritative for audit drops. If you want belt-and-suspenders, (b) is the cheapest defense-in-depth option since it just adds an input to an existing transform.

Which way?


Belt and suspenders it is — `drop_audit` becomes a multi-input transform.

**Q4 — `datadog-agent` module's new variable shape.**

To redirect DD Agent logs to Vector, the module needs to render `logs_config.use_http: true`, `logs_config.logs_no_ssl: true`, and `logs_config.logs_dd_url: <vector-url>:<port>` in its values template. Three shapes for the consumer-facing variable:

a) **Minimal — one variable `logs_intake_url` (string, default `null`).** When set, the module flips `use_http`/`logs_no_ssl: true` automatically and writes `logs_dd_url`. When `null`, DD Agent ships direct to DD SaaS exactly as today. Consumer passes the URL string only.

b) **Structured object** — `logs_intake = { url = "...", use_http = true, no_ssl = true }`. Surfaces all three knobs but they're effectively forced for cluster-internal Vector (TLS off, HTTP forced).

c) **Decomposed** — three separate variables. More idiomatic, more lines, no real benefit over (a).

My recommendation is **(a)**. The `use_http` / `logs_no_ssl` flags only matter when `logs_dd_url` is overridden to a non-Datadog target; exposing them as user knobs invites misconfiguration (TLS off when pointing at the real DD endpoint = broken). One variable, derived booleans, one decision point.

OK with (a)?


Good, that's the last question I need. Time to present the design.


Updated task #3 status


Updated task #4 status


## Design — Section 1 of 5: Architecture

**Pipeline (logs path) before:**
```
DD Agent (DaemonSet) → [agent-side processing rules drop audits] → datadoghq.com:443
```

**Pipeline (logs path) after:**
```
DD Agent (DaemonSet)
  → [agent-side processing rules drop audits]
  → vector.vector.svc.cluster.local:8080  (DD Agent intake protocol over HTTP, no TLS)
  → Vector dd_agent_in source
  → drop_audit (now multi-input: otlp_in.logs + dd_agent_in.logs — defense-in-depth)
  → redact_emails_logs (when enabled)
  → datadog_logs sink → datadoghq.com:443
```

Traces and metrics are unaffected — they keep their existing path (apps → otel-collector → vector → DD).

**Three repos touched, additive only:**

1. **`internal-modules/kubernetes/monitoring/vector`** — bumps to `v1.1.0`. Adds a `datadog_agent` Vector source on port 8080, exposes the port in helm `service.ports`, opens the netpol ingress, makes `drop_audit` accept the new input, and adds an output `service_endpoint_dd_agent`. The new source is always declared (the receiver is harmless if no agent is pointed at it), so there's no new module variable for this — just the new output.

2. **`internal-modules/kubernetes/monitoring/datadog-agent`** — bumps to `v1.1.0`. Adds one variable `logs_intake_url` (string, default `null`). When non-null, the values template renders `logs_config.use_http: true`, `logs_config.logs_no_ssl: true`, `logs_config.logs_dd_url: "<url>"`. When null, rendered output is byte-identical to today (no churn on clusters that don't opt in).

3. **`eks-cluster-applications`** consumer — adds `vector_dd_agent_logs_enabled` flag (mirrors the existing `vector_enabled_cluster_types` per-cluster-type-map + override pattern, defaults true on cluster types where Vector is enabled). Wires `module.datadog_agent.logs_intake_url = local.vector_dd_agent_logs_enabled ? module.vector[0].service_endpoint_dd_agent : null`.

**Why three modules and not one:**
- Vector module owns "what protocols Vector accepts" — the new source belongs there.
- DD Agent module owns "where DD Agent ships logs" — the new variable belongs there.
- Consumer wires them together (same pattern as the existing OTel→Vector handoff).

Does the architecture look right so far? Any objections to port 8080, the always-on source on the Vector side, or the additive-only DD Agent module change?


yes this looks right


## Design — Section 2 of 5: Components

**Vector module (`v1.1.0`) deltas:**

1. `templates/values.yaml.tpl` — three changes:
   - `service.ports`: add `{ name: dd-agent, port: 8080, protocol: TCP }`.
   - `customConfig.sources`: add `dd_agent_in` block:
     ```yaml
     dd_agent_in:
       type: datadog_agent
       address: 0.0.0.0:8080
       multiple_outputs: true
       disable_metrics: true
       disable_traces: true
       store_api_key: false
       acknowledgements:
         enabled: true
     ```
     — `multiple_outputs: true` so the `.logs` named output exists. `disable_metrics`/`disable_traces` because DD Agent only forwards logs; metrics/traces still come via OTel. `store_api_key: false` since the Vector sink supplies the key from its own ExternalSecret. `acknowledgements: true` so DD Agent retries on transient Vector failures (no silent log loss).
   - `customConfig.transforms.drop_audit.inputs`: change from `["otlp_in.logs"]` to `["otlp_in.logs", "dd_agent_in.logs"]`.

2. `netpol.tf` — add `port = "8080"` to the existing `fromEntities = ["cluster"]` ingress rule's `toPorts.ports` list (alongside 4317/4318). DD Agent pods are cluster-internal so no new from-selector needed.

3. `outputs.tf` — add:
   ```hcl
   output "service_endpoint_dd_agent" {
     value = "http://vector.${var.namespace}.svc.cluster.local:8080"
   }
   ```

4. README — add a Datadog-Agent-intake row to the inputs/outputs tables and a one-paragraph description of the new source.

**DD Agent module (`v1.1.0`) deltas:**

1. `variables.tf` — add `variable "logs_intake_url"` (string, nullable, default `null`, description explaining "when set, redirects logs to a non-Datadog HTTP intake; for cluster-internal Vector use `module.vector.service_endpoint_dd_agent`").

2. `values.yaml.tftpl` — wrap a new block with the templatefile `%{ if logs_intake_url != null ~}` directive and append to the existing `datadog.logs` block:
   ```yaml
   datadog:
     logs:
       enabled: true
       containerCollectAll: true
       autoMultiLineDetection: true
   %{ if logs_intake_url != null ~}
     logs_config:
       use_http: true
       logs_no_ssl: true
       logs_dd_url: "${logs_intake_url}"
   %{ endif ~}
   ```
   Rendered template is unchanged when `logs_intake_url` is null (additive directive only).

3. `main.tf` — pass `logs_intake_url = var.logs_intake_url` into the templatefile call.

4. **No netpol change.** The existing `egress: toEntities = ["cluster"]` catch-all already permits cluster-internal egress to Vector. The `*.datadoghq.com:443` FQDN rule still applies for metrics/traces and remains the path for logs when `logs_intake_url = null`.

5. README — document the new variable.

**Consumer (`eks-cluster-applications`) deltas:**

1. New file (or addition to `datadog_agent.tf` if it exists): a `vector_dd_agent_logs_enabled` flag with cluster-type-map and override, mirroring the `vector_enabled` pattern.
2. Wire `logs_intake_url = local.vector_dd_agent_logs_enabled ? module.vector[0].service_endpoint_dd_agent : null` into the existing `module.datadog_agent` call.
3. Bump the `module.datadog_agent` source ref to `v1.1.0` and `module.vector` source ref to `v1.1.0`.
4. README — add a row to the inputs table + note the dual log-redaction layers (agent rules drop audits, Vector applies email redaction).

Does the component breakdown look right? Anything missing — particularly on the Vector source config (acknowledgements, store_api_key, disable_metrics/traces) or on the DD Agent template gating?


Goal: route DD Agent container logs through Vector to apply email redaction before they reach Datadog SaaS. We're mid-design, just presented Section 2 of 5 (Components). Next: get your sign-off on Section 2 so I can move to Data Flow. (disable recaps in /config)


Can we change the way this will be done, specifically we want to use the OPW configuration in datadog (https://docs.datadoghq.com/observability_pipelines/sources/datadog_agent/?tab=secretsmanagement#connect-the-datadog-agent-to-the-observability-pipelines-worker) through setting the env vars DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED and DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL to the local in-cluster vector service


Smart pivot — OPW is the officially supported path and it's literally Vector under the hood (same `datadog_agent` source wire protocol, just packaged as a Datadog product). Let me verify the env-var names and TLS expectations against the docs you linked before revising the design.


ToolSearch

Question: select:WebFetch


Question: select:WebFetch


WebFetch

Prompt: Extract the exact environment variables set on the Datadog Agent to redirect logs to an Observability Pipelines Worker (OPW). I need: (1) the exact env variable names — confirm they are DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED and DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL, (2) whether the URL must use https:// or if http:// is accepted for cluster-internal worker endpoints, (3) whether processing rules (DD_LOGS_CONFIG_PROCESSING_RULES) still execute when the agent is configured to send to the worker, (4) any other related env vars (e.g. for metrics or traces) that would be relevant, (5) the default port for the datadog_agent source on the worker side. Quote exact text from the docs.


Can we change the way this will be done, specifically we want to use the OPW configuration in datadog (https://docs.datadoghq.com/observability_pipelines/sources/datadog_agent/?tab=secretsmanagement#c…


# Datadog Agent to OPW Environment Variables

## 1. **Exact Environment Variable Names (Logs)**

Confirmed. The documentation states:

> "DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED" and "DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL"

## 2. **HTTP vs HTTPS for Cluster-Internal Endpoints**

The docs show `http://` is accepted:

> "http://<OPW_HOST>:8282" and "http://opw-observability-pipelines-worker.default.svc.cluster.local:<PORT>"

No requirement for HTTPS is mentioned for internal Kubernetes endpoints.

## 3. **Processing Rules Execution**

The documentation provided does **not address** whether `DD_LOGS_CONFIG_PROCESSING_RULES` executes when the Agent routes to the Worker. This information is absent from the source material.

## 4. **Other Related Environment Variables**

**For Metrics:**
- `DD_OBSERVABILITY_PIPELINES_WORKER_METRICS_ENABLED`
- `DD_OBSERVABILITY_PIPELINES_WORKER_METRICS_URL`

**For Log Filtering:**
- `DD_LOGS_CONFIG_EXPECTED_TAGS_DURATION` (ensures host-level tags are attached)

## 5. **Default Port for Datadog Agent Source**

The documentation specifies:
- Logs: port **8282**
- Metrics: port **8383** (> "you must use another port for metrics, such as 8383")


Verified. Three things change from the previous Section 2:

- **Port becomes 8282**, not 8080 — that's the OPW documented default. (Metrics OPW uses 8383, but we're not enabling that path; metrics keep their OTel route.)
- **DD Agent module writes two env vars**, not a `logs_config` block. Cleaner — env vars don't need any conditional YAML directive contortions, just appended to the existing `datadog.env` list.
- **`logs_no_ssl` / `use_http` flags go away entirely.** OPW URL scheme (`http://...`) implies plaintext; the agent reads it from the URL.

## Design — Section 2 of 5: Components (revised — OPW)

**Vector module (`v1.1.0`) deltas:**

1. `templates/values.yaml.tpl`:
   - `service.ports`: add `{ name: dd-agent, port: 8282, protocol: TCP }`.
   - `customConfig.sources.dd_agent_in`:
     ```yaml
     dd_agent_in:
       type: datadog_agent
       address: 0.0.0.0:8282
       multiple_outputs: true
       disable_metrics: true
       disable_traces: true
       store_api_key: false
       acknowledgements:
         enabled: true
     ```
   - `customConfig.transforms.drop_audit.inputs`: `["otlp_in.logs", "dd_agent_in.logs"]`.

2. `netpol.tf` — add `port = "8282"` to the existing `fromEntities = ["cluster"]` ingress rule's `toPorts.ports` list.

3. `outputs.tf`:
   ```hcl
   output "service_endpoint_dd_agent" {
     description = "OPW-compatible Datadog Agent intake endpoint (http, cluster-internal)"
     value       = "http://vector.${var.namespace}.svc.cluster.local:8282"
   }
   ```

4. README — note that Vector functions as the OPW logs intake on 8282; describe the `datadog_agent` source.

**DD Agent module (`v1.1.0`) deltas:**

1. `variables.tf` — add:
   ```hcl
   variable "logs_intake_url" {
     description = "OPW-compatible logs intake URL. When set, agent ships logs to this endpoint via DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL. For cluster-internal Vector use module.vector.service_endpoint_dd_agent. When null (default), logs ship direct to Datadog SaaS."
     type        = string
     default     = null
   }
   ```

2. `values.yaml.tftpl` — append to the existing `datadog.env` list, gated by `%{ if logs_intake_url != null ~}`:
   ```yaml
   %{ if logs_intake_url != null ~}
       - name: DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED
         value: "true"
       - name: DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL
         value: "${logs_intake_url}"
   %{ endif ~}
   ```
   When `logs_intake_url` is null, rendered output is byte-identical to today.

3. `main.tf` — pass `logs_intake_url = var.logs_intake_url` into the templatefile call.

4. **No netpol change.** Existing `egress: toEntities = ["cluster"]` catch-all already permits cluster-internal egress to Vector.

5. **Agent-side processing rules unchanged.** `DD_LOGS_CONFIG_PROCESSING_RULES` continues to drop audit lines at the agent before transmission — destination doesn't affect when those rules execute.

6. README — document `logs_intake_url`.

**Consumer (`eks-cluster-applications`) deltas:**

1. New `vector_dd_agent_logs_enabled` flag (cluster-type-map + override, mirrors `vector_enabled` pattern).
2. Wire `logs_intake_url = local.vector_dd_agent_logs_enabled ? module.vector[0].service_endpoint_dd_agent : null` on the existing `module.datadog_agent` call.
3. Bump `module.datadog_agent` ref to `v1.1.0` and `module.vector` ref to `v1.1.0`.
4. README — inputs row + note about dual-layer log filtering (agent rules drop audits, Vector applies email redaction).

Does the revised component list look right? Particularly the port flip to 8282 and the env-var approach in the DD Agent template?


this looks good


## Design — Section 3 of 5: Data Flow

**Logs path (per pod, per line):**

1. Container writes to stdout/stderr.
2. DD Agent's containerd integration tails the log line, attaches Kubernetes metadata (namespace, pod, container, service, ddtags).
3. Agent applies `DD_LOGS_CONFIG_PROCESSING_RULES` from `processing-rules/{env}/rules.json`. If the line matches an `exclude_at_match` audit pattern, **drop here** — it never leaves the node.
4. Agent batches surviving log lines and POSTs them to `DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL` (= `http://vector.vector.svc.cluster.local:8282`) using the OPW HTTP intake protocol. The `DD-API-KEY` header is sent but ignored (Vector's `store_api_key: false`).
5. Service routes to one of the 3 Vector pods (round-robin via ClusterIP).
6. Vector's `dd_agent_in` source decodes the request, emits events on the `dd_agent_in.logs` named output (with `multiple_outputs: true`).
7. `drop_audit` filter receives both `otlp_in.logs` and `dd_agent_in.logs`. Same regex check applied in Vector — defense-in-depth catches anything the agent rules would miss.
8. `redact_emails_logs` (when enabled) substitutes matching email patterns with `[REDACTED-EMAIL]` in `.message` and string-typed `.attributes` values.
9. `datadog_logs` sink batches and POSTs to `https://*.datadoghq.com:443` using the API key from the Vector ExternalSecret.
10. Vector ack flows back: sink → transforms → source → DD Agent (via `acknowledgements: enabled: true`). DD Agent retries on transient Vector failures.

**Traces path (unchanged):** apps → otel-collector-lb → otel-collector-processor → vector `otlp_in.traces` → `redact_emails_traces` → `datadog_traces` sink → DD SaaS.

**Metrics path (unchanged):** apps → otel-collector → vector `otlp_in.metrics` → `redact_emails_metrics` → `datadog_metrics` sink → DD SaaS. (Datadog APM-derived metrics from the `datadog/connector` in OTel still flow this way.)

**Cutover semantics:** when `vector_dd_agent_logs_enabled` flips from false to true on a cluster, terraform apply re-renders the DD Agent helm release with the two new env vars. The DaemonSet rolls (one pod per node, default RollingUpdate strategy). During the rolling window, some agent pods ship to DD SaaS direct, others ship to Vector. Both paths converge to the same DD Logs index; no data loss, possible brief duplication if a single line spans the rollover. Acceptable.

**Failure modes:**

- **Vector unhealthy / pod unreachable:** DD Agent retries via OPW protocol. If retries exhaust, agent buffers in its on-disk log buffer (existing behavior). Once Vector recovers, queued logs flush.
- **Vector backpressure (sink rejecting):** acknowledgements propagate back to DD Agent, which slows ingestion. No silent loss.
- **Email redact VRL panic on a malformed event:** Vector's `remap` transform drops the bad event by default — guard already in place from the existing implementation. Visible on `vector_component_errors_total{component_id="redact_emails_logs"}`.
- **Bad rollout / Vector misbehaving:** flip `vector_dd_agent_logs_enabled = false` for the affected cluster, terraform apply, agent re-rolls back to direct-DD shipping in ~minutes.

Does the data flow + failure mode handling look right? Anything you want me to push further on (e.g., the rolling-update brief-duplication acceptance, or the acknowledgement back-pressure semantics)?


this seems good


## Design — Section 4 of 5: Error Handling (terraform-side & diagnostics)

Runtime failure modes are covered in Section 3. This section is the design-time / terraform-side error surface.

**Module input validation:**

1. **DD Agent module — `logs_intake_url`:** validate that when set, the value matches `^https?://` (must be a URL, not bare host:port). Reject empty string. The full validation block:
   ```hcl
   validation {
     condition     = var.logs_intake_url == null || can(regex("^https?://", var.logs_intake_url))
     error_message = "logs_intake_url must be a URL starting with http:// or https://, or null."
   }
   ```

2. **Consumer — `vector_dd_agent_logs_enabled`:** mirror the existing `vector_validate` `data "external"` guard pattern. Add a precondition that fires at plan time: if `vector_dd_agent_logs_enabled = true` but `local.vector_enabled = false`, fail with "vector_dd_agent_logs_enabled requires vector_enabled". Ensures the DD Agent isn't pointed at a non-existent Vector service.

3. **Vector module — no new validation needed.** The `datadog_agent` source is harmless when no agent is pointed at it; it just listens.

**Misconfiguration paths and how they surface:**

- **Wrong port in URL** (e.g., 8080 instead of 8282): DD Agent connection refused → agent buffers + retries → visible on the agent's `datadog.agent.logs.bytes_sent` metric flatlining.
- **Vector source misconfigured** (e.g., `disable_metrics: false` accidentally — would emit unwanted metrics events to nowhere): Vector startup config validation catches at helm release time. Module's existing `data "external" "vector_validate"` guard pattern doesn't cover this — would surface as Vector pods erroring at startup; visible via `kubectl logs`.
- **Stale `module.vector` ref** when bumping consumer: terraform plan diffs the `service_endpoint_dd_agent` output (new in v1.1.0); if module isn't bumped, the output doesn't exist and the consumer plan fails fast with "Unsupported attribute".

**Diagnostic surfaces (already-existing, no new infra needed):**

- `vector_component_received_events_total{component_id="dd_agent_in"}` — sanity check ingest is happening.
- `vector_component_sent_events_total{component_id="datadog_logs"}` — sanity check sink is shipping.
- `vector_component_errors_total{component_id=~"dd_agent_in|drop_audit|redact_emails_logs|datadog_logs"}` — error rate per stage.
- DD Agent's own log telemetry: `datadog.agent.logs.bytes_sent`, `datadog.agent.logs.errors_count`.

**Rollback procedure:**

Single-knob: set `vector_dd_agent_logs_enabled = false` for the cluster, `terraform apply`. DD Agent helm release re-renders without the OPW env vars; DaemonSet rolls; agents resume direct shipping to DD SaaS. ~3-5 minute rollback per cluster.

Does the error handling section look right? Want me to add anything on the validation or diagnostics surface?


## Design — Section 5 of 5: Testing

**Unit-level (per module, in-repo):**

1. **Vector module — VRL tests** (existing harness): the 4 existing VRL tests cover the OTLP-in path. Add a 5th test `tests/vrl/redact_emails_dd_agent_logs/` with input shape matching what the `datadog_agent` source emits (top-level `.message` string with embedded email + `ddtags`/`service`/`hostname` Datadog Agent metadata), expected output redacted. Same VRL program as the existing logs test (the transform doesn't care which source produced the event).

2. **Vector module — `terraform fmt` / `validate`** at module root and `examples/basic/`. Already part of the existing CI; the new source/port/transform-input change must round-trip cleanly.

3. **DD Agent module — template render check.** Add a quick render verification in the module's existing test harness (or via ad-hoc `terraform apply -auto-approve` against a tempdir, mirroring the Vector module's pattern):
   - Render with `logs_intake_url = null` → assert output is byte-identical to the pre-change render (no churn for non-opted-in clusters).
   - Render with `logs_intake_url = "http://vector.vector.svc.cluster.local:8282"` → assert the two OPW env vars appear in the `datadog.env` block.

4. **Consumer — `terraform fmt` / `validate`.** Standard.

**Integration verification (in-cluster, ad-hoc, after deploy):**

The smoke test from the prior Vector deploy session adapted for the DD Agent path:

1. Deploy a one-shot test pod that emits a UUID-tagged log line containing an email:
   ```bash
   kubectl run dd-redact-test-$RANDOM \
     --image=busybox --restart=Never --rm -it \
     -- sh -c 'echo "dd-e2e-$(cat /proc/sys/kernel/random/uuid): contact alice@example.com"; sleep 5'
   ```

2. Watch Vector's `dd_agent_in` ingest counter:
   ```bash
   kubectl exec -n vector deploy/vector -- curl -s http://localhost:8686/metrics | grep 'vector_component_received_events_total{component_id="dd_agent_in"'
   ```
   Should tick up by ≥1.

3. Query Datadog for the UUID — expect the log to appear with `[REDACTED-EMAIL]` substituted.

4. **Audit-drop verification**: emit an `Audit - test-<UUID>` line. Expect zero results in DD (dropped at the agent before it leaves the node — agent rules still authoritative). Vector's `dd_agent_in` received counter should also NOT increment for this line, because it's filtered before transmission.

5. **Rollback verification**: flip `vector_dd_agent_logs_enabled = false`, `terraform apply`, repeat step 1, expect log to appear in DD un-redacted (proving the agent went back to direct shipping). Then flip back.

**Soak (per spec §7.4 of the prior Vector spec, applied here):**

- One sandbox/dev cluster: 1 day, full smoke suite + Vector self-metrics watch.
- One `mgmt` cluster: 3-7 day soak. Watch `vector_component_errors_total{component_id="dd_agent_in"}`, agent buffer depth (`datadog.agent.logs.bytes_buffered`), DD Logs ingest delta vs. baseline.
- Remaining `mgmt` + `cps` clusters.
- `saas` clusters region-by-region.

**Cross-cutting checks:**

- **DD Logs UI metadata preservation.** After cutover, verify `service`, `host`, `kubernetes.namespace`, `kubernetes.pod_name`, `ddtags` are still attached to logs in DD UI. The `datadog_agent` source preserves them; the `datadog_logs` sink forwards them; redaction transforms only touch `.message` and string `.attributes` so should be safe — but verify via a known-shape log line.
- **Redaction does not corrupt JSON-structured logs.** Many container logs are JSON; the email regex runs against the string `.message`. Confirm a JSON line like `{"user":"alice@example.com","level":"info"}` becomes `{"user":"[REDACTED-EMAIL]","level":"info"}` and the JSON remains parseable in DD.

Does the testing plan look right? Anything to add — particularly on the soak duration or the JSON-shape check?


All five sections approved. Writing the spec doc now.


Updated task #5 status


Bash

ls /Users/steve.briggs/projects/IAC/docs/superpowers/specs/ 2>&1 | head -20

Verify specs directory exists


.rw-r--r-- steve.briggs staff  28 KB Tue May  5 12:11:52 2026 2026-05-05-vector-redaction-design.md
drwxr-xr-x steve.briggs staff  96 B  Tue May  5 12:11:52 2026 .
drwxr-xr-x steve.briggs staff 128 B  Tue May  5 12:21:22 2026 ..


Write

# Datadog Agent Logs via Vector (OPW Intake) — Design

**Date:** 2026-05-06
**Author:** Sir Charles of St Cheese (with Claudebob)
**Status:** Draft
**Depends on:** [2026-05-05 Vector + OpenTelemetry Redaction Pipeline](./2026-05-05-vector-redaction-design.md)

## 1. Goal

Apply Vector's existing email-redaction transform to the container logs the Datadog Agent collects today. Logs currently bypass Vector entirely (DD Agent ships direct to `datadoghq.com:443`); after this change, DD-Agent-collected logs traverse Vector before reaching Datadog SaaS.

**Out of scope:**
- Migrating the agent-side audit-drop processing rules (`processing-rules/{env}/rules.json`) into Vector. The agent rules continue to drop audit lines before transmission; Vector adds a defense-in-depth `drop_audit` filter on the new path but does not replace the agent rules.
- Routing DD Agent metrics or APM through Vector. Metrics and traces keep their existing OTel-based path.
- Any change to the OTel pipeline.

## 2. Approach

Use the **Datadog Observability Pipelines Worker (OPW)** intake protocol. Vector's native `datadog_agent` source is the same wire protocol Datadog's OPW product uses (Datadog acquired Vector; OPW is Vector with a curated config). Configuring the agent to ship to Vector is the standard, documented OPW deployment shape.

Two environment variables on the DD Agent — `DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED=true` and `DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL=<vector-url>` — redirect the agent's logs intake to Vector. No `logs_config.use_http` / `logs_no_ssl` / `logs_dd_url` block needed; the URL scheme implies plaintext for `http://` cluster-internal endpoints.

Reference: <https://docs.datadoghq.com/observability_pipelines/sources/datadog_agent/?tab=secretsmanagement>

## 3. Architecture

### 3.1 Pipeline (logs path)

**Before:**
```
DD Agent (DaemonSet)
  → [agent-side processing rules drop audits]
  → datadoghq.com:443
```

**After:**
```
DD Agent (DaemonSet)
  → [agent-side processing rules drop audits]
  → vector.vector.svc.cluster.local:8282  (OPW HTTP intake, no TLS)
  → Vector dd_agent_in source
  → drop_audit  (multi-input: otlp_in.logs + dd_agent_in.logs — defense-in-depth)
  → redact_emails_logs  (when enabled)
  → datadog_logs sink → datadoghq.com:443
```

Traces and metrics are unaffected — they keep their existing path (apps → otel-collector → vector → DD).

### 3.2 Module split

Three repos touched, additive only:

1. **`internal-modules/kubernetes/monitoring/vector`** — bumps to `v1.1.0`. Adds the `datadog_agent` source on port 8282, exposes the port in helm `service.ports`, opens netpol ingress, makes `drop_audit` accept the new input, and adds an output `service_endpoint_dd_agent`. The source is always declared (the receiver is harmless if no agent is pointed at it), so there is no new module variable.
2. **`internal-modules/kubernetes/monitoring/datadog-agent`** — bumps to `v1.1.0`. Adds one variable `logs_intake_url` (string, default `null`). When non-null, the values template renders the two OPW env vars in the `datadog.env` block. When null, rendered output is byte-identical to today.
3. **`eks-cluster-applications`** consumer — adds `vector_dd_agent_logs_enabled` flag (cluster-type-map + override, mirrors the existing `vector_enabled` pattern). Wires the URL across modules.

## 4. Components

### 4.1 Vector module deltas

**`templates/values.yaml.tpl`:**

- `service.ports`: add `{ name: dd-agent, port: 8282, protocol: TCP }`.
- `customConfig.sources.dd_agent_in`:
  ```yaml
  dd_agent_in:
    type: datadog_agent
    address: 0.0.0.0:8282
    multiple_outputs: true
    disable_metrics: true
    disable_traces: true
    store_api_key: false
    acknowledgements:
      enabled: true
  ```
  - `multiple_outputs: true` — exposes the `.logs` named output for downstream transforms.
  - `disable_metrics`/`disable_traces: true` — DD Agent only forwards logs through this path; metrics/traces still come via OTel.
  - `store_api_key: false` — the Vector sink supplies its own API key from the existing ExternalSecret; storing the per-event key is unnecessary.
  - `acknowledgements.enabled: true` — DD Agent retries on transient Vector failures, no silent log loss.
- `customConfig.transforms.drop_audit.inputs`: change `["otlp_in.logs"]` → `["otlp_in.logs", "dd_agent_in.logs"]`.

**`netpol.tf`:** add `port = "8282"` to the existing `fromEntities = ["cluster"]` ingress rule's `toPorts.ports` list (alongside 4317/4318). DD Agent pods are cluster-internal, no new from-selector needed.

**`outputs.tf`:**
```hcl
output "service_endpoint_dd_agent" {
  description = "OPW-compatible Datadog Agent intake endpoint (http, cluster-internal)"
  value       = "http://vector.${var.namespace}.svc.cluster.local:8282"
}
```

**README:** note that Vector functions as an OPW logs intake on 8282; describe the `datadog_agent` source.

### 4.2 DD Agent module deltas

**`variables.tf`:**
```hcl
variable "logs_intake_url" {
  description = "OPW-compatible logs intake URL. When set, the agent ships logs to this endpoint via DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL. For cluster-internal Vector use module.vector.service_endpoint_dd_agent. When null (default), logs ship direct to Datadog SaaS."
  type        = string
  default     = null

  validation {
    condition     = var.logs_intake_url == null || can(regex("^https?://", var.logs_intake_url))
    error_message = "logs_intake_url must be a URL starting with http:// or https://, or null."
  }
}
```

**`values.yaml.tftpl`:** append to the existing `datadog.env` list, gated by `%{ if logs_intake_url != null ~}`:
```yaml
%{ if logs_intake_url != null ~}
    - name: DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED
      value: "true"
    - name: DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL
      value: "${logs_intake_url}"
%{ endif ~}
```

When `logs_intake_url` is null, rendered output is byte-identical to today.

**`main.tf`:** pass `logs_intake_url = var.logs_intake_url` into the `templatefile()` call's variables map.

**No netpol change.** The existing `egress: toEntities = ["cluster"]` catch-all already permits cluster-internal egress to Vector. The `*.datadoghq.com:443` FQDN rule still applies for metrics/traces and remains the path for logs when `logs_intake_url = null`.

**Agent-side processing rules unchanged.** `DD_LOGS_CONFIG_PROCESSING_RULES` continues to drop audit lines before transmission, regardless of destination.

**README:** document `logs_intake_url`.

### 4.3 Consumer (`eks-cluster-applications`) deltas

- New `vector_dd_agent_logs_enabled` flag (cluster-type-map + override, mirrors `vector_enabled`'s pattern). Default-true on cluster types where Vector is enabled (`mgmt`, `cps`, `saas`); default-false elsewhere; explicit override variable.
- Local-precondition guard: if `vector_dd_agent_logs_enabled = true` but `local.vector_enabled = false`, fail at plan time. Mirrors the existing `vector_validate` `data "external"` pattern.
- Wire on the existing `module.datadog_agent` call:
  ```hcl
  logs_intake_url = local.vector_dd_agent_logs_enabled ? module.vector[0].service_endpoint_dd_agent : null
  ```
- Bump `module.datadog_agent` source ref to `v1.1.0` and `module.vector` source ref to `v1.1.0`.
- README — inputs row + note about dual-layer log filtering (agent rules drop audits, Vector applies email redaction).

## 5. Data Flow

Per pod, per log line, after cutover:

1. Container writes to stdout/stderr.
2. DD Agent's containerd integration tails the line, attaches Kubernetes metadata (namespace, pod, container, service, ddtags).
3. Agent applies `DD_LOGS_CONFIG_PROCESSING_RULES`. If the line matches an `exclude_at_match` audit pattern, **drop here** — never leaves the node.
4. Agent batches surviving lines and POSTs them to `DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL` (= `http://vector.vector.svc.cluster.local:8282`) using the OPW HTTP intake protocol. The `DD-API-KEY` header is sent but ignored by Vector (`store_api_key: false`).
5. Service routes to one of the 3 Vector pods (round-robin via ClusterIP).
6. Vector's `dd_agent_in` source decodes the request, emits events on the `dd_agent_in.logs` named output.
7. `drop_audit` filter receives both `otlp_in.logs` and `dd_agent_in.logs`. Same regex check applied — defense-in-depth catches anything the agent rules would miss.
8. `redact_emails_logs` (when enabled) substitutes matching email patterns with `[REDACTED-EMAIL]` in `.message` and string-typed `.attributes` values.
9. `datadog_logs` sink batches and POSTs to `https://*.datadoghq.com:443` using the API key from the Vector ExternalSecret.
10. Acknowledgement flows back: sink → transforms → source → DD Agent. Agent retries on transient Vector failures.

**Cutover semantics:** when `vector_dd_agent_logs_enabled` flips false→true, terraform apply re-renders the DD Agent helm release with the two new env vars. The DaemonSet rolls (default RollingUpdate). During the rolling window, some agent pods ship to DD SaaS direct, others ship to Vector. Both paths converge to the same DD Logs index; no data loss, possible brief duplication if a single line spans the rollover. Acceptable.

## 6. Error Handling

### 6.1 Module input validation

- **DD Agent module — `logs_intake_url`:** validation regex `^https?://`, rejects empty string. Null is allowed (means: ship direct).
- **Consumer — `vector_dd_agent_logs_enabled`:** plan-time guard. If true but `local.vector_enabled = false`, fail with "vector_dd_agent_logs_enabled requires vector_enabled". Ensures the agent isn't pointed at a non-existent service.
- **Vector module — no new validation.** The `datadog_agent` source is harmless when no agent connects.

### 6.2 Misconfiguration paths

- Wrong port in URL (e.g., 8080 instead of 8282): DD Agent connection refused → agent buffers + retries → visible on `datadog.agent.logs.bytes_sent` flatlining and `datadog.agent.logs.errors_count` increasing.
- Vector source misconfigured (e.g., accidental `disable_metrics: false`): Vector startup config validation rejects at helm release time; pods crash-loop visibly via `kubectl logs`.
- Stale `module.vector` ref when bumping consumer: terraform plan diffs the new `service_endpoint_dd_agent` output; if module isn't bumped, the output doesn't exist and the consumer plan fails fast with "Unsupported attribute".

### 6.3 Runtime failure modes

- **Vector unhealthy / pod unreachable:** DD Agent retries via OPW protocol. If retries exhaust, agent buffers in its on-disk log buffer (existing behavior). Once Vector recovers, queued logs flush.
- **Vector backpressure (sink rejecting):** acknowledgements propagate back to DD Agent, slowing ingestion. No silent loss.
- **Email redact VRL panic on a malformed event:** Vector's `remap` transform drops the bad event by default. Visible on `vector_component_errors_total{component_id="redact_emails_logs"}`.
- **Bad rollout / Vector misbehaving:** flip `vector_dd_agent_logs_enabled = false` for the affected cluster, terraform apply, agent re-rolls back to direct-DD shipping in ~3-5 minutes.

### 6.4 Diagnostic surfaces

Use existing infrastructure:

- `vector_component_received_events_total{component_id="dd_agent_in"}` — ingest sanity.
- `vector_component_sent_events_total{component_id="datadog_logs"}` — sink shipping sanity.
- `vector_component_errors_total{component_id=~"dd_agent_in|drop_audit|redact_emails_logs|datadog_logs"}` — error rate per stage.
- DD Agent telemetry: `datadog.agent.logs.bytes_sent`, `datadog.agent.logs.errors_count`, `datadog.agent.logs.bytes_buffered`.

## 7. Testing

### 7.1 Unit-level (per module, in-repo)

1. **Vector module — VRL tests.** Add a 5th VRL test `tests/vrl/redact_emails_dd_agent_logs/` with input shape matching `datadog_agent` source events (top-level `.message` string with embedded email + DD agent metadata: `ddtags`, `service`, `hostname`). Same VRL program as the existing logs test. Confirms the redaction transform is source-agnostic.
2. **Vector module — `terraform fmt` / `validate`** at module root and `examples/basic/`. Standard.
3. **DD Agent module — template render check.** Two assertions via ad-hoc render:
   - `logs_intake_url = null` → output byte-identical to pre-change render (no churn).
   - `logs_intake_url = "http://vector.vector.svc.cluster.local:8282"` → both OPW env vars appear in `datadog.env`.
4. **Consumer — `terraform fmt` / `validate`.** Standard.

### 7.2 Integration verification (in-cluster, ad-hoc, after deploy)

1. Deploy a one-shot test pod that emits a UUID-tagged log line containing an email:
   ```bash
   kubectl run dd-redact-test-$RANDOM \
     --image=busybox --restart=Never --rm -it \
     -- sh -c 'echo "dd-e2e-$(cat /proc/sys/kernel/random/uuid): contact alice@example.com"; sleep 5'
   ```
2. Watch Vector's `dd_agent_in` ingest counter:
   ```bash
   kubectl exec -n vector deploy/vector -- curl -s http://localhost:8686/metrics \
     | grep 'vector_component_received_events_total{component_id="dd_agent_in"'
   ```
   Should tick up by ≥1.
3. Query Datadog for the UUID — expect the log to appear with `[REDACTED-EMAIL]` substituted for the address.
4. **Audit-drop verification:** emit an `Audit - test-<UUID>` line. Expect zero results in DD (dropped at the agent before transmission). Vector's `dd_agent_in` received counter should also NOT increment for this line.
5. **Rollback verification:** flip `vector_dd_agent_logs_enabled = false`, `terraform apply`, repeat step 1, expect log to appear in DD un-redacted (proving the agent went back to direct shipping). Then flip back.

### 7.3 Cross-cutting checks

- **DD Logs UI metadata preservation.** After cutover, verify `service`, `host`, `kubernetes.namespace`, `kubernetes.pod_name`, `ddtags` are still attached to logs in DD UI. The `datadog_agent` source preserves them; the `datadog_logs` sink forwards them; redaction transforms only touch `.message` and string `.attributes`. Verify via a known-shape log line.
- **Redaction does not corrupt JSON-structured logs.** A JSON line like `{"user":"alice@example.com","level":"info"}` should become `{"user":"[REDACTED-EMAIL]","level":"info"}` and remain parseable in DD.

### 7.4 Soak

- One sandbox/dev cluster: 1 day, full smoke suite + Vector self-metrics watch.
- One `mgmt` cluster: 3-7 day soak. Watch `vector_component_errors_total{component_id="dd_agent_in"}`, agent buffer depth (`datadog.agent.logs.bytes_buffered`), DD Logs ingest delta vs. baseline.
- Remaining `mgmt` + all `cps` clusters.
- `saas` clusters region-by-region.

## 8. Open Questions

1. **Default for `vector_dd_agent_logs_enabled` per cluster type.** Current proposal mirrors `vector_enabled`: default-true on `mgmt`/`cps`/`saas`, default-false on `infra`/`dev`. Could decouple (e.g., default-false everywhere, opt-in cluster-by-cluster during initial rollout). Calling this out for confirmation during plan-writing.
2. **Agent buffer behavior under sustained Vector outage.** DD Agent's on-disk log buffer is bounded; if Vector is down for hours, oldest buffered logs may drop. Worth confirming the buffer cap and whether existing alerting catches this case (likely already covered by `datadog.agent.logs.bytes_buffered` rate alerts in the spec §6.3 monitor table).
3. **Audit-drop rule consolidation.** The agent-side `processing-rules/{env}/rules.json` and Vector's `drop_audit` regex now both exist. Drift between them is silent. Long-term, may want to make one authoritative — but explicitly out of scope for this design.

## 9. Next Steps (after design approval)

1. Hand off to `writing-plans` skill to produce the implementation plan.
2. Plan should sequence: Vector module change → DD Agent module change → consumer wire-up → ad-hoc deploy verification.
3. Sir Charles handles git operations (tag `vector v1.1.0`, `datadog-agent v1.1.0`, MRs for all three repos) per established working pattern.