1	# ---------------------------------------------------------------------------------------------------------------------
2	# VECTOR
3	# ---------------------------------------------------------------------------------------------------------------------
4	locals {
5	  vector_enabled = (
6	    var.vector_enabled != null
7	    ? var.vector_enabled
8	    : lookup(var.vector_enabled_cluster_types, local.cluster_type, true)
9	  )
10	}
11
12	module "vector" {
13	  #source = "crow.infrastructure.immuta.io/internal/monitoring-vector/kubernetes"
14	  #version = "1.2.1"
15	  source = "/Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector"
16
17	  count  = local.vector_enabled ? 1 : 0
18
19	  providers = {
20	    kubernetes = kubernetes.local-cluster
21	    helm       = helm.local-cluster
22	  }
23
24	  chart_version               = var.vector_chart_version
25	  image                       = var.vector_image
26	  image_tag_version           = var.vector_image_tag
27	  replicas                    = var.vector_replicas
28	  resources_requests          = var.vector_resource_requests
29	  resources_limits            = var.vector_resource_limits
30	  datadog_site                = var.vector_datadog_site
31	  datadog_api_key_vault_path  = var.vector_datadog_api_key_vault_path
32	  audit_drop_patterns         = var.vector_audit_drop_patterns
33	  email_redaction_enabled     = var.vector_email_redaction_enabled
34	  email_redaction_replacement = var.vector_email_redaction_replacement
35	  email_redaction_dry_run     = var.vector_email_redaction_dry_run
36
37	  bearer_token_redaction_enabled            = var.vector_bearer_token_redaction_enabled
38	  bearer_token_redaction_dry_run            = var.vector_bearer_token_redaction_dry_run
39	  bearer_token_redaction_replacement        = var.vector_bearer_token_redaction_replacement
40	  bearer_token_redaction_services           = var.vector_bearer_token_redaction_services
41	  bearer_token_redaction_exclusion_patterns = var.vector_bearer_token_redaction_exclusion_patterns
42
43	  aws_access_key_redaction_enabled              = var.vector_aws_access_key_redaction_enabled
44	  aws_access_key_redaction_dry_run              = var.vector_aws_access_key_redaction_dry_run
45	  aws_access_key_redaction_replacement          = var.vector_aws_access_key_redaction_replacement
46	  aws_access_key_redaction_services             = var.vector_aws_access_key_redaction_services
47	  aws_access_key_redaction_exclusion_patterns   = var.vector_aws_access_key_redaction_exclusion_patterns
48	  aws_access_key_redaction_exclusion_attributes = var.vector_aws_access_key_redaction_exclusion_attributes
49
50	  jwt_redaction_enabled            = var.vector_jwt_redaction_enabled
51	  jwt_redaction_dry_run            = var.vector_jwt_redaction_dry_run
52	  jwt_redaction_replacement        = var.vector_jwt_redaction_replacement
53	  jwt_redaction_services           = var.vector_jwt_redaction_services
54	  jwt_redaction_exclusion_patterns = var.vector_jwt_redaction_exclusion_patterns
55
56	  additional_config = var.vector_additional_config
57
58	  external_secrets_cluster_store_name = var.vector_external_secrets_cluster_store_name
59
60	  depends_on = [
61	    module.cilium,
62	    module.external_secrets,
63	  ]
64	}
65
66	resource "terraform_data" "vector_validate" {
67	  count = local.vector_enabled ? 1 : 0
68
69	  lifecycle {
70	    precondition {
71	      condition     = local.external_secrets_enabled
72	      error_message = "vector can only be enabled if external-secrets is also enabled"
73	    }
74	  }
75	}
76
77	# ---------------------------------------------------------------------------------------------------------------------
78	# REQUIRED PARAMETERS
79	# These variables must be set.
80	# ---------------------------------------------------------------------------------------------------------------------
81	variable "vector_chart_version" {
82	  description = "Version of vector helm chart"
83	  type        = string
84	  default     = ""
85	  validation {
86	    condition     = var.vector_chart_version == "" && local.vector_enabled ? false : true
87	    error_message = "vector_chart_version must be set when vector is enabled"
88	  }
89	}
90
91	variable "vector_image_tag" {
92	  description = "Vector image tag"
93	  type        = string
94	  default     = ""
95	  validation {
96	    condition     = var.vector_image_tag == "" && local.vector_enabled ? false : true
97	    error_message = "vector_image_tag must be set when vector is enabled"
98	  }
99	}
100
101	variable "vector_datadog_site" {
102	  description = "Datadog site for vector sinks (e.g. datadoghq.com)"
103	  type        = string
104	  default     = "datadoghq.com"
105	  validation {
106	    condition     = var.vector_datadog_site == "" && local.vector_enabled ? false : true
107	    error_message = "vector_datadog_site must be set when vector is enabled"
108	  }
109	}
110
111	variable "vector_datadog_api_key_vault_path" {
112	  description = "Vault path for the Datadog API key (consumed by ExternalSecret in vector ns)"
113	  type        = string
114	  default     = "secret/data/saas/infrastructure/datadog-api-key"
115	  validation {
116	    condition     = var.vector_datadog_api_key_vault_path == "" && local.vector_enabled ? false : true
117	    error_message = "vector_datadog_api_key_vault_path must be set when vector is enabled"
118	  }
119	}
120
121	variable "vector_external_secrets_cluster_store_name" {
122	  description = "Name of the ClusterSecretStore to use for vector ExternalSecret resources. Pass the name created by the external-secrets module (typically 'vault-cluster-store'). Required when vector is enabled."
123	  type        = string
124	  default     = "vault"
125	  validation {
126	    condition     = var.vector_external_secrets_cluster_store_name == "" && local.vector_enabled ? false : true
127	    error_message = "vector_external_secrets_cluster_store_name must be set when vector is enabled"
128	  }
129	}
130
131	# ---------------------------------------------------------------------------------------------------------------------
132	# OPTIONAL PARAMETERS
133	# These variables are configurable, but have reasonable defaults set.
134	# ---------------------------------------------------------------------------------------------------------------------
135	variable "vector_enabled_cluster_types" {
136	  description = "Types of clusters Vector should be enabled on by default"
137	  type        = map(string)
138	  default = {
139	    "mgmt"  = true
140	    "cps"   = true
141	    "saas"  = true
142	    "infra" = false
143	    "dev"   = false
144	  }
145	}
146
147	variable "vector_enabled" {
148	  description = "Override for explicitly enabling or disabling Vector"
149	  type        = bool
150	  default     = null
151	}
152
153	variable "vector_image" {
154	  description = "Image for Vector"
155	  type        = string
156	  default     = "registry.ic-ops.net/cache-docker-hub/timberio/vector"
157	}
158
159	variable "vector_replicas" {
160	  description = "Number of Vector aggregator replicas"
161	  type        = number
162	  default     = 3
163	}
164
165	variable "vector_audit_drop_patterns" {
166	  description = "Regex patterns; matching log message bodies are dropped"
167	  type        = list(string)
168	  default = [
169	    "Audit\\s-\\s.+",
170	    "\"level\":\"audit\"",
171	  ]
172	}
173
174	variable "vector_email_redaction_enabled" {
175	  description = "Toggle the email-redact VRL transform across all signals"
176	  type        = bool
177	  default     = true
178	}
179
180	variable "vector_email_redaction_replacement" {
181	  description = "String that replaces matched email addresses"
182	  type        = string
183	  default     = "[REDACTED-EMAIL]"
184	}
185
186	variable "vector_email_redaction_dry_run" {
187	  description = "When true, tag logs that would have been redacted instead of replacing. Takes priority over vector_email_redaction_enabled."
188	  type        = bool
189	  default     = false
190	}
191
192	variable "vector_bearer_token_redaction_enabled" {
193	  description = "Toggle bearer token redact VRL transform on logs"
194	  type        = bool
195	  default     = false
196	}
197
198	variable "vector_bearer_token_redaction_dry_run" {
199	  description = "When true, tag logs that would have been redacted with vector_bearer_token_detected:true instead of replacing. Takes priority over vector_bearer_token_redaction_enabled."
200	  type        = bool
201	  default     = false
202	}
203
204	variable "vector_bearer_token_redaction_replacement" {
205	  description = "String that replaces matched bearer tokens"
206	  type        = string
207	  default     = "Bearer [REDACTED-BEARER-TOKEN]"
208	}
209
210	variable "vector_bearer_token_redaction_services" {
211	  description = "Service names to apply bearer token redaction to"
212	  type        = list(string)
213	  default     = ["service", "secure-metadata-database"]
214	}
215
216	variable "vector_bearer_token_redaction_exclusion_patterns" {
217	  description = "Skip bearer token redaction for any log whose message or ddtags contains one of these strings"
218	  type        = list(string)
219	  default = [
220	    "internal-webhook",
221	    "starting crawl",
222	    "Databricks automatic subscription",
223	    "audit/enrich",
224	    "Background Job",
225	    "community-organization-id:immuta",
226	    "pg - pool.connect",
227	  ]
228	}
229
230	variable "vector_aws_access_key_redaction_enabled" {
231	  description = "Toggle AWS Access Key ID redact VRL transform on logs"
232	  type        = bool
233	  default     = false
234	}
235
236	variable "vector_aws_access_key_redaction_dry_run" {
237	  description = "When true, tag logs that would have been redacted with vector_aws_key_detected:true instead of replacing. Takes priority over vector_aws_access_key_redaction_enabled."
238	  type        = bool
239	  default     = false
240	}
241
242	variable "vector_aws_access_key_redaction_replacement" {
243	  description = "String that replaces matched AWS Access Key IDs"
244	  type        = string
245	  default     = "[REDACTED-AWS-ACCESS-KEY]"
246	}
247
248	variable "vector_aws_access_key_redaction_services" {
249	  description = "Service names to apply AWS Access Key ID redaction to"
250	  type        = list(string)
251	  default     = ["marketplace-api"]
252	}
253
254	variable "vector_aws_access_key_redaction_exclusion_patterns" {
255	  description = "Skip AWS Access Key ID redaction for any log whose message or ddtags contains one of these strings"
256	  type        = list(string)
257	  default     = []
258	}
259
260	variable "vector_aws_access_key_redaction_exclusion_attributes" {
261	  description = "Skip AWS Access Key ID redaction for any log where the named attribute equals the given value (keys are attribute paths, values are string values to match)"
262	  type        = map(string)
263	  default = {
264	    "req.headers.x-synthetic-test" = "true"
265	  }
266	}
267
268	variable "vector_jwt_redaction_enabled" {
269	  description = "Toggle JSON Web Token redact VRL transform on logs"
270	  type        = bool
271	  default     = false
272	}
273
274	variable "vector_jwt_redaction_dry_run" {
275	  description = "When true, tag logs that would have been redacted with vector_jwt_detected:true instead of replacing. Takes priority over vector_jwt_redaction_enabled."
276	  type        = bool
277	  default     = false
278	}
279
280	variable "vector_jwt_redaction_replacement" {
281	  description = "String that replaces matched JSON Web Tokens"
282	  type        = string
283	  default     = "[REDACTED-JWT]"
284	}
285
286	variable "vector_jwt_redaction_services" {
287	  description = "Service names to apply JWT redaction to"
288	  type        = list(string)
289	  default     = ["service", "secure-metadata-database"]
290	}
291
292	variable "vector_jwt_redaction_exclusion_patterns" {
293	  description = "Skip JWT redaction for any log whose message or ddtags contains one of these strings"
294	  type        = list(string)
295	  default = [
296	    "internal-webhook",
297	    "starting crawl",
298	    "Databricks automatic subscription",
299	    "audit/enrich",
300	    "Background Job",
301	    "community-organization-id:immuta",
302	    "pg - pool.connect",
303	  ]
304	}
305
306	variable "vector_resource_requests" {
307	  description = "Resource resource requests"
308	  type = map(string)
309	  default = {
310	    cpu = "200m"
311	    memory = "256Mi"
312	  }
313	}
314
315	variable "vector_resource_limits" {
316	  description = "Resource limits for vector"
317	  type = map(string)
318	  default = {
319	    cpu = "1"
320	    memory = "2Gi"
321	  }
322	}
323
324	variable "vector_additional_config" {
325	  description = "Alternate helm chart customConfig (overwrites entire helm config)"
326	  type        = any
327	  default     = null
328	}
329

# ---------------------------------------------------------------------------------------------------------------------
# VECTOR
# ---------------------------------------------------------------------------------------------------------------------
locals {
  vector_enabled = (
    var.vector_enabled != null
    ? var.vector_enabled
    : lookup(var.vector_enabled_cluster_types, local.cluster_type, true)
  )
}

module "vector" {
  #source = "crow.infrastructure.immuta.io/internal/monitoring-vector/kubernetes"
  #version = "1.2.1"
  source = "/Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector"

  count  = local.vector_enabled ? 1 : 0

  providers = {
    kubernetes = kubernetes.local-cluster
    helm       = helm.local-cluster
  }

  chart_version               = var.vector_chart_version
  image                       = var.vector_image
  image_tag_version           = var.vector_image_tag
  replicas                    = var.vector_replicas
  resources_requests          = var.vector_resource_requests
  resources_limits            = var.vector_resource_limits
  datadog_site                = var.vector_datadog_site
  datadog_api_key_vault_path  = var.vector_datadog_api_key_vault_path
  audit_drop_patterns         = var.vector_audit_drop_patterns
  email_redaction_enabled     = var.vector_email_redaction_enabled
  email_redaction_replacement = var.vector_email_redaction_replacement
  email_redaction_dry_run     = var.vector_email_redaction_dry_run

  bearer_token_redaction_enabled            = var.vector_bearer_token_redaction_enabled
  bearer_token_redaction_dry_run            = var.vector_bearer_token_redaction_dry_run
  bearer_token_redaction_replacement        = var.vector_bearer_token_redaction_replacement
  bearer_token_redaction_services           = var.vector_bearer_token_redaction_services
  bearer_token_redaction_exclusion_patterns = var.vector_bearer_token_redaction_exclusion_patterns

  aws_access_key_redaction_enabled              = var.vector_aws_access_key_redaction_enabled
  aws_access_key_redaction_dry_run              = var.vector_aws_access_key_redaction_dry_run
  aws_access_key_redaction_replacement          = var.vector_aws_access_key_redaction_replacement
  aws_access_key_redaction_services             = var.vector_aws_access_key_redaction_services
  aws_access_key_redaction_exclusion_patterns   = var.vector_aws_access_key_redaction_exclusion_patterns
  aws_access_key_redaction_exclusion_attributes = var.vector_aws_access_key_redaction_exclusion_attributes

  jwt_redaction_enabled            = var.vector_jwt_redaction_enabled
  jwt_redaction_dry_run            = var.vector_jwt_redaction_dry_run
  jwt_redaction_replacement        = var.vector_jwt_redaction_replacement
  jwt_redaction_services           = var.vector_jwt_redaction_services
  jwt_redaction_exclusion_patterns = var.vector_jwt_redaction_exclusion_patterns

  additional_config = var.vector_additional_config

  external_secrets_cluster_store_name = var.vector_external_secrets_cluster_store_name

  depends_on = [
    module.cilium,
    module.external_secrets,
  ]
}

resource "terraform_data" "vector_validate" {
  count = local.vector_enabled ? 1 : 0

  lifecycle {
    precondition {
      condition     = local.external_secrets_enabled
      error_message = "vector can only be enabled if external-secrets is also enabled"
    }
  }
}

# ---------------------------------------------------------------------------------------------------------------------
# REQUIRED PARAMETERS
# These variables must be set.
# ---------------------------------------------------------------------------------------------------------------------
variable "vector_chart_version" {
  description = "Version of vector helm chart"
  type        = string
  default     = ""
  validation {
    condition     = var.vector_chart_version == "" && local.vector_enabled ? false : true
    error_message = "vector_chart_version must be set when vector is enabled"
  }
}

variable "vector_image_tag" {
  description = "Vector image tag"
  type        = string
  default     = ""
  validation {
    condition     = var.vector_image_tag == "" && local.vector_enabled ? false : true
    error_message = "vector_image_tag must be set when vector is enabled"
  }
}

variable "vector_datadog_site" {
  description = "Datadog site for vector sinks (e.g. datadoghq.com)"
  type        = string
  default     = "datadoghq.com"
  validation {
    condition     = var.vector_datadog_site == "" && local.vector_enabled ? false : true
    error_message = "vector_datadog_site must be set when vector is enabled"
  }
}

variable "vector_datadog_api_key_vault_path" {
  description = "Vault path for the Datadog API key (consumed by ExternalSecret in vector ns)"
  type        = string
  default     = "secret/data/saas/infrastructure/datadog-api-key"
  validation {
    condition     = var.vector_datadog_api_key_vault_path == "" && local.vector_enabled ? false : true
    error_message = "vector_datadog_api_key_vault_path must be set when vector is enabled"
  }
}

variable "vector_external_secrets_cluster_store_name" {
  description = "Name of the ClusterSecretStore to use for vector ExternalSecret resources. Pass the name created by the external-secrets module (typically 'vault-cluster-store'). Required when vector is enabled."
  type        = string
  default     = "vault"
  validation {
    condition     = var.vector_external_secrets_cluster_store_name == "" && local.vector_enabled ? false : true
    error_message = "vector_external_secrets_cluster_store_name must be set when vector is enabled"
  }
}

# ---------------------------------------------------------------------------------------------------------------------
# OPTIONAL PARAMETERS
# These variables are configurable, but have reasonable defaults set.
# ---------------------------------------------------------------------------------------------------------------------
variable "vector_enabled_cluster_types" {
  description = "Types of clusters Vector should be enabled on by default"
  type        = map(string)
  default = {
    "mgmt"  = true
    "cps"   = true
    "saas"  = true
    "infra" = false
    "dev"   = false
  }
}

variable "vector_enabled" {
  description = "Override for explicitly enabling or disabling Vector"
  type        = bool
  default     = null
}

variable "vector_image" {
  description = "Image for Vector"
  type        = string
  default     = "registry.ic-ops.net/cache-docker-hub/timberio/vector"
}

variable "vector_replicas" {
  description = "Number of Vector aggregator replicas"
  type        = number
  default     = 3
}

variable "vector_audit_drop_patterns" {
  description = "Regex patterns; matching log message bodies are dropped"
  type        = list(string)
  default = [
    "Audit\\s-\\s.+",
    "\"level\":\"audit\"",
  ]
}

variable "vector_email_redaction_enabled" {
  description = "Toggle the email-redact VRL transform across all signals"
  type        = bool
  default     = true
}

variable "vector_email_redaction_replacement" {
  description = "String that replaces matched email addresses"
  type        = string
  default     = "[REDACTED-EMAIL]"
}

variable "vector_email_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted instead of replacing. Takes priority over vector_email_redaction_enabled."
  type        = bool
  default     = false
}

variable "vector_bearer_token_redaction_enabled" {
  description = "Toggle bearer token redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "vector_bearer_token_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_bearer_token_detected:true instead of replacing. Takes priority over vector_bearer_token_redaction_enabled."
  type        = bool
  default     = false
}

variable "vector_bearer_token_redaction_replacement" {
  description = "String that replaces matched bearer tokens"
  type        = string
  default     = "Bearer [REDACTED-BEARER-TOKEN]"
}

variable "vector_bearer_token_redaction_services" {
  description = "Service names to apply bearer token redaction to"
  type        = list(string)
  default     = ["service", "secure-metadata-database"]
}

variable "vector_bearer_token_redaction_exclusion_patterns" {
  description = "Skip bearer token redaction for any log whose message or ddtags contains one of these strings"
  type        = list(string)
  default = [
    "internal-webhook",
    "starting crawl",
    "Databricks automatic subscription",
    "audit/enrich",
    "Background Job",
    "community-organization-id:immuta",
    "pg - pool.connect",
  ]
}

variable "vector_aws_access_key_redaction_enabled" {
  description = "Toggle AWS Access Key ID redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "vector_aws_access_key_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_aws_key_detected:true instead of replacing. Takes priority over vector_aws_access_key_redaction_enabled."
  type        = bool
  default     = false
}

variable "vector_aws_access_key_redaction_replacement" {
  description = "String that replaces matched AWS Access Key IDs"
  type        = string
  default     = "[REDACTED-AWS-ACCESS-KEY]"
}

variable "vector_aws_access_key_redaction_services" {
  description = "Service names to apply AWS Access Key ID redaction to"
  type        = list(string)
  default     = ["marketplace-api"]
}

variable "vector_aws_access_key_redaction_exclusion_patterns" {
  description = "Skip AWS Access Key ID redaction for any log whose message or ddtags contains one of these strings"
  type        = list(string)
  default     = []
}

variable "vector_aws_access_key_redaction_exclusion_attributes" {
  description = "Skip AWS Access Key ID redaction for any log where the named attribute equals the given value (keys are attribute paths, values are string values to match)"
  type        = map(string)
  default = {
    "req.headers.x-synthetic-test" = "true"
  }
}

variable "vector_jwt_redaction_enabled" {
  description = "Toggle JSON Web Token redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "vector_jwt_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_jwt_detected:true instead of replacing. Takes priority over vector_jwt_redaction_enabled."
  type        = bool
  default     = false
}

variable "vector_jwt_redaction_replacement" {
  description = "String that replaces matched JSON Web Tokens"
  type        = string
  default     = "[REDACTED-JWT]"
}

variable "vector_jwt_redaction_services" {
  description = "Service names to apply JWT redaction to"
  type        = list(string)
  default     = ["service", "secure-metadata-database"]
}

variable "vector_jwt_redaction_exclusion_patterns" {
  description = "Skip JWT redaction for any log whose message or ddtags contains one of these strings"
  type        = list(string)
  default = [
    "internal-webhook",
    "starting crawl",
    "Databricks automatic subscription",
    "audit/enrich",
    "Background Job",
    "community-organization-id:immuta",
    "pg - pool.connect",
  ]
}

variable "vector_resource_requests" {
  description = "Resource resource requests"
  type = map(string)
  default = {
    cpu = "200m"
    memory = "256Mi"
  }
}

variable "vector_resource_limits" {
  description = "Resource limits for vector"
  type = map(string)
  default = {
    cpu = "1"
    memory = "2Gi"
  }
}

variable "vector_additional_config" {
  description = "Alternate helm chart customConfig (overwrites entire helm config)"
  type        = any
  default     = null
}


can we update the resource variables to be an object with the requests and limits instead of two separate variables like the following pattern? variable "argocd_application_controller_resources" {   d…


Fix YAML template strip marker merging entries


I have the full picture. Three files need updating:
1. `variables.tf` (internal module) — replace two `map(string)` vars with one `object`
2. `main.tf` (internal module) — update `templatefile` arg references
3. `vector.tf` (eks-cluster-applications) — collapse the two wrapper vars + update the module call


Edit

variable "resources_requests" {
  description = "Pod resource requests"
  type        = map(string)
  default = {
    cpu    = "200m"
    memory = "256Mi"
  }
}

variable "resources_limits" {
  description = "Pod resource limits"
  type        = map(string)
  default = {
    cpu    = "2"
    memory = "1Gi"
  }
}

variable "resources" {
  description = "Pod resource requests and limits"
  type = object({
    requests = object({
      cpu    = optional(string, "200m")
      memory = optional(string, "256Mi")
    })
    limits = object({
      cpu    = optional(string, null)
      memory = optional(string, null)
    })
  })
  default = {
    requests = {
      cpu    = "200m"
      memory = "256Mi"
    }
    limits = {
      cpu    = "2"
      memory = "1Gi"
    }
  }
}


The file /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/variables.tf has been updated successfully. (file state is current in your context — no need to Read it back)

variable "resources" {
  description = "Pod resource requests and limits"
  type = object({
    requests = object({
      cpu    = optional(string, "200m")
      memory = optional(string, "256Mi")
    })
    limits = object({
      cpu    = optional(string, null)
      memory = optional(string, null)
    })
  })
  default = {
    requests = {
      cpu    = "200m"
      memory = "256Mi"
    }
    limits = {
      cpu    = "2"
      memory = "1Gi"
    }
  }
}

# ---------------------------------------------------------------------------------------------------------------------
# REQUIRED PARAMETERS
# These variables must be set.
# ---------------------------------------------------------------------------------------------------------------------

variable "chart_version" {
  description = "Vector helm chart version pin"
  type        = string
}

variable "image_tag_version" {
  description = "Vector image tag"
  type        = string
}

# ---------------------------------------------------------------------------------------------------------------------
# OPTIONAL PARAMETERS
# These variables are configurable, but have reasonable defaults set.
# ---------------------------------------------------------------------------------------------------------------------
variable "datadog_site" {
  description = "Datadog site (e.g. datadoghq.com, us3.datadoghq.com)"
  type        = string
  default     = "datadoghq.com"
}

variable "image" {
  description = "Vector image source"
  type        = string
  default     = "registry.ic-ops.net/cache-docker-hub/timberio/vector"
}

variable "replicas" {
  description = "Number of Vector aggregator replicas"
  type        = number
  default     = 3
}

variable "audit_drop_patterns" {
  description = "Regex patterns; matching log message bodies are dropped"
  type        = list(string)
  default = [
    "Audit\\s-\\s.+",
    "\"level\":\"audit\"",
  ]
}

variable "datadog_api_key_vault_path" {
  description = "Vault path for the Datadog API key, consumed by ExternalSecret"
  type        = string
  default     = "secret/data/saas/infrastructure/datadog-api-key"
}

variable "external_secrets_cluster_store_name" {
  description = "Name of the ClusterSecretStore created by external-secrets module"
  type        = string
  default     = "vault"
}

variable "email_redaction_enabled" {
  description = "Toggle the email-redact VRL transform across all signals"
  type        = bool
  default     = true
}

variable "email_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_redaction:true instead of replacing. Takes priority over email_redaction_enabled."
  type        = bool
  default     = false
}

variable "email_redaction_replacement" {
  description = "String that replaces matched email addresses"
  type        = string
  default     = "[REDACTED-EMAIL]"
}

variable "additional_config" {
  description = "Escape hatch — overwrites helm customConfig wholesale (mirrors otel_collector_additional_config semantics)"
  type        = any
  default     = null
}

variable "resources_requests" {
  description = "Pod resource requests"
  type        = map(string)
  default = {
    cpu    = "200m"
    memory = "256Mi"
  }
}

variable "resources_limits" {
  description = "Pod resource limits"
  type        = map(string)
  default = {
    cpu    = "2"
    memory = "1Gi"
  }
}

variable "namespace" {
  description = "Kubernetes namespace for Vector"
  type        = string
  default     = "vector"
}

variable "service_monitor_enabled" {
  description = "Create a ServiceMonitor for Vector self-metrics. Requires kube-prometheus-stack CRDs in cluster."
  type        = bool
  default     = false
}

variable "bearer_token_redaction_enabled" {
  description = "Toggle bearer token redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "bearer_token_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_redaction:true instead of replacing. Takes priority over bearer_token_redaction_enabled."
  type        = bool
  default     = false
}

variable "aws_access_key_redaction_enabled" {
  description = "Toggle AWS Access Key ID redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "aws_access_key_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_redaction:true instead of replacing. Takes priority over aws_access_key_redaction_enabled."
  type        = bool
  default     = false
}

variable "aws_access_key_redaction_replacement" {
  description = "String that replaces matched AWS Access Key IDs"
  type        = string
  default     = "[REDACTED-AWS-ACCESS-KEY]"
}

variable "aws_access_key_redaction_services" {
  description = "Service names (.service field on DD agent logs) to apply AWS Access Key ID redaction to"
  type        = list(string)
  default     = ["marketplace-api"]
}

variable "aws_access_key_redaction_exclusion_patterns" {
  description = "Skip AWS Access Key ID redaction for any log whose message or ddtags contains one of these strings"
  type        = list(string)
  default     = []
}

variable "aws_access_key_redaction_exclusion_attributes" {
  description = "Skip AWS Access Key ID redaction for any log where the named attribute equals the given value (keys are attribute paths, values are string values to match)"
  type        = map(string)
  default = {
    "req.headers.x-synthetic-test" = "true"
  }
}

variable "jwt_redaction_enabled" {
  description = "Toggle JSON Web Token redact VRL transform on logs"
  type        = bool
  default     = false
}

variable "jwt_redaction_dry_run" {
  description = "When true, tag logs that would have been redacted with vector_redaction:true instead of replacing. Takes priority over jwt_redaction_enabled."
  type        = bool
  default     = false
}

variable "jwt_redaction_replacement" {
  description = "String that replaces matched JSON Web Tokens"
  type        = string
  default     = "[REDACTED-JWT]"
}

variable "jwt_redaction_services" {
  description = "Service names (.service field on DD agent logs) to apply JWT redaction to"
  type        = list(string)
  default     = ["service", "secure-metadata-database"]
}

variable "jwt_redaction_exclusion_patterns" {
  description = "Skip JWT redaction for any log whose message or ddtags contains one of these strings"
  type        = list(string)
  default = [
    "internal-webhook",
    "starting crawl",
    "Databricks automatic subscription",
    "audit/enrich",
    "Background Job",
    "community-organization-id:immuta",
    "pg - pool.connect",
  ]
}

variable "bearer_token_redaction_replacement" {
  description = "String that replaces matched bearer tokens (replaces the full 'Bearer <token>' match)"
  type        = string
  default     = "Bearer [REDACTED-BEARER-TOKEN]"
}

variable "bearer_token_redaction_services" {
  description = "Service names (.service field on DD agent logs) to apply bearer token redaction to"
  type        = list(string)
  default     = ["service", "secure-metadata-database"]
}

variable "bearer_token_redaction_exclusion_patterns" {
  description = "Skip bearer token redaction for any log whose message or ddtags contains one of these strings"
  type        = list(string)
  default = [
    "internal-webhook",
    "starting crawl",
    "Databricks automatic subscription",
    "audit/enrich",
    "Background Job",
    "community-organization-id:immuta",
    "pg - pool.connect",
  ]
}

-  description = "Pod resource requests"

+  description = "Pod resource requests and limits"

+      cpu    = optional(string, "200m")

+      memory = optional(string, "256Mi")


      requests_cpu                          = var.resources_requests.cpu
      requests_memory                       = var.resources_requests.memory
      limits_cpu                            = var.resources_limits.cpu
      limits_memory                         = var.resources_limits.memory

      requests_cpu                          = var.resources.requests.cpu
      requests_memory                       = var.resources.requests.memory
      limits_cpu                            = var.resources.limits.cpu
      limits_memory                         = var.resources.limits.memory


The file /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/main.tf has been updated successfully. (file state is current in your context — no need to Read it back)

      requests_cpu                          = var.resources.requests.cpu
      requests_memory                       = var.resources.requests.memory
      limits_cpu                            = var.resources.limits.cpu
      limits_memory                         = var.resources.limits.memory

terraform {
  required_version = ">= 1.9.0"

  required_providers {
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = ">= 2.20"
    }
    helm = {
      source  = "hashicorp/helm"
      version = ">= 2.10"
    }
  }
}

locals {
  audit_drop_condition = length(var.audit_drop_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.audit_drop_patterns : "!match(msg, r'${p}')"],
  )

  bearer_token_exclusion_condition = length(var.bearer_token_redaction_exclusion_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.bearer_token_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
  )

  bearer_token_services_arr = "[\"${join("\", \"", var.bearer_token_redaction_services)}\"]"

  # dry_run always wins over enabled for both scanner types
  email_do_redact = var.email_redaction_enabled && !var.email_redaction_dry_run
  email_do_tag    = var.email_redaction_dry_run

  bearer_do_redact = var.bearer_token_redaction_enabled && !var.bearer_token_redaction_dry_run
  bearer_do_tag    = var.bearer_token_redaction_dry_run

  # named output of each stage used to wire transform inputs and the final sink
  email_stage_output = (
    local.email_do_redact ? "redact_emails_logs" :
    local.email_do_tag    ? "tag_emails_logs" :
    "normalize_logs"
  )

  bearer_stage_active   = local.bearer_do_redact || local.bearer_do_tag
  bearer_stage_output   = local.bearer_do_redact ? "redact_bearer_tokens_logs" : "tag_bearer_tokens_logs"
  bearer_needs_combined = length(var.bearer_token_redaction_exclusion_patterns) > 0

  bearer_token_logs_input = local.email_stage_output

  # AWS Access Key ID scanner
  aws_key_do_redact = var.aws_access_key_redaction_enabled && !var.aws_access_key_redaction_dry_run
  aws_key_do_tag    = var.aws_access_key_redaction_dry_run

  aws_key_services_arr = "[\"${join("\", \"", var.aws_access_key_redaction_services)}\"]"

  aws_key_string_exclusion_condition = length(var.aws_access_key_redaction_exclusion_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.aws_access_key_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
  )

  # Each entry generates a VRL boolean: true when the attribute matches and should be excluded.
  # Uses quoted path access (.attributes."dotted.key") which is infallible; returns null for missing paths.
  aws_key_attr_exclusion_condition = length(var.aws_access_key_redaction_exclusion_attributes) == 0 ? "false" : join(
    " || ",
    [for k, v in var.aws_access_key_redaction_exclusion_attributes :
      "(to_string(.attributes.\"${k}\") ?? \"\") == \"${v}\""
    ],
  )

  aws_key_stage_active   = local.aws_key_do_redact || local.aws_key_do_tag
  aws_key_stage_output   = local.aws_key_do_redact ? "redact_aws_access_keys_logs" : "tag_aws_access_keys_logs"
  aws_key_needs_combined = length(var.aws_access_key_redaction_exclusion_patterns) > 0

  aws_key_logs_input = local.bearer_stage_active ? local.bearer_stage_output : local.email_stage_output

  # JWT scanner
  jwt_do_redact = var.jwt_redaction_enabled && !var.jwt_redaction_dry_run
  jwt_do_tag    = var.jwt_redaction_dry_run

  jwt_services_arr = "[\"${join("\", \"", var.jwt_redaction_services)}\"]"

  jwt_exclusion_condition = length(var.jwt_redaction_exclusion_patterns) == 0 ? "true" : join(
    " && ",
    [for p in var.jwt_redaction_exclusion_patterns : "!contains(combined, \"${p}\")"],
  )

  jwt_stage_active   = local.jwt_do_redact || local.jwt_do_tag
  jwt_stage_output   = local.jwt_do_redact ? "redact_jwts_logs" : "tag_jwts_logs"
  jwt_needs_combined = length(var.jwt_redaction_exclusion_patterns) > 0

  jwt_logs_input = (
    local.aws_key_stage_active ? local.aws_key_stage_output :
    local.bearer_stage_active  ? local.bearer_stage_output :
    local.email_stage_output
  )

  logs_sink_input = (
    local.jwt_stage_active     ? local.jwt_stage_output :
    local.aws_key_stage_active ? local.aws_key_stage_output :
    local.bearer_stage_active  ? local.bearer_stage_output :
    local.email_stage_output
  )

  rendered_values = var.additional_config != null ? yamlencode({ customConfig = var.additional_config }) : templatefile(
    "${path.module}/templates/values.yaml.tpl",
    {
      replicas                              = var.replicas
      requests_cpu                          = var.resources_requests.cpu
      requests_memory                       = var.resources_requests.memory
      limits_cpu                            = var.resources_limits.cpu
      limits_memory                         = var.resources_limits.memory
      datadog_site                          = var.datadog_site
      audit_drop_condition                  = local.audit_drop_condition
      email_do_redact                       = local.email_do_redact
      email_do_tag                          = local.email_do_tag
      email_redaction_replacement           = var.email_redaction_replacement
      service_monitor_enabled               = var.service_monitor_enabled
      bearer_do_redact                      = local.bearer_do_redact
      bearer_do_tag                         = local.bearer_do_tag
      bearer_needs_combined                 = local.bearer_needs_combined
      bearer_token_redaction_replacement    = var.bearer_token_redaction_replacement
      bearer_token_services_arr             = local.bearer_token_services_arr
      bearer_token_exclusion_condition      = local.bearer_token_exclusion_condition
      bearer_token_logs_input               = local.bearer_token_logs_input
      aws_key_do_redact                     = local.aws_key_do_redact
      aws_key_do_tag                        = local.aws_key_do_tag
      aws_key_needs_combined                = local.aws_key_needs_combined
      aws_access_key_redaction_replacement  = var.aws_access_key_redaction_replacement
      aws_key_services_arr                  = local.aws_key_services_arr
      aws_key_string_exclusion_condition    = local.aws_key_string_exclusion_condition
      aws_key_attr_exclusion_condition      = local.aws_key_attr_exclusion_condition
      aws_key_logs_input                    = local.aws_key_logs_input
      jwt_do_redact                         = local.jwt_do_redact
      jwt_do_tag                            = local.jwt_do_tag
      jwt_needs_combined                    = local.jwt_needs_combined
      jwt_redaction_replacement             = var.jwt_redaction_replacement
      jwt_services_arr                      = local.jwt_services_arr
      jwt_exclusion_condition               = local.jwt_exclusion_condition
      jwt_logs_input                        = local.jwt_logs_input
      logs_sink_input                       = local.logs_sink_input
    },
  )
}

resource "kubernetes_namespace_v1" "vector" {
  metadata {
    name = var.namespace
    labels = {
      "app.kubernetes.io/name"      = "vector"
      "app.kubernetes.io/component" = "telemetry-aggregator"
    }
  }
}

resource "kubernetes_manifest" "datadog_external_secret" {
  manifest = {
    apiVersion = "external-secrets.io/v1"
    kind       = "ExternalSecret"
    metadata = {
      name      = "vector-datadog"
      namespace = kubernetes_namespace_v1.vector.metadata[0].name
    }
    spec = {
      refreshInterval = "1h"
      secretStoreRef = {
        kind = "ClusterSecretStore"
        name = var.external_secrets_cluster_store_name
      }
      target = {
        name           = "vector-datadog"
        creationPolicy = "Owner"
      }
      data = [{
        secretKey = "api_key"
        remoteRef = {
          key      = var.datadog_api_key_vault_path
          property = "apiKey"
        }
      }]
    }
  }

  wait {
    fields = {
      "status.conditions[0].type"   = "Ready"
      "status.conditions[0].status" = "True"
    }
  }

  timeouts {
    create = "5m"
    update = "5m"
  }
}

resource "helm_release" "vector" {
  name       = "vector"
  namespace  = kubernetes_namespace_v1.vector.metadata[0].name
  repository = "https://helm.vector.dev"
  chart      = "vector"
  version    = var.chart_version

  values = [local.rendered_values]

  set {
    name  = "image.repository"
    value = var.image
  }
  set {
    name  = "image.tag"
    value = var.image_tag_version
  }

  depends_on = [
    kubernetes_manifest.datadog_external_secret,
  ]
}

module "netpol" {
  source = "crow.infrastructure.immuta.io/internal/security-network-policy/kubernetes"
  version = "1.1.0"

  name             = "vector-netpol"
  namespace        = var.namespace
  network_policies = local.netpols

  depends_on = [
    kubernetes_namespace_v1.vector,
  ]
}

replicas                              = var.replicas

-      requests_cpu                          = var.resources_requests.cpu

-      requests_memory                       = var.resources_requests.memory

-      limits_cpu                            = var.resources_limits.cpu

-      limits_memory                         = var.resources_limits.memory

+      requests_cpu                          = var.resources.requests.cpu

+      requests_memory                       = var.resources.requests.memory

+      limits_cpu                            = var.resources.limits.cpu

+      limits_memory                         = var.resources.limits.memory

datadog_site                          = var.datadog_site

email_do_redact                       = local.email_do_redact


Now update the eks-cluster-applications wrapper — module call first, then the two variables:


  resources_requests          = var.vector_resource_requests
  resources_limits            = var.vector_resource_limits

resources                   = var.vector_resources


The file /Users/steve.briggs/projects/operations/terraform/iac-library/aws/compute/eks-cluster-applications/vector.tf has been updated successfully. (file state is current in your context — no need to Read it back)

  resources_requests          = var.vector_resource_requests
  resources_limits            = var.vector_resource_limits

resources                   = var.vector_resources

image                       = var.vector_image

image_tag_version           = var.vector_image_tag

replicas                    = var.vector_replicas

-  resources_requests          = var.vector_resource_requests

-  resources_limits            = var.vector_resource_limits

+  resources                   = var.vector_resources

datadog_site                = var.vector_datadog_site

datadog_api_key_vault_path  = var.vector_datadog_api_key_vault_path

audit_drop_patterns         = var.vector_audit_drop_patterns


variable "vector_resource_requests" {
  description = "Resource resource requests"
  type = map(string)
  default = {
    cpu = "200m"
    memory = "256Mi"
  }
}

variable "vector_resource_limits" {
  description = "Resource limits for vector"
  type = map(string)
  default = {
    cpu = "1"
    memory = "2Gi"
  }
}

variable "vector_resources" {
  description = "Resource requests and limits for Vector"
  type = object({
    requests = object({
      cpu    = optional(string, "200m")
      memory = optional(string, "256Mi")
    })
    limits = object({
      cpu    = optional(string, null)
      memory = optional(string, null)
    })
  })
  default = {
    requests = {
      cpu    = "200m"
      memory = "256Mi"
    }
    limits = {
      cpu    = "1"
      memory = "2Gi"
    }
  }
}


variable "vector_resource_requests" {
  description = "Resource resource requests"
  type = map(string)
  default = {
    cpu = "200m"
    memory = "256Mi"
  }
}

variable "vector_resource_limits" {
  description = "Resource limits for vector"
  type = map(string)
  default = {
    cpu = "1"
    memory = "2Gi"
  }
}

variable "vector_resources" {
  description = "Resource requests and limits for Vector"
  type = object({
    requests = object({
      cpu    = optional(string, "200m")
      memory = optional(string, "256Mi")
    })
    limits = object({
      cpu    = optional(string, null)
      memory = optional(string, null)
    })
  })
  default = {
    requests = {
      cpu    = "200m"
      memory = "256Mi"
    }
    limits = {
      cpu    = "1"
      memory = "2Gi"
    }
  }
}

-  description = "Resource resource requests"

+  description = "Resource requests and limits for Vector"

+      cpu    = optional(string, "200m")

+      memory = optional(string, "256Mi")

-  description = "Resource limits for vector"

description = "Alternate helm chart customConfig (overwrites entire helm config)"


Let me verify the final state of the three changed sections:


Bash

grep -n "resources" /Users/steve.briggs/projects/operations/terraform/internal-modules/kubernetes/monitoring/vector/main.tf | grep -E "requests|limits"

Verify main.tf references


grep -n "resources\|vector_resource" /Users/steve.briggs/projects/operations/terraform/iac-library/aws/compute/eks-cluster-applications/vector.tf | head -20

Verify vector.tf references


107:      requests_cpu                          = var.resources.requests.cpu
108:      requests_memory                       = var.resources.requests.memory
109:      limits_cpu                            = var.resources.limits.cpu
110:      limits_memory                         = var.resources.limits.memory


28:  resources                   = var.vector_resources
121:  description = "Name of the ClusterSecretStore to use for vector ExternalSecret resources. Pass the name created by the external-secrets module (typically 'vault-cluster-store'). Required when vector is enabled."
305:variable "vector_resources" {


All clean. Here's a summary of what changed, Sir Charles:

**`variables.tf` (internal module)**: Replaced `resources_requests` (`map(string)`) and `resources_limits` (`map(string)`) with a single `resources` object. Limits fields use `optional(string, null)` so individual CPU/memory limits can be omitted without touching the whole block. Defaults preserved (requests: 200m/256Mi, limits: 2 CPU/1Gi).

**`main.tf` (internal module)**: Four templatefile argument references updated from `var.resources_requests.cpu` → `var.resources.requests.cpu` etc. The template itself and its `limits_cpu != null` checks are unaffected.

**`vector.tf` (eks-cluster-applications)**: Collapsed `vector_resource_requests` + `vector_resource_limits` into `vector_resources` with the same object shape, preserving its existing defaults (requests: 200m/256Mi, limits: 1 CPU/2Gi which differ from the inner module defaults). Module call updated to `resources = var.vector_resources`.


afplay ~/.claude/sounds/mario/stop/stomp.wav 2>/dev/null || true


Fixed a YAML rendering bug in the Vector Helm values template and refactored the resource variables from two separate maps into a single typed object. Ready to commit. (disable recaps in /config)