diff --git a/.trivyignore.yaml b/.trivyignore.yaml index 5079ce5db1..9073eb7ead 100644 --- a/.trivyignore.yaml +++ b/.trivyignore.yaml @@ -14,6 +14,8 @@ misconfigurations: - docker/osv-scanner/Dockerfile - docker/semgrep/Dockerfile - docker/trivy/Dockerfile + # Container runs as non-root (user 65532) via the ECS task definition; the image is a thin version-pin for dependabot. + - docker/victoriametrics/Dockerfile - docker/zap/Dockerfile - id: AVD-AWS-0053 # Public ALB intentional for nest.owasp.dev paths: @@ -43,6 +45,9 @@ misconfigurations: - id: AVD-AWS-0176 # IAM DB auth deferred; app uses Secrets Manager credentials today paths: - infrastructure/modules/database/main.tf + - id: AVD-AWS-0178 # Throwaway VPC fixture for module tests; flow logs are not applicable + paths: + - infrastructure/modules/observability/tests/setup/main.tf - id: AVD-AWS-0342 # EventBridge PassRole required to run ECS tasks paths: - infrastructure/modules/tasks/main.tf diff --git a/docker-compose/infrastructure/compose.yaml b/docker-compose/infrastructure/compose.yaml index 1aeda3d1b4..35ad30005b 100644 --- a/docker-compose/infrastructure/compose.yaml +++ b/docker-compose/infrastructure/compose.yaml @@ -31,6 +31,7 @@ services: network_mode: service:localstack pull_policy: never volumes: + - ../../docker:/home/owasp/docker:ro - ../../infrastructure/bootstrap:/home/owasp/infrastructure/bootstrap - ../../infrastructure/modules:/home/owasp/infrastructure/modules - ../../infrastructure/scripts:/home/owasp/infrastructure/scripts:ro diff --git a/docker-compose/local/compose.o11y.yaml b/docker-compose/local/compose.o11y.yaml index 3873f69c7f..ccd2acc025 100644 --- a/docker-compose/local/compose.o11y.yaml +++ b/docker-compose/local/compose.o11y.yaml @@ -35,7 +35,9 @@ services: command: - -retentionPeriod=5y - -storageDataPath=/data - image: victoriametrics/victoria-metrics:v1.145.0@sha256:c014fb5a711d38cb24fd0673197592cd1394bb903dbb16aea565620c9c8a3d70 + build: + context: ../../docker/victoriametrics + dockerfile: Dockerfile healthcheck: interval: 5s retries: 5 diff --git a/docker/victoriametrics/Dockerfile b/docker/victoriametrics/Dockerfile new file mode 100644 index 0000000000..6fbce0da07 --- /dev/null +++ b/docker/victoriametrics/Dockerfile @@ -0,0 +1 @@ +FROM victoriametrics/victoria-metrics:v1.145.0@sha256:c014fb5a711d38cb24fd0673197592cd1394bb903dbb16aea565620c9c8a3d70 diff --git a/infrastructure/live/README.md b/infrastructure/live/README.md index 7f6ac2ac43..ebf0a2ccfa 100644 --- a/infrastructure/live/README.md +++ b/infrastructure/live/README.md @@ -60,6 +60,7 @@ No providers. | [frontend\_build\_cache](#module\_frontend\_build\_cache) | ../modules/ecr-cache | n/a | | [kms](#module\_kms) | ../modules/kms | n/a | | [networking](#module\_networking) | ../modules/networking | n/a | +| [observability](#module\_observability) | ../modules/observability | n/a | | [parameters](#module\_parameters) | ../modules/parameters | n/a | | [security](#module\_security) | ../modules/security | n/a | | [storage](#module\_storage) | ../modules/storage | n/a | @@ -102,6 +103,7 @@ No resources. | [domain\_name](#input\_domain\_name) | The domain name for the site. | `string` | n/a | yes | | [enable\_additional\_parameters](#input\_enable\_additional\_parameters) | Whether to enable additional parameters (e.g. for production). | `bool` | `false` | no | | [enable\_cron\_tasks](#input\_enable\_cron\_tasks) | Whether to enable scheduled cron tasks. | `bool` | n/a | yes | +| [enable\_observability](#input\_enable\_observability) | Whether to create the observability stack. | `bool` | `false` | no | | [enable\_rds\_proxy](#input\_enable\_rds\_proxy) | Whether to create an RDS proxy. | `bool` | `false` | no | | [enable\_vpc\_cloudwatch\_logs\_endpoint](#input\_enable\_vpc\_cloudwatch\_logs\_endpoint) | Whether to create CloudWatch Logs VPC endpoint. | `bool` | `false` | no | | [enable\_vpc\_ecr\_api\_endpoint](#input\_enable\_vpc\_ecr\_api\_endpoint) | Whether to create ECR API VPC endpoint. | `bool` | `false` | no | diff --git a/infrastructure/live/main.tf b/infrastructure/live/main.tf index a56ca59346..b393de97eb 100644 --- a/infrastructure/live/main.tf +++ b/infrastructure/live/main.tf @@ -21,6 +21,7 @@ locals { Project = var.project_name } fixtures_bucket_name = coalesce(var.fixtures_bucket_name, "${var.project_name}-${var.environment}-fixtures") + observability_image = regex("(?m)^FROM (victoriametrics/victoria-metrics:\\S+)", file("${path.root}/../../docker/victoriametrics/Dockerfile"))[0] } module "alb" { @@ -175,6 +176,26 @@ module "networking" { vpc_cidr = var.vpc_cidr } +module "observability" { + count = var.enable_observability ? 1 : 0 + source = "../modules/observability" + + app_security_group_ids = [ + module.security.backend_sg_id, + module.security.frontend_sg_id, + module.security.tasks_sg_id, + ] + assign_public_ip = false + aws_region = var.aws_region + common_tags = local.common_tags + environment = var.environment + kms_key_arn = module.kms.key_arn + project_name = var.project_name + subnet_ids = module.networking.private_subnet_ids + image = local.observability_image + vpc_id = module.networking.vpc_id +} + module "parameters" { source = "../modules/parameters" diff --git a/infrastructure/live/variables.tf b/infrastructure/live/variables.tf index ed7af5ff60..7533d089b1 100644 --- a/infrastructure/live/variables.tf +++ b/infrastructure/live/variables.tf @@ -193,6 +193,12 @@ variable "enable_cron_tasks" { type = bool } +variable "enable_observability" { + description = "Whether to create the observability stack." + type = bool + default = false +} + variable "enable_rds_proxy" { description = "Whether to create an RDS proxy." type = bool diff --git a/infrastructure/make/test.mk b/infrastructure/make/test.mk index cc39eae293..f8f82693b5 100644 --- a/infrastructure/make/test.mk +++ b/infrastructure/make/test.mk @@ -37,6 +37,7 @@ infrastructure-test-image-build: infrastructure-test-unit: @$(MAKE) infrastructure-test-image-build @docker run --rm \ + -v "$(CURDIR)/docker:/home/owasp/docker:ro" \ -v "$(CURDIR)/infrastructure/bootstrap:/home/owasp/infrastructure/bootstrap" \ -v "$(CURDIR)/infrastructure/live:/home/owasp/infrastructure/live" \ -v "$(CURDIR)/infrastructure/modules:/home/owasp/infrastructure/modules" \ diff --git a/infrastructure/modules/observability/.terraform.lock.hcl b/infrastructure/modules/observability/.terraform.lock.hcl new file mode 100644 index 0000000000..36c63c7eb3 --- /dev/null +++ b/infrastructure/modules/observability/.terraform.lock.hcl @@ -0,0 +1,28 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.58.0" + constraints = "~> 6.58.0" + hashes = [ + "h1:2kpake4zZKRX5437QVIRU3qFYH6Bjw/QE1fgVCPOrUg=", + "h1:OWl47Bo8Vzlf5srTUCmA6v4kvQGfah/P1joRtIYUUMc=", + "h1:UFot9S97tuAPvjKvoxm08sDG/gKYdDK+lMwsZKtLieY=", + "zh:1221253beee5629fb503d79cebc9bc661279cbc4be5d01db9ab4c1b702108250", + "zh:132bd0925bdc4b72446ac750b7ccb1e19b9ba8fbb6df57b2c1423314d2195d4f", + "zh:18cda250b9e82b753808715893c8927f132273c00ffae7a697d65ac1cb577e48", + "zh:204c944f1fb7f440a335bb2083c9691a9d1f677aea9701025dd5816aee41f0ba", + "zh:2dc41df289f2b10a01e650cdd73699955f0ab0645d09cfb114a8cd0f4cc4ede7", + "zh:345633dfa9a234659d52aadd126e6dce658518c3ab5cbf6d871221287ed5ec56", + "zh:4dadcced73e742903158bc9838936d911f3fa4c2c37b5591c1a28f8f2a1902a6", + "zh:5bc60cc2b8c093da98b211d9f6c21c9ecec0f21b944d9ecbe3961fba33086e80", + "zh:6cc8f084938b0033a9c0c910989919dad6b1683e76e0afa1a5c604e39f398a75", + "zh:7db214647f79de9a033b5dfd6cbfaa42d53c4d056b32cb3acc7ad99306dd548a", + "zh:9078589ec881cee7ed9403af262c98ff257fb3e1baae72ff6429a398b1c730af", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:bd5bce6aec4d4922b1127b8575688bd4bc4279670ee28d198ede404709826c7c", + "zh:cd900ecf56d21023873898b06e40234f3f4d350f2343b7d9b980d6c5cb604fae", + "zh:dbe93b276a84421026b956c3c5b4eb8897da6cbb54b93cb89f5d6ebbd30805ca", + "zh:f6b6c7bb2dbf04ee085e5c22f7a65b3ccaebf368ed95584dc0dfee8a22771056", + ] +} diff --git a/infrastructure/modules/observability/README.md b/infrastructure/modules/observability/README.md new file mode 100644 index 0000000000..455e85e428 --- /dev/null +++ b/infrastructure/modules/observability/README.md @@ -0,0 +1,69 @@ + +## Requirements + +| Name | Version | +| ---- | ------- | +| [terraform](#requirement\_terraform) | ~> 1.15.0 | +| [aws](#requirement\_aws) | ~> 6.58.0 | + +## Providers + +| Name | Version | +| ---- | ------- | +| [aws](#provider\_aws) | 6.58.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +| ---- | ---- | +| [aws_cloudwatch_log_group.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_ecs_cluster.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_cluster) | resource | +| [aws_ecs_cluster_capacity_providers.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_cluster_capacity_providers) | resource | +| [aws_ecs_service.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource | +| [aws_ecs_task_definition.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource | +| [aws_efs_access_point.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_access_point) | resource | +| [aws_efs_file_system.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_file_system) | resource | +| [aws_efs_mount_target.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_mount_target) | resource | +| [aws_iam_policy.ecs_task_execution_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.ecs_task_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.ecs_task_execution_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_security_group.efs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group_rule.efs_from_vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.vm_egress_https](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.vm_ingest_from_apps](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.vm_to_efs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +| ---- | ----------- | ---- | ------- | :------: | +| [app\_security\_group\_ids](#input\_app\_security\_group\_ids) | Security group IDs of the application tasks allowed to send metrics to the observability backend. | `list(string)` | n/a | yes | +| [assign\_public\_ip](#input\_assign\_public\_ip) | Whether to assign a public IP to the observability task. | `bool` | `false` | no | +| [aws\_region](#input\_aws\_region) | The AWS region where the module is deployed. | `string` | n/a | yes | +| [common\_tags](#input\_common\_tags) | A map of common tags to apply to all resources. | `map(string)` | `{}` | no | +| [cpu](#input\_cpu) | The CPU units for the observability Fargate task. | `number` | `512` | no | +| [desired\_count](#input\_desired\_count) | The number of observability tasks to run (0 or 1; the current backend is a single-node store). | `number` | `1` | no | +| [environment](#input\_environment) | The environment (e.g., staging, production). | `string` | n/a | yes | +| [image](#input\_image) | The observability backend container image (including digest). | `string` | n/a | yes | +| [kms\_key\_arn](#input\_kms\_key\_arn) | The ARN of the KMS key used to encrypt the EFS file system. | `string` | n/a | yes | +| [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | The number of days to retain observability container logs. | `number` | `90` | no | +| [memory](#input\_memory) | The memory (in MiB) for the observability Fargate task. | `number` | `1024` | no | +| [port](#input\_port) | The port the observability backend listens on for ingest and queries. | `number` | `8428` | no | +| [project\_name](#input\_project\_name) | The name of the project. | `string` | n/a | yes | +| [retention\_period](#input\_retention\_period) | The VictoriaMetrics data retention period. A value without a suffix is in months, so the default "12" means 12 months (duration suffixes like 1y, 30d, 1w are also supported). | `string` | `"12"` | no | +| [subnet\_ids](#input\_subnet\_ids) | The private subnet IDs for the EFS mount targets and the observability task. | `list(string)` | n/a | yes | +| [vpc\_id](#input\_vpc\_id) | The VPC ID where the observability backend security group is created. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +| ---- | ----------- | +| [cluster\_name](#output\_cluster\_name) | The name of the ECS cluster running the observability backend. | +| [efs\_file\_system\_id](#output\_efs\_file\_system\_id) | The ID of the EFS file system backing observability storage. | +| [security\_group\_id](#output\_security\_group\_id) | The ID of the observability backend security group. | + diff --git a/infrastructure/modules/observability/main.tf b/infrastructure/modules/observability/main.tf new file mode 100644 index 0000000000..04cd7fa453 --- /dev/null +++ b/infrastructure/modules/observability/main.tf @@ -0,0 +1,287 @@ +terraform { + required_version = "~> 1.15.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.58.0" + } + } +} + +locals { + name_prefix = "${var.project_name}-${var.environment}-observability" + + container_definition = { + command = [ + "-storageDataPath=/data", + "-retentionPeriod=${var.retention_period}", + "-httpListenAddr=:${var.port}", + ] + essential = true + healthCheck = { + command = ["CMD-SHELL", "wget --spider -q http://localhost:${var.port}/health || exit 1"] + interval = 30 + retries = 3 + startPeriod = 30 + timeout = 5 + } + image = var.image + logConfiguration = { + logDriver = "awslogs" + options = { + "awslogs-group" = aws_cloudwatch_log_group.vm.name + "awslogs-region" = var.aws_region + "awslogs-stream-prefix" = "ecs" + } + } + mountPoints = [ + { + containerPath = "/data" + readOnly = false + sourceVolume = "vm-data" + } + ] + name = "victoriametrics" + portMappings = [ + { + containerPort = var.port + hostPort = var.port + protocol = "tcp" + } + ] + user = "65532" + } +} + +resource "aws_security_group" "vm" { + description = "Security group for the observability backend task" + name = "${local.name_prefix}-vm-sg" + tags = merge(var.common_tags, { + Name = "${local.name_prefix}-vm-sg" + }) + vpc_id = var.vpc_id +} + +resource "aws_security_group_rule" "vm_ingest_from_apps" { + count = length(var.app_security_group_ids) + + description = "Allow metrics ingest and queries from application tasks" + from_port = var.port + protocol = "tcp" + security_group_id = aws_security_group.vm.id + source_security_group_id = var.app_security_group_ids[count.index] + to_port = var.port + type = "ingress" +} + +resource "aws_security_group_rule" "vm_egress_https" { + cidr_blocks = ["0.0.0.0/0"] + description = "Allow HTTPS egress for container image pulls" + from_port = 443 + protocol = "tcp" + security_group_id = aws_security_group.vm.id + to_port = 443 + type = "egress" +} + +resource "aws_security_group_rule" "vm_to_efs" { + description = "Allow NFS to the observability EFS" + from_port = 2049 + protocol = "tcp" + security_group_id = aws_security_group.vm.id + source_security_group_id = aws_security_group.efs.id + to_port = 2049 + type = "egress" +} + +resource "aws_security_group" "efs" { + description = "Security group for the observability EFS file system" + name = "${local.name_prefix}-efs-sg" + tags = merge(var.common_tags, { + Name = "${local.name_prefix}-efs-sg" + }) + vpc_id = var.vpc_id +} + +resource "aws_security_group_rule" "efs_from_vm" { + description = "Allow NFS from the observability backend task" + from_port = 2049 + protocol = "tcp" + security_group_id = aws_security_group.efs.id + source_security_group_id = aws_security_group.vm.id + to_port = 2049 + type = "ingress" +} + +resource "aws_efs_file_system" "vm" { + encrypted = true + kms_key_id = var.kms_key_arn + tags = merge(var.common_tags, { + Name = "${local.name_prefix}-vm" + }) + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_efs_mount_target" "vm" { + count = length(var.subnet_ids) + + file_system_id = aws_efs_file_system.vm.id + security_groups = [aws_security_group.efs.id] + subnet_id = var.subnet_ids[count.index] +} + +resource "aws_efs_access_point" "vm" { + file_system_id = aws_efs_file_system.vm.id + + posix_user { + gid = 65532 + uid = 65532 + } + + root_directory { + path = "/victoriametrics" + + creation_info { + owner_gid = 65532 + owner_uid = 65532 + permissions = "0755" + } + } + + tags = merge(var.common_tags, { + Name = "${local.name_prefix}-vm" + }) +} + +resource "aws_cloudwatch_log_group" "vm" { + kms_key_id = var.kms_key_arn + name = "/aws/ecs/${local.name_prefix}" + retention_in_days = var.log_retention_in_days + tags = merge(var.common_tags, { + Name = "${local.name_prefix}-logs" + }) +} + +resource "aws_ecs_cluster" "vm" { + name = "${local.name_prefix}-cluster" + tags = merge(var.common_tags, { + Name = "${local.name_prefix}-cluster" + }) + + setting { + name = "containerInsights" + value = "enabled" + } +} + +resource "aws_ecs_cluster_capacity_providers" "vm" { + capacity_providers = ["FARGATE"] + cluster_name = aws_ecs_cluster.vm.name + + default_capacity_provider_strategy { + base = 0 + capacity_provider = "FARGATE" + weight = 1 + } +} + +resource "aws_iam_role" "ecs_task_execution_role" { + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "ecs-tasks.amazonaws.com" + } + } + ] + }) + name = "${local.name_prefix}-execution-role" + tags = var.common_tags +} + +resource "aws_iam_policy" "ecs_task_execution_policy" { + description = "Policy for observability ECS task execution - CloudWatch Logs access." + name = "${local.name_prefix}-execution-policy" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Effect = "Allow" + Resource = "${aws_cloudwatch_log_group.vm.arn}:*" + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "ecs_task_execution_policy_attachment" { + policy_arn = aws_iam_policy.ecs_task_execution_policy.arn + role = aws_iam_role.ecs_task_execution_role.name +} + +resource "aws_ecs_task_definition" "vm" { + container_definitions = jsonencode([local.container_definition]) + cpu = var.cpu + execution_role_arn = aws_iam_role.ecs_task_execution_role.arn + family = local.name_prefix + memory = var.memory + network_mode = "awsvpc" + requires_compatibilities = ["FARGATE"] + runtime_platform { + cpu_architecture = "ARM64" + operating_system_family = "LINUX" + } + tags = merge(var.common_tags, { + Name = "${local.name_prefix}-task-def" + }) + + volume { + name = "vm-data" + + efs_volume_configuration { + file_system_id = aws_efs_file_system.vm.id + transit_encryption = "ENABLED" + + authorization_config { + access_point_id = aws_efs_access_point.vm.id + } + } + } +} + +resource "aws_ecs_service" "vm" { + cluster = aws_ecs_cluster.vm.id + deployment_maximum_percent = 100 + deployment_minimum_healthy_percent = 0 + desired_count = var.desired_count + name = "${local.name_prefix}-service" + tags = merge(var.common_tags, { + Name = "${local.name_prefix}-service" + }) + task_definition = aws_ecs_task_definition.vm.arn + + capacity_provider_strategy { + base = 0 + capacity_provider = "FARGATE" + weight = 1 + } + + network_configuration { + assign_public_ip = var.assign_public_ip + security_groups = [aws_security_group.vm.id] + subnets = var.subnet_ids + } + + depends_on = [aws_efs_mount_target.vm] +} diff --git a/infrastructure/modules/observability/outputs.tf b/infrastructure/modules/observability/outputs.tf new file mode 100644 index 0000000000..6556045257 --- /dev/null +++ b/infrastructure/modules/observability/outputs.tf @@ -0,0 +1,14 @@ +output "efs_file_system_id" { + description = "The ID of the EFS file system backing observability storage." + value = aws_efs_file_system.vm.id +} + +output "cluster_name" { + description = "The name of the ECS cluster running the observability backend." + value = aws_ecs_cluster.vm.name +} + +output "security_group_id" { + description = "The ID of the observability backend security group." + value = aws_security_group.vm.id +} diff --git a/infrastructure/modules/observability/tests/integration.tftest.hcl b/infrastructure/modules/observability/tests/integration.tftest.hcl new file mode 100644 index 0000000000..5a7dda1f02 --- /dev/null +++ b/infrastructure/modules/observability/tests/integration.tftest.hcl @@ -0,0 +1,77 @@ +provider "aws" { + access_key = "test" + region = "us-east-1" + s3_use_path_style = true + secret_key = "test" + skip_credentials_validation = true + skip_metadata_api_check = true + skip_requesting_account_id = true +} + +variables { + aws_region = "us-east-1" + common_tags = { Environment = "test", Project = "nest" } + environment = "test" + project_name = "nest" + image = regex("(?m)^FROM (victoriametrics/victoria-metrics:\\S+)", file("../../../docker/victoriametrics/Dockerfile"))[0] +} + +run "setup" { + command = apply + + module { + source = "./tests/setup" + } +} + +run "observability_integration_apply" { + command = apply + + variables { + app_security_group_ids = run.setup.app_security_group_ids + kms_key_arn = run.setup.kms_key_arn + subnet_ids = run.setup.subnet_ids + desired_count = 0 + vpc_id = run.setup.vpc_id + } + + assert { + condition = can(aws_efs_file_system.vm.id) + error_message = "EFS file system was not created." + } + + assert { + condition = aws_efs_file_system.vm.encrypted == true + error_message = "EFS must be encrypted at rest." + } + + assert { + condition = length(aws_efs_mount_target.vm) == length(var.subnet_ids) + error_message = "There must be one EFS mount target per subnet." + } + + assert { + condition = aws_security_group_rule.efs_from_vm.source_security_group_id == aws_security_group.vm.id + error_message = "EFS ingress must come only from the VictoriaMetrics security group." + } + + assert { + condition = one([for v in aws_ecs_task_definition.vm.volume : v if v.name == "vm-data"]).efs_volume_configuration[0].file_system_id == aws_efs_file_system.vm.id + error_message = "The vm-data volume must reference the module's EFS file system." + } + + assert { + condition = can(aws_efs_access_point.vm.id) + error_message = "EFS access point was not created." + } + + assert { + condition = one([for v in aws_ecs_task_definition.vm.volume : v if v.name == "vm-data"]).efs_volume_configuration[0].authorization_config[0].access_point_id == aws_efs_access_point.vm.id + error_message = "The vm-data volume must mount through the EFS access point (for non-root UID enforcement)." + } + + assert { + condition = can(aws_ecs_service.vm.id) + error_message = "ECS service was not created." + } +} diff --git a/infrastructure/modules/observability/tests/setup/.terraform.lock.hcl b/infrastructure/modules/observability/tests/setup/.terraform.lock.hcl new file mode 100644 index 0000000000..36c63c7eb3 --- /dev/null +++ b/infrastructure/modules/observability/tests/setup/.terraform.lock.hcl @@ -0,0 +1,28 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.58.0" + constraints = "~> 6.58.0" + hashes = [ + "h1:2kpake4zZKRX5437QVIRU3qFYH6Bjw/QE1fgVCPOrUg=", + "h1:OWl47Bo8Vzlf5srTUCmA6v4kvQGfah/P1joRtIYUUMc=", + "h1:UFot9S97tuAPvjKvoxm08sDG/gKYdDK+lMwsZKtLieY=", + "zh:1221253beee5629fb503d79cebc9bc661279cbc4be5d01db9ab4c1b702108250", + "zh:132bd0925bdc4b72446ac750b7ccb1e19b9ba8fbb6df57b2c1423314d2195d4f", + "zh:18cda250b9e82b753808715893c8927f132273c00ffae7a697d65ac1cb577e48", + "zh:204c944f1fb7f440a335bb2083c9691a9d1f677aea9701025dd5816aee41f0ba", + "zh:2dc41df289f2b10a01e650cdd73699955f0ab0645d09cfb114a8cd0f4cc4ede7", + "zh:345633dfa9a234659d52aadd126e6dce658518c3ab5cbf6d871221287ed5ec56", + "zh:4dadcced73e742903158bc9838936d911f3fa4c2c37b5591c1a28f8f2a1902a6", + "zh:5bc60cc2b8c093da98b211d9f6c21c9ecec0f21b944d9ecbe3961fba33086e80", + "zh:6cc8f084938b0033a9c0c910989919dad6b1683e76e0afa1a5c604e39f398a75", + "zh:7db214647f79de9a033b5dfd6cbfaa42d53c4d056b32cb3acc7ad99306dd548a", + "zh:9078589ec881cee7ed9403af262c98ff257fb3e1baae72ff6429a398b1c730af", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:bd5bce6aec4d4922b1127b8575688bd4bc4279670ee28d198ede404709826c7c", + "zh:cd900ecf56d21023873898b06e40234f3f4d350f2343b7d9b980d6c5cb604fae", + "zh:dbe93b276a84421026b956c3c5b4eb8897da6cbb54b93cb89f5d6ebbd30805ca", + "zh:f6b6c7bb2dbf04ee085e5c22f7a65b3ccaebf368ed95584dc0dfee8a22771056", + ] +} diff --git a/infrastructure/modules/observability/tests/setup/main.tf b/infrastructure/modules/observability/tests/setup/main.tf new file mode 100644 index 0000000000..6d7c7d179c --- /dev/null +++ b/infrastructure/modules/observability/tests/setup/main.tf @@ -0,0 +1,38 @@ +terraform { + required_version = "~> 1.15.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.58.0" + } + } +} + +data "aws_availability_zones" "available" { + state = "available" +} + +resource "aws_vpc" "main" { + cidr_block = "10.0.0.0/16" +} + +resource "aws_subnet" "main" { + count = 2 + + availability_zone = data.aws_availability_zones.available.names[count.index] + cidr_block = cidrsubnet(aws_vpc.main.cidr_block, 8, count.index) + vpc_id = aws_vpc.main.id +} + +resource "aws_security_group" "app" { + count = 3 + + name = "nest-test-app-${count.index}" + vpc_id = aws_vpc.main.id +} + +resource "aws_kms_key" "main" { + description = "Test key for the observability integration tests." + enable_key_rotation = true +} diff --git a/infrastructure/modules/observability/tests/setup/outputs.tf b/infrastructure/modules/observability/tests/setup/outputs.tf new file mode 100644 index 0000000000..50efb66319 --- /dev/null +++ b/infrastructure/modules/observability/tests/setup/outputs.tf @@ -0,0 +1,19 @@ +output "app_security_group_ids" { + description = "The IDs of the stand-in application security groups." + value = aws_security_group.app[*].id +} + +output "kms_key_arn" { + description = "The ARN of the stand-in KMS key." + value = aws_kms_key.main.arn +} + +output "subnet_ids" { + description = "The IDs of the stand-in subnets." + value = aws_subnet.main[*].id +} + +output "vpc_id" { + description = "The ID of the stand-in VPC." + value = aws_vpc.main.id +} diff --git a/infrastructure/modules/observability/tests/setup/variables.tf b/infrastructure/modules/observability/tests/setup/variables.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/infrastructure/modules/observability/tests/unit.tftest.hcl b/infrastructure/modules/observability/tests/unit.tftest.hcl new file mode 100644 index 0000000000..ea373243cc --- /dev/null +++ b/infrastructure/modules/observability/tests/unit.tftest.hcl @@ -0,0 +1,173 @@ +mock_provider "aws" {} + +variables { + app_security_group_ids = ["sg-backend", "sg-frontend", "sg-tasks"] + aws_region = "us-east-2" + common_tags = { Environment = "test", Project = "nest" } + environment = "test" + kms_key_arn = "arn:aws:kms:us-east-2:123456789012:key/12345678-1234-1234-1234-123456789012" + project_name = "nest" + subnet_ids = ["subnet-1", "subnet-2"] + image = regex("(?m)^FROM (victoriametrics/victoria-metrics:\\S+)", file("../../../docker/victoriametrics/Dockerfile"))[0] + port = 8428 + vpc_id = "vpc-12345" +} + +run "test_efs_encryption_enabled" { + command = plan + + assert { + condition = aws_efs_file_system.vm.encrypted == true + error_message = "EFS must be encrypted at rest." + } +} + +run "test_efs_uses_kms_key" { + command = plan + + assert { + condition = aws_efs_file_system.vm.kms_key_id == var.kms_key_arn + error_message = "EFS must be encrypted with the provided KMS key." + } +} + +run "test_efs_mount_target_per_subnet" { + command = plan + + assert { + condition = length(aws_efs_mount_target.vm) == length(var.subnet_ids) + error_message = "There must be one EFS mount target per subnet." + } +} + +run "test_vm_ingest_rule_per_app_security_group" { + command = plan + + assert { + condition = length(aws_security_group_rule.vm_ingest_from_apps) == length(var.app_security_group_ids) + error_message = "There must be one VM ingest rule per application security group." + } +} + +run "test_vm_ingest_from_source_security_group_only" { + command = plan + + assert { + condition = aws_security_group_rule.vm_ingest_from_apps[0].source_security_group_id == var.app_security_group_ids[0] + error_message = "VM ingest must be restricted to application security groups, not public CIDRs." + } + + assert { + condition = aws_security_group_rule.vm_ingest_from_apps[0].from_port == var.port + error_message = "VM ingest must be allowed on the configured VictoriaMetrics port." + } +} + +run "test_efs_ingress_from_vm_only" { + command = plan + + assert { + condition = aws_security_group_rule.efs_from_vm.from_port == 2049 && aws_security_group_rule.efs_from_vm.type == "ingress" + error_message = "EFS must only allow NFS ingress on port 2049." + } +} + +run "test_vm_service_is_single_task" { + command = plan + + assert { + condition = aws_ecs_service.vm.desired_count == 1 + error_message = "VictoriaMetrics must run as a single task." + } +} + +run "test_vm_service_stops_before_starting" { + command = plan + + assert { + condition = aws_ecs_service.vm.deployment_minimum_healthy_percent == 0 && aws_ecs_service.vm.deployment_maximum_percent == 100 + error_message = "Deployments must stop the old task before starting the new one to avoid two writers on EFS." + } +} + +run "test_vm_uses_on_demand_fargate_only" { + command = plan + + assert { + condition = length(aws_ecs_cluster_capacity_providers.vm.capacity_providers) == 1 && contains(aws_ecs_cluster_capacity_providers.vm.capacity_providers, "FARGATE") + error_message = "VictoriaMetrics must use on-demand FARGATE only, never FARGATE_SPOT." + } +} + +run "test_task_uses_arm64" { + command = plan + + assert { + condition = aws_ecs_task_definition.vm.runtime_platform[0].cpu_architecture == "ARM64" + error_message = "The VictoriaMetrics task must run on ARM64." + } +} + +run "test_container_runs_as_non_root" { + command = plan + + assert { + condition = jsondecode(aws_ecs_task_definition.vm.container_definitions)[0].user == "65532" + error_message = "The VictoriaMetrics container must run as the non-root user 65532." + } +} + +run "test_access_point_enforces_non_root_owner" { + command = plan + + assert { + condition = aws_efs_access_point.vm.posix_user[0].uid == 65532 && aws_efs_access_point.vm.posix_user[0].gid == 65532 + error_message = "The EFS access point must enforce the non-root POSIX user 65532." + } +} + +run "test_task_mounts_encrypted_efs_volume" { + command = plan + + assert { + condition = one([for v in aws_ecs_task_definition.vm.volume : v if v.name == "vm-data"]).efs_volume_configuration[0].transit_encryption == "ENABLED" + error_message = "The vm-data volume must enable transit encryption." + } +} + +run "test_log_group_name_and_retention" { + command = plan + + assert { + condition = aws_cloudwatch_log_group.vm.name == "/aws/ecs/${var.project_name}-${var.environment}-observability" + error_message = "CloudWatch log group name must follow the /aws/ecs/{project}-{environment}-observability format." + } + + assert { + condition = aws_cloudwatch_log_group.vm.retention_in_days == var.log_retention_in_days + error_message = "CloudWatch log group must use the configured retention." + } +} + +run "test_cluster_name_format" { + command = plan + + assert { + condition = aws_ecs_cluster.vm.name == "${var.project_name}-${var.environment}-observability-cluster" + error_message = "ECS cluster name must follow the {project}-{environment}-observability-cluster format." + } +} + +run "test_common_tags_applied" { + command = plan + + assert { + condition = alltrue([for k, v in var.common_tags : lookup(aws_efs_file_system.vm.tags, k, null) == v]) + error_message = "common_tags must be applied to the EFS file system." + } + + assert { + condition = alltrue([for k, v in var.common_tags : lookup(aws_ecs_cluster.vm.tags, k, null) == v]) + error_message = "common_tags must be applied to the ECS cluster." + } +} diff --git a/infrastructure/modules/observability/variables.tf b/infrastructure/modules/observability/variables.tf new file mode 100644 index 0000000000..2a637f315d --- /dev/null +++ b/infrastructure/modules/observability/variables.tf @@ -0,0 +1,112 @@ +variable "app_security_group_ids" { + description = "Security group IDs of the application tasks allowed to send metrics to the observability backend." + type = list(string) + + validation { + condition = length(var.app_security_group_ids) > 0 + error_message = "app_security_group_ids must contain at least one security group." + } +} + +variable "assign_public_ip" { + description = "Whether to assign a public IP to the observability task." + type = bool + default = false +} + +variable "aws_region" { + description = "The AWS region where the module is deployed." + type = string +} + +variable "common_tags" { + description = "A map of common tags to apply to all resources." + type = map(string) + default = {} +} + +variable "environment" { + description = "The environment (e.g., staging, production)." + type = string +} + +variable "kms_key_arn" { + description = "The ARN of the KMS key used to encrypt the EFS file system." + type = string +} + +variable "log_retention_in_days" { + description = "The number of days to retain observability container logs." + type = number + default = 90 +} + +variable "project_name" { + description = "The name of the project." + type = string +} + +variable "subnet_ids" { + description = "The private subnet IDs for the EFS mount targets and the observability task." + type = list(string) + + validation { + condition = length(var.subnet_ids) > 0 + error_message = "subnet_ids must contain at least one subnet." + } +} + +variable "cpu" { + description = "The CPU units for the observability Fargate task." + type = number + default = 512 +} + +variable "desired_count" { + description = "The number of observability tasks to run (0 or 1; the current backend is a single-node store)." + type = number + default = 1 + + validation { + condition = contains([0, 1], var.desired_count) + error_message = "desired_count must be 0 or 1 because the current observability backend is a single-node store." + } +} + +variable "image" { + description = "The observability backend container image (including digest)." + type = string + + validation { + condition = can(regex("^[^@]+@sha256:[0-9a-f]{64}$", var.image)) + error_message = "image must be an image reference pinned to an immutable digest (e.g., repo:tag@sha256:...)." + } +} + +variable "memory" { + description = "The memory (in MiB) for the observability Fargate task." + type = number + default = 1024 +} + +variable "port" { + description = "The port the observability backend listens on for ingest and queries." + type = number + default = 8428 + + validation { + condition = var.port > 0 && var.port < 65536 && floor(var.port) == var.port + error_message = "port must be a whole number between 1 and 65535." + } +} + +variable "retention_period" { + description = "The VictoriaMetrics data retention period. A value without a suffix is in months, so the default \"12\" means 12 months (duration suffixes like 1y, 30d, 1w are also supported)." + type = string + default = "12" # 12 months +} + +variable "vpc_id" { + description = "The VPC ID where the observability backend security group is created." + type = string +} diff --git a/infrastructure/scripts/localstack.py b/infrastructure/scripts/localstack.py index a793b0f1da..ee6f82feb9 100644 --- a/infrastructure/scripts/localstack.py +++ b/infrastructure/scripts/localstack.py @@ -256,6 +256,10 @@ class OverrideManager: # Temporary Terraform overrides that disable prevent_destroy during integration tests. OVERRIDES: list[tuple[str, str]] = [ + ( + "infrastructure/modules/observability/test_override.tf", + "aws_efs_file_system.vm", + ), ( "infrastructure/modules/storage/modules/s3-bucket/test_override.tf", "aws_s3_bucket.this",