diff --git a/misc/roll_apps.rb b/misc/roll_apps.rb index 880f966..7098049 100755 --- a/misc/roll_apps.rb +++ b/misc/roll_apps.rb @@ -6,12 +6,13 @@ gem 'aws-sdk-ecr' end +FILES = %W[k8s/**/*.{j,lib}sonnet tf/**/*.tf] REGISTRIES = %w[005216166247] TAG_RE = %r{(?\d+)\.dkr\.ecr\.(?[\w-]+)\.amazonaws\.com/(?[\w/-]+):\K(?[0-9a-f]{40})} @ecr = Hash.new {|h, region| h[region] = Aws::ECR::Client.new(region:) } -Pathname(__dir__).glob('../k8s/**/*.{j,lib}sonnet') do |path| +Pathname(__dir__).parent.glob(FILES) do |path| content = path.read updated = content.gsub!(TAG_RE) do diff --git a/tf/acme-responder/ec2.tf b/tf/acme-responder/ec2.tf index 319c6d4..d3404da 100644 --- a/tf/acme-responder/ec2.tf +++ b/tf/acme-responder/ec2.tf @@ -37,7 +37,6 @@ resource "aws_instance" "acme-responder-apne1c" { user_data = local.user_data - associate_public_ip_address = false ipv6_addresses = [ cidrhost(data.aws_subnet.main-public-c.ipv6_cidr_block, 4294945854) # 0xFFFF_AC3E ] @@ -86,7 +85,6 @@ resource "aws_instance" "acme-responder-apne1d" { user_data = local.user_data - associate_public_ip_address = false ipv6_addresses = [ cidrhost(data.aws_subnet.main-public-d.ipv6_cidr_block, 4294945854) # 0xFFFF_AC3E ] diff --git a/tf/acmesmith/.terraform.lock.hcl b/tf/acmesmith/.terraform.lock.hcl new file mode 100644 index 0000000..902d05d --- /dev/null +++ b/tf/acmesmith/.terraform.lock.hcl @@ -0,0 +1,63 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.39.0" + constraints = ">= 5.0.0" + hashes = [ + "h1:jweey4Iefm/DuuBg84saQ8vz5IO3vC6hDFTU/eGdmBI=", + "zh:00c3e3c38063ff629d6fdbce04e9ac2e241566e0f5ad5399c335f0abdefd7bff", + "zh:148f95b62791080537d926b9d2f5d8457cca45921d9b1019d03ceb3ab93bf9db", + "zh:203da629ed5191dd5d7aa3427a5d1d1a83eed5c1b0114166897206973f0d0fd0", + "zh:21923eedbc60b4f68c8d717b951d16b0b1bbf31d66330c7be228869bec18f7ce", + "zh:26226f02e3661b3d071c01601b654a308b29d21758b75692bec66f70c6f6b945", + "zh:271c7c6fadcd8ac7ed37c11e61c0f374773eaaa5293703499f8a0f75830060e0", + "zh:46e319a8888dc50ed8d26a1cbee9637f529112a88f5d44decc8f1d10ef968ffe", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a3c3ca09cdbf3b9a3f892a23c000ff04772bdf19f626959ea83d0803c8fd2350", + "zh:a5fa6515ffc3c815e0d2204d67e838f5bad8635009dab85211d166c7ae729d2c", + "zh:c0807566b4ddde8390f50c5475464103f066bc7f511a6c0be762d75cb6d1a078", + "zh:da754a529fd0e06ac372f62d88566f85a8c4bcec7ee9a231b65e0a0148165e63", + "zh:dcb768e48363a9f4dffaf2dc7d01f1877285528925ec50de6335286298e37e1d", + "zh:eac9de9d123c679ea3035199fb9c588a08cda281cbabf948dc696e2a1a1b9063", + "zh:fef276b6331c663ca0e60dc7f637b2b8244825b8c9bc721481957e58f74ffb4f", + ] +} + +provider "registry.terraform.io/hashicorp/external" { + version = "2.3.5" + hashes = [ + "h1:smKSos4zs57pJjQrNuvGBpSWth2el9SgePPbPHo0aps=", + "zh:6e89509d056091266532fa64de8c06950010498adf9070bf6ff85bc485a82562", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:86868aec05b58dc0aa1904646a2c26b9367d69b890c9ad70c33c0d3aa7b1485a", + "zh:a2ce38fda83a62fa5fb5a70e6ca8453b168575feb3459fa39803f6f40bd42154", + "zh:a6c72798f4a9a36d1d1433c0372006cc9b904e8cfd60a2ae03ac5b7d2abd2398", + "zh:a8a3141d2fc71c86bf7f3c13b0b3be8a1b0f0144a47572a15af4dfafc051e28a", + "zh:aa20a1242eb97445ad26ebcfb9babf2cd675bdb81cac5f989268ebefa4ef278c", + "zh:b58a22445fb8804e933dcf835ab06c29a0f33148dce61316814783ee7f4e4332", + "zh:cb5626a661ee761e0576defb2a2d75230a3244799d380864f3089c66e99d0dcc", + "zh:d1acb00d20445f682c4e705c965e5220530209c95609194c2dc39324f3d4fcce", + "zh:d91a254ba77b69a29d8eae8ed0e9367cbf0ea6ac1a85b58e190f8cb096a40871", + "zh:f6592327673c9f85cdb6f20336faef240abae7621b834f189c4a62276ea5db41", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "3.0.1" + hashes = [ + "h1:vyHdH0p6bf9xp1NPePObAJkXTJb/I09FQQmmevTzZe0=", + "zh:02d55b0b2238fd17ffa12d5464593864e80f402b90b31f6e1bd02249b9727281", + "zh:20b93a51bfeed82682b3c12f09bac3031f5bdb4977c47c97a042e4df4fb2f9ba", + "zh:6e14486ecfaee38c09ccf33d4fdaf791409f90795c1b66e026c226fad8bc03c7", + "zh:8d0656ff422df94575668e32c310980193fccb1c28117e5c78dd2d4050a760a6", + "zh:9795119b30ec0c1baa99a79abace56ac850b6e6fbce60e7f6067792f6eb4b5f4", + "zh:b388c87acc40f6bd9620f4e23f01f3c7b41d9b88a68d5255dec0a72f0bdec249", + "zh:b59abd0a980649c2f97f172392f080eaeb18e486b603f83bf95f5d93aeccc090", + "zh:ba6e3060fddf4a022087d8f09e38aa0001c705f21170c2ded3d1c26c12f70d97", + "zh:c12626d044b1d5501cf95ca78cbe507c13ad1dd9f12d4736df66eb8e5f336eb8", + "zh:c55203240d50f4cdeb3df1e1760630d677679f5b1a6ffd9eba23662a4ad05119", + "zh:ea206a5a32d6e0d6e32f1849ad703da9a28355d9c516282a8458b5cf1502b2a1", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/tf/acmesmith/acmesmith.tf b/tf/acmesmith/acmesmith.tf new file mode 100644 index 0000000..db67f03 --- /dev/null +++ b/tf/acmesmith/acmesmith.tf @@ -0,0 +1,241 @@ +locals { + image = "005216166247.dkr.ecr.ap-northeast-1.amazonaws.com/acmesmith:aec356de54ac789e482af0043cdd002b4e5ba9ab" + + hosted_zones = [data.aws_route53_zone.rubykaigi_net] + hosted_zone_map = { + for z in local.hosted_zones : "${z.name}." => "/hostedzone/${z.zone_id}" + } +} + +data "aws_route53_zone" "rubykaigi_net" { + name = "rubykaigi.net." + private_zone = false +} + +data "aws_s3_bucket" "rubykaigi-public" { + bucket = "rubykaigi-public" +} + +data "external" "letsencrypt-staging" { + program = ["${path.module}/../jsonnet.rb"] + query = { + path = "${path.module}/letsencrypt.jsonnet" + args = jsonencode({ + staging = true + hosted_zone_map = local.hosted_zone_map + bucket = data.aws_s3_bucket.rubykaigi-public.bucket + }) + } +} + +data "external" "letsencrypt" { + program = ["${path.module}/../jsonnet.rb"] + query = { + path = "${path.module}/letsencrypt.jsonnet" + args = jsonencode({ + staging = false + hosted_zone_map = local.hosted_zone_map + bucket = data.aws_s3_bucket.rubykaigi-public.bucket + }) + } +} + +resource "kubernetes_config_map_v1" "letsencrypt-staging" { + metadata { + name = "acmesmith-letsencrypt-staging" + namespace = "default" + } + data = { + "acmesmith.yml" = data.external.letsencrypt-staging.result.json + } +} + +resource "kubernetes_config_map_v1" "letsencrypt" { + metadata { + name = "acmesmith-letsencrypt" + namespace = "default" + } + data = { + "acmesmith.yml" = data.external.letsencrypt.result.json + } +} + +resource "kubernetes_service_account_v1" "acmesmith" { + metadata { + name = "acmesmith" + namespace = "default" + annotations = { + "eks.amazonaws.com/role-arn" = aws_iam_role.acmesmith.arn + "eks.amazonaws.com/sts-regional-endpoints" = true + } + } +} + +resource "kubernetes_role_v1" "acmesmith" { + metadata { + name = "acmesmith" + namespace = "default" + } + rule { + api_groups = [""] + resources = ["secrets"] + verbs = ["get", "list", "watch", "create", "update", "patch", "delete"] + } + rule { + api_groups = ["apps"] + resources = ["deployments"] + verbs = ["get", "list", "patch", "update"] + } +} + +resource "kubernetes_role_binding_v1" "acmesmith" { + metadata { + name = "acmesmith" + namespace = "default" + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "Role" + name = kubernetes_role_v1.acmesmith.metadata[0].name + } + subject { + kind = "ServiceAccount" + name = kubernetes_service_account_v1.acmesmith.metadata[0].name + namespace = "default" + } +} + +resource "kubernetes_job_v1" "new-account" { + for_each = tomap({ + letsencrypt-staging = kubernetes_config_map_v1.letsencrypt-staging + letsencrypt = kubernetes_config_map_v1.letsencrypt + }) + + metadata { + name = "acmesmith-new-account-${each.key}" + namespace = "default" + } + spec { + template { + metadata {} + spec { + container { + name = "acmesmith" + image = local.image + args = [ + "new-account", + "-c", "/config/acmesmith/acmesmith.yml", + "mailto:info@rubykaigi.org", + ] + volume_mount { + name = "config" + mount_path = "/config/acmesmith" + read_only = true + } + } + volume { + name = "config" + config_map { + name = each.value.metadata[0].name + } + } + service_account_name = kubernetes_service_account_v1.acmesmith.metadata[0].name + restart_policy = "Never" + } + } + } + + wait_for_completion = false +} + +resource "kubernetes_cron_job_v1" "autorenew" { + for_each = tomap({ + letsencrypt-staging = kubernetes_config_map_v1.letsencrypt-staging + letsencrypt = kubernetes_config_map_v1.letsencrypt + }) + + metadata { + name = "acmesmith-autorenew-${each.key}" + namespace = "default" + } + spec { + schedule = "0 * * * *" + job_template { + metadata {} + spec { + template { + metadata {} + spec { + container { + name = "acmesmith" + image = local.image + args = [ + "autorenew", + "-c", "/config/acmesmith/acmesmith.yml", + "-r", "1/2", + ] + volume_mount { + name = "config" + mount_path = "/config/acmesmith" + read_only = true + } + } + volume { + name = "config" + config_map { + name = each.value.metadata[0].name + } + } + service_account_name = kubernetes_service_account_v1.acmesmith.metadata[0].name + restart_policy = "Never" + } + } + } + } + } +} + +resource "kubernetes_job_v1" "order-resolver-rubykaigi-net" { + for_each = tomap({ + letsencrypt = kubernetes_config_map_v1.letsencrypt + }) + + metadata { + name = "acmesmith-order-resolver-rubykaigi-net-${each.key}" + namespace = "default" + } + spec { + template { + metadata {} + spec { + container { + name = "acmesmith" + image = local.image + args = [ + "order", + "-c", "/config/acmesmith/acmesmith.yml", + "--key-type", "ec", "--elliptic-curve", "prime256v1", + "resolver.rubykaigi.net", + "192.50.220.164", "192.50.220.165", + "2001:df0:8500:ca6d:53::0:c", "2001:df0:8500:ca6d:53:0::d", + ] + volume_mount { + name = "config" + mount_path = "/config/acmesmith" + read_only = true + } + } + volume { + name = "config" + config_map { + name = each.value.metadata[0].name + } + } + service_account_name = kubernetes_service_account_v1.acmesmith.metadata[0].name + restart_policy = "Never" + } + } + } + + wait_for_completion = false +} diff --git a/tf/acmesmith/aws.tf b/tf/acmesmith/aws.tf new file mode 100644 index 0000000..fdfa053 --- /dev/null +++ b/tf/acmesmith/aws.tf @@ -0,0 +1,12 @@ +provider "aws" { + region = "ap-northeast-1" + allowed_account_ids = ["005216166247"] + default_tags { + tags = { + Project = "rk26net" + Component = "acmesmith" + } + } +} + +data "aws_caller_identity" "current" {} diff --git a/tf/acmesmith/backend.tf b/tf/acmesmith/backend.tf new file mode 100644 index 0000000..d18075a --- /dev/null +++ b/tf/acmesmith/backend.tf @@ -0,0 +1,17 @@ +terraform { + backend "s3" { + bucket = "rk-infra" + region = "ap-northeast-1" + key = "terraform/nw-acmesmith.tfstate" + dynamodb_table = "rk-terraform" + } +} + +data "terraform_remote_state" "k8s" { + backend = "s3" + config = { + bucket = "rk-infra" + region = "ap-northeast-1" + key = "terraform/nw-k8s.tfstate" + } +} diff --git a/tf/acmesmith/ecr.tf b/tf/acmesmith/ecr.tf new file mode 100644 index 0000000..0fa1c17 --- /dev/null +++ b/tf/acmesmith/ecr.tf @@ -0,0 +1,23 @@ +resource "aws_ecr_repository" "acmesmith" { + name = "acmesmith" +} + +resource "aws_ecr_lifecycle_policy" "acmesmith" { + repository = aws_ecr_repository.acmesmith.name + policy = jsonencode({ + rules = [ + { + rulePriority = 10 + description = "expire old images" + selection = { + tagStatus = "any" + countType = "imageCountMoreThan" + countNumber = 10 + } + action = { + type = "expire" + } + } + ] + }) +} diff --git a/tf/acmesmith/iam.tf b/tf/acmesmith/iam.tf new file mode 100644 index 0000000..623cc70 --- /dev/null +++ b/tf/acmesmith/iam.tf @@ -0,0 +1,52 @@ +data "aws_iam_policy" "nocadmin-base" { + name = "NocAdminBase" +} + +resource "aws_iam_role" "acmesmith" { + name = "NwAcmesmith" + description = "k8s acmesmith" + assume_role_policy = data.aws_iam_policy_document.acmesmith-trust.json + permissions_boundary = data.aws_iam_policy.nocadmin-base.arn +} + +resource "aws_iam_role_policy" "acmesmith" { + role = aws_iam_role.acmesmith.name + name = "acmesmith" + policy = data.aws_iam_policy_document.acmesmith-policy.json +} + +data "aws_iam_policy_document" "acmesmith-policy" { + statement { + effect = "Allow" + actions = ["route53:ChangeResourceRecordSets", "route53:ListResourceRecordSets"] + resources = ["arn:aws:route53:::hostedzone/${data.aws_route53_zone.rubykaigi_net.zone_id}"] + } + + statement { + effect = "Allow" + actions = ["s3:PutObject", "s3:DeleteObject"] + resources = ["${data.aws_s3_bucket.rubykaigi-public.arn}/*"] + } + + statement { + effect = "Allow" + actions = ["route53:ListHostedZones", "route53:GetChange"] + resources = ["*"] + } +} + +data "aws_iam_policy_document" "acmesmith-trust" { + statement { + actions = ["sts:AssumeRoleWithWebIdentity"] + effect = "Allow" + principals { + type = "Federated" + identifiers = [local.cluster_oidc_config.arn] + } + condition { + test = "StringEquals" + variable = local.cluster_oidc_config.condition + values = ["system:serviceaccount:default:acmesmith"] + } + } +} diff --git a/tf/acmesmith/kubernetes.tf b/tf/acmesmith/kubernetes.tf new file mode 100644 index 0000000..302eb19 --- /dev/null +++ b/tf/acmesmith/kubernetes.tf @@ -0,0 +1,14 @@ +locals { + cluster_config = data.terraform_remote_state.k8s.outputs.cluster_config + cluster_oidc_config = data.terraform_remote_state.k8s.outputs.cluster_oidc_config +} + +provider "kubernetes" { + host = local.cluster_config.endpoint + cluster_ca_certificate = base64decode(local.cluster_config.ca_data) + exec { + api_version = "client.authentication.k8s.io/v1beta1" + args = ["eks", "get-token", "--region", "ap-northeast-1", "--cluster-name", local.cluster_config.name] + command = "aws" + } +} diff --git a/tf/acmesmith/letsencrypt.jsonnet b/tf/acmesmith/letsencrypt.jsonnet new file mode 100644 index 0000000..9272b7c --- /dev/null +++ b/tf/acmesmith/letsencrypt.jsonnet @@ -0,0 +1,63 @@ +function(args) { + staging:: args.staging, + + directory: ( + if $.staging + then 'https://acme-staging-v02.api.letsencrypt.org/directory' + else 'https://acme-v02.api.letsencrypt.org/directory' + ), + + storage: { + type: 'kubernetes_secrets', + instance: if $.staging then 'letsencrypt-staging' else 'letsencrypt', + }, + + challenge_responders: [ + { + s3: { + bucket: args.bucket, + prefix: 'rknet/acme-http-01/', + }, + filter: { + subject_name_cidr: [ + '192.50.220.164/31', + '2001:df0:8500:ca6d:53::c/127', + ], + }, + }, + { + route53: { + hosted_zone_map: args.hosted_zone_map, + }, + }, + ], + profiles: [ + { name: 'shortlived' }, + ], + + post_issuing_hooks: ( + if $.staging + then {} + else { + 'resolver.rubykaigi.net': [ + { + kubernetes_secret: { + name: 'cert-resolver-rubykaigi-net', + }, + }, + { + kubernetes_rollout: { + kind: 'Deployment', + selector: 'rubykaigi.org/app=unbound', + }, + }, + { + kubernetes_rollout: { + kind: 'Deployment', + selector: 'rubykaigi.org/app=unbound-envoy', + }, + }, + ], + } + ), +} diff --git a/tf/acmesmith/versions.tf b/tf/acmesmith/versions.tf new file mode 100644 index 0000000..f3dca06 --- /dev/null +++ b/tf/acmesmith/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 3" + } + } + required_version = ">= 0.13" +} diff --git a/tf/core/iam_gha_docker-push.tf b/tf/core/iam_gha_docker-push.tf index 971ded6..784a24f 100644 --- a/tf/core/iam_gha_docker-push.tf +++ b/tf/core/iam_gha_docker-push.tf @@ -72,6 +72,7 @@ data "aws_iam_policy_document" "GhaDockerPush" { "ecr:UploadLayerPart", ] resources = [ + "arn:aws:ecr:ap-northeast-1:${data.aws_caller_identity.current.account_id}:repository/acmesmith", "arn:aws:ecr:ap-northeast-1:${data.aws_caller_identity.current.account_id}:repository/kea", "arn:aws:ecr:ap-northeast-1:${data.aws_caller_identity.current.account_id}:repository/fluentd", "arn:aws:ecr:ap-northeast-1:${data.aws_caller_identity.current.account_id}:repository/unbound",