From 1326ce8fdf3353344d534edf22629cfed286708e Mon Sep 17 00:00:00 2001 From: David Whittington Date: Thu, 23 Jul 2026 01:03:41 +0000 Subject: [PATCH 1/4] feat(cloudbuild): run gce containers from startup scripts --- cloudbuild/create-instance-template.sh | 43 +++++ cloudbuild/gce-startup.sh | 98 ++++++++++++ test/cloudbuild/gce_scripts_test.sh | 207 +++++++++++++++++++++++++ 3 files changed, 348 insertions(+) create mode 100755 cloudbuild/create-instance-template.sh create mode 100755 cloudbuild/gce-startup.sh create mode 100755 test/cloudbuild/gce_scripts_test.sh diff --git a/cloudbuild/create-instance-template.sh b/cloudbuild/create-instance-template.sh new file mode 100755 index 0000000000..74c4f52631 --- /dev/null +++ b/cloudbuild/create-instance-template.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +set -euo pipefail + +if [[ $# -lt 2 ]]; then + echo "Usage: $0 PROJECT TEMPLATE [GCLOUD_CREATE_ARGS...]" >&2 + exit 2 +fi + +readonly PROJECT="$1" +readonly TEMPLATE="$2" +readonly GCLOUD_BIN="${GCLOUD_BIN:-gcloud}" +shift 2 + +metadata_has_key() { + local description="$1" + local key="$2" + + grep -Eq "\"key\"[[:space:]]*:[[:space:]]*\"${key}\"" <<<"${description}" +} + +if description="$("${GCLOUD_BIN}" compute instance-templates describe "${TEMPLATE}" \ + --project="${PROJECT}" \ + --format=json 2>/dev/null)"; then + if metadata_has_key "${description}" "gce-container-declaration"; then + echo "Existing template ${TEMPLATE} still uses gce-container-declaration" >&2 + exit 1 + fi + + for key in startup-script shutdown-script logflare-container-env logflare-container-image; do + if ! metadata_has_key "${description}" "${key}"; then + echo "Existing template ${TEMPLATE} is missing ${key} metadata" >&2 + exit 1 + fi + done + + echo "Instance template already exists with startup-script metadata: ${TEMPLATE}" + exit 0 +fi + +"${GCLOUD_BIN}" compute instance-templates create "${TEMPLATE}" \ + --project="${PROJECT}" \ + "$@" diff --git a/cloudbuild/gce-startup.sh b/cloudbuild/gce-startup.sh new file mode 100755 index 0000000000..b06d2298bf --- /dev/null +++ b/cloudbuild/gce-startup.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +set -euo pipefail + +readonly METADATA_URL="http://metadata.google.internal/computeMetadata/v1/instance/attributes" +readonly CONTAINER_NAME="logflare" +readonly CONTAINER_ENV_FILE="${LOGFLARE_CONTAINER_ENV_FILE:-/run/logflare-container.env}" +readonly DOCKER_HOME="${LOGFLARE_DOCKER_HOME:-/home/logflare}" + +metadata() { + local key="$1" + + curl \ + --fail \ + --silent \ + --show-error \ + --connect-timeout 5 \ + --max-time 30 \ + --retry 10 \ + --retry-connrefused \ + --retry-delay 2 \ + --header "Metadata-Flavor: Google" \ + "${METADATA_URL}/${key}" +} + +wait_for_docker() { + local attempt + + for attempt in {1..30}; do + if docker info >/dev/null 2>&1; then + return 0 + fi + + sleep 2 + done + + echo "Docker did not become ready after 60 seconds" >&2 + return 1 +} + +configure_firewall() { + local chain + local protocol + + # Konlet opened these host firewall paths before starting the container. + # Keep the rules idempotent because startup scripts can be rerun. + for protocol in tcp udp icmp; do + for chain in INPUT FORWARD; do + if ! iptables -C "${chain}" -p "${protocol}" -j ACCEPT 2>/dev/null; then + iptables -A "${chain}" -p "${protocol}" -j ACCEPT + fi + done + done +} + +main() { + local image + + umask 077 + + image="$(metadata logflare-container-image)" + if [[ -z "${image}" ]]; then + echo "logflare-container-image metadata is empty" >&2 + return 1 + fi + + metadata logflare-container-env >"${CONTAINER_ENV_FILE}" + if [[ ! -s "${CONTAINER_ENV_FILE}" ]]; then + echo "logflare-container-env metadata is empty" >&2 + return 1 + fi + + export HOME="${DOCKER_HOME}" + install -d -m 0700 "${HOME}" + docker-credential-gcr configure-docker --registries=gcr.io + + configure_firewall + wait_for_docker + + # Keep the existing container available if the registry is temporarily + # unavailable during a manual startup-script rerun. + docker pull "${image}" + docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true + + docker run \ + --name="${CONTAINER_NAME}" \ + --privileged \ + --restart=always \ + --network=host \ + --detach \ + --log-driver=json-file \ + --log-opt max-size=500m \ + --log-opt max-file=3 \ + --env-file "${CONTAINER_ENV_FILE}" \ + "${image}" +} + +main "$@" diff --git a/test/cloudbuild/gce_scripts_test.sh b/test/cloudbuild/gce_scripts_test.sh new file mode 100755 index 0000000000..11df29f593 --- /dev/null +++ b/test/cloudbuild/gce_scripts_test.sh @@ -0,0 +1,207 @@ +#!/bin/bash + +set -euo pipefail + +readonly REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +readonly STARTUP_SCRIPT="${REPO_ROOT}/cloudbuild/gce-startup.sh" +readonly CREATE_SCRIPT="${REPO_ROOT}/cloudbuild/create-instance-template.sh" +readonly TEST_DIR="$(mktemp -d)" + +trap 'rm -rf "${TEST_DIR}"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +assert_contains() { + local file="$1" + local expected="$2" + + grep -Fq -- "${expected}" "${file}" || fail "${file} does not contain: ${expected}" +} + +assert_not_contains() { + local file="$1" + local unexpected="$2" + + if grep -Fq -- "${unexpected}" "${file}"; then + fail "${file} unexpectedly contains: ${unexpected}" + fi +} + +install_startup_stubs() { + local bin_dir="$1" + + mkdir -p "${bin_dir}" + + cat >"${bin_dir}/curl" <<'STUB' +#!/bin/bash +url="${!#}" +printf 'curl %s\n' "${url}" >>"${CALL_LOG}" +case "${url}" in + */logflare-container-image) + printf '%s' 'gcr.io/test-project/logflare_app:test-tag' + ;; + */logflare-container-env) + printf '%s\n' \ + 'LOGFLARE_GRPC_PORT=50051' \ + 'RELEASE_COOKIE=test-cookie' \ + 'LOGFLARE_METADATA_CLUSTER=test-cluster' + ;; + *) + exit 22 + ;; +esac +STUB + + cat >"${bin_dir}/docker-credential-gcr" <<'STUB' +#!/bin/bash +printf 'credential HOME=%s %s\n' "${HOME}" "$*" >>"${CALL_LOG}" +STUB + + cat >"${bin_dir}/iptables" <<'STUB' +#!/bin/bash +printf 'iptables %s\n' "$*" >>"${CALL_LOG}" +rule="$2 $4" +case "$1" in + -C) + grep -Fxq -- "${rule}" "${IPTABLES_STATE}" 2>/dev/null + ;; + -A) + printf '%s\n' "${rule}" >>"${IPTABLES_STATE}" + ;; + *) + exit 2 + ;; +esac +STUB + + cat >"${bin_dir}/docker" <<'STUB' +#!/bin/bash +printf 'docker %s\n' "$*" >>"${CALL_LOG}" +if [[ "$1" == "pull" && "${DOCKER_PULL_FAIL:-false}" == "true" ]]; then + exit 1 +fi +STUB + + chmod +x "${bin_dir}"/* +} + +run_startup_script() { + local call_log="$1" + local iptables_state="$2" + local docker_home="$3" + local env_file="$4" + local pull_fail="${5:-false}" + + CALL_LOG="${call_log}" \ + IPTABLES_STATE="${iptables_state}" \ + DOCKER_PULL_FAIL="${pull_fail}" \ + LOGFLARE_DOCKER_HOME="${docker_home}" \ + LOGFLARE_CONTAINER_ENV_FILE="${env_file}" \ + PATH="${TEST_DIR}/startup-bin:${PATH}" \ + bash "${STARTUP_SCRIPT}" +} + +test_startup_script() { + local call_log="${TEST_DIR}/startup-calls" + local iptables_state="${TEST_DIR}/iptables-state" + local docker_home="${TEST_DIR}/docker-home" + local env_file="${TEST_DIR}/container.env" + local first_add_count + local pull_line + local remove_line + + : >"${call_log}" + : >"${iptables_state}" + install_startup_stubs "${TEST_DIR}/startup-bin" + + run_startup_script "${call_log}" "${iptables_state}" "${docker_home}" "${env_file}" + + [[ "$(stat -c '%a' "${env_file}")" == "600" ]] || fail "container env file is not mode 600" + assert_contains "${env_file}" "RELEASE_COOKIE=test-cookie" + assert_contains "${call_log}" "credential HOME=${docker_home} configure-docker --registries=gcr.io" + assert_contains "${call_log}" "docker run --name=logflare --privileged --restart=always --network=host --detach --log-driver=json-file --log-opt max-size=500m --log-opt max-file=3 --env-file ${env_file} gcr.io/test-project/logflare_app:test-tag" + + first_add_count="$(grep -c '^iptables -A ' "${call_log}")" + [[ "${first_add_count}" == "6" ]] || fail "expected six firewall rules, got ${first_add_count}" + + pull_line="$(grep -n '^docker pull ' "${call_log}" | head -1 | cut -d: -f1)" + remove_line="$(grep -n '^docker rm ' "${call_log}" | head -1 | cut -d: -f1)" + ((pull_line < remove_line)) || fail "container was removed before the image pull completed" + + run_startup_script "${call_log}" "${iptables_state}" "${docker_home}" "${env_file}" + [[ "$(grep -c '^iptables -A ' "${call_log}")" == "6" ]] || fail "firewall rules were duplicated on rerun" + + : >"${call_log}" + if run_startup_script "${call_log}" "${iptables_state}" "${docker_home}" "${env_file}" true; then + fail "startup script succeeded when docker pull failed" + fi + assert_not_contains "${call_log}" "docker rm -f logflare" +} + +install_gcloud_stub() { + local path="$1" + + cat >"${path}" <<'STUB' +#!/bin/bash +printf 'gcloud %s\n' "$*" >>"${CALL_LOG}" +if [[ "$3" == "describe" ]]; then + case "${GCLOUD_DESCRIBE_MODE}" in + missing) + exit 1 + ;; + good) + printf '%s\n' '{"properties":{"metadata":{"items":[{"key":"startup-script"},{"key":"shutdown-script"},{"key":"logflare-container-env"},{"key":"logflare-container-image"}]}}}' + ;; + deprecated) + printf '%s\n' '{"properties":{"metadata":{"items":[{"key":"gce-container-declaration"}]}}}' + ;; + incomplete) + printf '%s\n' '{"properties":{"metadata":{"items":[{"key":"startup-script"}]}}}' + ;; + esac +fi +STUB + chmod +x "${path}" +} + +run_create_script() { + local mode="$1" + local call_log="$2" + + CALL_LOG="${call_log}" \ + GCLOUD_DESCRIBE_MODE="${mode}" \ + GCLOUD_BIN="${TEST_DIR}/gcloud" \ + bash "${CREATE_SCRIPT}" test-project test-template --machine-type=test-machine +} + +test_create_script() { + local call_log="${TEST_DIR}/gcloud-calls" + + install_gcloud_stub "${TEST_DIR}/gcloud" + + : >"${call_log}" + run_create_script missing "${call_log}" + assert_contains "${call_log}" "gcloud compute instance-templates create test-template --project=test-project --machine-type=test-machine" + + : >"${call_log}" + run_create_script good "${call_log}" + assert_not_contains "${call_log}" "instance-templates create" + + : >"${call_log}" + if run_create_script deprecated "${call_log}"; then + fail "deprecated existing template was accepted" + fi + + : >"${call_log}" + if run_create_script incomplete "${call_log}"; then + fail "incomplete existing template was accepted" + fi +} + +test_startup_script +test_create_script + +echo "gce script tests passed" From c20ae42e7ba7953e5f1adcc291cb52db1cd99bd2 Mon Sep 17 00:00:00 2001 From: David Whittington Date: Thu, 23 Jul 2026 01:06:24 +0000 Subject: [PATCH 2/4] refactor(cloudbuild): remove gce container declarations --- cloudbuild/gce-startup.sh | 7 ++-- cloudbuild/prod/pre-deploy.yaml | 35 ++++++++++++------ cloudbuild/staging/deploy.yaml | 31 ++++++++++------ scripts/create-instance-templates.sh | 43 ++++++++++++++-------- test/cloudbuild/gce_scripts_test.sh | 53 ++++++++++++++++++++++++++-- 5 files changed, 127 insertions(+), 42 deletions(-) diff --git a/cloudbuild/gce-startup.sh b/cloudbuild/gce-startup.sh index b06d2298bf..6c3a832fc7 100755 --- a/cloudbuild/gce-startup.sh +++ b/cloudbuild/gce-startup.sh @@ -24,9 +24,7 @@ metadata() { } wait_for_docker() { - local attempt - - for attempt in {1..30}; do + for _ in {1..30}; do if docker info >/dev/null 2>&1; then return 0 fi @@ -71,7 +69,8 @@ main() { fi export HOME="${DOCKER_HOME}" - install -d -m 0700 "${HOME}" + mkdir -p "${HOME}" + chmod 0700 "${HOME}" docker-credential-gcr configure-docker --registries=gcr.io configure_firewall diff --git a/cloudbuild/prod/pre-deploy.yaml b/cloudbuild/prod/pre-deploy.yaml index 3e2a0e2426..35964284cb 100644 --- a/cloudbuild/prod/pre-deploy.yaml +++ b/cloudbuild/prod/pre-deploy.yaml @@ -1,28 +1,41 @@ steps: + # generate the per-cluster container environment metadata + - name: gcr.io/cloud-builders/gcloud + entrypoint: bash + args: + - -ceu + - | + umask 077 + { + printf '%s\n' 'LOGFLARE_GRPC_PORT=50051' + printf '%s\n' 'LOGFLARE_MIN_CLUSTER_SIZE=2' + printf 'RELEASE_COOKIE=%s\n' "$$1" + printf '%s\n' 'LOGFLARE_PUBSUB_POOL_SIZE=32' + printf 'LOGFLARE_METADATA_CLUSTER=%s\n' "$$2" + printf 'LOGFLARE_ALERTS_ENABLED=%s\n' "$$3" + } > gce-container.env + - -- + - ${_COOKIE} + - ${_CLUSTER} + - ${_LOGFLARE_ALERTS_ENABLED} # create instance template - name: gcr.io/cloud-builders/gcloud - allowExitCodes: [1] + entrypoint: bash args: - - compute - - instance-templates - - create-with-container + - ./cloudbuild/create-instance-template.sh + - logflare-232118 - $_TEMPLATE_NAME - --boot-disk-size=10GB - --boot-disk-type=pd-balanced - --machine-type=c2d-highcpu-32 - - --project=logflare-232118 - --network-interface=network=global,network-tier=PREMIUM,no-address - --maintenance-policy=TERMINATE - --service-account=compute-engine-2022@logflare-232118.iam.gserviceaccount.com - --scopes=https://www.googleapis.com/auth/cloud-platform - --tags=phoenix-http,https-server - - --metadata-from-file=shutdown-script=./cloudbuild/shutdown.sh + - --metadata-from-file=startup-script=./cloudbuild/gce-startup.sh,shutdown-script=./cloudbuild/shutdown.sh,logflare-container-env=./gce-container.env # needed for enabling node-problem-detector, explicitly enable logging - - --metadata=google-monitoring-enabled=true,google-logging-enabled=true - - --container-image=${_CONTAINER_IMAGE} - - --container-privileged - - --container-restart-policy=always - - --container-env=LOGFLARE_GRPC_PORT=50051,LOGFLARE_MIN_CLUSTER_SIZE=2,RELEASE_COOKIE=${_COOKIE},LOGFLARE_PUBSUB_POOL_SIZE=32,LOGFLARE_METADATA_CLUSTER=${_CLUSTER},LOGFLARE_ALERTS_ENABLED=${_LOGFLARE_ALERTS_ENABLED} + - --metadata=google-monitoring-enabled=true,google-logging-enabled=true,logflare-container-image=${_CONTAINER_IMAGE} - --no-shielded-secure-boot - --shielded-vtpm - --shielded-integrity-monitoring diff --git a/cloudbuild/staging/deploy.yaml b/cloudbuild/staging/deploy.yaml index 77b2a68c96..fd7282b78f 100644 --- a/cloudbuild/staging/deploy.yaml +++ b/cloudbuild/staging/deploy.yaml @@ -1,28 +1,37 @@ steps: + # generate the per-cluster container environment metadata + - name: gcr.io/cloud-builders/gcloud + entrypoint: bash + args: + - -ceu + - | + umask 077 + { + printf '%s\n' 'LOGFLARE_GRPC_PORT=50051' + printf 'RELEASE_COOKIE=%s\n' "$$1" + printf 'LOGFLARE_METADATA_CLUSTER=%s\n' "$$2" + } > gce-container.env + - -- + - ${_COOKIE} + - ${_CLUSTER} # create instance template - dedicated - name: gcr.io/cloud-builders/gcloud - allowExitCodes: [1] + entrypoint: bash args: - - compute - - instance-templates - - create-with-container + - ./cloudbuild/create-instance-template.sh + - logflare-staging - $_TEMPLATE_NAME - --boot-disk-size=10GB - --boot-disk-type=pd-balanced - --machine-type=${_INSTANCE_TYPE} - - --project=logflare-staging - --network-interface=network=default,network-tier=PREMIUM,no-address - --maintenance-policy=TERMINATE - --service-account=compute-engine-2022@logflare-staging.iam.gserviceaccount.com - --scopes=https://www.googleapis.com/auth/cloud-platform - --tags=phoenix-http,https-server - - --container-image=${_CONTAINER_IMAGE} - - --metadata-from-file=shutdown-script=./cloudbuild/shutdown.sh + - --metadata-from-file=startup-script=./cloudbuild/gce-startup.sh,shutdown-script=./cloudbuild/shutdown.sh,logflare-container-env=./gce-container.env # needed for enabling node-problem-detector, explicitly enable logging - - --metadata=google-monitoring-enabled=true,google-logging-enabled=true - - --container-privileged - - --container-restart-policy=always - - --container-env=LOGFLARE_GRPC_PORT=50051,RELEASE_COOKIE=${_COOKIE},LOGFLARE_METADATA_CLUSTER=${_CLUSTER} + - --metadata=google-monitoring-enabled=true,google-logging-enabled=true,logflare-container-image=${_CONTAINER_IMAGE} - --no-shielded-secure-boot - --shielded-vtpm - --shielded-integrity-monitoring diff --git a/scripts/create-instance-templates.sh b/scripts/create-instance-templates.sh index bc3752ad05..da7420d61e 100755 --- a/scripts/create-instance-templates.sh +++ b/scripts/create-instance-templates.sh @@ -12,6 +12,8 @@ set -euo pipefail +umask 077 + # ── Argument validation ─────────────────────────────────────────────────────── if [[ $# -lt 1 || $# -gt 2 ]]; then @@ -30,6 +32,10 @@ if [[ "${ENV}" != "prod" && "${ENV}" != "staging" ]]; then fi NORMALIZED_VERSION="${VERSION//./-}" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TEMP_DIR="$(mktemp -d)" + +trap 'rm -rf "${TEMP_DIR}"' EXIT # ── Environment-specific config ─────────────────────────────────────────────── @@ -72,6 +78,8 @@ for cluster in "${CLUSTERS[@]}"; do # Cookie derived per cluster: "default-" (matches _COOKIE in cloudbuild yamls) cluster_cookie="default-${cluster}" + container_env_file="${TEMP_DIR}/${cluster}.env" + # Container env differs per env if [[ "${ENV}" == "prod" ]]; then # Only prod-a has alerting enabled (matches deploy.prod.versioned in Makefile) @@ -80,15 +88,26 @@ for cluster in "${CLUSTERS[@]}"; do else alerts_enabled="false" fi - container_env="LOGFLARE_GRPC_PORT=50051,LOGFLARE_MIN_CLUSTER_SIZE=2,RELEASE_COOKIE=${cluster_cookie},LOGFLARE_PUBSUB_POOL_SIZE=32,LOGFLARE_METADATA_CLUSTER=${cluster},LOGFLARE_ALERTS_ENABLED=${alerts_enabled}" + + { + printf '%s\n' 'LOGFLARE_GRPC_PORT=50051' + printf '%s\n' 'LOGFLARE_MIN_CLUSTER_SIZE=2' + printf 'RELEASE_COOKIE=%s\n' "${cluster_cookie}" + printf '%s\n' 'LOGFLARE_PUBSUB_POOL_SIZE=32' + printf 'LOGFLARE_METADATA_CLUSTER=%s\n' "${cluster}" + printf 'LOGFLARE_ALERTS_ENABLED=%s\n' "${alerts_enabled}" + } >"${container_env_file}" else - container_env="LOGFLARE_GRPC_PORT=50051,RELEASE_COOKIE=${cluster_cookie},LOGFLARE_METADATA_CLUSTER=${cluster}" + { + printf '%s\n' 'LOGFLARE_GRPC_PORT=50051' + printf 'RELEASE_COOKIE=%s\n' "${cluster_cookie}" + printf 'LOGFLARE_METADATA_CLUSTER=%s\n' "${cluster}" + } >"${container_env_file}" fi printf " %-55s " "${template}" - err_output=$(gcloud compute instance-templates create-with-container "${template}" \ - --project="${PROJECT}" \ + err_output=$("${REPO_ROOT}/cloudbuild/create-instance-template.sh" "${PROJECT}" "${template}" \ --boot-disk-size=10GB \ --boot-disk-type=pd-balanced \ --machine-type="${MACHINE_TYPE}" \ @@ -97,12 +116,8 @@ for cluster in "${CLUSTERS[@]}"; do --service-account="${SERVICE_ACCOUNT}" \ --scopes=https://www.googleapis.com/auth/cloud-platform \ --tags=phoenix-http,https-server \ - --metadata-from-file=shutdown-script=./cloudbuild/shutdown.sh \ - --metadata=google-monitoring-enabled=true,google-logging-enabled=true \ - --container-image="${CONTAINER_IMAGE}" \ - --container-privileged \ - --container-restart-policy=always \ - --container-env="${container_env}" \ + --metadata-from-file="startup-script=${REPO_ROOT}/cloudbuild/gce-startup.sh,shutdown-script=${REPO_ROOT}/cloudbuild/shutdown.sh,logflare-container-env=${container_env_file}" \ + --metadata="google-monitoring-enabled=true,google-logging-enabled=true,logflare-container-image=${CONTAINER_IMAGE}" \ --no-shielded-secure-boot \ --shielded-vtpm \ --shielded-integrity-monitoring \ @@ -110,12 +125,12 @@ for cluster in "${CLUSTERS[@]}"; do --image-project=cos-cloud \ 2>&1) && rc=0 || rc=$? - if [[ $rc -eq 0 ]]; then - echo "created" - created+=("${template}") - elif echo "${err_output}" | grep -qiE "already exists|alreadyExists"; then + if [[ $rc -eq 0 ]] && grep -qi "already exists" <<<"${err_output}"; then echo "already exists (skipped)" already_exists+=("${template}") + elif [[ $rc -eq 0 ]]; then + echo "created" + created+=("${template}") else echo "ERROR" echo " ${err_output}" >&2 diff --git a/test/cloudbuild/gce_scripts_test.sh b/test/cloudbuild/gce_scripts_test.sh index 11df29f593..9f686bcb46 100755 --- a/test/cloudbuild/gce_scripts_test.sh +++ b/test/cloudbuild/gce_scripts_test.sh @@ -2,10 +2,13 @@ set -euo pipefail -readonly REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +readonly REPO_ROOT readonly STARTUP_SCRIPT="${REPO_ROOT}/cloudbuild/gce-startup.sh" readonly CREATE_SCRIPT="${REPO_ROOT}/cloudbuild/create-instance-template.sh" -readonly TEST_DIR="$(mktemp -d)" +readonly TEMPLATE_SCRIPT="${REPO_ROOT}/scripts/create-instance-templates.sh" +TEST_DIR="$(mktemp -d)" +readonly TEST_DIR trap 'rm -rf "${TEST_DIR}"' EXIT @@ -162,6 +165,16 @@ if [[ "$3" == "describe" ]]; then printf '%s\n' '{"properties":{"metadata":{"items":[{"key":"startup-script"}]}}}' ;; esac +elif [[ "$3" == "create" && -n "${ENV_CAPTURE_DIR:-}" ]]; then + template="$4" + for arg in "$@"; do + case "${arg}" in + --metadata-from-file=*logflare-container-env=*) + env_file="${arg##*logflare-container-env=}" + cp "${env_file}" "${ENV_CAPTURE_DIR}/${template}.env" + ;; + esac + done fi STUB chmod +x "${path}" @@ -201,7 +214,43 @@ test_create_script() { fi } +test_manual_template_script() { + local call_log="${TEST_DIR}/manual-gcloud-calls" + local capture_dir="${TEST_DIR}/captured-env" + local output="${TEST_DIR}/manual-output" + + mkdir -p "${capture_dir}" + : >"${call_log}" + + CALL_LOG="${call_log}" \ + ENV_CAPTURE_DIR="${capture_dir}" \ + GCLOUD_DESCRIBE_MODE=missing \ + GCLOUD_BIN="${TEST_DIR}/gcloud" \ + bash "${TEMPLATE_SCRIPT}" 1.2.3 staging >"${output}" + + assert_contains "${output}" "Created : 2" + assert_contains "${capture_dir}/logflare-staging-main-cluster-1-2-3.env" "RELEASE_COOKIE=default-main" + assert_contains "${capture_dir}/logflare-staging-versioned-cluster-1-2-3.env" "LOGFLARE_METADATA_CLUSTER=versioned" + assert_contains "${call_log}" "logflare-container-image=gcr.io/logflare-staging/logflare_app:1.2.3" + assert_not_contains "${call_log}" "create-with-container" + + rm -rf "${capture_dir}" + mkdir -p "${capture_dir}" + : >"${call_log}" + + CALL_LOG="${call_log}" \ + ENV_CAPTURE_DIR="${capture_dir}" \ + GCLOUD_DESCRIBE_MODE=missing \ + GCLOUD_BIN="${TEST_DIR}/gcloud" \ + bash "${TEMPLATE_SCRIPT}" 1.2.3 >"${output}" + + assert_contains "${output}" "Created : 8" + assert_contains "${capture_dir}/logflare-prod-1-2-3-prod-a.env" "LOGFLARE_ALERTS_ENABLED=true" + assert_contains "${capture_dir}/logflare-prod-1-2-3-prod-b.env" "LOGFLARE_ALERTS_ENABLED=false" +} + test_startup_script test_create_script +test_manual_template_script echo "gce script tests passed" From 6ee283bbb869fbca370bfe64e3440020fd3ad604 Mon Sep 17 00:00:00 2001 From: David Whittington Date: Fri, 24 Jul 2026 15:18:11 +0000 Subject: [PATCH 3/4] test(cloudbuild): simplify GCE script assertions --- test/cloudbuild/gce_scripts_test.sh | 257 +++++----------------------- 1 file changed, 42 insertions(+), 215 deletions(-) diff --git a/test/cloudbuild/gce_scripts_test.sh b/test/cloudbuild/gce_scripts_test.sh index 9f686bcb46..40d8b0f807 100755 --- a/test/cloudbuild/gce_scripts_test.sh +++ b/test/cloudbuild/gce_scripts_test.sh @@ -6,11 +6,9 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" readonly REPO_ROOT readonly STARTUP_SCRIPT="${REPO_ROOT}/cloudbuild/gce-startup.sh" readonly CREATE_SCRIPT="${REPO_ROOT}/cloudbuild/create-instance-template.sh" +readonly PROD_CONFIG="${REPO_ROOT}/cloudbuild/prod/pre-deploy.yaml" +readonly STAGING_CONFIG="${REPO_ROOT}/cloudbuild/staging/deploy.yaml" readonly TEMPLATE_SCRIPT="${REPO_ROOT}/scripts/create-instance-templates.sh" -TEST_DIR="$(mktemp -d)" -readonly TEST_DIR - -trap 'rm -rf "${TEST_DIR}"' EXIT fail() { echo "FAIL: $*" >&2 @@ -33,224 +31,53 @@ assert_not_contains() { fi } -install_startup_stubs() { - local bin_dir="$1" - - mkdir -p "${bin_dir}" - - cat >"${bin_dir}/curl" <<'STUB' -#!/bin/bash -url="${!#}" -printf 'curl %s\n' "${url}" >>"${CALL_LOG}" -case "${url}" in - */logflare-container-image) - printf '%s' 'gcr.io/test-project/logflare_app:test-tag' - ;; - */logflare-container-env) - printf '%s\n' \ - 'LOGFLARE_GRPC_PORT=50051' \ - 'RELEASE_COOKIE=test-cookie' \ - 'LOGFLARE_METADATA_CLUSTER=test-cluster' - ;; - *) - exit 22 - ;; -esac -STUB - - cat >"${bin_dir}/docker-credential-gcr" <<'STUB' -#!/bin/bash -printf 'credential HOME=%s %s\n' "${HOME}" "$*" >>"${CALL_LOG}" -STUB - - cat >"${bin_dir}/iptables" <<'STUB' -#!/bin/bash -printf 'iptables %s\n' "$*" >>"${CALL_LOG}" -rule="$2 $4" -case "$1" in - -C) - grep -Fxq -- "${rule}" "${IPTABLES_STATE}" 2>/dev/null - ;; - -A) - printf '%s\n' "${rule}" >>"${IPTABLES_STATE}" - ;; - *) - exit 2 - ;; -esac -STUB - - cat >"${bin_dir}/docker" <<'STUB' -#!/bin/bash -printf 'docker %s\n' "$*" >>"${CALL_LOG}" -if [[ "$1" == "pull" && "${DOCKER_PULL_FAIL:-false}" == "true" ]]; then - exit 1 -fi -STUB - - chmod +x "${bin_dir}"/* -} - -run_startup_script() { - local call_log="$1" - local iptables_state="$2" - local docker_home="$3" - local env_file="$4" - local pull_fail="${5:-false}" - - CALL_LOG="${call_log}" \ - IPTABLES_STATE="${iptables_state}" \ - DOCKER_PULL_FAIL="${pull_fail}" \ - LOGFLARE_DOCKER_HOME="${docker_home}" \ - LOGFLARE_CONTAINER_ENV_FILE="${env_file}" \ - PATH="${TEST_DIR}/startup-bin:${PATH}" \ - bash "${STARTUP_SCRIPT}" -} - -test_startup_script() { - local call_log="${TEST_DIR}/startup-calls" - local iptables_state="${TEST_DIR}/iptables-state" - local docker_home="${TEST_DIR}/docker-home" - local env_file="${TEST_DIR}/container.env" - local first_add_count - local pull_line - local remove_line - - : >"${call_log}" - : >"${iptables_state}" - install_startup_stubs "${TEST_DIR}/startup-bin" - - run_startup_script "${call_log}" "${iptables_state}" "${docker_home}" "${env_file}" - - [[ "$(stat -c '%a' "${env_file}")" == "600" ]] || fail "container env file is not mode 600" - assert_contains "${env_file}" "RELEASE_COOKIE=test-cookie" - assert_contains "${call_log}" "credential HOME=${docker_home} configure-docker --registries=gcr.io" - assert_contains "${call_log}" "docker run --name=logflare --privileged --restart=always --network=host --detach --log-driver=json-file --log-opt max-size=500m --log-opt max-file=3 --env-file ${env_file} gcr.io/test-project/logflare_app:test-tag" - - first_add_count="$(grep -c '^iptables -A ' "${call_log}")" - [[ "${first_add_count}" == "6" ]] || fail "expected six firewall rules, got ${first_add_count}" - - pull_line="$(grep -n '^docker pull ' "${call_log}" | head -1 | cut -d: -f1)" - remove_line="$(grep -n '^docker rm ' "${call_log}" | head -1 | cut -d: -f1)" - ((pull_line < remove_line)) || fail "container was removed before the image pull completed" - - run_startup_script "${call_log}" "${iptables_state}" "${docker_home}" "${env_file}" - [[ "$(grep -c '^iptables -A ' "${call_log}")" == "6" ]] || fail "firewall rules were duplicated on rerun" - - : >"${call_log}" - if run_startup_script "${call_log}" "${iptables_state}" "${docker_home}" "${env_file}" true; then - fail "startup script succeeded when docker pull failed" - fi - assert_not_contains "${call_log}" "docker rm -f logflare" -} - -install_gcloud_stub() { - local path="$1" - - cat >"${path}" <<'STUB' -#!/bin/bash -printf 'gcloud %s\n' "$*" >>"${CALL_LOG}" -if [[ "$3" == "describe" ]]; then - case "${GCLOUD_DESCRIBE_MODE}" in - missing) - exit 1 - ;; - good) - printf '%s\n' '{"properties":{"metadata":{"items":[{"key":"startup-script"},{"key":"shutdown-script"},{"key":"logflare-container-env"},{"key":"logflare-container-image"}]}}}' - ;; - deprecated) - printf '%s\n' '{"properties":{"metadata":{"items":[{"key":"gce-container-declaration"}]}}}' - ;; - incomplete) - printf '%s\n' '{"properties":{"metadata":{"items":[{"key":"startup-script"}]}}}' - ;; - esac -elif [[ "$3" == "create" && -n "${ENV_CAPTURE_DIR:-}" ]]; then - template="$4" - for arg in "$@"; do - case "${arg}" in - --metadata-from-file=*logflare-container-env=*) - env_file="${arg##*logflare-container-env=}" - cp "${env_file}" "${ENV_CAPTURE_DIR}/${template}.env" - ;; - esac +# Expected snippets intentionally include literal shell substitutions. +# shellcheck disable=SC2016 +test_startup_script_config() { + local option + + assert_contains "${STARTUP_SCRIPT}" 'image="$(metadata logflare-container-image)"' + assert_contains "${STARTUP_SCRIPT}" 'metadata logflare-container-env >"${CONTAINER_ENV_FILE}"' + assert_contains "${STARTUP_SCRIPT}" "docker-credential-gcr configure-docker --registries=gcr.io" + assert_contains "${STARTUP_SCRIPT}" 'iptables -C "${chain}" -p "${protocol}" -j ACCEPT' + assert_contains "${STARTUP_SCRIPT}" 'iptables -A "${chain}" -p "${protocol}" -j ACCEPT' + + for option in \ + '--privileged' \ + '--restart=always' \ + '--network=host' \ + '--log-driver=json-file' \ + '--log-opt max-size=500m' \ + '--log-opt max-file=3' \ + '--env-file "${CONTAINER_ENV_FILE}"'; do + assert_contains "${STARTUP_SCRIPT}" "${option}" done -fi -STUB - chmod +x "${path}" -} - -run_create_script() { - local mode="$1" - local call_log="$2" - - CALL_LOG="${call_log}" \ - GCLOUD_DESCRIBE_MODE="${mode}" \ - GCLOUD_BIN="${TEST_DIR}/gcloud" \ - bash "${CREATE_SCRIPT}" test-project test-template --machine-type=test-machine -} - -test_create_script() { - local call_log="${TEST_DIR}/gcloud-calls" - - install_gcloud_stub "${TEST_DIR}/gcloud" - - : >"${call_log}" - run_create_script missing "${call_log}" - assert_contains "${call_log}" "gcloud compute instance-templates create test-template --project=test-project --machine-type=test-machine" - - : >"${call_log}" - run_create_script good "${call_log}" - assert_not_contains "${call_log}" "instance-templates create" - - : >"${call_log}" - if run_create_script deprecated "${call_log}"; then - fail "deprecated existing template was accepted" - fi - - : >"${call_log}" - if run_create_script incomplete "${call_log}"; then - fail "incomplete existing template was accepted" - fi } -test_manual_template_script() { - local call_log="${TEST_DIR}/manual-gcloud-calls" - local capture_dir="${TEST_DIR}/captured-env" - local output="${TEST_DIR}/manual-output" - - mkdir -p "${capture_dir}" - : >"${call_log}" - - CALL_LOG="${call_log}" \ - ENV_CAPTURE_DIR="${capture_dir}" \ - GCLOUD_DESCRIBE_MODE=missing \ - GCLOUD_BIN="${TEST_DIR}/gcloud" \ - bash "${TEMPLATE_SCRIPT}" 1.2.3 staging >"${output}" +# shellcheck disable=SC2016 +test_instance_template_config() { + local config - assert_contains "${output}" "Created : 2" - assert_contains "${capture_dir}/logflare-staging-main-cluster-1-2-3.env" "RELEASE_COOKIE=default-main" - assert_contains "${capture_dir}/logflare-staging-versioned-cluster-1-2-3.env" "LOGFLARE_METADATA_CLUSTER=versioned" - assert_contains "${call_log}" "logflare-container-image=gcr.io/logflare-staging/logflare_app:1.2.3" - assert_not_contains "${call_log}" "create-with-container" + assert_contains "${CREATE_SCRIPT}" '"${GCLOUD_BIN}" compute instance-templates create "${TEMPLATE}"' + assert_contains "${CREATE_SCRIPT}" 'metadata_has_key "${description}" "gce-container-declaration"' - rm -rf "${capture_dir}" - mkdir -p "${capture_dir}" - : >"${call_log}" + for config in "${PROD_CONFIG}" "${STAGING_CONFIG}"; do + assert_contains "${config}" "./cloudbuild/create-instance-template.sh" + assert_contains "${config}" "> gce-container.env" + assert_contains "${config}" "--metadata-from-file=startup-script=./cloudbuild/gce-startup.sh,shutdown-script=./cloudbuild/shutdown.sh,logflare-container-env=./gce-container.env" + assert_contains "${config}" '--metadata=google-monitoring-enabled=true,google-logging-enabled=true,logflare-container-image=${_CONTAINER_IMAGE}' + done - CALL_LOG="${call_log}" \ - ENV_CAPTURE_DIR="${capture_dir}" \ - GCLOUD_DESCRIBE_MODE=missing \ - GCLOUD_BIN="${TEST_DIR}/gcloud" \ - bash "${TEMPLATE_SCRIPT}" 1.2.3 >"${output}" + assert_contains "${TEMPLATE_SCRIPT}" '"${REPO_ROOT}/cloudbuild/create-instance-template.sh"' + assert_contains "${TEMPLATE_SCRIPT}" '--metadata-from-file="startup-script=${REPO_ROOT}/cloudbuild/gce-startup.sh,shutdown-script=${REPO_ROOT}/cloudbuild/shutdown.sh,logflare-container-env=${container_env_file}"' + assert_contains "${TEMPLATE_SCRIPT}" '--metadata="google-monitoring-enabled=true,google-logging-enabled=true,logflare-container-image=${CONTAINER_IMAGE}"' - assert_contains "${output}" "Created : 8" - assert_contains "${capture_dir}/logflare-prod-1-2-3-prod-a.env" "LOGFLARE_ALERTS_ENABLED=true" - assert_contains "${capture_dir}/logflare-prod-1-2-3-prod-b.env" "LOGFLARE_ALERTS_ENABLED=false" + for config in "${PROD_CONFIG}" "${STAGING_CONFIG}" "${TEMPLATE_SCRIPT}"; do + assert_not_contains "${config}" "create-with-container" + done } -test_startup_script -test_create_script -test_manual_template_script +test_startup_script_config +test_instance_template_config -echo "gce script tests passed" +echo "gce script config tests passed" From f2f830b788a75b6167ae7991cc49e7c77b0d3662 Mon Sep 17 00:00:00 2001 From: David Whittington Date: Fri, 24 Jul 2026 15:30:43 +0000 Subject: [PATCH 4/4] test(cloudbuild): assert GCE configuration contracts --- test/cloudbuild/gce_scripts_test.sh | 73 ++++++++++++++--------------- 1 file changed, 35 insertions(+), 38 deletions(-) diff --git a/test/cloudbuild/gce_scripts_test.sh b/test/cloudbuild/gce_scripts_test.sh index 40d8b0f807..c661002ea7 100755 --- a/test/cloudbuild/gce_scripts_test.sh +++ b/test/cloudbuild/gce_scripts_test.sh @@ -31,53 +31,50 @@ assert_not_contains() { fi } -# Expected snippets intentionally include literal shell substitutions. -# shellcheck disable=SC2016 -test_startup_script_config() { - local option - - assert_contains "${STARTUP_SCRIPT}" 'image="$(metadata logflare-container-image)"' - assert_contains "${STARTUP_SCRIPT}" 'metadata logflare-container-env >"${CONTAINER_ENV_FILE}"' - assert_contains "${STARTUP_SCRIPT}" "docker-credential-gcr configure-docker --registries=gcr.io" - assert_contains "${STARTUP_SCRIPT}" 'iptables -C "${chain}" -p "${protocol}" -j ACCEPT' - assert_contains "${STARTUP_SCRIPT}" 'iptables -A "${chain}" -p "${protocol}" -j ACCEPT' - - for option in \ - '--privileged' \ - '--restart=always' \ - '--network=host' \ - '--log-driver=json-file' \ - '--log-opt max-size=500m' \ - '--log-opt max-file=3' \ - '--env-file "${CONTAINER_ENV_FILE}"'; do - assert_contains "${STARTUP_SCRIPT}" "${option}" +test_startup_script_contract() { + local required + + for required in \ + "logflare-container-image" \ + "logflare-container-env" \ + "docker-credential-gcr configure-docker" \ + "iptables -C" \ + "iptables -A" \ + "docker run" \ + "--privileged" \ + "--restart=always" \ + "--network=host" \ + "--log-driver=json-file" \ + "--log-opt max-size=500m" \ + "--log-opt max-file=3" \ + "--env-file"; do + assert_contains "${STARTUP_SCRIPT}" "${required}" done } -# shellcheck disable=SC2016 -test_instance_template_config() { +test_instance_template_contract() { local config + local metadata_key - assert_contains "${CREATE_SCRIPT}" '"${GCLOUD_BIN}" compute instance-templates create "${TEMPLATE}"' - assert_contains "${CREATE_SCRIPT}" 'metadata_has_key "${description}" "gce-container-declaration"' - - for config in "${PROD_CONFIG}" "${STAGING_CONFIG}"; do - assert_contains "${config}" "./cloudbuild/create-instance-template.sh" - assert_contains "${config}" "> gce-container.env" - assert_contains "${config}" "--metadata-from-file=startup-script=./cloudbuild/gce-startup.sh,shutdown-script=./cloudbuild/shutdown.sh,logflare-container-env=./gce-container.env" - assert_contains "${config}" '--metadata=google-monitoring-enabled=true,google-logging-enabled=true,logflare-container-image=${_CONTAINER_IMAGE}' - done - - assert_contains "${TEMPLATE_SCRIPT}" '"${REPO_ROOT}/cloudbuild/create-instance-template.sh"' - assert_contains "${TEMPLATE_SCRIPT}" '--metadata-from-file="startup-script=${REPO_ROOT}/cloudbuild/gce-startup.sh,shutdown-script=${REPO_ROOT}/cloudbuild/shutdown.sh,logflare-container-env=${container_env_file}"' - assert_contains "${TEMPLATE_SCRIPT}" '--metadata="google-monitoring-enabled=true,google-logging-enabled=true,logflare-container-image=${CONTAINER_IMAGE}"' + assert_contains "${CREATE_SCRIPT}" "compute instance-templates create" for config in "${PROD_CONFIG}" "${STAGING_CONFIG}" "${TEMPLATE_SCRIPT}"; do + assert_contains "${config}" "create-instance-template.sh" + assert_contains "${config}" "--metadata-from-file=" + assert_contains "${config}" "--metadata=" assert_not_contains "${config}" "create-with-container" + + for metadata_key in \ + "startup-script" \ + "shutdown-script" \ + "logflare-container-env" \ + "logflare-container-image"; do + assert_contains "${config}" "${metadata_key}" + done done } -test_startup_script_config -test_instance_template_config +test_startup_script_contract +test_instance_template_contract -echo "gce script config tests passed" +echo "gce script contract tests passed"