refactor(cloudbuild): replace GCE container declarations with startup scripts

cloudbuild/create-instance-template.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [[ $# -lt 2 ]]; then
+  echo "Usage: $0 PROJECT TEMPLATE [GCLOUD_CREATE_ARGS...]" >&2
+  exit 2
+fi
+
+readonly PROJECT="$1"
+readonly TEMPLATE="$2"
+readonly GCLOUD_BIN="${GCLOUD_BIN:-gcloud}"
+shift 2
+
+metadata_has_key() {
+  local description="$1"
+  local key="$2"
+
+  grep -Eq "\"key\"[[:space:]]*:[[:space:]]*\"${key}\"" <<<"${description}"
+}
+
+if description="$("${GCLOUD_BIN}" compute instance-templates describe "${TEMPLATE}" \
+  --project="${PROJECT}" \
+  --format=json 2>/dev/null)"; then
+  if metadata_has_key "${description}" "gce-container-declaration"; then
+    echo "Existing template ${TEMPLATE} still uses gce-container-declaration" >&2
+    exit 1
+  fi
+
+  for key in startup-script shutdown-script logflare-container-env logflare-container-image; do
+    if ! metadata_has_key "${description}" "${key}"; then
+      echo "Existing template ${TEMPLATE} is missing ${key} metadata" >&2
+      exit 1
+    fi
+  done
+
+  echo "Instance template already exists with startup-script metadata: ${TEMPLATE}"
+  exit 0
+fi
+
+"${GCLOUD_BIN}" compute instance-templates create "${TEMPLATE}" \
+  --project="${PROJECT}" \
+  "$@"

cloudbuild/gce-startup.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+
+set -euo pipefail
+
+readonly METADATA_URL="http://metadata.google.internal/computeMetadata/v1/instance/attributes"
+readonly CONTAINER_NAME="logflare"
+readonly CONTAINER_ENV_FILE="${LOGFLARE_CONTAINER_ENV_FILE:-/run/logflare-container.env}"
+readonly DOCKER_HOME="${LOGFLARE_DOCKER_HOME:-/home/logflare}"
+
+metadata() {
+  local key="$1"
+
+  curl \
+    --fail \
+    --silent \
+    --show-error \
+    --connect-timeout 5 \
+    --max-time 30 \
+    --retry 10 \
+    --retry-connrefused \
+    --retry-delay 2 \
+    --header "Metadata-Flavor: Google" \
+    "${METADATA_URL}/${key}"
+}
+
+wait_for_docker() {
+  for _ in {1..30}; do
+    if docker info >/dev/null 2>&1; then
+      return 0
+    fi
+
+    sleep 2
+  done
+
+  echo "Docker did not become ready after 60 seconds" >&2
+  return 1
+}
+
+configure_firewall() {
+  local chain
+  local protocol
+
+  # Konlet opened these host firewall paths before starting the container.
+  # Keep the rules idempotent because startup scripts can be rerun.
+  for protocol in tcp udp icmp; do
+    for chain in INPUT FORWARD; do
+      if ! iptables -C "${chain}" -p "${protocol}" -j ACCEPT 2>/dev/null; then
+        iptables -A "${chain}" -p "${protocol}" -j ACCEPT
+      fi
+    done
+  done
+}
+
+main() {
+  local image
+
+  umask 077
+
+  image="$(metadata logflare-container-image)"
+  if [[ -z "${image}" ]]; then
+    echo "logflare-container-image metadata is empty" >&2
+    return 1
+  fi
+
+  metadata logflare-container-env >"${CONTAINER_ENV_FILE}"
+  if [[ ! -s "${CONTAINER_ENV_FILE}" ]]; then
+    echo "logflare-container-env metadata is empty" >&2
+    return 1
+  fi
+
+  export HOME="${DOCKER_HOME}"
+  mkdir -p "${HOME}"
+  chmod 0700 "${HOME}"
+  docker-credential-gcr configure-docker --registries=gcr.io
+
+  configure_firewall
+  wait_for_docker
+
+  # Keep the existing container available if the registry is temporarily
+  # unavailable during a manual startup-script rerun.
+  docker pull "${image}"
+  docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true
+
+  docker run \
+    --name="${CONTAINER_NAME}" \
+    --privileged \
+    --restart=always \
+    --network=host \
+    --detach \
+    --log-driver=json-file \
+    --log-opt max-size=500m \
+    --log-opt max-file=3 \
+    --env-file "${CONTAINER_ENV_FILE}" \
+    "${image}"
+}
+
+main "$@"

cloudbuild/prod/pre-deploy.yaml

@@ -1,28 +1,41 @@
 steps:
+  # generate the per-cluster container environment metadata
+  - name: gcr.io/cloud-builders/gcloud
+    entrypoint: bash
+    args:
+      - -ceu
+      - |
+        umask 077
+        {
+          printf '%s\n' 'LOGFLARE_GRPC_PORT=50051'
+          printf '%s\n' 'LOGFLARE_MIN_CLUSTER_SIZE=2'
+          printf 'RELEASE_COOKIE=%s\n' "$$1"
+          printf '%s\n' 'LOGFLARE_PUBSUB_POOL_SIZE=32'
+          printf 'LOGFLARE_METADATA_CLUSTER=%s\n' "$$2"
+          printf 'LOGFLARE_ALERTS_ENABLED=%s\n' "$$3"
+        } > gce-container.env
+      - --
+      - ${_COOKIE}
+      - ${_CLUSTER}
+      - ${_LOGFLARE_ALERTS_ENABLED}
   # create instance template
   - name: gcr.io/cloud-builders/gcloud
-    allowExitCodes: [1]
+    entrypoint: bash
     args:
-      - compute
-      - instance-templates
-      - create-with-container
+      - ./cloudbuild/create-instance-template.sh
+      - logflare-232118
       - $_TEMPLATE_NAME
       - --boot-disk-size=10GB
       - --boot-disk-type=pd-balanced
       - --machine-type=c2d-highcpu-32
-      - --project=logflare-232118
       - --network-interface=network=global,network-tier=PREMIUM,no-address
       - --maintenance-policy=TERMINATE
       - --service-account=compute-engine-2022@logflare-232118.iam.gserviceaccount.com
       - --scopes=https://www.googleapis.com/auth/cloud-platform
       - --tags=phoenix-http,https-server
-      - --metadata-from-file=shutdown-script=./cloudbuild/shutdown.sh
+      - --metadata-from-file=startup-script=./cloudbuild/gce-startup.sh,shutdown-script=./cloudbuild/shutdown.sh,logflare-container-env=./gce-container.env
       # needed for enabling node-problem-detector, explicitly enable logging
-      - --metadata=google-monitoring-enabled=true,google-logging-enabled=true
-      - --container-image=${_CONTAINER_IMAGE}
-      - --container-privileged
-      - --container-restart-policy=always
-      - --container-env=LOGFLARE_GRPC_PORT=50051,LOGFLARE_MIN_CLUSTER_SIZE=2,RELEASE_COOKIE=${_COOKIE},LOGFLARE_PUBSUB_POOL_SIZE=32,LOGFLARE_METADATA_CLUSTER=${_CLUSTER},LOGFLARE_ALERTS_ENABLED=${_LOGFLARE_ALERTS_ENABLED}
+      - --metadata=google-monitoring-enabled=true,google-logging-enabled=true,logflare-container-image=${_CONTAINER_IMAGE}
       - --no-shielded-secure-boot
       - --shielded-vtpm
       - --shielded-integrity-monitoring

cloudbuild/staging/deploy.yaml

@@ -1,28 +1,37 @@
 steps:
+  # generate the per-cluster container environment metadata
+  - name: gcr.io/cloud-builders/gcloud
+    entrypoint: bash
+    args:
+      - -ceu
+      - |
+        umask 077
+        {
+          printf '%s\n' 'LOGFLARE_GRPC_PORT=50051'
+          printf 'RELEASE_COOKIE=%s\n' "$$1"
+          printf 'LOGFLARE_METADATA_CLUSTER=%s\n' "$$2"
+        } > gce-container.env
+      - --
+      - ${_COOKIE}
+      - ${_CLUSTER}
   # create instance template - dedicated
   - name: gcr.io/cloud-builders/gcloud
-    allowExitCodes: [1]
+    entrypoint: bash
     args:
-      - compute
-      - instance-templates
-      - create-with-container
+      - ./cloudbuild/create-instance-template.sh
+      - logflare-staging
       - $_TEMPLATE_NAME
       - --boot-disk-size=10GB
       - --boot-disk-type=pd-balanced
       - --machine-type=${_INSTANCE_TYPE}
-      - --project=logflare-staging
       - --network-interface=network=default,network-tier=PREMIUM,no-address
       - --maintenance-policy=TERMINATE
       - --service-account=compute-engine-2022@logflare-staging.iam.gserviceaccount.com
       - --scopes=https://www.googleapis.com/auth/cloud-platform
       - --tags=phoenix-http,https-server
-      - --container-image=${_CONTAINER_IMAGE}
-      - --metadata-from-file=shutdown-script=./cloudbuild/shutdown.sh
+      - --metadata-from-file=startup-script=./cloudbuild/gce-startup.sh,shutdown-script=./cloudbuild/shutdown.sh,logflare-container-env=./gce-container.env
       # needed for enabling node-problem-detector, explicitly enable logging
-      - --metadata=google-monitoring-enabled=true,google-logging-enabled=true
-      - --container-privileged
-      - --container-restart-policy=always
-      - --container-env=LOGFLARE_GRPC_PORT=50051,RELEASE_COOKIE=${_COOKIE},LOGFLARE_METADATA_CLUSTER=${_CLUSTER}
+      - --metadata=google-monitoring-enabled=true,google-logging-enabled=true,logflare-container-image=${_CONTAINER_IMAGE}
       - --no-shielded-secure-boot
       - --shielded-vtpm
       - --shielded-integrity-monitoring

scripts/create-instance-templates.sh

@@ -12,6 +12,8 @@
 
 set -euo pipefail
 
+umask 077
+
 # ── Argument validation ───────────────────────────────────────────────────────
 
 if [[ $# -lt 1 || $# -gt 2 ]]; then
@@ -30,6 +32,10 @@ if [[ "${ENV}" != "prod" && "${ENV}" != "staging" ]]; then
 fi
 
 NORMALIZED_VERSION="${VERSION//./-}"
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+TEMP_DIR="$(mktemp -d)"
+
+trap 'rm -rf "${TEMP_DIR}"' EXIT
 
 # ── Environment-specific config ───────────────────────────────────────────────
 
@@ -72,6 +78,8 @@ for cluster in "${CLUSTERS[@]}"; do
   # Cookie derived per cluster: "default-<cluster>" (matches _COOKIE in cloudbuild yamls)
   cluster_cookie="default-${cluster}"
 
+  container_env_file="${TEMP_DIR}/${cluster}.env"
+
   # Container env differs per env
   if [[ "${ENV}" == "prod" ]]; then
     # Only prod-a has alerting enabled (matches deploy.prod.versioned in Makefile)
@@ -80,15 +88,26 @@ for cluster in "${CLUSTERS[@]}"; do
     else
       alerts_enabled="false"
     fi
-    container_env="LOGFLARE_GRPC_PORT=50051,LOGFLARE_MIN_CLUSTER_SIZE=2,RELEASE_COOKIE=${cluster_cookie},LOGFLARE_PUBSUB_POOL_SIZE=32,LOGFLARE_METADATA_CLUSTER=${cluster},LOGFLARE_ALERTS_ENABLED=${alerts_enabled}"
+
+    {
+      printf '%s\n' 'LOGFLARE_GRPC_PORT=50051'
+      printf '%s\n' 'LOGFLARE_MIN_CLUSTER_SIZE=2'
+      printf 'RELEASE_COOKIE=%s\n' "${cluster_cookie}"
+      printf '%s\n' 'LOGFLARE_PUBSUB_POOL_SIZE=32'
+      printf 'LOGFLARE_METADATA_CLUSTER=%s\n' "${cluster}"
+      printf 'LOGFLARE_ALERTS_ENABLED=%s\n' "${alerts_enabled}"
+    } >"${container_env_file}"
   else
-    container_env="LOGFLARE_GRPC_PORT=50051,RELEASE_COOKIE=${cluster_cookie},LOGFLARE_METADATA_CLUSTER=${cluster}"
+    {
+      printf '%s\n' 'LOGFLARE_GRPC_PORT=50051'
+      printf 'RELEASE_COOKIE=%s\n' "${cluster_cookie}"
+      printf 'LOGFLARE_METADATA_CLUSTER=%s\n' "${cluster}"
+    } >"${container_env_file}"
   fi
 
   printf "  %-55s " "${template}"
 
-  err_output=$(gcloud compute instance-templates create-with-container "${template}" \
-    --project="${PROJECT}" \
+  err_output=$("${REPO_ROOT}/cloudbuild/create-instance-template.sh" "${PROJECT}" "${template}" \
     --boot-disk-size=10GB \
     --boot-disk-type=pd-balanced \
     --machine-type="${MACHINE_TYPE}" \
@@ -97,25 +116,21 @@ for cluster in "${CLUSTERS[@]}"; do
     --service-account="${SERVICE_ACCOUNT}" \
     --scopes=https://www.googleapis.com/auth/cloud-platform \
     --tags=phoenix-http,https-server \
-    --metadata-from-file=shutdown-script=./cloudbuild/shutdown.sh \
-    --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
-    --container-image="${CONTAINER_IMAGE}" \
-    --container-privileged \
-    --container-restart-policy=always \
-    --container-env="${container_env}" \
+    --metadata-from-file="startup-script=${REPO_ROOT}/cloudbuild/gce-startup.sh,shutdown-script=${REPO_ROOT}/cloudbuild/shutdown.sh,logflare-container-env=${container_env_file}" \
+    --metadata="google-monitoring-enabled=true,google-logging-enabled=true,logflare-container-image=${CONTAINER_IMAGE}" \
     --no-shielded-secure-boot \
     --shielded-vtpm \
     --shielded-integrity-monitoring \
     --image=cos-stable-109-17800-147-54 \
     --image-project=cos-cloud \
     2>&1) && rc=0 || rc=$?
 
-  if [[ $rc -eq 0 ]]; then
-    echo "created"
-    created+=("${template}")
-  elif echo "${err_output}" | grep -qiE "already exists|alreadyExists"; then
+  if [[ $rc -eq 0 ]] && grep -qi "already exists" <<<"${err_output}"; then
     echo "already exists (skipped)"
     already_exists+=("${template}")
+  elif [[ $rc -eq 0 ]]; then
+    echo "created"
+    created+=("${template}")
   else
     echo "ERROR"
     echo "    ${err_output}" >&2