diff --git a/.gitignore b/.gitignore index b4b1acd..38a8334 100644 --- a/.gitignore +++ b/.gitignore @@ -27,5 +27,6 @@ livid-bot .mise.toml .jj logs/ +deploy/k3s/output/ .claude/settings.local.json diff --git a/README.md b/README.md index f4831fb..98c045f 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,17 @@ docker compose up --build > 앱 시작 시 DB 마이그레이션이 자동 실행됩니다. +### 3) k3s 배포 +`deploy/k3s/` 아래에 Helm chart와 컷오버 스크립트가 준비되어 있습니다. + +```bash +./deploy/k3s/bin/apply-secrets.sh +./deploy/k3s/bin/deploy.sh +``` + +데이터를 유지한 채 Docker Compose에서 k3s로 넘길 때는 +`./deploy/k3s/bin/cutover.sh`를 사용합니다. + ## 슬래시 명령어 ### `/help` diff --git a/deploy/k3s/README.md b/deploy/k3s/README.md new file mode 100644 index 0000000..0bfe211 --- /dev/null +++ b/deploy/k3s/README.md @@ -0,0 +1,71 @@ +# livid-bot on k3s + +`livid-bot`의 Docker Compose 런타임을 k3s로 옮기기 위한 Helm chart와 +데이터 보존형 컷오버 스크립트 모음이다. + +## Layout + +- `charts/livid-bot`: PostgreSQL + Discord bot Helm chart +- `bin/apply-secrets.sh`: `.env` 기반 Kubernetes Secret 반영 +- `bin/deploy.sh`: Helm 배포 +- `bin/verify-migration.sh`: Docker 원본 DB와 k3s DB row count 비교 +- `bin/cutover.sh`: dump/restore + 로그 PVC 이관 + bot scale-up + +## Runtime Contract + +- Namespace: `livid-bot` +- Helm release: `livid-bot` +- Secret names: + - `livid-bot-db-credentials` + - `livid-bot-bot-secrets` +- PVC names: + - `pgdata-livid-bot-db-0` + - `livid-bot-logs` + +## First Deploy + +Prerequisite: `quant` 저장소의 `deploy/k3s/setup/04-bootstrap-cluster.sh`를 먼저 실행해 +`livid-bot` namespace와 `livid-bot-app` ServiceAccount를 만들어 둔다. + +```bash +cd /Users/haril/projects/livid-bot +./deploy/k3s/bin/apply-secrets.sh +./deploy/k3s/bin/deploy.sh +``` + +## Data-Preserving Cutover + +컷오버는 짧은 점검창을 전제로 한다. `bot` 컨테이너를 멈춘 뒤 최종 +`pg_dump`를 수행하고, 복구 가능한 Docker 원본을 그대로 남겨 둔다. + +```bash +cd /Users/haril/projects/livid-bot +./deploy/k3s/bin/cutover.sh +``` + +worktree에서 실행할 때는 원본 Docker Compose 체크아웃 경로를 지정한다. + +```bash +SOURCE_PROJECT_DIR=/Users/haril/projects/livid-bot ./deploy/k3s/bin/cutover.sh +``` + +실행 내용: + +1. Secrets 적용 +2. Helm 배포 (`bot.replicas=0`) +3. Docker 원본 DB pre-cutover dump +4. Docker bot 정지 후 final dump +5. k3s PostgreSQL restore +6. `./logs`를 PVC로 복사 +7. row count / `schema_migrations` 검증 +8. `bot.replicas=1`로 scale-up + +## Rollback + +원본 Docker DB 볼륨과 `./logs`는 컷오버 후에도 유지된다. + +```bash +cd /Users/haril/projects/livid-bot +helm uninstall livid-bot -n livid-bot +docker compose up -d db bot +``` diff --git a/deploy/k3s/bin/apply-secrets.sh b/deploy/k3s/bin/apply-secrets.sh new file mode 100755 index 0000000..917503c --- /dev/null +++ b/deploy/k3s/bin/apply-secrets.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +SOURCE_PROJECT_DIR="${SOURCE_PROJECT_DIR:-${PROJECT_DIR}}" +ENV_FILE="${ENV_FILE:-${SOURCE_PROJECT_DIR}/.env}" + +RELEASE_NAME="${RELEASE_NAME:-livid-bot}" +NAMESPACE="${NAMESPACE:-livid-bot}" + +if [[ -f "${ENV_FILE}" ]]; then + set -a + # shellcheck disable=SC1090 + source "${ENV_FILE}" + set +a +fi + +: "${DISCORD_BOT_TOKEN:?DISCORD_BOT_TOKEN is required}" +: "${DISCORD_APPLICATION_ID:?DISCORD_APPLICATION_ID is required}" +: "${DISCORD_GUILD_ID:?DISCORD_GUILD_ID is required}" + +POSTGRES_USER="${POSTGRES_USER:-livid}" +POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-livid}" +POSTGRES_DB="${POSTGRES_DB:-livid}" + +kubectl create secret generic "${RELEASE_NAME}-db-credentials" \ + --namespace "${NAMESPACE}" \ + --from-literal=POSTGRES_USER="${POSTGRES_USER}" \ + --from-literal=POSTGRES_PASSWORD="${POSTGRES_PASSWORD}" \ + --from-literal=POSTGRES_DB="${POSTGRES_DB}" \ + --dry-run=client \ + -o yaml | kubectl apply -n "${NAMESPACE}" -f - + +kubectl create secret generic "${RELEASE_NAME}-bot-secrets" \ + --namespace "${NAMESPACE}" \ + --from-literal=DISCORD_BOT_TOKEN="${DISCORD_BOT_TOKEN}" \ + --from-literal=DISCORD_APPLICATION_ID="${DISCORD_APPLICATION_ID}" \ + --from-literal=DISCORD_GUILD_ID="${DISCORD_GUILD_ID}" \ + --dry-run=client \ + -o yaml | kubectl apply -n "${NAMESPACE}" -f - diff --git a/deploy/k3s/bin/cutover.sh b/deploy/k3s/bin/cutover.sh new file mode 100755 index 0000000..659bbe2 --- /dev/null +++ b/deploy/k3s/bin/cutover.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +SOURCE_PROJECT_DIR="${SOURCE_PROJECT_DIR:-${PROJECT_DIR}}" +OUTPUT_DIR="${PROJECT_DIR}/deploy/k3s/output" + +RELEASE_NAME="${RELEASE_NAME:-livid-bot}" +NAMESPACE="${NAMESPACE:-livid-bot}" +SOURCE_POSTGRES_USER="${SOURCE_POSTGRES_USER:-livid}" +SOURCE_POSTGRES_PASSWORD="${SOURCE_POSTGRES_PASSWORD:-livid}" +SOURCE_POSTGRES_DB="${SOURCE_POSTGRES_DB:-livid}" +TIMESTAMP="$(date +%Y%m%d-%H%M%S)" + +PRE_DUMP="${OUTPUT_DIR}/${TIMESTAMP}-pre-cutover.dump" +FINAL_DUMP="${OUTPUT_DIR}/${TIMESTAMP}-final.dump" +HELPER_POD="${RELEASE_NAME}-logs-sync" + +mkdir -p "${OUTPUT_DIR}" + +target_secret_key() { + local key="$1" + kubectl get secret "${RELEASE_NAME}-db-credentials" \ + -n "${NAMESPACE}" \ + -o "jsonpath={.data.${key}}" | python3 -c 'import base64, sys; print(base64.b64decode(sys.stdin.read()).decode(), end="")' +} + +cleanup_helper() { + kubectl delete pod "${HELPER_POD}" -n "${NAMESPACE}" --ignore-not-found >/dev/null 2>&1 || true +} + +trap cleanup_helper EXIT + +"${SCRIPT_DIR}/apply-secrets.sh" +"${SCRIPT_DIR}/deploy.sh" --set bot.replicas=0 + +kubectl rollout status statefulset/"${RELEASE_NAME}-db" -n "${NAMESPACE}" --timeout=180s + +( + cd "${SOURCE_PROJECT_DIR}" + docker compose exec -T \ + -e PGPASSWORD="${SOURCE_POSTGRES_PASSWORD}" \ + db \ + pg_dump -U "${SOURCE_POSTGRES_USER}" -d "${SOURCE_POSTGRES_DB}" -Fc +) > "${PRE_DUMP}" + +( + cd "${SOURCE_PROJECT_DIR}" + docker compose stop bot +) + +( + cd "${SOURCE_PROJECT_DIR}" + docker compose exec -T \ + -e PGPASSWORD="${SOURCE_POSTGRES_PASSWORD}" \ + db \ + pg_dump -U "${SOURCE_POSTGRES_USER}" -d "${SOURCE_POSTGRES_DB}" -Fc +) > "${FINAL_DUMP}" + +TARGET_POSTGRES_USER="$(target_secret_key POSTGRES_USER)" +TARGET_POSTGRES_PASSWORD="$(target_secret_key POSTGRES_PASSWORD)" +TARGET_POSTGRES_DB="$(target_secret_key POSTGRES_DB)" + +kubectl exec -i -n "${NAMESPACE}" "${RELEASE_NAME}-db-0" -- env \ + PGPASSWORD="${TARGET_POSTGRES_PASSWORD}" \ + pg_restore \ + --clean \ + --if-exists \ + --no-owner \ + --no-privileges \ + -U "${TARGET_POSTGRES_USER}" \ + -d "${TARGET_POSTGRES_DB}" < "${FINAL_DUMP}" + +kubectl apply -n "${NAMESPACE}" -f - </dev/null 2>&1; then + echo "Missing ${SERVICE_ACCOUNT_NAME} in namespace ${NAMESPACE}." >&2 + echo "Run quant/deploy/k3s/setup/04-bootstrap-cluster.sh first." >&2 + exit 1 +fi + +helm upgrade --install \ + "${RELEASE_NAME}" \ + "${PROJECT_DIR}/deploy/k3s/charts/livid-bot" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + "$@" diff --git a/deploy/k3s/bin/verify-migration.sh b/deploy/k3s/bin/verify-migration.sh new file mode 100755 index 0000000..80265e1 --- /dev/null +++ b/deploy/k3s/bin/verify-migration.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +SOURCE_PROJECT_DIR="${SOURCE_PROJECT_DIR:-${PROJECT_DIR}}" + +RELEASE_NAME="${RELEASE_NAME:-livid-bot}" +NAMESPACE="${NAMESPACE:-livid-bot}" +SOURCE_POSTGRES_USER="${SOURCE_POSTGRES_USER:-livid}" +SOURCE_POSTGRES_PASSWORD="${SOURCE_POSTGRES_PASSWORD:-livid}" +SOURCE_POSTGRES_DB="${SOURCE_POSTGRES_DB:-livid}" +TARGET_POD="${TARGET_POD:-${RELEASE_NAME}-db-0}" + +target_secret_key() { + local key="$1" + kubectl get secret "${RELEASE_NAME}-db-credentials" \ + -n "${NAMESPACE}" \ + -o "jsonpath={.data.${key}}" | python3 -c 'import base64, sys; print(base64.b64decode(sys.stdin.read()).decode(), end="")' +} + +source_psql() { + local sql="$1" + ( + cd "${SOURCE_PROJECT_DIR}" + docker compose exec -T \ + -e PGPASSWORD="${SOURCE_POSTGRES_PASSWORD}" \ + db \ + psql -U "${SOURCE_POSTGRES_USER}" -d "${SOURCE_POSTGRES_DB}" -At -c "${sql}" + ) +} + +target_psql() { + local sql="$1" + kubectl exec -n "${NAMESPACE}" "${TARGET_POD}" -- env \ + PGPASSWORD="${TARGET_POSTGRES_PASSWORD}" \ + psql -U "${TARGET_POSTGRES_USER}" -d "${TARGET_POSTGRES_DB}" -At -c "${sql}" +} + +TARGET_POSTGRES_USER="$(target_secret_key POSTGRES_USER)" +TARGET_POSTGRES_PASSWORD="$(target_secret_key POSTGRES_PASSWORD)" +TARGET_POSTGRES_DB="$(target_secret_key POSTGRES_DB)" + +mapfile -t tables < <(source_psql "SELECT tablename FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename;") + +if [[ "${#tables[@]}" -eq 0 ]]; then + echo "No public tables found in source database." >&2 + exit 1 +fi + +for table in "${tables[@]}"; do + source_count="$(source_psql "SELECT COUNT(*) FROM \"${table}\";")" + target_count="$(target_psql "SELECT COUNT(*) FROM \"${table}\";")" + printf '%-32s source=%s target=%s\n' "${table}" "${source_count}" "${target_count}" + if [[ "${source_count}" != "${target_count}" ]]; then + echo "Count mismatch detected for ${table}." >&2 + exit 1 + fi +done + +source_migrations="$(source_psql "SELECT filename FROM schema_migrations ORDER BY filename;")" +target_migrations="$(target_psql "SELECT filename FROM schema_migrations ORDER BY filename;")" + +if [[ "${source_migrations}" != "${target_migrations}" ]]; then + echo "schema_migrations mismatch detected." >&2 + exit 1 +fi + +echo "Migration verification passed." diff --git a/mise.toml b/mise.toml index 7676d45..2a5ba6d 100644 --- a/mise.toml +++ b/mise.toml @@ -4,3 +4,15 @@ golangci-lint = "latest" [env] _.file = ".env" + +[tasks.apply-secrets] +description = "Apply livid-bot runtime secrets to Kubernetes" +run = "./deploy/k3s/bin/apply-secrets.sh" + +[tasks.deploy-k3s] +description = "Deploy or update livid-bot on k3s" +run = "./deploy/k3s/bin/deploy.sh" + +[tasks.cutover-k3s] +description = "Migrate livid-bot data from Docker Compose to k3s" +run = "./deploy/k3s/bin/cutover.sh"