Repository navigation
fix(backups): leave the .git history of cloned checkouts out of web-server backups #5734
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| import { execSync } from "node:child_process"; | ||
| import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; | ||
| import { tmpdir } from "node:os"; | ||
| import { dirname, join } from "node:path"; | ||
| import { getWebServerBackupFilters } from "@dokploy/server/utils/backups/web-server-filters"; | ||
| import { afterEach, describe, expect, it } from "vitest"; | ||
|
|
||
| const hasRsync = (() => { | ||
| try { | ||
| execSync("rsync --version", { stdio: "ignore" }); | ||
| return true; | ||
| } catch { | ||
| return false; | ||
| } | ||
| })(); | ||
|
|
||
| let root: string | undefined; | ||
| afterEach(() => { | ||
| if (root) rmSync(root, { recursive: true, force: true }); | ||
| root = undefined; | ||
| }); | ||
|
|
||
| const backedUpFiles = (files: string[], dropAppNames: string[]) => { | ||
| root = mkdtempSync(join(tmpdir(), "web-server-filters-")); | ||
| const src = join(root, "src"); | ||
| const dst = join(root, "dst"); | ||
| mkdirSync(dst); | ||
| for (const file of files) { | ||
| mkdirSync(dirname(join(src, file)), { recursive: true }); | ||
| writeFileSync(join(src, file), "x"); | ||
| } | ||
| const filters = getWebServerBackupFilters(dropAppNames).join(" "); | ||
| execSync(`rsync -a ${filters} ${src}/ ${dst}/`); | ||
| return execSync("find . -type f", { cwd: dst }) | ||
| .toString() | ||
| .trim() | ||
| .split("\n") | ||
| .map((file) => file.replace(/^\.\//, "")) | ||
| .sort(); | ||
| }; | ||
|
|
||
| describe("getWebServerBackupFilters", () => { | ||
| it("puts drop-app includes before the code/ excludes", () => { | ||
| expect(getWebServerBackupFilters(["site-abc"])).toEqual([ | ||
| "--include=applications/site-abc/code/\\*\\*\\*", | ||
| "--exclude='applications/*/code/'", | ||
| "--exclude='compose/*/code/'", | ||
| ]); | ||
| }); | ||
|
|
||
| it.skipIf(!hasRsync)( | ||
| "skips git checkouts but keeps drop uploads and service files", | ||
| () => { | ||
| const files = backedUpFiles( | ||
| [ | ||
| "applications/git-app/code/.git/HEAD", | ||
| "applications/git-app/code/src/index.ts", | ||
| "applications/git-app/files/config.yml", | ||
| "applications/drop-app/code/index.html", | ||
| "compose/stack/code/docker-compose.yml", | ||
| "compose/stack/files/config.yml", | ||
| "traefik/traefik.yml", | ||
| ], | ||
| ["drop-app"], | ||
| ); | ||
|
|
||
| expect(files).toEqual([ | ||
| "applications/drop-app/code/index.html", | ||
| "applications/git-app/files/config.yml", | ||
| "compose/stack/files/config.yml", | ||
| "traefik/traefik.yml", | ||
| ]); | ||
| }, | ||
| ); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| import { quote } from "shell-quote"; | ||
|
|
||
| // rsync filter rules for the web-server backup. Git and raw sources are | ||
| // re-created under */code/ on every deploy, so those checkouts are skipped; | ||
| // drop (zip upload) apps keep theirs, since it is the only copy of their code. | ||
| // Includes must come before the excludes: rsync applies the first match. | ||
| export const getWebServerBackupFilters = (dropAppNames: string[]) => [ | ||
| ...dropAppNames.map( | ||
| (appName) => `--include=${quote([`applications/${appName}/code/***`])}`, | ||
| ), | ||
| "--exclude='applications/*/code/'", | ||
| "--exclude='compose/*/code/'", | ||
| ]; | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,6 +3,8 @@ import { mkdtemp, rm, stat, writeFile } from "node:fs/promises"; | |
| import { tmpdir } from "node:os"; | ||
| import { join } from "node:path"; | ||
| import { IS_CLOUD, paths } from "@dokploy/server/constants"; | ||
| import { db } from "@dokploy/server/db"; | ||
| import { applications } from "@dokploy/server/db/schema"; | ||
| import { | ||
| ENCRYPTION_KEY_BACKUP_FILE, | ||
| exportEncryptionKeys, | ||
|
|
@@ -13,10 +15,12 @@ import { | |
| updateDeploymentStatus, | ||
| } from "@dokploy/server/services/deployment"; | ||
| import { findDestinationById } from "@dokploy/server/services/destination"; | ||
| import { eq } from "drizzle-orm"; | ||
| import { sendDokployBackupNotifications } from "../notifications/dokploy-backup"; | ||
| import { execAsync } from "../process/execAsync"; | ||
| import { redactRcloneCredentials } from "./redact"; | ||
| import { getBackupTimestamp, getS3Credentials, normalizeS3Path } from "./utils"; | ||
| import { getWebServerBackupFilters } from "./web-server-filters"; | ||
|
|
||
| function formatBytes(bytes?: number) { | ||
| if (bytes === undefined) return "Unknown size"; | ||
|
|
@@ -81,8 +85,15 @@ export const runWebServerBackup = async (backup: BackupSchedule) => { | |
| writeStream.write(`Cleaning up temp file: ${cleanupCommand}\n`); | ||
| await execAsync(cleanupCommand); | ||
|
|
||
| const dropApps = await db.query.applications.findMany({ | ||
| columns: { appName: true }, | ||
| where: eq(applications.sourceType, "drop"), | ||
| }); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The drop-app list is read after Use the dump's snapshot for the include list, or conservatively keep code for apps whose source changes during the backup.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Resolved by fd7533f: with only |
||
| const filters = getWebServerBackupFilters( | ||
| dropApps.map(({ appName }) => appName), | ||
| ).join(" "); | ||
| await execAsync( | ||
| `rsync -a --ignore-errors --no-specials --no-devices --exclude='volume-backups/' --exclude='${ENCRYPTION_KEY_BACKUP_FILE}' ${BASE_PATH}/ ${tempDir}/filesystem/`, | ||
| `rsync -a --ignore-errors --no-specials --no-devices --exclude='volume-backups/' --exclude='${ENCRYPTION_KEY_BACKUP_FILE}' ${filters} ${BASE_PATH}/ ${tempDir}/filesystem/`, | ||
| ); | ||
|
|
||
| writeStream.write("Copied filesystem to temp directory\n"); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The blanket Compose exclusion removes live files, not just cloned source. A raw Compose service with
./data:/var/lib/postgresql/datastores its database undercompose/<appName>/code/data. Those files cannot be rebuilt from the saved Compose text. Restore deletes the existing data path before copying the archive, so this change loses that database on restore.Preserve bind-mounted paths under
code/instead of excluding the whole directory.Knowledge Base Used: Backups and restore
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You're right. Restore runs
rm -rf "$BASE_PATH/"*before copying, so excludingcompose/*/code/would delete a running service's./dataon a same-server restore. Fixed in fd7533f: the backup now keeps every working tree and only excludesapplications/*/code/.git/andcompose/*/code/.git/, which every deploy re-clones. The rsync test now asserts that acompose/stack/code/datafile is kept. I updated the PR description to explain the narrower scope.