Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 75 additions & 0 deletions apps/dokploy/__test__/backups/web-server-filters.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import { execSync } from "node:child_process";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { getWebServerBackupFilters } from "@dokploy/server/utils/backups/web-server-filters";
import { afterEach, describe, expect, it } from "vitest";

const hasRsync = (() => {
try {
execSync("rsync --version", { stdio: "ignore" });
return true;
} catch {
return false;
}
})();

let root: string | undefined;
afterEach(() => {
if (root) rmSync(root, { recursive: true, force: true });
root = undefined;
});

const backedUpFiles = (files: string[], dropAppNames: string[]) => {
root = mkdtempSync(join(tmpdir(), "web-server-filters-"));
const src = join(root, "src");
const dst = join(root, "dst");
mkdirSync(dst);
for (const file of files) {
mkdirSync(dirname(join(src, file)), { recursive: true });
writeFileSync(join(src, file), "x");
}
const filters = getWebServerBackupFilters(dropAppNames).join(" ");
execSync(`rsync -a ${filters} ${src}/ ${dst}/`);
return execSync("find . -type f", { cwd: dst })
.toString()
.trim()
.split("\n")
.map((file) => file.replace(/^\.\//, ""))
.sort();
};

describe("getWebServerBackupFilters", () => {
it("puts drop-app includes before the code/ excludes", () => {
expect(getWebServerBackupFilters(["site-abc"])).toEqual([
"--include=applications/site-abc/code/\\*\\*\\*",
"--exclude='applications/*/code/'",
"--exclude='compose/*/code/'",
]);
});

it.skipIf(!hasRsync)(
"skips git checkouts but keeps drop uploads and service files",
() => {
const files = backedUpFiles(
[
"applications/git-app/code/.git/HEAD",
"applications/git-app/code/src/index.ts",
"applications/git-app/files/config.yml",
"applications/drop-app/code/index.html",
"compose/stack/code/docker-compose.yml",
"compose/stack/files/config.yml",
"traefik/traefik.yml",
],
["drop-app"],
);

expect(files).toEqual([
"applications/drop-app/code/index.html",
"applications/git-app/files/config.yml",
"compose/stack/files/config.yml",
"traefik/traefik.yml",
]);
},
);
});
13 changes: 13 additions & 0 deletions packages/server/src/utils/backups/web-server-filters.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import { quote } from "shell-quote";

// rsync filter rules for the web-server backup. Git and raw sources are
// re-created under */code/ on every deploy, so those checkouts are skipped;
// drop (zip upload) apps keep theirs, since it is the only copy of their code.
// Includes must come before the excludes: rsync applies the first match.
export const getWebServerBackupFilters = (dropAppNames: string[]) => [
...dropAppNames.map(
(appName) => `--include=${quote([`applications/${appName}/code/***`])}`,
),
"--exclude='applications/*/code/'",
"--exclude='compose/*/code/'",

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Compose backups omit live data

The blanket Compose exclusion removes live files, not just cloned source. A raw Compose service with ./data:/var/lib/postgresql/data stores its database under compose/<appName>/code/data. Those files cannot be rebuilt from the saved Compose text. Restore deletes the existing data path before copying the archive, so this change loses that database on restore.

Preserve bind-mounted paths under code/ instead of excluding the whole directory.

Knowledge Base Used: Backups and restore

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right. Restore runs rm -rf "$BASE_PATH/"* before copying, so excluding compose/*/code/ would delete a running service's ./data on a same-server restore. Fixed in fd7533f: the backup now keeps every working tree and only excludes applications/*/code/.git/ and compose/*/code/.git/, which every deploy re-clones. The rsync test now asserts that a compose/stack/code/data file is kept. I updated the PR description to explain the narrower scope.

];
13 changes: 12 additions & 1 deletion packages/server/src/utils/backups/web-server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ import { mkdtemp, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { IS_CLOUD, paths } from "@dokploy/server/constants";
import { db } from "@dokploy/server/db";
import { applications } from "@dokploy/server/db/schema";
import {
ENCRYPTION_KEY_BACKUP_FILE,
exportEncryptionKeys,
Expand All @@ -13,10 +15,12 @@ import {
updateDeploymentStatus,
} from "@dokploy/server/services/deployment";
import { findDestinationById } from "@dokploy/server/services/destination";
import { eq } from "drizzle-orm";
import { sendDokployBackupNotifications } from "../notifications/dokploy-backup";
import { execAsync } from "../process/execAsync";
import { redactRcloneCredentials } from "./redact";
import { getBackupTimestamp, getS3Credentials, normalizeS3Path } from "./utils";
import { getWebServerBackupFilters } from "./web-server-filters";

function formatBytes(bytes?: number) {
if (bytes === undefined) return "Unknown size";
Expand Down Expand Up @@ -81,8 +85,15 @@ export const runWebServerBackup = async (backup: BackupSchedule) => {
writeStream.write(`Cleaning up temp file: ${cleanupCommand}\n`);
await execAsync(cleanupCommand);

const dropApps = await db.query.applications.findMany({
columns: { appName: true },
where: eq(applications.sourceType, "drop"),
});

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Drop backups can omit uploads

The drop-app list is read after pg_dump, so it can disagree with the saved database. If a user switches a drop app to Git while the dump runs, the dump can still contain sourceType: "drop" while this query omits the app. The new exclusion then skips its uploaded code even if those files remain untouched. Restoring that backup brings back a drop app with no source to redeploy.

Use the dump's snapshot for the include list, or conservatively keep code for apps whose source changes during the backup.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Resolved by fd7533f: with only .git excluded there's no drop-app lookup anymore, so nothing depends on database state and there's no race with pg_dump. Drop apps' code is kept like everything else.

const filters = getWebServerBackupFilters(
dropApps.map(({ appName }) => appName),
).join(" ");
await execAsync(
`rsync -a --ignore-errors --no-specials --no-devices --exclude='volume-backups/' --exclude='${ENCRYPTION_KEY_BACKUP_FILE}' ${BASE_PATH}/ ${tempDir}/filesystem/`,
`rsync -a --ignore-errors --no-specials --no-devices --exclude='volume-backups/' --exclude='${ENCRYPTION_KEY_BACKUP_FILE}' ${filters} ${BASE_PATH}/ ${tempDir}/filesystem/`,
);

writeStream.write("Copied filesystem to temp directory\n");
Expand Down