From 665b8d54b6f5098dff57163744bce03140a567df Mon Sep 17 00:00:00 2001 From: minwoox Date: Mon, 22 Aug 2022 21:37:21 +0900 Subject: [PATCH 01/11] Support local-to-remote mirroring Motivation We currently support only remote-to-local mirroring. We should support the opposite direction as well. Modifications: - Implement `GitMirror#mirrorToLocalRemote()` which threw `UnsupportedOperationException` before. Result: - Close #53 - You can now enable mirroring from Central Dogma to a remote Git server. --- .../it/mirror/git/GitMirrorTest.java | 20 +- .../git/LocalToRemoteGitMirrorTest.java | 462 ++++++++++++++++++ .../server/internal/mirror/GitMirror.java | 456 +++++++++++++++-- .../server/internal/mirror/MirrorState.java | 5 +- .../testing/internal/TestUtil.java | 3 +- 5 files changed, 882 insertions(+), 64 deletions(-) create mode 100644 it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java diff --git a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java index 00fda5c96..fe2e3537f 100644 --- a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java +++ b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java @@ -46,6 +46,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -63,7 +64,6 @@ import com.linecorp.centraldogma.server.MirrorException; import com.linecorp.centraldogma.server.MirroringService; import com.linecorp.centraldogma.server.storage.project.Project; -import com.linecorp.centraldogma.testing.internal.TemporaryFolderExtension; import com.linecorp.centraldogma.testing.internal.TestUtil; import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension; @@ -93,14 +93,8 @@ static void init() { mirroringService = dogma.mirroringService(); } - @RegisterExtension - final TemporaryFolderExtension gitRepoDir = new TemporaryFolderExtension() { - @Override - protected boolean runForEachTest() { - return true; - } - }; - + @TempDir + File gitRepoDir; private Git git; private File gitWorkTree; private String gitUri; @@ -110,7 +104,7 @@ protected boolean runForEachTest() { @BeforeEach void initGitRepo(TestInfo testInfo) throws Exception { final String repoName = TestUtil.normalizedDisplayName(testInfo); - gitWorkTree = new File(gitRepoDir.getRoot().toFile(), repoName).getAbsoluteFile(); + gitWorkTree = new File(gitRepoDir, repoName).getAbsoluteFile(); final Repository gitRepo = new FileRepositoryBuilder().setWorkTree(gitWorkTree).build(); createGitRepo(gitRepo); @@ -334,7 +328,7 @@ void remoteToLocal_submodule(TestInfo testInfo) throws Exception { // Create a new repository for a submodule. final String submoduleName = TestUtil.normalizedDisplayName(testInfo) + ".submodule"; final File gitSubmoduleWorkTree = - new File(gitRepoDir.getRoot().toFile(), submoduleName).getAbsoluteFile(); + new File(gitRepoDir, submoduleName).getAbsoluteFile(); final Repository gitSubmoduleRepo = new FileRepositoryBuilder().setWorkTree(gitSubmoduleWorkTree).build(); createGitRepo(gitSubmoduleRepo); @@ -453,8 +447,8 @@ private void addToGitIndex(String path, String content) throws IOException, GitA addToGitIndex(git, gitWorkTree, path, content); } - private static void addToGitIndex(Git git, File gitWorkTree, - String path, String content) throws IOException, GitAPIException { + static void addToGitIndex(Git git, File gitWorkTree, + String path, String content) throws IOException, GitAPIException { final File file = Paths.get(gitWorkTree.getAbsolutePath(), path.split("/")).toFile(); file.getParentFile().mkdirs(); Files.asCharSink(file, StandardCharsets.UTF_8).write(content); diff --git a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java new file mode 100644 index 000000000..d7b655070 --- /dev/null +++ b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java @@ -0,0 +1,462 @@ +/* + * Copyright 2022 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.it.mirror.git; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.linecorp.centraldogma.it.mirror.git.GitMirrorTest.addToGitIndex; +import static com.linecorp.centraldogma.server.internal.mirror.GitMirror.LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_COMMIT_SECTION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_GPGSIGN; +import static org.eclipse.jgit.lib.Constants.R_HEADS; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Set; + +import javax.annotation.Nullable; + +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.storage.file.FileRepositoryBuilder; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import com.google.common.base.Strings; + +import com.linecorp.centraldogma.client.CentralDogma; +import com.linecorp.centraldogma.common.CentralDogmaException; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.server.CentralDogmaBuilder; +import com.linecorp.centraldogma.server.MirrorException; +import com.linecorp.centraldogma.server.MirroringService; +import com.linecorp.centraldogma.server.internal.mirror.MirrorState; +import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.testing.internal.TestUtil; +import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension; + +class LocalToRemoteGitMirrorTest { + + private static final int MAX_NUM_FILES = 32; + private static final long MAX_NUM_BYTES = 1048576; // 1 MiB + + private static final String REPO_FOO = "foo"; + + @RegisterExtension + static final CentralDogmaExtension dogma = new CentralDogmaExtension() { + @Override + protected void configure(CentralDogmaBuilder builder) { + builder.mirroringEnabled(true); + builder.maxNumFilesPerMirror(MAX_NUM_FILES); + builder.maxNumBytesPerMirror(MAX_NUM_BYTES); + } + }; + + private static CentralDogma client; + private static MirroringService mirroringService; + + @BeforeAll + static void init() { + client = dogma.client(); + mirroringService = dogma.mirroringService(); + } + + @TempDir + File gitRepoDir; + + private Git git; + private File gitWorkTree; + private String gitUri; + + private String projName; + + @BeforeEach + void initGitRepo(TestInfo testInfo) throws Exception { + final String repoName = TestUtil.normalizedDisplayName(testInfo); + gitWorkTree = new File(gitRepoDir, repoName).getAbsoluteFile(); + final Repository gitRepo = new FileRepositoryBuilder().setWorkTree(gitWorkTree).build(); + createGitRepo(gitRepo); + + git = Git.wrap(gitRepo); + gitUri = "git+file://" + + (gitWorkTree.getPath().startsWith(File.separator) ? "" : '/') + + gitWorkTree.getPath().replace(File.separatorChar, '/') + + "/.git"; + // Start the master branch with an empty commit. + git.commit().setMessage("Initial commit").call(); + } + + private static void createGitRepo(Repository gitRepo) throws IOException { + gitRepo.create(); + + // Disable GPG signing. + final StoredConfig config = gitRepo.getConfig(); + config.setBoolean(CONFIG_COMMIT_SECTION, null, CONFIG_KEY_GPGSIGN, false); + config.save(); + } + + @BeforeEach + void initDogmaRepo(TestInfo testInfo) { + projName = TestUtil.normalizedDisplayName(testInfo); + client.createProject(projName).join(); + client.createRepository(projName, REPO_FOO).join(); + } + + @AfterEach + void destroyDogmaRepo() { + client.removeProject(projName).join(); + client.purgeProject(projName).join(); + } + + @ParameterizedTest + @CsvSource({ + "'', ''", + "/local/foo, /remote", + "/local, /remote/foo", + "/local/foo, /remote/foo" + }) + void localToRemote(String localPath, String remotePath) throws Exception { + pushMirrorSettings(localPath, remotePath, null); + + final ObjectId commitId = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(getFileContent(commitId, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME)) + .isNull(); + // Mirror an empty Central Dogma repository, which will; + // - Create /.mirror_state.json + mirroringService.mirror().join(); + + final ObjectId commitId1 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(commitId).isNotEqualTo(commitId1); + byte[] content = getFileContent(commitId1, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + MirrorState mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("1"); + + // Mirror once again without adding a commit. + mirroringService.mirror().join(); + + // Make sure no commit was added thus the source revision wasn't changed. + final ObjectId commitId2 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(commitId2).isEqualTo(commitId1); + content = getFileContent(commitId2, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("1"); + + // Create a new commit + client.forRepo(projName, REPO_FOO) + .commit("Add a commit", + Change.ofJsonUpsert(localPath + "/foo.json", "{\"a\":\"b\"}"), + Change.ofJsonUpsert(localPath + "/bar/foo.json", "{\"a\":\"c\"}"), + Change.ofTextUpsert(localPath + "/baz/foo.txt", "\"a\": \"b\"\n")) + .push().join(); + + mirroringService.mirror().join(); + final ObjectId commitId3 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(commitId3).isNotEqualTo(commitId2); + content = getFileContent(commitId3, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("2"); + assertThat(Jackson.writeValueAsString(Jackson.readTree( + getFileContent(commitId3, remotePath + "/foo.json")))).isEqualTo("{\"a\":\"b\"}"); + assertThat(Jackson.writeValueAsString(Jackson.readTree( + getFileContent(commitId3, remotePath + "/bar/foo.json")))).isEqualTo("{\"a\":\"c\"}"); + assertThat(new String(getFileContent(commitId3, remotePath + "/baz/foo.txt"))) + .isEqualTo("\"a\": \"b\"\n"); + + // Mirror once again without adding a commit. + mirroringService.mirror().join(); + + // Make sure no commit was added thus the source revision wasn't changed. + final ObjectId commitId4 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(commitId4).isEqualTo(commitId3); + content = getFileContent(commitId4, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("2"); + + // Create a new commit + client.forRepo(projName, REPO_FOO) + .commit("Remove foo.json and foo.txt", + Change.ofRemoval(localPath + "/foo.json"), + Change.ofRemoval(localPath + "/baz/foo.txt")) + .push().join(); + + mirroringService.mirror().join(); + final ObjectId commitId5 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(commitId5).isNotEqualTo(commitId4); + content = getFileContent(commitId5, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("3"); + assertThat(getFileContent(commitId5, remotePath + "/foo.json")).isNull(); + assertThat(getFileContent(commitId5, remotePath + "/baz/foo.txt")).isNull(); + assertThat(Jackson.writeValueAsString(Jackson.readTree( + getFileContent(commitId5, remotePath + "/bar/foo.json")))).isEqualTo("{\"a\":\"c\"}"); + + addToGitIndex(git, gitWorkTree, (remotePath + "/bar/foo.json").substring(1), "{\"a\":\"d\"}"); + git.commit().setMessage("Change the file arbitrarily").call(); + final ObjectId commitId6 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(Jackson.writeValueAsString(Jackson.readTree( + getFileContent(commitId6, remotePath + "/bar/foo.json")))).isEqualTo("{\"a\":\"d\"}"); + + mirroringService.mirror().join(); + final ObjectId commitId7 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(commitId7).isNotEqualTo(commitId6); + content = getFileContent(commitId7, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("3"); + // The arbitrarily changed file is overwritten. + assertThat(Jackson.writeValueAsString(Jackson.readTree( + getFileContent(commitId7, remotePath + "/bar/foo.json")))).isEqualTo("{\"a\":\"c\"}"); + } + + @Nullable + private byte[] getFileContent(ObjectId commitId, String fileName) throws IOException { + try (ObjectReader reader = git.getRepository().newObjectReader(); + TreeWalk treeWalk = new TreeWalk(reader); + RevWalk revWalk = new RevWalk(reader)) { + treeWalk.addTree(revWalk.parseTree(commitId).getId()); + + while (treeWalk.next()) { + if (treeWalk.getFileMode() == FileMode.TREE) { + treeWalk.enterSubtree(); + continue; + } + if (fileName.equals('/' + treeWalk.getPathString())) { + final ObjectId objectId = treeWalk.getObjectId(0); + return reader.open(objectId).getBytes(); + } + } + } + return null; + } + + @ParameterizedTest + @CsvSource({ + "'', ''", + "/local/foo, /remote", + "/local, /remote/foo", + "/local/foo, /remote/foo" + }) + void LocalToRemote_gitignore(String localPath, String remotePath) throws Exception { + pushMirrorSettings(localPath, remotePath, "\"/exclude_if_root.txt\\n**/exclude_dir\""); + checkGitignore(localPath, remotePath); + } + + @ParameterizedTest + @CsvSource({ + "'', ''", + "/local/foo, /remote", + "/local, /remote/foo", + "/local/foo, /remote/foo" + }) + void localToRemote_gitignore_with_array(String localPath, String remotePath) throws Exception { + pushMirrorSettings(localPath, remotePath, "[\"/exclude_if_root.txt\", \"exclude_dir\"]"); + checkGitignore(localPath, remotePath); + } + + @Test + void localToRemote_subdirectory() throws Exception { + pushMirrorSettings("/source/main", "/target", null); + + client.forRepo(projName, REPO_FOO) + .commit("Add a file that's not part of mirror", Change.ofTextUpsert("/not_mirrored.txt", "")) + .push().join(); + + // Mirror an empty git repository, which will; + // - Create /target/mirror_state.json + mirroringService.mirror().join(); + + final ObjectId commitId = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + byte[] content = getFileContent(commitId, "/target/" + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + MirrorState mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("2"); + + Set files = listFiles(commitId); + assertThat(files.size()).isOne(); // mirror state file. + + // Now, add some files to the git repository and mirror. + // Note that the files not under '/source' should not be mirrored. + client.forRepo(projName, REPO_FOO) + .commit("Add the release dates of the 'Infamous' series", + Change.ofTextUpsert("/source/main/first/light.txt", "26-Aug-2014"), // mirrored + Change.ofJsonUpsert("/second/son.json", "{\"release\": \"21-Mar-2014\"}")) // not mirrored + .push().join(); + mirroringService.mirror().join(); + + final ObjectId commitId1 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + content = getFileContent(commitId1, "/target/" + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("3"); + + files = listFiles(commitId1); + assertThat(files.size()).isSameAs(2); // mirror state file and target/first/light.txt + // Make sure 'target/first/light.txt' is mirrored. + assertThat(new String(getFileContent(commitId1, "/target/first/light.txt"))) + .isEqualTo("26-Aug-2014\n"); + } + + @Test + void localToRemote_tooManyFiles() throws Exception { + pushMirrorSettings(null, null, null); + + // Add more than allowed number of filed. + final ArrayList> changes = new ArrayList<>(); + for (int i = 0; i <= MAX_NUM_FILES; i++) { + changes.add(Change.ofTextUpsert("/" + i + ".txt", String.valueOf(i))); + } + client.forRepo(projName, REPO_FOO).commit("Add a bunch of numbered files", changes).push().join(); + + // Perform mirroring, which should fail. + assertThatThrownBy(() -> mirroringService.mirror().join()) + .hasCauseInstanceOf(MirrorException.class) + .hasMessageContaining("contains more than") + .hasMessageContaining("file"); + } + + @Test + void localToRemote_tooManyBytes() throws Exception { + pushMirrorSettings(null, null, null); + + // Add files whose total size exceeds the allowed maximum. + long remainder = MAX_NUM_BYTES + 1; + final int defaultFileSize = (int) (MAX_NUM_BYTES / MAX_NUM_FILES * 2); + final ArrayList> changes = new ArrayList<>(); + for (int i = 0;; i++) { + final int fileSize; + if (remainder > defaultFileSize) { + remainder -= defaultFileSize; + fileSize = defaultFileSize; + } else { + fileSize = (int) remainder; + remainder = 0; + } + + changes.add(Change.ofTextUpsert("/" + i + ".txt", Strings.repeat("*", fileSize))); + + if (remainder == 0) { + break; + } + } + client.forRepo(projName, REPO_FOO).commit("Add a bunch of numbered asterisks", changes).push().join(); + + // Perform mirroring, which should fail. + assertThatThrownBy(() -> mirroringService.mirror().join()) + .hasCauseInstanceOf(MirrorException.class) + .hasMessageContaining("contains more than") + .hasMessageContaining("byte"); + } + + @CsvSource({ "meta", "dogma" }) + @ParameterizedTest + void cannotMirrorInternalRepositories(String localRepo) { + assertThatThrownBy(() -> pushMirrorSettings(localRepo, "/", "/", null)) + .hasCauseInstanceOf(CentralDogmaException.class) + .hasMessageContaining("invalid localRepo:"); + } + + private void pushMirrorSettings(@Nullable String localPath, @Nullable String remotePath, + @Nullable String gitignore) { + pushMirrorSettings(REPO_FOO, localPath, remotePath, gitignore); + } + + private void pushMirrorSettings(String localRepo, @Nullable String localPath, @Nullable String remotePath, + @Nullable String gitignore) { + client.forRepo(projName, Project.REPO_META) + .commit("Add /mirrors.json", + Change.ofJsonUpsert("/mirrors.json", + "[{" + + " \"type\": \"single\"," + + " \"direction\": \"LOCAL_TO_REMOTE\"," + + " \"localRepo\": \"" + localRepo + "\"," + + (localPath != null ? "\"localPath\": \"" + localPath + "\"," : "") + + " \"remoteUri\": \"" + gitUri + firstNonNull(remotePath, "") + '"' + + ",\"gitignore\": " + firstNonNull(gitignore, "\"\"") + + "}]")) + .push().join(); + } + + private Set listFiles(ObjectId commitId) throws IOException { + try (ObjectReader reader = git.getRepository().newObjectReader(); + TreeWalk treeWalk = new TreeWalk(reader); + RevWalk revWalk = new RevWalk(reader)) { + treeWalk.addTree(revWalk.parseTree(commitId).getId()); + + final HashSet files = new HashSet<>(); + while (treeWalk.next()) { + if (treeWalk.getFileMode() == FileMode.TREE) { + treeWalk.enterSubtree(); + continue; + } + files.add('/' + treeWalk.getPathString()); + } + return files; + } + } + + private void checkGitignore(String localPath, String remotePath) throws IOException, GitAPIException { + // Mirror an empty git repository, which will; + // - Create /mirror_state.json + mirroringService.mirror().join(); + + // Make sure /mirror_state.json exists + final ObjectId commitId = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + byte[] content = getFileContent(commitId, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + MirrorState mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("1"); + + // Now, add files to the local repository and mirror. + client.forRepo(projName, REPO_FOO) + .commit("Add the release dates of the 'Infamous' series", + Change.ofTextUpsert(localPath + "/light.txt", "26-Aug-2014"), + Change.ofTextUpsert(localPath + "/exclude_if_root.txt", "26-Aug-2014"), // excluded + Change.ofTextUpsert(localPath + "/subdir/exclude_if_root.txt", "26-Aug-2014"), + Change.ofTextUpsert(localPath + "/subdir/exclude_dir/foo.txt", "26-Aug-2014")) // excluded + .push().join(); + + mirroringService.mirror().join(); + + final ObjectId commitId1 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(commitId1).isNotEqualTo(commitId); + content = getFileContent(commitId1, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("2"); + // Remove first directory because it's localPath(). + assertThat(new String(getFileContent(commitId1, remotePath + "/light.txt"))).isEqualTo("26-Aug-2014\n"); + assertThat(new String(getFileContent(commitId1, remotePath + "/subdir/exclude_if_root.txt"))) + .isEqualTo("26-Aug-2014\n"); + + // Make sure the files that match gitignore are not mirrored. + assertThat(getFileContent(commitId1, remotePath + "/exclude_if_root.txt")).isNull(); + assertThat(getFileContent(commitId1, remotePath + "/subdir/exclude_dir/foo.txt")).isNull(); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java index bf69926cb..72430c533 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java @@ -16,17 +16,22 @@ package com.linecorp.centraldogma.server.internal.mirror; +import static com.google.common.base.MoreObjects.firstNonNull; import static com.linecorp.centraldogma.server.mirror.MirrorSchemes.SCHEME_GIT_SSH; import static com.linecorp.centraldogma.server.storage.repository.FindOptions.FIND_ALL_WITHOUT_CONTENT; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.net.URI; -import java.nio.charset.StandardCharsets; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; +import java.util.Objects; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nullable; @@ -34,13 +39,24 @@ import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.RemoteSetUrlCommand; import org.eclipse.jgit.api.RemoteSetUrlCommand.UriType; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.dircache.DirCache; +import org.eclipse.jgit.dircache.DirCacheBuilder; +import org.eclipse.jgit.dircache.DirCacheEditor; +import org.eclipse.jgit.dircache.DirCacheEditor.DeletePath; +import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit; +import org.eclipse.jgit.dircache.DirCacheEntry; import org.eclipse.jgit.ignore.IgnoreNode; import org.eclipse.jgit.ignore.IgnoreNode.MatchResult; +import org.eclipse.jgit.lib.CommitBuilder; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.RefUpdate; +import org.eclipse.jgit.lib.RefUpdate.Result; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.transport.FetchResult; import org.eclipse.jgit.transport.RefSpec; @@ -51,8 +67,10 @@ import org.slf4j.LoggerFactory; import com.cronutils.model.Cron; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.TreeNode; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.linecorp.centraldogma.common.Change; import com.linecorp.centraldogma.common.Entry; @@ -68,12 +86,21 @@ import com.linecorp.centraldogma.server.internal.mirror.credential.PublicKeyMirrorCredential; import com.linecorp.centraldogma.server.mirror.MirrorCredential; import com.linecorp.centraldogma.server.mirror.MirrorDirection; +import com.linecorp.centraldogma.server.storage.StorageException; import com.linecorp.centraldogma.server.storage.repository.Repository; public final class GitMirror extends AbstractMirror { private static final Logger logger = LoggerFactory.getLogger(GitMirror.class); + public static final String LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME = ".mirror_state.json"; + + private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL); + + private static final byte[] EMPTY_BYTE = new byte[0]; + + private static final String MIRROR_STATE_FILE_NAME = "mirror_state.json"; + private static final Pattern DISALLOWED_CHARS = Pattern.compile("[^-_a-zA-Z]"); private static final Pattern CONSECUTIVE_UNDERSCORES = Pattern.compile("_+"); @@ -101,7 +128,63 @@ public GitMirror(Cron schedule, MirrorDirection direction, MirrorCredential cred @Override protected void mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes) throws Exception { - throw new UnsupportedOperationException(); + try (Git git = openGit(workDir)) { + final String headBranchRefName = Constants.R_HEADS + remoteBranch(); + final ObjectId headCommitId = fetchRemoteHeadAndGetCommitId(git, headBranchRefName); + + final org.eclipse.jgit.lib.Repository gitRepository = git.getRepository(); + try (ObjectReader reader = gitRepository.newObjectReader(); + TreeWalk treeWalk = new TreeWalk(reader); + RevWalk revWalk = new RevWalk(reader)) { + + // Prepare to traverse the tree. We can get the tree ID by parsing the object ID. + final ObjectId headTreeId = revWalk.parseTree(headCommitId).getId(); + treeWalk.addTree(headTreeId); + + final String mirrorStatePath = remotePath() + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME; + final Revision localHead = localRepo().normalizeNow(Revision.HEAD); + final Revision remoteCurrentRevision = remoteCurrentRevision(reader, treeWalk, mirrorStatePath); + if (localHead.equals(remoteCurrentRevision)) { + // The remote repository is up-to date. + logger.debug("The remote repository '{}#{}' already at {}. Local repository: '{}'", + remoteRepoUri(), remoteBranch(), localHead, localRepo().name()); + return; + } + + // Reset to traverse the tree from the first. + treeWalk.reset(headTreeId); + + // The staging area that keeps the entries of the new tree. + // It starts with the entries of the tree at the current head and then this method will apply + // the requested changes to build the new tree. + final DirCache dirCache = DirCache.newInCore(); + final DirCacheBuilder builder = dirCache.builder(); + builder.addTree(EMPTY_BYTE, 0, reader, headTreeId); + builder.finish(); + + try (ObjectInserter inserter = gitRepository.newObjectInserter()) { + addModifiedEntryToCache(localHead, dirCache, reader, inserter, + treeWalk, mirrorStatePath, maxNumFiles, maxNumBytes); + // Add the mirror state file. + final MirrorState mirrorState = new MirrorState(localHead.text()); + applyPathEdit( + dirCache, new InsertText(mirrorStatePath.substring(1), // Strip the leading '/'. + inserter, + Jackson.writeValueAsPrettyString(mirrorState) + '\n')); + } + + final ObjectId nextCommitId = + commit(gitRepository, dirCache, headCommitId, localHead); + updateRef(gitRepository, revWalk, headBranchRefName, nextCommitId); + + git.push() + .setRefSpecs(new RefSpec(headBranchRefName)) + .setForce(true) + .setAtomic(true) + .setTimeout(GIT_TIMEOUT_SECS) + .call(); + } + } } @Override @@ -112,20 +195,8 @@ protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor, final String summary; try (Git git = openGit(workDir)) { - final FetchCommand fetch = git.fetch(); - final String refName = Constants.R_HEADS + remoteBranch(); - final FetchResult fetchResult = fetch.setRefSpecs(new RefSpec(refName)) - .setCheckFetchedObjects(true) - .setRemoveDeletedRefs(true) - .setTagOpt(TagOpt.NO_TAGS) - .setTimeout(GIT_TIMEOUT_SECS) - .call(); - - final ObjectId id = fetchResult.getAdvertisedRef(refName).getObjectId(); - final RefUpdate refUpdate = git.getRepository().updateRef(refName); - refUpdate.setNewObjectId(id); - refUpdate.update(); - + final String headBranchRefName = Constants.R_HEADS + remoteBranch(); + final ObjectId id = fetchRemoteHeadAndGetCommitId(git, headBranchRefName); final Revision localRev = localRepo().normalizeNow(Revision.HEAD); try (ObjectReader reader = git.getRepository().newObjectReader(); @@ -136,7 +207,7 @@ protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor, treeWalk.addTree(revWalk.parseTree(id).getId()); // Check if local repository needs update. - final String mirrorStatePath = localPath() + "mirror_state.json"; + final String mirrorStatePath = localPath() + MIRROR_STATE_FILE_NAME; final Entry mirrorState = localRepo().getOrNull(localRev, mirrorStatePath).join(); final String localSourceRevision; if (mirrorState == null || mirrorState.type() != EntryType.JSON) { @@ -175,37 +246,8 @@ protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor, continue; } - // Recurse into a directory if necessary. if (fileMode == FileMode.TREE) { - // Enter if the directory is under remotePath. - // e.g. - // path == /foo/bar - // remotePath == /foo/ - if (path.startsWith(remotePath())) { - treeWalk.enterSubtree(); - continue; - } - - // Enter if the directory is equal to remotePath. - // e.g. - // path == /foo - // remotePath == /foo/ - final int pathLen = path.length() + 1; // Include the trailing '/'. - if (pathLen == remotePath().length() && remotePath().startsWith(path)) { - treeWalk.enterSubtree(); - continue; - } - - // Enter if the directory is parent of remotePath. - // e.g. - // path == /foo - // remotePath == /foo/bar/ - if (pathLen < remotePath().length() && remotePath().startsWith(path + '/')) { - treeWalk.enterSubtree(); - continue; - } - - // Skip the directory that are not under the remote path. + maybeEnterSubtree(treeWalk, remotePath(), path, false); continue; } @@ -244,7 +286,7 @@ protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor, changes.putIfAbsent(localPath, Change.ofJsonUpsert(localPath, jsonNode)); break; case TEXT: - final String strVal = new String(content, StandardCharsets.UTF_8); + final String strVal = new String(content, UTF_8); changes.putIfAbsent(localPath, Change.ofTextUpsert(localPath, strVal)); break; } @@ -333,4 +375,322 @@ private Git openGit(File workDir) throws Exception { } } } + + @Nullable + private Revision remoteCurrentRevision( + ObjectReader reader, TreeWalk treeWalk, String mirrorStatePath) { + try { + while (treeWalk.next()) { + final FileMode fileMode = treeWalk.getFileMode(); + final String path = '/' + treeWalk.getPathString(); + + // Recurse into a directory if necessary. + if (fileMode == FileMode.TREE) { + maybeEnterSubtree(treeWalk, remotePath(), path, true); + continue; + } + + if (!path.equals(mirrorStatePath)) { + continue; + } + + final byte[] content = currentEntryContent(reader, treeWalk); + final MirrorState mirrorState = Jackson.readValue(content, MirrorState.class); + return new Revision(mirrorState.sourceRevision()); + } + // There's no mirror state file which means this is the first mirroring or the file is removed. + return null; + } catch (Exception e) { + logger.warn("Unexpected exception while retrieving the remote source revision", e); + return null; + } + } + + private static ObjectId fetchRemoteHeadAndGetCommitId( + Git git, String headBranchRefName) throws GitAPIException, IOException { + final FetchCommand fetch = git.fetch(); + final FetchResult fetchResult = fetch.setRefSpecs(new RefSpec(headBranchRefName)) + .setCheckFetchedObjects(true) + .setRemoveDeletedRefs(true) + .setTagOpt(TagOpt.NO_TAGS) + .setTimeout(GIT_TIMEOUT_SECS) + .call(); + + final ObjectId commitId = fetchResult.getAdvertisedRef(headBranchRefName).getObjectId(); + final RefUpdate refUpdate = git.getRepository().updateRef(headBranchRefName); + refUpdate.setNewObjectId(commitId); + refUpdate.update(); + return commitId; + } + + private Map> localHeadEntries(Revision localHead) { + final Map> localRawHeadEntries = localRepo().find(localHead, localPath() + "**") + .join(); + + final Stream>> filteredStream = + localRawHeadEntries.entrySet() + .stream() + .filter(e -> e.getKey().startsWith(localPath())); + if (ignoreNode == null) { + // Use HashMap to manipulate it. + return filteredStream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + final Map> sortedMap = + filteredStream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, + (v1, v2) -> v1, LinkedHashMap::new)); + // Use HashMap to manipulate it. + final HashMap> result = new HashMap<>(sortedMap.size()); + String lastIgnoredDirectory = null; + for (Map.Entry> entry : sortedMap.entrySet()) { + final String path = entry.getKey(); + final boolean isDirectory = entry.getValue().type() == EntryType.DIRECTORY; + final MatchResult ignoreResult = ignoreNode.isIgnored( + path.substring(localPath().length()), isDirectory); + if (ignoreResult == MatchResult.IGNORED) { + if (isDirectory) { + lastIgnoredDirectory = path; + } + continue; + } + if (ignoreResult == MatchResult.CHECK_PARENT) { + if (lastIgnoredDirectory != null && path.startsWith(lastIgnoredDirectory)) { + continue; + } + } + result.put(path, entry.getValue()); + } + + return result; + } + + private void addModifiedEntryToCache(Revision localHead, DirCache dirCache, ObjectReader reader, + ObjectInserter inserter, TreeWalk treeWalk, + String mirrorStatePath, int maxNumFiles, + long maxNumBytes) throws IOException { + final Map> localHeadEntries = localHeadEntries(localHead); + long numFiles = 0; + long numBytes = 0; + while (treeWalk.next()) { + final FileMode fileMode = treeWalk.getFileMode(); + final String pathString = treeWalk.getPathString(); + final String path = '/' + pathString; + + if (path.equals(mirrorStatePath)) { + continue; + } + + // Recurse into a directory if necessary. + if (fileMode == FileMode.TREE) { + maybeEnterSubtree(treeWalk, remotePath(), path, false); + continue; + } + + // Skip the entries that are not under the remote path. + if (!path.startsWith(remotePath())) { + continue; + } + + if (fileMode != FileMode.REGULAR_FILE && fileMode != FileMode.EXECUTABLE_FILE) { + // Remove non-file entries. + applyPathEdit(dirCache, new DeletePath(pathString)); + localHeadEntries.remove(path); + continue; + } + + final String localFilePath = localPath() + path.substring(remotePath().length()); + final Entry entry = localHeadEntries.remove(localFilePath); + if (entry == null) { + // Remove a deleted entry. + applyPathEdit(dirCache, new DeletePath(pathString)); + continue; + } + + if (++numFiles > maxNumFiles) { + throw new MirrorException("mirror contains more than " + maxNumFiles + " file(s)"); + } + + final byte[] oldContent = currentEntryContent(reader, treeWalk); + final long contentLength = applyPathEdit(dirCache, inserter, pathString, entry, oldContent); + numBytes += contentLength; + if (numBytes > maxNumBytes) { + throw new MirrorException("mirror contains more than " + maxNumBytes + " byte(s)"); + } + } + + // Add newly added entries. + for (Map.Entry> entry : localHeadEntries.entrySet()) { + final Entry value = entry.getValue(); + if (value.type() == EntryType.DIRECTORY) { + continue; + } + if (++numFiles > maxNumFiles) { + throw new MirrorException("mirror contains more than " + maxNumFiles + " file(s)"); + } + + final String convertedPath = remotePath().substring(1) + // Strip the leading '/' + entry.getKey().substring(localPath().length()); + final long contentLength = applyPathEdit(dirCache, inserter, convertedPath, entry.getValue(), null); + numBytes += contentLength; + if (numBytes > maxNumBytes) { + throw new MirrorException("mirror contains more than " + maxNumBytes + " byte(s)"); + } + } + } + + private static long applyPathEdit(DirCache dirCache, ObjectInserter inserter, String pathString, + Entry entry, @Nullable byte[] oldContent) + throws JsonProcessingException { + switch (EntryType.guessFromPath(pathString)) { + case JSON: + final JsonNode oldJsonNode = oldContent != null ? Jackson.readTree(oldContent) : null; + final JsonNode newJsonNode = firstNonNull((JsonNode) entry.content(), + JsonNodeFactory.instance.nullNode()); + + // Upsert only when the contents are really different. + if (!Objects.equals(newJsonNode, oldJsonNode)) { + // Use InsertText to store the content in pretty format + final String newContent = newJsonNode.toPrettyString() + '\n'; + applyPathEdit(dirCache, new InsertText(pathString, inserter, newContent)); + return newContent.length(); + } + break; + case TEXT: + final String sanitizedOldText = oldContent != null ? + sanitizeText(new String(oldContent, UTF_8)) : null; + final String sanitizedNewText = sanitizeText(entry.contentAsText()); + // Upsert only when the contents are really different. + if (!sanitizedNewText.equals(sanitizedOldText)) { + applyPathEdit(dirCache, new InsertText(pathString, inserter, sanitizedNewText)); + return sanitizedNewText.length(); + } + break; + } + return 0; + } + + private static byte[] currentEntryContent(ObjectReader reader, TreeWalk treeWalk) throws IOException { + final ObjectId objectId = treeWalk.getObjectId(0); + return reader.open(objectId).getBytes(); + } + + private static void maybeEnterSubtree( + TreeWalk treeWalk, String remotePath, + String path, boolean findingMirrorStateFile) throws IOException { + // Enter if the directory is under the remote path. + // e.g. + // path == /foo/bar + // remotePath == /foo/ + if (path.startsWith(remotePath)) { + if (findingMirrorStateFile) { + // The mirror state file isn't placed under a subtree. + return; + } + treeWalk.enterSubtree(); + return; + } + + // Enter if the directory is equal to the remote path. + // e.g. + // path == /foo + // remotePath == /foo/ + final int pathLen = path.length() + 1; // Include the trailing '/'. + if (pathLen == remotePath.length() && remotePath.startsWith(path)) { + treeWalk.enterSubtree(); + return; + } + + // Enter if the directory is the parent of the remote path. + // e.g. + // path == /foo + // remotePath == /foo/bar/ + if (pathLen < remotePath.length() && remotePath.startsWith(path + '/')) { + treeWalk.enterSubtree(); + } + } + + private static void applyPathEdit(DirCache dirCache, PathEdit edit) { + final DirCacheEditor e = dirCache.editor(); + e.add(edit); + e.finish(); + } + + /** + * Removes {@code \r} and appends {@code \n} on the last line if it does not end with {@code \n}. + */ + private static String sanitizeText(String text) { + if (text.indexOf('\r') >= 0) { + text = CR.matcher(text).replaceAll(""); + } + if (!text.isEmpty() && !text.endsWith("\n")) { + text += "\n"; + } + return text; + } + + private ObjectId commit(org.eclipse.jgit.lib.Repository gitRepository, DirCache dirCache, + ObjectId headCommitId, Revision localHead) throws IOException { + try (ObjectInserter inserter = gitRepository.newObjectInserter()) { + // flush the current index to repository and get the result tree object id. + final ObjectId nextTreeId = dirCache.writeTree(inserter); + // build a commit object + final PersonIdent personIdent = + new PersonIdent(MIRROR_AUTHOR.name(), MIRROR_AUTHOR.email(), + System.currentTimeMillis() / 1000L * 1000L, // Drop the milliseconds + 0); + + final CommitBuilder commitBuilder = new CommitBuilder(); + commitBuilder.setAuthor(personIdent); + commitBuilder.setCommitter(personIdent); + commitBuilder.setTreeId(nextTreeId); + commitBuilder.setEncoding(UTF_8); + commitBuilder.setParentId(headCommitId); + + final String summary = "Mirror '" + localRepo().name() + "' at " + localHead + + " to the repository '" + remoteRepoUri() + '#' + remoteBranch() + '\''; + logger.debug(summary); + commitBuilder.setMessage(summary); + + final ObjectId nextCommitId = inserter.insert(commitBuilder); + inserter.flush(); + return nextCommitId; + } + } + + static void updateRef(org.eclipse.jgit.lib.Repository jGitRepository, RevWalk revWalk, + String ref, ObjectId commitId) throws IOException { + final RefUpdate refUpdate = jGitRepository.updateRef(ref); + refUpdate.setNewObjectId(commitId); + + final Result res = refUpdate.update(revWalk); + switch (res) { + case NEW: + case FAST_FORWARD: + // Expected + break; + default: + throw new StorageException("unexpected refUpdate state: " + res); + } + } + + private static final class InsertText extends PathEdit { + private final ObjectInserter inserter; + private final String text; + + InsertText(String entryPath, ObjectInserter inserter, String text) { + super(entryPath); + this.inserter = inserter; + this.text = text; + } + + @Override + public void apply(DirCacheEntry ent) { + try { + ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, text.getBytes(UTF_8))); + ent.setFileMode(FileMode.REGULAR_FILE); + } catch (IOException e) { + throw new StorageException("failed to create a new text blob", e); + } + } + } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorState.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorState.java index a69a7a988..81cf445b8 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorState.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorState.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -final class MirrorState { +public final class MirrorState { private final String sourceRevision; @@ -30,7 +30,8 @@ final class MirrorState { this.sourceRevision = requireNonNull(sourceRevision, "sourceRevision"); } - String sourceRevision() { + @JsonProperty("sourceRevision") + public String sourceRevision() { return sourceRevision; } } diff --git a/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/TestUtil.java b/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/TestUtil.java index ace83be7f..c18872450 100644 --- a/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/TestUtil.java +++ b/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/TestUtil.java @@ -47,7 +47,8 @@ public static void assertJsonConversion(T value, Class valueType, String } public static String normalizedDisplayName(TestInfo testInfo) { - return DISALLOWED_CHARS.matcher(testInfo.getDisplayName()).replaceAll(""); + return DISALLOWED_CHARS.matcher(testInfo.getDisplayName() + testInfo.getTestMethod().get().getName()) + .replaceAll(""); } private TestUtil() {} From 665f77c1a88b1426fdb2731116bc48d4bdcb02cc Mon Sep 17 00:00:00 2001 From: minwoox Date: Mon, 22 Aug 2022 23:43:17 +0900 Subject: [PATCH 02/11] Update document --- site/src/sphinx/mirroring.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/site/src/sphinx/mirroring.rst b/site/src/sphinx/mirroring.rst index d7e7e4d56..2de09eec8 100644 --- a/site/src/sphinx/mirroring.rst +++ b/site/src/sphinx/mirroring.rst @@ -217,6 +217,11 @@ If everything was configured correctly, the repository you specified in ``localR "sourceRevision": "22fb176e4d8096d709d34ffe985c5f3acea83ef2" } +Setting up a CD-to-Git mirror +----------------------------- +It's exactly the same as setting up a Git-to-CD mirror which is described above, except you need to specify +``direction`` with ``LOCAL_TO_REMOTE``. + Mirror limit settings --------------------- Central Dogma limits the number of files and the total size of the files in a mirror for its reliability. From 7085f030a89c684cd340289cafcfa3a253347ea7 Mon Sep 17 00:00:00 2001 From: minwoox Date: Mon, 26 Sep 2022 16:36:41 +0900 Subject: [PATCH 03/11] Address comment by @ikhoon and add more test --- .../git/LocalToRemoteGitMirrorTest.java | 69 +++++++++++++++-- .../mirror/DefaultMirroringService.java | 4 +- .../server/internal/mirror/GitMirror.java | 75 +++++++++---------- site/src/sphinx/concepts.rst | 2 +- site/src/sphinx/setup-configuration.rst | 2 +- 5 files changed, 104 insertions(+), 48 deletions(-) diff --git a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java index d7b655070..52b79acd9 100644 --- a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java +++ b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java @@ -18,7 +18,6 @@ import static com.google.common.base.MoreObjects.firstNonNull; import static com.linecorp.centraldogma.it.mirror.git.GitMirrorTest.addToGitIndex; -import static com.linecorp.centraldogma.server.internal.mirror.GitMirror.LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_COMMIT_SECTION; @@ -29,6 +28,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; +import java.util.Map; import java.util.Set; import javax.annotation.Nullable; @@ -58,17 +58,23 @@ import com.linecorp.centraldogma.client.CentralDogma; import com.linecorp.centraldogma.common.CentralDogmaException; import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.PathPattern; +import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.internal.Jackson; import com.linecorp.centraldogma.server.CentralDogmaBuilder; import com.linecorp.centraldogma.server.MirrorException; import com.linecorp.centraldogma.server.MirroringService; import com.linecorp.centraldogma.server.internal.mirror.MirrorState; +import com.linecorp.centraldogma.server.mirror.MirrorDirection; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.testing.internal.TestUtil; import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension; class LocalToRemoteGitMirrorTest { + private static final String LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME = ".mirror_state.json"; + private static final int MAX_NUM_FILES = 32; private static final long MAX_NUM_BYTES = 1048576; // 1 MiB @@ -267,7 +273,7 @@ private byte[] getFileContent(ObjectId commitId, String fileName) throws IOExcep "/local, /remote/foo", "/local/foo, /remote/foo" }) - void LocalToRemote_gitignore(String localPath, String remotePath) throws Exception { + void localToRemote_gitignore(String localPath, String remotePath) throws Exception { pushMirrorSettings(localPath, remotePath, "\"/exclude_if_root.txt\\n**/exclude_dir\""); checkGitignore(localPath, remotePath); } @@ -379,24 +385,24 @@ void localToRemote_tooManyBytes() throws Exception { @CsvSource({ "meta", "dogma" }) @ParameterizedTest void cannotMirrorInternalRepositories(String localRepo) { - assertThatThrownBy(() -> pushMirrorSettings(localRepo, "/", "/", null)) + assertThatThrownBy(() -> pushMirrorSettings(localRepo, "/", "/", null, MirrorDirection.LOCAL_TO_REMOTE)) .hasCauseInstanceOf(CentralDogmaException.class) .hasMessageContaining("invalid localRepo:"); } private void pushMirrorSettings(@Nullable String localPath, @Nullable String remotePath, @Nullable String gitignore) { - pushMirrorSettings(REPO_FOO, localPath, remotePath, gitignore); + pushMirrorSettings(REPO_FOO, localPath, remotePath, gitignore, MirrorDirection.LOCAL_TO_REMOTE); } private void pushMirrorSettings(String localRepo, @Nullable String localPath, @Nullable String remotePath, - @Nullable String gitignore) { + @Nullable String gitignore, MirrorDirection direction) { client.forRepo(projName, Project.REPO_META) .commit("Add /mirrors.json", Change.ofJsonUpsert("/mirrors.json", "[{" + " \"type\": \"single\"," + - " \"direction\": \"LOCAL_TO_REMOTE\"," + + " \"direction\": \"" + direction + "\"," + " \"localRepo\": \"" + localRepo + "\"," + (localPath != null ? "\"localPath\": \"" + localPath + "\"," : "") + " \"remoteUri\": \"" + gitUri + firstNonNull(remotePath, "") + '"' + @@ -459,4 +465,55 @@ private void checkGitignore(String localPath, String remotePath) throws IOExcept assertThat(getFileContent(commitId1, remotePath + "/exclude_if_root.txt")).isNull(); assertThat(getFileContent(commitId1, remotePath + "/subdir/exclude_dir/foo.txt")).isNull(); } + + @Test + void changeDirection() throws Exception { + pushMirrorSettings(null, null, null); + + // Mirror an empty Central Dogma repository, which will; + // - Create /.mirror_state.json + mirroringService.mirror().join(); + + final ObjectId commitId1 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + byte[] content = getFileContent(commitId1, '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + MirrorState mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("1"); + + // Create a new commit + client.forRepo(projName, REPO_FOO) + .commit("Add a commit", + Change.ofJsonUpsert("/foo.json", "{\"a\":\"b\"}"), + Change.ofJsonUpsert("/bar/foo.json", "{\"a\":\"c\"}"), + Change.ofTextUpsert("/baz/foo.txt", "\"a\": \"b\"\n")) + .push().join(); + + mirroringService.mirror().join(); + + final ObjectId commitId2 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + content = getFileContent(commitId2, '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("2"); + assertThat(Jackson.writeValueAsString(Jackson.readTree( + getFileContent(commitId2, "/foo.json")))).isEqualTo("{\"a\":\"b\"}"); + assertThat(Jackson.writeValueAsString(Jackson.readTree( + getFileContent(commitId2, "/bar/foo.json")))).isEqualTo("{\"a\":\"c\"}"); + assertThat(new String(getFileContent(commitId2, "/baz/foo.txt"))) + .isEqualTo("\"a\": \"b\"\n"); + + // Change the direction + pushMirrorSettings(REPO_FOO, null, null, null, MirrorDirection.REMOTE_TO_LOCAL); + addToGitIndex(git, gitWorkTree, "foo.json", "{\"a\":\"foo\"}"); + git.commit().setMessage("Modify foo.json").call(); + mirroringService.mirror().join(); + + final Map> entries = client.forRepo(projName, REPO_FOO) + .file(PathPattern.all()) + .get() + .join(); + assertThat(entries.size()).isEqualTo(2); + assertThat(entries.get("/foo.json")).isEqualTo( + Entry.ofJson(new Revision(3), "/foo.json", "{\"a\":\"foo\"}")); + + assertThat(entries.get("/mirror_state.json")).isNotNull(); + } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringService.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringService.java index 18a7a51ba..e78011f03 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringService.java @@ -130,7 +130,7 @@ public void onSuccess(@Nullable Object result) {} @Override public void onFailure(Throwable cause) { - logger.error("Git-to-CD mirroring scheduler stopped due to an unexpected exception:", cause); + logger.error("Git mirroring scheduler stopped due to an unexpected exception:", cause); } }, MoreExecutors.directExecutor()); } @@ -199,7 +199,7 @@ public void onSuccess(@Nullable Object result) {} @Override public void onFailure(Throwable cause) { - logger.warn("Unexpected Git-to-CD mirroring failure: {}", m, cause); + logger.warn("Unexpected Git mirroring failure: {}", m, cause); } }, MoreExecutors.directExecutor()); }); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java index 72430c533..0daf4c49d 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java @@ -16,7 +16,6 @@ package com.linecorp.centraldogma.server.internal.mirror; -import static com.google.common.base.MoreObjects.firstNonNull; import static com.linecorp.centraldogma.server.mirror.MirrorSchemes.SCHEME_GIT_SSH; import static com.linecorp.centraldogma.server.storage.repository.FindOptions.FIND_ALL_WITHOUT_CONTENT; import static java.nio.charset.StandardCharsets.UTF_8; @@ -70,7 +69,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.TreeNode; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.linecorp.centraldogma.common.Change; import com.linecorp.centraldogma.common.Entry; @@ -93,14 +91,16 @@ public final class GitMirror extends AbstractMirror { private static final Logger logger = LoggerFactory.getLogger(GitMirror.class); - public static final String LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME = ".mirror_state.json"; + // We are going to hide this file from CD UI after we implement UI for mirroring. + private static final String MIRROR_STATE_FILE_NAME = "mirror_state.json"; + + // Prepend '.' because this file is a metadata. + private static final String LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME = '.' + MIRROR_STATE_FILE_NAME; private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL); private static final byte[] EMPTY_BYTE = new byte[0]; - private static final String MIRROR_STATE_FILE_NAME = "mirror_state.json"; - private static final Pattern DISALLOWED_CHARS = Pattern.compile("[^-_a-zA-Z]"); private static final Pattern CONSECUTIVE_UNDERSCORES = Pattern.compile("_+"); @@ -139,7 +139,7 @@ protected void mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumByt // Prepare to traverse the tree. We can get the tree ID by parsing the object ID. final ObjectId headTreeId = revWalk.parseTree(headCommitId).getId(); - treeWalk.addTree(headTreeId); + treeWalk.reset(headTreeId); final String mirrorStatePath = remotePath() + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME; final Revision localHead = localRepo().normalizeNow(Revision.HEAD); @@ -164,7 +164,7 @@ protected void mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumByt try (ObjectInserter inserter = gitRepository.newObjectInserter()) { addModifiedEntryToCache(localHead, dirCache, reader, inserter, - treeWalk, mirrorStatePath, maxNumFiles, maxNumBytes); + treeWalk, maxNumFiles, maxNumBytes); // Add the mirror state file. final MirrorState mirrorState = new MirrorState(localHead.text()); applyPathEdit( @@ -247,7 +247,7 @@ protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor, } if (fileMode == FileMode.TREE) { - maybeEnterSubtree(treeWalk, remotePath(), path, false); + maybeEnterSubtree(treeWalk, remotePath(), path); continue; } @@ -386,7 +386,9 @@ private Revision remoteCurrentRevision( // Recurse into a directory if necessary. if (fileMode == FileMode.TREE) { - maybeEnterSubtree(treeWalk, remotePath(), path, true); + if (remotePath().startsWith(path + '/')) { + treeWalk.enterSubtree(); + } continue; } @@ -415,7 +417,7 @@ private static ObjectId fetchRemoteHeadAndGetCommitId( .setTagOpt(TagOpt.NO_TAGS) .setTimeout(GIT_TIMEOUT_SECS) .call(); - + System.err.println(fetchResult.getMessages()); final ObjectId commitId = fetchResult.getAdvertisedRef(headBranchRefName).getObjectId(); final RefUpdate refUpdate = git.getRepository().updateRef(headBranchRefName); refUpdate.setNewObjectId(commitId); @@ -466,8 +468,7 @@ private Map> localHeadEntries(Revision localHead) { private void addModifiedEntryToCache(Revision localHead, DirCache dirCache, ObjectReader reader, ObjectInserter inserter, TreeWalk treeWalk, - String mirrorStatePath, int maxNumFiles, - long maxNumBytes) throws IOException { + int maxNumFiles, long maxNumBytes) throws IOException { final Map> localHeadEntries = localHeadEntries(localHead); long numFiles = 0; long numBytes = 0; @@ -476,13 +477,14 @@ private void addModifiedEntryToCache(Revision localHead, DirCache dirCache, Obje final String pathString = treeWalk.getPathString(); final String path = '/' + pathString; - if (path.equals(mirrorStatePath)) { + // Recurse into a directory if necessary. + if (fileMode == FileMode.TREE) { + maybeEnterSubtree(treeWalk, remotePath(), path); continue; } - // Recurse into a directory if necessary. - if (fileMode == FileMode.TREE) { - maybeEnterSubtree(treeWalk, remotePath(), path, false); + if (fileMode != FileMode.REGULAR_FILE && fileMode != FileMode.EXECUTABLE_FILE) { + // Skip non-file entries. continue; } @@ -491,14 +493,13 @@ private void addModifiedEntryToCache(Revision localHead, DirCache dirCache, Obje continue; } - if (fileMode != FileMode.REGULAR_FILE && fileMode != FileMode.EXECUTABLE_FILE) { - // Remove non-file entries. - applyPathEdit(dirCache, new DeletePath(pathString)); - localHeadEntries.remove(path); + final String localFilePath = localPath() + path.substring(remotePath().length()); + + // Skip the entry whose path does not conform to CD's path rule. + if (!Util.isValidFilePath(localFilePath)) { continue; } - final String localFilePath = localPath() + path.substring(remotePath().length()); final Entry entry = localHeadEntries.remove(localFilePath); if (entry == null) { // Remove a deleted entry. @@ -524,13 +525,17 @@ private void addModifiedEntryToCache(Revision localHead, DirCache dirCache, Obje if (value.type() == EntryType.DIRECTORY) { continue; } + if (entry.getKey().endsWith(MIRROR_STATE_FILE_NAME)) { + continue; + } + if (++numFiles > maxNumFiles) { throw new MirrorException("mirror contains more than " + maxNumFiles + " file(s)"); } final String convertedPath = remotePath().substring(1) + // Strip the leading '/' entry.getKey().substring(localPath().length()); - final long contentLength = applyPathEdit(dirCache, inserter, convertedPath, entry.getValue(), null); + final long contentLength = applyPathEdit(dirCache, inserter, convertedPath, value, null); numBytes += contentLength; if (numBytes > maxNumBytes) { throw new MirrorException("mirror contains more than " + maxNumBytes + " byte(s)"); @@ -544,8 +549,7 @@ private static long applyPathEdit(DirCache dirCache, ObjectInserter inserter, St switch (EntryType.guessFromPath(pathString)) { case JSON: final JsonNode oldJsonNode = oldContent != null ? Jackson.readTree(oldContent) : null; - final JsonNode newJsonNode = firstNonNull((JsonNode) entry.content(), - JsonNodeFactory.instance.nullNode()); + final JsonNode newJsonNode = (JsonNode) entry.content(); // Upsert only when the contents are really different. if (!Objects.equals(newJsonNode, oldJsonNode)) { @@ -569,23 +573,24 @@ private static long applyPathEdit(DirCache dirCache, ObjectInserter inserter, St return 0; } + private static void applyPathEdit(DirCache dirCache, PathEdit edit) { + final DirCacheEditor e = dirCache.editor(); + e.add(edit); + e.finish(); + } + private static byte[] currentEntryContent(ObjectReader reader, TreeWalk treeWalk) throws IOException { final ObjectId objectId = treeWalk.getObjectId(0); return reader.open(objectId).getBytes(); } private static void maybeEnterSubtree( - TreeWalk treeWalk, String remotePath, - String path, boolean findingMirrorStateFile) throws IOException { + TreeWalk treeWalk, String remotePath, String path) throws IOException { // Enter if the directory is under the remote path. // e.g. // path == /foo/bar // remotePath == /foo/ if (path.startsWith(remotePath)) { - if (findingMirrorStateFile) { - // The mirror state file isn't placed under a subtree. - return; - } treeWalk.enterSubtree(); return; } @@ -609,12 +614,6 @@ private static void maybeEnterSubtree( } } - private static void applyPathEdit(DirCache dirCache, PathEdit edit) { - final DirCacheEditor e = dirCache.editor(); - e.add(edit); - e.finish(); - } - /** * Removes {@code \r} and appends {@code \n} on the last line if it does not end with {@code \n}. */ @@ -647,8 +646,8 @@ private ObjectId commit(org.eclipse.jgit.lib.Repository gitRepository, DirCache commitBuilder.setParentId(headCommitId); final String summary = "Mirror '" + localRepo().name() + "' at " + localHead + - " to the repository '" + remoteRepoUri() + '#' + remoteBranch() + '\''; - logger.debug(summary); + " to the repository '" + remoteRepoUri() + '#' + remoteBranch() + "'\n"; + logger.info(summary); commitBuilder.setMessage(summary); final ObjectId nextCommitId = inserter.insert(commitBuilder); diff --git a/site/src/sphinx/concepts.rst b/site/src/sphinx/concepts.rst index 296220486..a31e2c1cb 100644 --- a/site/src/sphinx/concepts.rst +++ b/site/src/sphinx/concepts.rst @@ -42,7 +42,7 @@ Concepts - Meta repository - A *meta repository* is a repository whose name is ``meta`` in a project. It is dedicated to store the - metadata related with the project it belongs to, such as Git-to-CD mirroring settings. + metadata related with the project it belongs to, such as Git mirroring settings. - Commit diff --git a/site/src/sphinx/setup-configuration.rst b/site/src/sphinx/setup-configuration.rst index ddbaaba7c..cee560db3 100644 --- a/site/src/sphinx/setup-configuration.rst +++ b/site/src/sphinx/setup-configuration.rst @@ -182,7 +182,7 @@ Core properties - ``mirroringEnabled`` (boolean) - - whether to enable Git-to-CD mirroring. It's enabled by default. For more information about mirroring, + - whether to enable Git mirroring. It's enabled by default. For more information about mirroring, refer to :ref:`mirroring`. - ``numMirroringThreads`` (integer) From 6f9dfae6fed29b25065e41d386e08d17d7e6b890 Mon Sep 17 00:00:00 2001 From: minwoox Date: Mon, 26 Sep 2022 16:49:42 +0900 Subject: [PATCH 04/11] Add visible for testing annotation --- .../centraldogma/server/internal/mirror/MirrorState.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorState.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorState.java index 81cf445b8..136fb2dfc 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorState.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorState.java @@ -20,7 +20,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; +@VisibleForTesting public final class MirrorState { private final String sourceRevision; From 065b28e535bffd9bbee1e64a3908e89d810d019a Mon Sep 17 00:00:00 2001 From: minwoox Date: Mon, 26 Sep 2022 17:49:13 +0900 Subject: [PATCH 05/11] Close git --- .../centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java index 52b79acd9..97f097d4b 100644 --- a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java +++ b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java @@ -144,6 +144,7 @@ void initDogmaRepo(TestInfo testInfo) { void destroyDogmaRepo() { client.removeProject(projName).join(); client.purgeProject(projName).join(); + git.close(); } @ParameterizedTest From 71ded38df33279d805115cf053a133afe1470fc5 Mon Sep 17 00:00:00 2001 From: minwoox Date: Mon, 26 Sep 2022 20:46:21 +0900 Subject: [PATCH 06/11] Test deleing file --- .../it/mirror/git/LocalToRemoteGitMirrorTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java index 97f097d4b..0b477ecc2 100644 --- a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java +++ b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java @@ -26,6 +26,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.util.ArrayList; import java.util.HashSet; import java.util.Map; @@ -141,10 +142,11 @@ void initDogmaRepo(TestInfo testInfo) { } @AfterEach - void destroyDogmaRepo() { + void destroyDogmaRepo() throws IOException { client.removeProject(projName).join(); client.purgeProject(projName).join(); git.close(); + Files.delete(gitWorkTree.toPath()); } @ParameterizedTest From fb6a18f5d4a00ab7e934cd17c80c7c182a060d03 Mon Sep 17 00:00:00 2001 From: minwoox Date: Mon, 26 Sep 2022 21:43:08 +0900 Subject: [PATCH 07/11] Removing purging --- .../it/mirror/git/LocalToRemoteGitMirrorTest.java | 4 ---- .../centraldogma/server/internal/mirror/GitMirror.java | 1 - 2 files changed, 5 deletions(-) diff --git a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java index 0b477ecc2..eaa3778be 100644 --- a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java +++ b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java @@ -26,7 +26,6 @@ import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.util.ArrayList; import java.util.HashSet; import java.util.Map; @@ -144,9 +143,6 @@ void initDogmaRepo(TestInfo testInfo) { @AfterEach void destroyDogmaRepo() throws IOException { client.removeProject(projName).join(); - client.purgeProject(projName).join(); - git.close(); - Files.delete(gitWorkTree.toPath()); } @ParameterizedTest diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java index 0daf4c49d..b329f2b47 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java @@ -417,7 +417,6 @@ private static ObjectId fetchRemoteHeadAndGetCommitId( .setTagOpt(TagOpt.NO_TAGS) .setTimeout(GIT_TIMEOUT_SECS) .call(); - System.err.println(fetchResult.getMessages()); final ObjectId commitId = fetchResult.getAdvertisedRef(headBranchRefName).getObjectId(); final RefUpdate refUpdate = git.getRepository().updateRef(headBranchRefName); refUpdate.setNewObjectId(commitId); From 9e4d6588415abc3083497459754a7aac7d633071 Mon Sep 17 00:00:00 2001 From: minwoox Date: Mon, 26 Sep 2022 22:35:11 +0900 Subject: [PATCH 08/11] Revert TempDir --- .../it/mirror/git/GitMirrorTest.java | 16 +++++++++++----- .../mirror/git/LocalToRemoteGitMirrorTest.java | 13 +++++++++---- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java index fe2e3537f..3bc17cb4c 100644 --- a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java +++ b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java @@ -46,7 +46,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -64,6 +63,7 @@ import com.linecorp.centraldogma.server.MirrorException; import com.linecorp.centraldogma.server.MirroringService; import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.testing.internal.TemporaryFolderExtension; import com.linecorp.centraldogma.testing.internal.TestUtil; import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension; @@ -93,8 +93,14 @@ static void init() { mirroringService = dogma.mirroringService(); } - @TempDir - File gitRepoDir; + @RegisterExtension + final TemporaryFolderExtension gitRepoDir = new TemporaryFolderExtension() { + @Override + protected boolean runForEachTest() { + return true; + } + }; + private Git git; private File gitWorkTree; private String gitUri; @@ -104,7 +110,7 @@ static void init() { @BeforeEach void initGitRepo(TestInfo testInfo) throws Exception { final String repoName = TestUtil.normalizedDisplayName(testInfo); - gitWorkTree = new File(gitRepoDir, repoName).getAbsoluteFile(); + gitWorkTree = new File(gitRepoDir.getRoot().toFile(), repoName).getAbsoluteFile(); final Repository gitRepo = new FileRepositoryBuilder().setWorkTree(gitWorkTree).build(); createGitRepo(gitRepo); @@ -328,7 +334,7 @@ void remoteToLocal_submodule(TestInfo testInfo) throws Exception { // Create a new repository for a submodule. final String submoduleName = TestUtil.normalizedDisplayName(testInfo) + ".submodule"; final File gitSubmoduleWorkTree = - new File(gitRepoDir, submoduleName).getAbsoluteFile(); + new File(gitRepoDir.getRoot().toFile(), submoduleName).getAbsoluteFile(); final Repository gitSubmoduleRepo = new FileRepositoryBuilder().setWorkTree(gitSubmoduleWorkTree).build(); createGitRepo(gitSubmoduleRepo); diff --git a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java index eaa3778be..d9f8f3125 100644 --- a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java +++ b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java @@ -49,7 +49,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -68,6 +67,7 @@ import com.linecorp.centraldogma.server.internal.mirror.MirrorState; import com.linecorp.centraldogma.server.mirror.MirrorDirection; import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.testing.internal.TemporaryFolderExtension; import com.linecorp.centraldogma.testing.internal.TestUtil; import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension; @@ -99,8 +99,13 @@ static void init() { mirroringService = dogma.mirroringService(); } - @TempDir - File gitRepoDir; + @RegisterExtension + final TemporaryFolderExtension gitRepoDir = new TemporaryFolderExtension() { + @Override + protected boolean runForEachTest() { + return true; + } + }; private Git git; private File gitWorkTree; @@ -111,7 +116,7 @@ static void init() { @BeforeEach void initGitRepo(TestInfo testInfo) throws Exception { final String repoName = TestUtil.normalizedDisplayName(testInfo); - gitWorkTree = new File(gitRepoDir, repoName).getAbsoluteFile(); + gitWorkTree = new File(gitRepoDir.getRoot().toFile(), repoName).getAbsoluteFile(); final Repository gitRepo = new FileRepositoryBuilder().setWorkTree(gitWorkTree).build(); createGitRepo(gitRepo); From e2c8db4a54d96553feb60adbeb41a6b03f853471 Mon Sep 17 00:00:00 2001 From: minwoox Date: Wed, 5 Oct 2022 15:18:51 +0900 Subject: [PATCH 09/11] Address comments by @ikhoon --- .../server/internal/mirror/GitMirror.java | 43 +++++++++++-------- .../server/mirror/MirrorUtil.java | 2 +- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java index b329f2b47..b9934b4a4 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java @@ -94,7 +94,7 @@ public final class GitMirror extends AbstractMirror { // We are going to hide this file from CD UI after we implement UI for mirroring. private static final String MIRROR_STATE_FILE_NAME = "mirror_state.json"; - // Prepend '.' because this file is a metadata. + // Prepend '.' because this file is metadata. private static final String LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME = '.' + MIRROR_STATE_FILE_NAME; private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL); @@ -269,13 +269,15 @@ protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor, } if (++numFiles > maxNumFiles) { - throw new MirrorException("mirror contains more than " + maxNumFiles + " file(s)"); + throwMirrorException(maxNumFiles, "files"); + return; } final ObjectId objectId = treeWalk.getObjectId(0); final long contentLength = reader.getObjectSize(objectId, ObjectReader.OBJ_ANY); if (numBytes > maxNumBytes - contentLength) { - throw new MirrorException("mirror contains more than " + maxNumBytes + " byte(s)"); + throwMirrorException(maxNumBytes, "bytes"); + return; } numBytes += contentLength; @@ -428,17 +430,16 @@ private Map> localHeadEntries(Revision localHead) { final Map> localRawHeadEntries = localRepo().find(localHead, localPath() + "**") .join(); - final Stream>> filteredStream = + final Stream>> entryStream = localRawHeadEntries.entrySet() - .stream() - .filter(e -> e.getKey().startsWith(localPath())); + .stream(); if (ignoreNode == null) { // Use HashMap to manipulate it. - return filteredStream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + return entryStream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } final Map> sortedMap = - filteredStream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, + entryStream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (v1, v2) -> v1, LinkedHashMap::new)); // Use HashMap to manipulate it. final HashMap> result = new HashMap<>(sortedMap.size()); @@ -474,11 +475,11 @@ private void addModifiedEntryToCache(Revision localHead, DirCache dirCache, Obje while (treeWalk.next()) { final FileMode fileMode = treeWalk.getFileMode(); final String pathString = treeWalk.getPathString(); - final String path = '/' + pathString; + final String remoteFilePath = '/' + pathString; // Recurse into a directory if necessary. if (fileMode == FileMode.TREE) { - maybeEnterSubtree(treeWalk, remotePath(), path); + maybeEnterSubtree(treeWalk, remotePath(), remoteFilePath); continue; } @@ -488,11 +489,11 @@ private void addModifiedEntryToCache(Revision localHead, DirCache dirCache, Obje } // Skip the entries that are not under the remote path. - if (!path.startsWith(remotePath())) { + if (!remoteFilePath.startsWith(remotePath())) { continue; } - final String localFilePath = localPath() + path.substring(remotePath().length()); + final String localFilePath = localPath() + remoteFilePath.substring(remotePath().length()); // Skip the entry whose path does not conform to CD's path rule. if (!Util.isValidFilePath(localFilePath)) { @@ -507,14 +508,16 @@ private void addModifiedEntryToCache(Revision localHead, DirCache dirCache, Obje } if (++numFiles > maxNumFiles) { - throw new MirrorException("mirror contains more than " + maxNumFiles + " file(s)"); + throwMirrorException(maxNumFiles, "files"); + return; } final byte[] oldContent = currentEntryContent(reader, treeWalk); final long contentLength = applyPathEdit(dirCache, inserter, pathString, entry, oldContent); numBytes += contentLength; if (numBytes > maxNumBytes) { - throw new MirrorException("mirror contains more than " + maxNumBytes + " byte(s)"); + throwMirrorException(maxNumBytes, "bytes"); + return; } } @@ -529,7 +532,8 @@ private void addModifiedEntryToCache(Revision localHead, DirCache dirCache, Obje } if (++numFiles > maxNumFiles) { - throw new MirrorException("mirror contains more than " + maxNumFiles + " file(s)"); + throwMirrorException(maxNumFiles, "files"); + return; } final String convertedPath = remotePath().substring(1) + // Strip the leading '/' @@ -537,7 +541,7 @@ private void addModifiedEntryToCache(Revision localHead, DirCache dirCache, Obje final long contentLength = applyPathEdit(dirCache, inserter, convertedPath, value, null); numBytes += contentLength; if (numBytes > maxNumBytes) { - throw new MirrorException("mirror contains more than " + maxNumBytes + " byte(s)"); + throwMirrorException(maxNumBytes, "bytes"); } } } @@ -561,7 +565,7 @@ private static long applyPathEdit(DirCache dirCache, ObjectInserter inserter, St case TEXT: final String sanitizedOldText = oldContent != null ? sanitizeText(new String(oldContent, UTF_8)) : null; - final String sanitizedNewText = sanitizeText(entry.contentAsText()); + final String sanitizedNewText = entry.contentAsText(); // Already sanitized when committing. // Upsert only when the contents are really different. if (!sanitizedNewText.equals(sanitizedOldText)) { applyPathEdit(dirCache, new InsertText(pathString, inserter, sanitizedNewText)); @@ -655,6 +659,11 @@ private ObjectId commit(org.eclipse.jgit.lib.Repository gitRepository, DirCache } } + private T throwMirrorException(long number, String filesOrBytes) { + throw new MirrorException("mirror (" + remoteRepoUri() + '#' + remoteBranch() + + ") contains more than " + number + ' ' + filesOrBytes); + } + static void updateRef(org.eclipse.jgit.lib.Repository jGitRepository, RevWalk revWalk, String ref, ObjectId commitId) throws IOException { final RefUpdate refUpdate = jGitRepository.updateRef(ref); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorUtil.java b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorUtil.java index 3d5a17e59..2fd6b8d2f 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorUtil.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorUtil.java @@ -61,7 +61,7 @@ public static String normalizePath(String path) { * *

e.g. git+ssh://foo.com/bar.git/some-path#master is split into: * - remoteRepoUri: git+ssh://foo.com/bar.git - * - remotePath: /some-path + * - remotePath: /some-path/ * - remoteBranch: master * *

e.g. dogma://foo.com/bar/qux.dogma is split into: From 72a29e0777a6469703535f56aa3ad54039f7fdba Mon Sep 17 00:00:00 2001 From: minwoox Date: Wed, 19 Oct 2022 22:19:09 +0900 Subject: [PATCH 10/11] Address comments by @jrhee17 --- .../centraldogma/it/mirror/git/GitMirrorTest.java | 9 ++++++--- .../it/mirror/git/LocalToRemoteGitMirrorTest.java | 9 ++++++--- .../centraldogma/server/internal/mirror/GitMirror.java | 1 - 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java index 3bc17cb4c..c27ef8fb8 100644 --- a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java +++ b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java @@ -426,6 +426,8 @@ private void pushMirrorSettings(@Nullable String localPath, @Nullable String rem private void pushMirrorSettings(String localRepo, @Nullable String localPath, @Nullable String remotePath, @Nullable String gitignore) { + final String localPath0 = localPath == null ? "/" : localPath; + final String remoteUri = gitUri + firstNonNull(remotePath, ""); client.forRepo(projName, Project.REPO_META) .commit("Add /mirrors.json", Change.ofJsonUpsert("/mirrors.json", @@ -433,9 +435,10 @@ private void pushMirrorSettings(String localRepo, @Nullable String localPath, @N " \"type\": \"single\"," + " \"direction\": \"REMOTE_TO_LOCAL\"," + " \"localRepo\": \"" + localRepo + "\"," + - (localPath != null ? "\"localPath\": \"" + localPath + "\"," : "") + - " \"remoteUri\": \"" + gitUri + firstNonNull(remotePath, "") + '"' + - ",\"gitignore\": " + firstNonNull(gitignore, "\"\"") + + " \"localPath\": \"" + localPath0 + "\"," + + " \"remoteUri\": \"" + remoteUri + "\"," + + " \"schedule\": \"0 0 0 1 1 ? 2099\"," + + " \"gitignore\": " + firstNonNull(gitignore, "\"\"") + "}]")) .push().join(); } diff --git a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java index d9f8f3125..aff7c611c 100644 --- a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java +++ b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java @@ -401,6 +401,8 @@ private void pushMirrorSettings(@Nullable String localPath, @Nullable String rem private void pushMirrorSettings(String localRepo, @Nullable String localPath, @Nullable String remotePath, @Nullable String gitignore, MirrorDirection direction) { + final String localPath0 = localPath == null ? "/" : localPath; + final String remoteUri = gitUri + firstNonNull(remotePath, ""); client.forRepo(projName, Project.REPO_META) .commit("Add /mirrors.json", Change.ofJsonUpsert("/mirrors.json", @@ -408,9 +410,10 @@ private void pushMirrorSettings(String localRepo, @Nullable String localPath, @N " \"type\": \"single\"," + " \"direction\": \"" + direction + "\"," + " \"localRepo\": \"" + localRepo + "\"," + - (localPath != null ? "\"localPath\": \"" + localPath + "\"," : "") + - " \"remoteUri\": \"" + gitUri + firstNonNull(remotePath, "") + '"' + - ",\"gitignore\": " + firstNonNull(gitignore, "\"\"") + + " \"localPath\": \"" + localPath0 + "\"," + + " \"remoteUri\": \"" + remoteUri + "\"," + + " \"schedule\": \"0 0 0 1 1 ? 2099\"," + + " \"gitignore\": " + firstNonNull(gitignore, "\"\"") + "}]")) .push().join(); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java index b9934b4a4..a3c4e2d18 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java @@ -179,7 +179,6 @@ dirCache, new InsertText(mirrorStatePath.substring(1), // Strip the leading '/'. git.push() .setRefSpecs(new RefSpec(headBranchRefName)) - .setForce(true) .setAtomic(true) .setTimeout(GIT_TIMEOUT_SECS) .call(); From 52187d1e47f3f06bd6ef60e637e311c3693768cc Mon Sep 17 00:00:00 2001 From: minwoox Date: Fri, 21 Oct 2022 23:17:42 +0900 Subject: [PATCH 11/11] Address the comment by @ikhoon --- .../it/mirror/git/LocalToRemoteGitMirrorTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java index aff7c611c..90d727ab2 100644 --- a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java +++ b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java @@ -154,8 +154,8 @@ void destroyDogmaRepo() throws IOException { @CsvSource({ "'', ''", "/local/foo, /remote", - "/local, /remote/foo", - "/local/foo, /remote/foo" + "/local, /remote/bar", + "/local/foo, /remote/bar" }) void localToRemote(String localPath, String remotePath) throws Exception { pushMirrorSettings(localPath, remotePath, null); @@ -274,8 +274,8 @@ private byte[] getFileContent(ObjectId commitId, String fileName) throws IOExcep @CsvSource({ "'', ''", "/local/foo, /remote", - "/local, /remote/foo", - "/local/foo, /remote/foo" + "/local, /remote/bar", + "/local/foo, /remote/bar" }) void localToRemote_gitignore(String localPath, String remotePath) throws Exception { pushMirrorSettings(localPath, remotePath, "\"/exclude_if_root.txt\\n**/exclude_dir\""); @@ -286,8 +286,8 @@ void localToRemote_gitignore(String localPath, String remotePath) throws Excepti @CsvSource({ "'', ''", "/local/foo, /remote", - "/local, /remote/foo", - "/local/foo, /remote/foo" + "/local, /remote/bar", + "/local/foo, /remote/bar" }) void localToRemote_gitignore_with_array(String localPath, String remotePath) throws Exception { pushMirrorSettings(localPath, remotePath, "[\"/exclude_if_root.txt\", \"exclude_dir\"]");