From b38014d9f342db4a210bfd136d1a470d4011a80b Mon Sep 17 00:00:00 2001 From: minwoox Date: Tue, 6 Jul 2021 21:19:18 +0900 Subject: [PATCH 01/14] Provide a way to remove old commits Motivation: Central Dogma uses jGit to store data. Due to the nature of Git that stores unlimited history, Central Dogma will eventually get in trouble managing disk usage. We can handle this by maintaing the primary and secondary Git repository internally. This works in this way: 1 Commits are pushed to the primary Git repository. 2 If the number of commits exceed the threshold (`minRetentionCommits`), then the secondary Git repository is created. 3 Commits are pushed to the both primary and secondary Git repositories. 4 If the secondary Git repository has the number of commits more than the threshold; - The secondary Git repository is promoted to the primary Git repository. - The primary Git repository is removed completely. - Another secondary Git repository is created. 5 Back to 3. Modifications: - TBD Result: - Close 575 - TBD Todo: - Provide a way to set `minRetentionCommits` and `minRetentionDay` for each repository. - Support mirroring from CD to external Git. --- .../git/GitRepositoryBenchmark.java | 6 +- .../server/CentralDogmaBuilder.java | 22 +- .../server/CentralDogmaConfig.java | 19 +- .../server/CommitRetentionConfig.java | 98 ++ .../storage/repository/RepositoryWrapper.java | 5 + .../repository/cache/CachingRepository.java | 24 +- .../git/CacheableCompareTreesCall.java | 2 +- .../repository/git/CommitIdDatabase.java | 141 +- .../git/CommitRetentionManagementPlugin.java | 152 ++ .../storage/repository/git/FailFastUtil.java | 41 + .../storage/repository/git/GitRepository.java | 17 +- .../repository/git/GitRepositoryManager.java | 8 +- .../repository/git/GitRepositoryUtil.java | 628 ++++++++ .../repository/git/GitRepositoryV2.java | 1388 +++++++++++++++++ .../repository/git/InternalRepository.java | 109 ++ .../git/RepositoryMetadataDatabase.java | 199 +++ .../server/storage/repository/Repository.java | 30 +- ...linecorp.centraldogma.server.plugin.Plugin | 1 + .../cache/CachingRepositoryTest.java | 7 - .../repository/git/CommitIdDatabaseTest.java | 105 +- .../git/GitRepositoryManagerTest.java | 10 +- .../git/GitRepositoryMigrationTest.java | 63 + .../repository/git/GitRepositoryTest.java | 56 +- .../repository/git/GitRepositoryUtilTest.java | 69 + .../git/GitRepositoryV2PromotionTest.java | 103 ++ .../git/RepositoryMetadataDatabaseTest.java | 84 + site/src/sphinx/setup-configuration.rst | 28 + 27 files changed, 3209 insertions(+), 206 deletions(-) create mode 100644 server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java create mode 100644 server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java create mode 100644 server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtil.java create mode 100644 server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java create mode 100644 server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java create mode 100644 server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java create mode 100644 server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java create mode 100644 server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtilTest.java create mode 100644 server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java create mode 100644 server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabaseTest.java diff --git a/server/src/jmh/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryBenchmark.java b/server/src/jmh/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryBenchmark.java index fa1b7520f..8e6b615e7 100644 --- a/server/src/jmh/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryBenchmark.java +++ b/server/src/jmh/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryBenchmark.java @@ -47,14 +47,14 @@ public class GitRepositoryBenchmark { private int previousCommits; private File repoDir; - private GitRepository repo; + private GitRepositoryV2 repo; private int currentRevision; @Setup public void init() throws Exception { repoDir = Files.createTempDirectory("jmh-gitrepository.").toFile(); - repo = new GitRepository(mock(Project.class), repoDir, ForkJoinPool.commonPool(), - System.currentTimeMillis(), AUTHOR, null); + repo = new GitRepositoryV2(mock(Project.class), repoDir, ForkJoinPool.commonPool(), + System.currentTimeMillis(), AUTHOR, null); currentRevision = 1; for (int i = 0; i < previousCommits; i++) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java index c32f8be68..cd7c4a5d6 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java @@ -122,6 +122,9 @@ public final class CentralDogmaBuilder { private int writeQuota; private int timeWindowSeconds; + @Nullable + private CommitRetentionConfig commitRetentionConfig; + /** * Creates a new builder with the specified data directory. */ @@ -520,6 +523,22 @@ public CentralDogmaBuilder writeQuotaPerRepository(int writeQuota, int timeWindo return this; } + /** + * Sets the configuration for retaining commits in a {@link Repository}. + * The {@link Repository} retains at least the number of {@code minRetentionCommits} when more than + * {@code minRetentionCommits} are made. + * The {@link Repository} also retains commits at least {@code minRetentionDays}. If both conditions are + * not satisfied, the commits are not removed. + * For example, when {@code minRetentionCommits} is set to 2000 and {@code minRetentionDays} is set to 14, + * the commits that are created more than 2000 are not removed until 14 days have passed. Set 0 to retain + * all commits. + */ + public CentralDogmaBuilder commitRetention(int minRetentionCommits, int minRetentionDays, + @Nullable String schedule) { + commitRetentionConfig = new CommitRetentionConfig(minRetentionCommits, minRetentionDays, schedule); + return this; + } + /** * Returns a newly-created {@link CentralDogma} server. */ @@ -553,6 +572,7 @@ private CentralDogmaConfig buildConfig() { maxRemovedRepositoryAgeMillis, gracefulShutdownTimeout, webAppEnabled, webAppTitle, mirroringEnabled, numMirroringThreads, maxNumFilesPerMirror, maxNumBytesPerMirror, replicationConfig, - null, accessLogFormat, authCfg, quotaConfig); + null, accessLogFormat, authCfg, quotaConfig, + commitRetentionConfig); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaConfig.java b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaConfig.java index bdd205843..d7fefe36b 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaConfig.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaConfig.java @@ -138,6 +138,9 @@ public final class CentralDogmaConfig { @Nullable private final QuotaConfig writeQuotaPerRepository; + @Nullable + private final CommitRetentionConfig commitRetentionConfig; + CentralDogmaConfig( @JsonProperty(value = "dataDir", required = true) File dataDir, @JsonProperty(value = "ports", required = true) @@ -165,7 +168,9 @@ public final class CentralDogmaConfig { @JsonProperty("csrfTokenRequiredForThrift") @Nullable Boolean csrfTokenRequiredForThrift, @JsonProperty("accessLogFormat") @Nullable String accessLogFormat, @JsonProperty("authentication") @Nullable AuthConfig authConfig, - @JsonProperty("writeQuotaPerRepository") @Nullable QuotaConfig writeQuotaPerRepository) { + @JsonProperty("writeQuotaPerRepository") @Nullable QuotaConfig writeQuotaPerRepository, + @JsonProperty("commitRetention") @Nullable + CommitRetentionConfig commitRetentionConfig) { this.dataDir = requireNonNull(dataDir, "dataDir"); this.ports = ImmutableList.copyOf(requireNonNull(ports, "ports")); @@ -219,6 +224,7 @@ public final class CentralDogmaConfig { ports.stream().anyMatch(ServerPort::hasProxyProtocol)); this.writeQuotaPerRepository = writeQuotaPerRepository; + this.commitRetentionConfig = commitRetentionConfig; } /** @@ -451,11 +457,20 @@ public AuthConfig authConfig() { * Returns the maximum allowed write quota per {@link Repository}. */ @Nullable - @JsonProperty("writeQuotaPerRepository") + @JsonProperty public QuotaConfig writeQuotaPerRepository() { return writeQuotaPerRepository; } + /** + * Returns the {@link CommitRetentionConfig}. + */ + @Nullable + @JsonProperty("commitRetention") + public CommitRetentionConfig commitRetentionConfig() { + return commitRetentionConfig; + } + @Override public String toString() { try { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java b/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java new file mode 100644 index 000000000..78d83fcfa --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java @@ -0,0 +1,98 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; + +import javax.annotation.Nullable; + +import com.cronutils.model.Cron; +import com.cronutils.model.CronType; +import com.cronutils.model.definition.CronDefinitionBuilder; +import com.cronutils.parser.CronParser; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import com.linecorp.centraldogma.server.storage.repository.Repository; + +/** + * A configuration for retaining commits in a {@link Repository}. + */ +public final class CommitRetentionConfig { + + // The minimum of minRetentionCommits + private static final int MINIMUM_MINIMUM_RETENTION_COMMITS = 1000; + + private static final String DEFAULT_SCHEDULE = "0 0 * * * ?"; // Every day + private static final CronParser cronParser = new CronParser(CronDefinitionBuilder.instanceDefinitionFor( + CronType.QUARTZ)); + + private final int minRetentionCommits; + private final int minRetentionDays; + private final Cron schedule; + + /** + * Creates a new instance. + */ + @JsonCreator + public CommitRetentionConfig(@JsonProperty("minRetentionCommits") int minRetentionCommits, + @JsonProperty("minRetentionDays") int minRetentionDays, + @JsonProperty("schedule") @Nullable String schedule) { + checkArgument(minRetentionCommits == 0 || minRetentionCommits >= MINIMUM_MINIMUM_RETENTION_COMMITS, + "minRetentionCommits: %s (expected: 0 || >= %s)", + minRetentionCommits, MINIMUM_MINIMUM_RETENTION_COMMITS); + checkArgument(minRetentionDays >= 0, + "minRetentionDays: %s (expected: >= 0)", minRetentionDays); + this.minRetentionCommits = minRetentionCommits; + this.minRetentionDays = minRetentionDays; + this.schedule = cronParser.parse(firstNonNull(schedule, DEFAULT_SCHEDULE)); + } + + /** + * Returns the minimum number of commits that a {@link Repository} should retain. 0 means that + * the number of commits are not taken into account when {@link Repository#removeOldCommits(int, int)} + * is called. + */ + public int minRetentionCommits() { + return minRetentionCommits; + } + + /** + * Returns the minimum number of days of a commit that a {@link Repository} should retain. 0 means that + * the number of retention days of commits are not taken into account when + * {@link Repository#removeOldCommits(int, int)} is called. + */ + public int minRetentionDays() { + return minRetentionDays; + } + + /** + * Returns the schedule of the job that removes old commits. + */ + public Cron schedule() { + return schedule; + } + + @Override + public String toString() { + return toStringHelper(this).add("minRetentionCommits", minRetentionCommits) + .add("minRetentionDays", minRetentionDays) + .add("schedule", schedule) + .toString(); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java index 2427e078d..68dc5321b 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java @@ -196,6 +196,11 @@ public CompletableFuture> mergeFiles(Revision revision, Merge return unwrap().mergeFiles(revision, query); } + @Override + public void removeOldCommits(int minRetentionCommits, int minRetentionDays) { + unwrap().removeOldCommits(minRetentionCommits, minRetentionDays); + } + @Override public String toString() { return Util.simpleTypeName(this) + '(' + unwrap() + ')'; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java index d73dc3b2f..28a2537a3 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java @@ -26,11 +26,8 @@ import java.util.Map; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; import java.util.concurrent.Executor; -import com.google.common.base.Throwables; - import com.linecorp.armeria.common.CommonPools; import com.linecorp.armeria.common.RequestContext; import com.linecorp.armeria.common.util.Exceptions; @@ -47,7 +44,6 @@ import com.linecorp.centraldogma.common.RevisionRange; import com.linecorp.centraldogma.server.command.CommitResult; import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache; -import com.linecorp.centraldogma.server.storage.StorageException; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.repository.FindOption; import com.linecorp.centraldogma.server.storage.repository.Repository; @@ -59,30 +55,20 @@ final class CachingRepository implements Repository { private final Repository repo; private final RepositoryCache cache; - private final Commit firstCommit; CachingRepository(Repository repo, RepositoryCache cache) { this.repo = requireNonNull(repo, "repo"); this.cache = requireNonNull(cache, "cache"); - - try { - final List history = repo.history(Revision.INIT, Revision.INIT, ALL_PATH, 1).join(); - firstCommit = history.get(0); - } catch (CompletionException e) { - final Throwable cause = Exceptions.peel(e); - Throwables.throwIfUnchecked(cause); - throw new StorageException("failed to retrieve the initial commit", cause); - } } @Override public long creationTimeMillis() { - return firstCommit.when(); + return repo.creationTimeMillis(); } @Override public Author author() { - return firstCommit.author(); + return repo.author(); } @Override @@ -310,11 +296,15 @@ public CompletableFuture commit(Revision baseRevision, long commit normalizing); } + @Override + public void removeOldCommits(int minRetentionCommits, int minRetentionDays) { + repo.removeOldCommits(minRetentionCommits, minRetentionDays); + } + @Override public String toString() { return toStringHelper(this) .add("repo", repo) - .add("firstCommit", firstCommit) .toString(); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CacheableCompareTreesCall.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CacheableCompareTreesCall.java index 3e068c28f..c1a86c731 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CacheableCompareTreesCall.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CacheableCompareTreesCall.java @@ -78,7 +78,7 @@ protected int weigh(List value) { } /** - * Never invoked because {@link GitRepository} produces the value of this call. + * Never invoked because {@link GitRepositoryV2} produces the value of this call. */ @Override public CompletableFuture> execute() { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java index 0b18d2681..8c659f49d 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java @@ -17,7 +17,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; -import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepository.R_HEADS_MASTER; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryV2.R_HEADS_MASTER; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; import java.io.EOFException; @@ -75,7 +75,10 @@ final class CommitIdDatabase implements AutoCloseable { private final Path path; private final FileChannel channel; private final boolean fsync; + @Nullable private volatile Revision headRevision; + @Nullable + private volatile Revision firstRevision; CommitIdDatabase(Repository repo) { // NB: We enable fsync only when our Git repository has been configured so, @@ -115,7 +118,13 @@ private CommitIdDatabase(File rootDir, boolean fsync) { } final int numRecords = (int) (size / RECORD_LEN); - headRevision = numRecords > 0 ? new Revision(numRecords) : null; + if (numRecords > 0) { + firstRevision = retrieveFirstRevision(); + headRevision = new Revision(numRecords + firstRevision.major() - 1); + } else { + firstRevision = null; + headRevision = null; + } success = true; } finally { if (!success) { @@ -124,21 +133,18 @@ private CommitIdDatabase(File rootDir, boolean fsync) { } } - @Nullable Revision headRevision() { - return headRevision; - } - - ObjectId get(Revision revision) { - final Revision headRevision = this.headRevision; - checkState(headRevision != null, "initial commit not available yet: %s", path); - checkArgument(!revision.isRelative(), "revision: %s (expected: an absolute revision)", revision); - if (revision.major() > headRevision.major()) { - throw new RevisionNotFoundException(revision); - } - + private Revision retrieveFirstRevision() { final ByteBuffer buf = threadLocalBuffer.get(); buf.clear(); - long pos = (long) (revision.major() - 1) * RECORD_LEN; + buf.limit(4); + readTo(buf, 0); + buf.flip(); + return new Revision(buf.getInt()); + // Do not have to call buf.clear() because clear method is called before the buffer is used. + } + + private void readTo(ByteBuffer buf, long startPosition) { + long pos = startPosition; try { do { final int readBytes = channel.read(buf, pos); @@ -150,6 +156,29 @@ ObjectId get(Revision revision) { } catch (IOException e) { throw new StorageException("failed to read the commit ID database: " + path, e); } + } + + @Nullable + Revision headRevision() { + return headRevision; + } + + @Nullable + Revision firstRevision() { + return firstRevision; + } + + ObjectId get(Revision revision) { + final Revision headRevision = this.headRevision; + checkState(headRevision != null, "initial commit not available yet: %s", path); + checkArgument(!revision.isRelative(), "revision: %s (expected: an absolute revision)", revision); + if (!(firstRevision.major() <= revision.major() && revision.major() <= headRevision.major())) { + throw new RevisionNotFoundException(revision); + } + + final ByteBuffer buf = threadLocalBuffer.get(); + buf.clear(); + readTo(buf, (long) (revision.major() - firstRevision.major()) * RECORD_LEN); buf.flip(); @@ -166,15 +195,15 @@ void put(Revision revision, ObjectId commitId) { put(revision, commitId, true); } - private synchronized void put(Revision revision, ObjectId commitId, boolean safeMode) { - if (safeMode) { - final Revision expected; - if (headRevision == null) { - expected = Revision.INIT; + private synchronized void put(Revision revision, ObjectId commitId, boolean isNewHeadRevision) { + if (isNewHeadRevision) { + if (headRevision != null) { + final Revision expected = headRevision.forward(1); + checkState(revision.equals(expected), "incorrect revision: %s (expected: %s)", + revision, expected); } else { - expected = headRevision.forward(1); + firstRevision = revision; } - checkState(revision.equals(expected), "incorrect revision: %s (expected: %s)", revision, expected); } // Build a record. @@ -185,20 +214,20 @@ private synchronized void put(Revision revision, ObjectId commitId, boolean safe buf.flip(); // Append a record to the file. - long pos = (long) (revision.major() - 1) * RECORD_LEN; + long pos = (long) (revision.major() - firstRevision.major()) * RECORD_LEN; try { do { pos += channel.write(buf, pos); } while (buf.hasRemaining()); - if (safeMode && fsync) { + if (isNewHeadRevision && fsync) { channel.force(true); } } catch (IOException e) { throw new StorageException("failed to update the commit ID database: " + path, e); } - if (safeMode || + if (isNewHeadRevision || headRevision == null || headRevision.major() < revision.major()) { headRevision = revision; @@ -224,29 +253,45 @@ void rebuild(Repository gitRepo) { throw new StorageException("failed to determine the HEAD: " + gitRepo.getDirectory()); } - RevCommit revCommit = revWalk.parseCommit(headCommitId); + final RevCommit revCommit = revWalk.parseCommit(headCommitId); headRevision = CommitUtil.extractRevision(revCommit.getFullMessage()); // NB: We did not store the last commit ID until all commit IDs are stored, // so that the partially built database always has mismatching head revision. - ObjectId currentId; - Revision previousRevision = headRevision; - loop: for (;;) { - switch (revCommit.getParentCount()) { - case 0: - // End of the history - break loop; - case 1: - currentId = revCommit.getParent(0); - break; - default: - throw new StorageException("found more than one parent: " + - gitRepo.getDirectory()); - } + rebuild(gitRepo, revWalk, headRevision, revCommit, true); + rebuild(gitRepo, revWalk, headRevision, revCommit, false); + + // All commit IDs except the head have been stored. Store the head finally. + put(headRevision, headCommitId); + } catch (Exception e) { + throw new StorageException("failed to rebuild the commit ID database", e); + } + + logger.info("Rebuilt the commit ID database. firstRevision: {}, headRevision: {}", + firstRevision, headRevision); + } + + private void rebuild(Repository gitRepo, RevWalk revWalk, Revision headRevision, + RevCommit revCommit, boolean findingFirstRevision) throws IOException { + ObjectId currentId; + Revision previousRevision = headRevision; + loop: for (;;) { + switch (revCommit.getParentCount()) { + case 0: + // End of the history + break loop; + case 1: + currentId = revCommit.getParent(0); + break; + default: + throw new StorageException("found more than one parent: " + + gitRepo.getDirectory()); + } - revCommit = revWalk.parseCommit(currentId); + revCommit = revWalk.parseCommit(currentId); + if (findingFirstRevision) { final Revision currentRevision = CommitUtil.extractRevision(revCommit.getFullMessage()); final Revision expectedRevision = previousRevision.backward(1); if (!currentRevision.equals(expectedRevision)) { @@ -254,18 +299,16 @@ void rebuild(Repository gitRepo) { " (actual: " + currentRevision.major() + ", expected: " + expectedRevision.major() + ')'); } - + previousRevision = currentRevision; + } else { + final Revision currentRevision = previousRevision.backward(1); put(currentRevision, currentId, false); previousRevision = currentRevision; } - - // All commit IDs except the head have been stored. Store the head finally. - put(headRevision, headCommitId); - } catch (Exception e) { - throw new StorageException("failed to rebuild the commit ID database", e); } - - logger.info("Rebuilt the commit ID database."); + if (findingFirstRevision) { + firstRevision = previousRevision; + } } @Override diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java new file mode 100644 index 000000000..bbadfc49f --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java @@ -0,0 +1,152 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import java.time.Duration; +import java.time.ZonedDateTime; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import javax.annotation.Nullable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cronutils.model.time.ExecutionTime; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableScheduledFuture; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.common.util.concurrent.MoreExecutors; + +import com.linecorp.centraldogma.server.CentralDogmaConfig; +import com.linecorp.centraldogma.server.CommitRetentionConfig; +import com.linecorp.centraldogma.server.plugin.Plugin; +import com.linecorp.centraldogma.server.plugin.PluginContext; +import com.linecorp.centraldogma.server.plugin.PluginTarget; +import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.project.ProjectManager; +import com.linecorp.centraldogma.server.storage.repository.Repository; + +import io.netty.util.concurrent.DefaultThreadFactory; + +public final class CommitRetentionManagementPlugin implements Plugin { + + private static final Logger logger = LoggerFactory.getLogger(CommitRetentionManagementPlugin.class); + + @Nullable + private ListeningScheduledExecutorService worker; + + private volatile boolean stopping; + @Nullable + private ListenableScheduledFuture scheduledFuture; + + @Override + public PluginTarget target() { + return PluginTarget.ALL_REPLICAS; + } + + @Override + public boolean isEnabled(CentralDogmaConfig config) { + return config.commitRetentionConfig() != null; + } + + @Override + public CompletionStage start(PluginContext context) { + final CommitRetentionConfig commitRetentionConfig = context.config().commitRetentionConfig(); + assert commitRetentionConfig != null; + worker = MoreExecutors.listeningDecorator( + Executors.newSingleThreadScheduledExecutor( + new DefaultThreadFactory("commit-retention-worker", true))); + scheduleRemovingOldCommits(context, commitRetentionConfig); + + return CompletableFuture.completedFuture(null); + } + + private void scheduleRemovingOldCommits(PluginContext context, CommitRetentionConfig config) { + if (stopping) { + return; + } + + final Duration nextExecution = ExecutionTime.forCron(config.schedule()) + .timeToNextExecution(ZonedDateTime.now()); + final ListeningScheduledExecutorService worker = this.worker; + assert worker != null; + scheduledFuture = worker.schedule(() -> removeOldCommits(context, config), nextExecution); + + Futures.addCallback(scheduledFuture, new FutureCallback() { + @Override + public void onSuccess(@Nullable Object result) {} + + @Override + public void onFailure(Throwable cause) { + if (!stopping) { + logger.warn("Commit retention scheduler stopped due to an unexpected exception:", + cause); + } + } + }, worker); + } + + private void removeOldCommits(PluginContext context, CommitRetentionConfig config) { + if (stopping) { + return; + } + + final ProjectManager pm = context.projectManager(); + for (Project project : pm.list().values()) { + for (Repository repo : project.repos().list().values()) { + repo.removeOldCommits(config.minRetentionCommits(), config.minRetentionDays()); + } + } + + scheduleRemovingOldCommits(context, config); + } + + @Override + public CompletionStage stop(PluginContext context) { + stopping = true; + if (scheduledFuture != null) { + scheduledFuture.cancel(false); + } + + try { + if (worker != null && !worker.isTerminated()) { + logger.info("Stopping the commit retention worker .."); + boolean interruptLater = false; + while (!worker.isTerminated()) { + worker.shutdownNow(); + try { + worker.awaitTermination(1, TimeUnit.SECONDS); + } catch (InterruptedException e) { + // Interrupt later. + interruptLater = true; + } + } + logger.info("Stopped the commit retention worker."); + + if (interruptLater) { + Thread.currentThread().interrupt(); + } + } + } catch (Throwable t) { + logger.warn("Failed to stop the commit retention worker:", t); + } + return CompletableFuture.completedFuture(null); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/FailFastUtil.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/FailFastUtil.java index d4274534a..b55fe7c11 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/FailFastUtil.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/FailFastUtil.java @@ -79,5 +79,46 @@ static void failFastIfTimedOut(GitRepository repo, Logger logger, @Nullable Serv } } + @SuppressWarnings("MethodParameterNamingConvention") + static void failFastIfTimedOut(GitRepositoryV2 repo, Logger logger, @Nullable ServiceRequestContext ctx, + String methodName, Object arg1) { + if (ctx != null && ctx.isTimedOut()) { + logger.info("{} Rejecting a request timed out already: repo={}/{}, method={}, args={}", + ctx, repo.parent().name(), repo.name(), methodName, arg1); + throw REQUEST_ALREADY_TIMED_OUT; + } + } + + @SuppressWarnings("MethodParameterNamingConvention") + static void failFastIfTimedOut(GitRepositoryV2 repo, Logger logger, @Nullable ServiceRequestContext ctx, + String methodName, Object arg1, Object arg2) { + if (ctx != null && ctx.isTimedOut()) { + logger.info("{} Rejecting a request timed out already: repo={}/{}, method={}, args=[{}, {}]", + ctx, repo.parent().name(), repo.name(), methodName, arg1, arg2); + throw REQUEST_ALREADY_TIMED_OUT; + } + } + + @SuppressWarnings("MethodParameterNamingConvention") + static void failFastIfTimedOut(GitRepositoryV2 repo, Logger logger, @Nullable ServiceRequestContext ctx, + String methodName, Object arg1, Object arg2, Object arg3) { + if (ctx != null && ctx.isTimedOut()) { + logger.info("{} Rejecting a request timed out already: repo={}/{}, method={}, args=[{}, {}, {}]", + ctx, repo.parent().name(), repo.name(), methodName, arg1, arg2, arg3); + throw REQUEST_ALREADY_TIMED_OUT; + } + } + + @SuppressWarnings("MethodParameterNamingConvention") + static void failFastIfTimedOut(GitRepositoryV2 repo, Logger logger, @Nullable ServiceRequestContext ctx, + String methodName, Object arg1, Object arg2, Object arg3, int arg4) { + if (ctx != null && ctx.isTimedOut()) { + logger.info( + "{} Rejecting a request timed out already: repo={}/{}, method={}, args=[{}, {}, {}, {}]", + ctx, repo.parent().name(), repo.name(), methodName, arg1, arg2, arg3, arg4); + throw REQUEST_ALREADY_TIMED_OUT; + } + } + private FailFastUtil() {} } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java index 2c1b3a7d3..5225d31dc 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java @@ -135,7 +135,7 @@ class GitRepository implements Repository { private static final Logger logger = LoggerFactory.getLogger(GitRepository.class); - static final String R_HEADS_MASTER = Constants.R_HEADS + Constants.MASTER; + private static final String R_HEADS_MASTER = Constants.R_HEADS + Constants.MASTER; private static final byte[] EMPTY_BYTE = new byte[0]; private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL); @@ -392,6 +392,16 @@ public String name() { return name; } + @Override + public long creationTimeMillis() { + return 0; + } + + @Override + public Author author() { + return null; + } + @Override public Revision normalizeNow(Revision revision) { return normalizeNow(revision, cachedHeadRevision().major()); @@ -1605,6 +1615,11 @@ private static void deleteCruft(File repoDir) { } } + @Override + public void removeOldCommits(int minRetentionCommits, int minRetentionDays) { + // Not supported. + } + @Override public String toString() { return MoreObjects.toStringHelper(this) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManager.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManager.java index 551acda55..0b745ee5a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManager.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManager.java @@ -69,7 +69,7 @@ public Project parent() { @Override protected Repository openChild(File childDir) throws Exception { - return new GitRepository(parent, childDir, repositoryWorker, cache); + return GitRepositoryV2.open(parent, childDir, repositoryWorker, cache); } private static void deleteCruft(File dir) throws IOException { @@ -80,14 +80,14 @@ private static void deleteCruft(File dir) throws IOException { @Override protected Repository createChild(File childDir, Author author, long creationTimeMillis) throws Exception { - return new GitRepository(parent, childDir, repositoryWorker, - creationTimeMillis, author, cache); + return new GitRepositoryV2(parent, childDir, repositoryWorker, + creationTimeMillis, author, cache); } @Override protected void closeChild(File childDir, Repository child, Supplier failureCauseSupplier) { - ((GitRepository) child).close(failureCauseSupplier); + ((GitRepositoryV2) child).close(failureCauseSupplier); } @Override diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtil.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtil.java new file mode 100644 index 000000000..b6ac6033b --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtil.java @@ -0,0 +1,628 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.linecorp.centraldogma.server.storage.repository.Repository.ALL_PATH; +import static com.linecorp.centraldogma.server.storage.repository.Repository.MAX_MAX_COMMITS; +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.StringJoiner; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import javax.annotation.Nullable; + +import org.eclipse.jgit.dircache.DirCache; +import org.eclipse.jgit.dircache.DirCacheBuilder; +import org.eclipse.jgit.dircache.DirCacheEditor; +import org.eclipse.jgit.dircache.DirCacheEditor.DeletePath; +import org.eclipse.jgit.dircache.DirCacheEditor.DeleteTree; +import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit; +import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectIdOwnerMap; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.RefUpdate; +import org.eclipse.jgit.lib.RefUpdate.Result; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.revwalk.TreeRevFilter; +import org.eclipse.jgit.revwalk.filter.RevFilter; +import org.eclipse.jgit.treewalk.filter.AndTreeFilter; +import org.eclipse.jgit.treewalk.filter.TreeFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.google.common.annotations.VisibleForTesting; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.CentralDogmaException; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.ChangeConflictException; +import com.linecorp.centraldogma.common.Commit; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.common.RevisionRange; +import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.internal.Util; +import com.linecorp.centraldogma.internal.jsonpatch.JsonPatch; +import com.linecorp.centraldogma.server.storage.StorageException; + +import difflib.DiffUtils; +import difflib.Patch; + +final class GitRepositoryUtil { + + private static final Logger logger = LoggerFactory.getLogger(GitRepositoryUtil.class); + + private static final byte[] EMPTY_BYTE = new byte[0]; + + private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL); + + private static final Field revWalkObjectsField; + + static { + Field field = null; + try { + field = RevWalk.class.getDeclaredField("objects"); + if (field.getType() != ObjectIdOwnerMap.class) { + throw new IllegalStateException( + RevWalk.class.getSimpleName() + ".objects is not an " + + ObjectIdOwnerMap.class.getSimpleName() + '.'); + } + field.setAccessible(true); + } catch (NoSuchFieldException e) { + throw new IllegalStateException( + RevWalk.class.getSimpleName() + ".objects does not exist."); + } + + revWalkObjectsField = field; + } + + static RevWalk newRevWalk(Repository jGitRepository) { + final RevWalk revWalk = new RevWalk(jGitRepository); + // Disable rewriteParents because otherwise `RevWalk` will load every commit into memory. + revWalk.setRewriteParents(false); + return revWalk; + } + + static RevWalk newRevWalk(ObjectReader reader) { + final RevWalk revWalk = new RevWalk(reader); + // Disable rewriteParents because otherwise `RevWalk` will load every commit into memory. + revWalk.setRewriteParents(false); + return revWalk; + } + + static RevTree toTree(CommitIdDatabase commitIdDatabase, RevWalk revWalk, Revision revision) { + final ObjectId commitId = commitIdDatabase.get(revision); + try { + return revWalk.parseCommit(commitId).getTree(); + } catch (IOException e) { + throw new StorageException("failed to parse a commit: " + commitId, e); + } + } + + static int applyChanges(Repository jGitRepo, @Nullable Revision baseRevision, + @Nullable ObjectId baseTreeId, DirCache dirCache, Iterable> changes) { + int numEdits = 0; + + try (ObjectInserter inserter = jGitRepo.newObjectInserter(); + ObjectReader reader = jGitRepo.newObjectReader()) { + + if (baseTreeId != null) { + // the DirCacheBuilder is to used for doing update operations on the given DirCache object + final DirCacheBuilder builder = dirCache.builder(); + + // Add the tree object indicated by the prevRevision to the temporary DirCache object. + builder.addTree(EMPTY_BYTE, 0, reader, baseTreeId); + builder.finish(); + } + + // loop over the specified changes. + for (Change change : changes) { + final String changePath = change.path().substring(1); // Strip the leading '/'. + final DirCacheEntry oldEntry = dirCache.getEntry(changePath); + final byte[] oldContent = oldEntry != null ? reader.open(oldEntry.getObjectId()).getBytes() + : null; + + switch (change.type()) { + case UPSERT_JSON: { + final JsonNode oldJsonNode = oldContent != null ? Jackson.readTree(oldContent) : null; + final JsonNode newJsonNode = firstNonNull((JsonNode) change.content(), + JsonNodeFactory.instance.nullNode()); + + // Upsert only when the contents are really different. + if (!Objects.equals(newJsonNode, oldJsonNode)) { + applyPathEdit(dirCache, new InsertJson(changePath, inserter, newJsonNode)); + numEdits++; + } + break; + } + case UPSERT_TEXT: { + final String sanitizedOldText; + if (oldContent != null) { + sanitizedOldText = sanitizeText(new String(oldContent, UTF_8)); + } else { + sanitizedOldText = null; + } + + final String sanitizedNewText = sanitizeText(change.contentAsText()); + + // Upsert only when the contents are really different. + if (!sanitizedNewText.equals(sanitizedOldText)) { + applyPathEdit(dirCache, new InsertText(changePath, inserter, sanitizedNewText)); + numEdits++; + } + break; + } + case REMOVE: + if (oldEntry != null) { + applyPathEdit(dirCache, new DeletePath(changePath)); + numEdits++; + break; + } + + // The path might be a directory. + if (applyDirectoryEdits(dirCache, changePath, null, change)) { + numEdits++; + } else { + // Was not a directory either; conflict. + reportNonExistentEntry(change); + break; + } + break; + case RENAME: { + final String newPath = + ((String) change.content()).substring(1); // Strip the leading '/'. + + if (dirCache.getEntry(newPath) != null) { + throw new ChangeConflictException("a file exists at the target path: " + change); + } + + if (oldEntry != null) { + if (changePath.equals(newPath)) { + // Redundant rename request - old path and new path are same. + break; + } + + final DirCacheEditor editor = dirCache.editor(); + editor.add(new DeletePath(changePath)); + editor.add(new CopyOldEntry(newPath, oldEntry)); + editor.finish(); + numEdits++; + break; + } + + // The path might be a directory. + if (applyDirectoryEdits(dirCache, changePath, newPath, change)) { + numEdits++; + } else { + // Was not a directory either; conflict. + reportNonExistentEntry(change); + } + break; + } + case APPLY_JSON_PATCH: { + final JsonNode oldJsonNode; + if (oldContent != null) { + oldJsonNode = Jackson.readTree(oldContent); + } else { + oldJsonNode = Jackson.nullNode; + } + + final JsonNode newJsonNode; + try { + newJsonNode = JsonPatch.fromJson((JsonNode) change.content()).apply(oldJsonNode); + } catch (Exception e) { + throw new ChangeConflictException("failed to apply JSON patch: " + change, e); + } + + // Apply only when the contents are really different. + if (!newJsonNode.equals(oldJsonNode)) { + applyPathEdit(dirCache, new InsertJson(changePath, inserter, newJsonNode)); + numEdits++; + } + break; + } + case APPLY_TEXT_PATCH: + final Patch patch = DiffUtils.parseUnifiedDiff( + Util.stringToLines(sanitizeText((String) change.content()))); + + final String sanitizedOldText; + final List sanitizedOldTextLines; + if (oldContent != null) { + sanitizedOldText = sanitizeText(new String(oldContent, UTF_8)); + sanitizedOldTextLines = Util.stringToLines(sanitizedOldText); + } else { + sanitizedOldText = null; + sanitizedOldTextLines = Collections.emptyList(); + } + + final String newText; + try { + final List newTextLines = DiffUtils.patch(sanitizedOldTextLines, patch); + if (newTextLines.isEmpty()) { + newText = ""; + } else { + final StringJoiner joiner = new StringJoiner("\n", "", "\n"); + for (String line : newTextLines) { + joiner.add(line); + } + newText = joiner.toString(); + } + } catch (Exception e) { + throw new ChangeConflictException("failed to apply text patch: " + change, e); + } + + // Apply only when the contents are really different. + if (!newText.equals(sanitizedOldText)) { + applyPathEdit(dirCache, new InsertText(changePath, inserter, newText)); + numEdits++; + } + break; + } + } + } catch (CentralDogmaException | IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new StorageException("failed to apply changes on revision " + baseRevision, e); + } + return numEdits; + } + + private static void applyPathEdit(DirCache dirCache, PathEdit edit) { + final DirCacheEditor e = dirCache.editor(); + e.add(edit); + e.finish(); + } + + /** + * Removes {@code \r} and appends {@code \n} on the last line if it does not end with {@code \n}. + */ + private static String sanitizeText(String text) { + if (text.indexOf('\r') >= 0) { + text = CR.matcher(text).replaceAll(""); + } + if (!text.isEmpty() && !text.endsWith("\n")) { + text += "\n"; + } + return text; + } + + /** + * Applies recursive directory edits. + * + * @param oldDir the path to the directory to make a recursive change + * @param newDir the path to the renamed directory, or {@code null} to remove the directory. + * + * @return {@code true} if any edits were made to {@code dirCache}, {@code false} otherwise + */ + private static boolean applyDirectoryEdits(DirCache dirCache, + String oldDir, @Nullable String newDir, Change change) { + + if (!oldDir.endsWith("/")) { + oldDir += '/'; + } + if (newDir != null && !newDir.endsWith("/")) { + newDir += '/'; + } + + final byte[] rawOldDir = Constants.encode(oldDir); + final byte[] rawNewDir = newDir != null ? Constants.encode(newDir) : null; + final int numEntries = dirCache.getEntryCount(); + DirCacheEditor editor = null; + + loop: + for (int i = 0; i < numEntries; i++) { + final DirCacheEntry e = dirCache.getEntry(i); + final byte[] rawPath = e.getRawPath(); + + // Ensure that there are no entries under the newDir; we have a conflict otherwise. + if (rawNewDir != null) { + boolean conflict = true; + if (rawPath.length > rawNewDir.length) { + // Check if there is a file whose path starts with 'newDir'. + for (int j = 0; j < rawNewDir.length; j++) { + if (rawNewDir[j] != rawPath[j]) { + conflict = false; + break; + } + } + } else if (rawPath.length == rawNewDir.length - 1) { + // Check if there is a file whose path is exactly same with newDir without trailing '/'. + for (int j = 0; j < rawNewDir.length - 1; j++) { + if (rawNewDir[j] != rawPath[j]) { + conflict = false; + break; + } + } + } else { + conflict = false; + } + + if (conflict) { + throw new ChangeConflictException("target directory exists already: " + change); + } + } + + // Skip the entries that do not belong to the oldDir. + if (rawPath.length <= rawOldDir.length) { + continue; + } + for (int j = 0; j < rawOldDir.length; j++) { + if (rawOldDir[j] != rawPath[j]) { + continue loop; + } + } + + // Do not create an editor until we find an entry to rename/remove. + // We can tell if there was any matching entries or not from the nullness of editor later. + if (editor == null) { + editor = dirCache.editor(); + editor.add(new DeleteTree(oldDir)); + if (newDir == null) { + // Recursive removal + break; + } + } + + assert newDir != null; // We should get here only when it's a recursive rename. + + final String oldPath = e.getPathString(); + final String newPath = newDir + oldPath.substring(oldDir.length()); + editor.add(new CopyOldEntry(newPath, e)); + } + + if (editor != null) { + editor.finish(); + return true; + } else { + return false; + } + } + + private static void reportNonExistentEntry(Change change) { + throw new ChangeConflictException("non-existent file/directory: " + change); + } + + @VisibleForTesting + static void doRefUpdate(Repository jGitRepo, RevWalk revWalk, + String ref, ObjectId commitId) throws IOException { + if (ref.startsWith(Constants.R_TAGS)) { + final Ref oldRef = jGitRepo.exactRef(ref); + if (oldRef != null) { + throw new StorageException("tag ref exists already: " + ref); + } + } + + final RefUpdate refUpdate = jGitRepo.updateRef(ref); + refUpdate.setNewObjectId(commitId); + + final Result res = refUpdate.update(revWalk); + switch (res) { + case NEW: + case FAST_FORWARD: + // Expected + break; + default: + throw new StorageException("unexpected refUpdate state: " + res); + } + } + + static List getCommits(InternalRepository repo, String pathPattern, int maxCommits, + RevisionRange range, RevisionRange descendingRange) { + try (RevWalk revWalk = newRevWalk(repo.jGitRepo())) { + final ObjectIdOwnerMap revWalkInternalMap = + (ObjectIdOwnerMap) revWalkObjectsField.get(revWalk); + + final CommitIdDatabase commitIdDatabase = repo.commitIdDatabase(); + final ObjectId fromCommitId = commitIdDatabase.get(descendingRange.from()); + final ObjectId toCommitId = commitIdDatabase.get(descendingRange.to()); + + revWalk.markStart(revWalk.parseCommit(fromCommitId)); + revWalk.setRetainBody(false); + + // Instead of relying on RevWalk to filter the commits, + // we let RevWalk yield all commits so we can: + // - Have more control on when iteration should be stopped. + // (A single Iterator.next() doesn't take long.) + // - Clean up the internal map as early as possible. + final RevFilter filter = new TreeRevFilter(revWalk, AndTreeFilter.create( + TreeFilter.ANY_DIFF, PathPatternFilter.of(pathPattern))); + + // Search up to 1000 commits when maxCommits <= 100. + // Search up to (maxCommits * 10) commits when 100 < maxCommits <= 1000. + final int maxNumProcessedCommits = Math.max(maxCommits * 10, MAX_MAX_COMMITS); + + final List commitList = new ArrayList<>(); + int numProcessedCommits = 0; + for (RevCommit revCommit : revWalk) { + numProcessedCommits++; + + if (filter.include(revWalk, revCommit)) { + revWalk.parseBody(revCommit); + commitList.add(toCommit(revCommit)); + revCommit.disposeBody(); + } + + if (revCommit.getId().equals(toCommitId) || + commitList.size() >= maxCommits || + // Prevent from iterating for too long. + numProcessedCommits >= maxNumProcessedCommits) { + break; + } + + // Clear the internal lookup table of RevWalk to reduce the memory usage. + // This is safe because we have linear history and traverse in one direction. + if (numProcessedCommits % 16 == 0) { + revWalkInternalMap.clear(); + } + } + + // Include the initial empty commit only when the caller specified + // the initial revision (1) in the range and the pathPattern contains '/**'. + if (commitList.size() < maxCommits && + descendingRange.to().major() == 1 && + pathPattern.contains(ALL_PATH)) { + try (RevWalk tmpRevWalk = newRevWalk(repo.jGitRepo())) { + final RevCommit lastRevCommit = tmpRevWalk.parseCommit(toCommitId); + commitList.add(toCommit(lastRevCommit)); + } + } + + if (!descendingRange.equals(range)) { // from and to is swapped so reverse the list. + Collections.reverse(commitList); + } + + return commitList; + } catch (CentralDogmaException e) { + throw e; + } catch (Exception e) { + throw new StorageException( + "failed to retrieve the history: " + repo.repoDir() + + " (" + pathPattern + ", " + range.from() + ".." + range.to() + ')', e); + } + } + + private static Commit toCommit(RevCommit revCommit) { + final Author author; + final PersonIdent committerIdent = revCommit.getCommitterIdent(); + final long when; + if (committerIdent == null) { + author = Author.UNKNOWN; + when = 0; + } else { + author = new Author(committerIdent.getName(), committerIdent.getEmailAddress()); + when = committerIdent.getWhen().getTime(); + } + + try { + return CommitUtil.newCommit(author, when, revCommit.getFullMessage()); + } catch (Exception e) { + throw new StorageException("failed to create a Commit", e); + } + } + + static boolean deleteDirectory(File dir) throws IOException { + try (Stream walk = Files.walk(dir.toPath())) { + return walk.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .map(File::delete) + // Return false if it fails to delete a file. + .reduce(true, (a, b) -> a && b); + } + } + + static boolean isEmpty(File dir) throws IOException { + if (!dir.isDirectory()) { + return false; + } + if (!dir.exists()) { + return true; + } + try (Stream entries = Files.list(dir.toPath())) { + return entries.findFirst().isPresent(); + } + } + + static void closeJGitRepo(Repository repository) { + try { + repository.close(); + } catch (Throwable t) { + logger.warn("Failed to close a Git repository: {}", repository.getDirectory(), t); + } + } + + // PathEdit implementations which is used when applying changes. + + private static final class InsertText extends PathEdit { + private final ObjectInserter inserter; + private final String text; + + InsertText(String entryPath, ObjectInserter inserter, String text) { + super(entryPath); + this.inserter = inserter; + this.text = text; + } + + @Override + public void apply(DirCacheEntry ent) { + try { + ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, text.getBytes(UTF_8))); + ent.setFileMode(FileMode.REGULAR_FILE); + } catch (IOException e) { + throw new StorageException("failed to create a new text blob", e); + } + } + } + + private static final class InsertJson extends PathEdit { + private final ObjectInserter inserter; + private final JsonNode jsonNode; + + InsertJson(String entryPath, ObjectInserter inserter, JsonNode jsonNode) { + super(entryPath); + this.inserter = inserter; + this.jsonNode = jsonNode; + } + + @Override + public void apply(DirCacheEntry ent) { + try { + ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, Jackson.writeValueAsBytes(jsonNode))); + ent.setFileMode(FileMode.REGULAR_FILE); + } catch (IOException e) { + throw new StorageException("failed to create a new JSON blob", e); + } + } + } + + private static final class CopyOldEntry extends PathEdit { + private final DirCacheEntry oldEntry; + + CopyOldEntry(String entryPath, DirCacheEntry oldEntry) { + super(entryPath); + this.oldEntry = oldEntry; + } + + @Override + public void apply(DirCacheEntry ent) { + ent.setFileMode(oldEntry.getFileMode()); + ent.setObjectId(oldEntry.getObjectId()); + } + } + + private GitRepositoryUtil() {} +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java new file mode 100644 index 000000000..7377588f9 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java @@ -0,0 +1,1388 @@ +/* + * Copyright 2017 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.context; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.failFastIfTimedOut; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.applyChanges; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.closeJGitRepo; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.deleteDirectory; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.doRefUpdate; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.getCommits; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.isEmpty; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.newRevWalk; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.toTree; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_COMMIT_SECTION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DIFF_SECTION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_ALGORITHM; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_FILEMODE; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_GPGSIGN; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_HIDEDOTFILES; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_RENAMES; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REPO_FORMAT_VERSION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_SYMLINKS; + +import java.io.File; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Supplier; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; + +import org.eclipse.jgit.diff.DiffEntry; +import org.eclipse.jgit.diff.DiffFormatter; +import org.eclipse.jgit.dircache.DirCache; +import org.eclipse.jgit.dircache.DirCacheIterator; +import org.eclipse.jgit.lib.CommitBuilder; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.CoreConfig.HideDotFiles; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.RefUpdate; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.RepositoryBuilder; +import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.treewalk.CanonicalTreeParser; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.TreeFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.ImmutableMap; + +import com.linecorp.armeria.server.ServiceRequestContext; +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.CentralDogmaException; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.ChangeConflictException; +import com.linecorp.centraldogma.common.Commit; +import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.EntryType; +import com.linecorp.centraldogma.common.Markup; +import com.linecorp.centraldogma.common.RedundantChangeException; +import com.linecorp.centraldogma.common.RepositoryNotFoundException; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.common.RevisionNotFoundException; +import com.linecorp.centraldogma.common.RevisionRange; +import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.internal.Util; +import com.linecorp.centraldogma.internal.jsonpatch.JsonPatch; +import com.linecorp.centraldogma.internal.jsonpatch.ReplaceMode; +import com.linecorp.centraldogma.server.command.CommitResult; +import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache; +import com.linecorp.centraldogma.server.storage.StorageException; +import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.repository.FindOption; +import com.linecorp.centraldogma.server.storage.repository.FindOptions; + +class GitRepositoryV2 implements com.linecorp.centraldogma.server.storage.repository.Repository { + + private static final Logger logger = LoggerFactory.getLogger(GitRepositoryV2.class); + + private static final String PRIMARY_REPOSITORY_SUFFIX = "_primary"; + private static final String FOLLOWER_REPOSITORY_SUFFIX = "_secondary"; + + static final String R_HEADS_MASTER = Constants.R_HEADS + Constants.MASTER; + + private static final byte[] EMPTY_BYTE = new byte[0]; + private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL); + + /** + * Opens an existing Git-backed repository. + * + * @param repositoryDir the location of this repository + * @param repositoryWorker the {@link Executor} which will perform the blocking repository operations + * + * @throws StorageException if failed to open the repository at the specified location + */ + static GitRepositoryV2 open(Project parent, File repositoryDir, + Executor repositoryWorker, @Nullable RepositoryCache cache) { + return new GitRepositoryV2(parent, repositoryDir, repositoryWorker, cache); + } + + private final ReadWriteLock rwLock = new ReentrantReadWriteLock(); + private final Project parent; + private final File repositoryDir; + private final String name; + private final Executor repositoryWorker; + + private final RepositoryMetadataDatabase repoMetadata; + @Nullable + @VisibleForTesting + final RepositoryCache cache; + @VisibleForTesting + final CommitWatchers commitWatchers = new CommitWatchers(); + private final AtomicReference> closePending = new AtomicReference<>(); + private final CompletableFuture closeFuture = new CompletableFuture<>(); + + @VisibleForTesting + InternalRepository primaryRepo; + + @Nullable + @VisibleForTesting + InternalRepository secondaryRepo; + + private final long creationTimeMillis; + private final Author author; + + // Guarded by the write lock. + private boolean isCreatingSecondaryRepo; + private final List laggedCommitsForSecondary = new ArrayList<>(); + + /** + * The current head revision. Initialized by the constructor and updated by commit(). + */ + private volatile Revision headRevision; + + GitRepositoryV2(Project parent, File repositoryDir, Executor repositoryWorker, + long creationTimeMillis, Author author, @Nullable RepositoryCache cache) { + + this.parent = requireNonNull(parent, "parent"); + this.repositoryDir = requireNonNull(repositoryDir, "repositoryDir"); + name = repositoryDir.getName(); + this.repositoryWorker = requireNonNull(repositoryWorker, "repositoryWorker"); + this.cache = cache; + + requireNonNull(author, "author"); + try { + if (repositoryDir.exists()) { + if (!isEmpty(repositoryDir)) { + throw new StorageException("failed to create a repository at: " + repositoryDir + + " (exists already)"); + } + } else if (!repositoryDir.mkdir()) { + throw new StorageException("failed to create a repository at: " + repositoryDir); + } + } catch (IOException e) { + throw new StorageException("failed to create a repository at: " + repositoryDir, e); + } + + repoMetadata = new RepositoryMetadataDatabase(repositoryDir, true); + final File primaryRepoDir = repoMetadata.primaryRepoDir(); + try { + primaryRepo = createInternalRepo(parent, name, primaryRepoDir, Revision.INIT, + creationTimeMillis, author, ImmutableList.of()); + } catch (Throwable t) { + repoMetadata.close(); + throw t; + } + headRevision = Revision.INIT; + this.creationTimeMillis = creationTimeMillis; + this.author = author; + } + + @VisibleForTesting + static InternalRepository createInternalRepo(Project parent, String originalRepoName, File repoDir, + Revision nextRevision, long commitTimeMillis, Author author, + Iterable> changes) { + boolean success = false; + InternalRepository internalRepo = null; + try { + createEmptyJGitRepo(repoDir); + + // Re-open the repository with the updated settings and format version. + final Repository jGitRepo = new RepositoryBuilder().setGitDir(repoDir) + .build(); + internalRepo = new InternalRepository(parent, originalRepoName, repoDir, + jGitRepo, new CommitIdDatabase(jGitRepo)); + + // Initialize the master branch. + final RefUpdate head = jGitRepo.updateRef(Constants.HEAD); + head.disableRefLog(); + head.link(R_HEADS_MASTER); + + // Insert the initial commit into the master branch. + commit0(internalRepo, null, nextRevision, commitTimeMillis, author, + "Create a new repository", "", Markup.PLAINTEXT, changes, true); + success = true; + return internalRepo; + } catch (IOException e) { + throw new StorageException("failed to create a repository at: " + repoDir, e); + } finally { + if (!success) { + closeInternalRepository(internalRepo); + // Failed to create a repository. Remove any cruft so that it is not loaded on the next run. + deleteCruft(repoDir); + } + } + } + + private static void createEmptyJGitRepo(File repositoryDir) throws IOException { + try (org.eclipse.jgit.lib.Repository initRepository = buildjGitRepo(repositoryDir)) { + initRepository.create(true); + + final StoredConfig config = initRepository.getConfig(); + // Update the repository settings to upgrade to format version 1 and reftree. + config.setInt(CONFIG_CORE_SECTION, null, CONFIG_KEY_REPO_FORMAT_VERSION, 1); + + // Disable hidden files, symlinks and file modes we do not use. + config.setEnum(CONFIG_CORE_SECTION, null, CONFIG_KEY_HIDEDOTFILES, HideDotFiles.FALSE); + config.setBoolean(CONFIG_CORE_SECTION, null, CONFIG_KEY_SYMLINKS, false); + config.setBoolean(CONFIG_CORE_SECTION, null, CONFIG_KEY_FILEMODE, false); + + // Disable GPG signing. + config.setBoolean(CONFIG_COMMIT_SECTION, null, CONFIG_KEY_GPGSIGN, false); + + // Set the diff algorithm. + config.setString(CONFIG_DIFF_SECTION, null, CONFIG_KEY_ALGORITHM, "histogram"); + + // Disable rename detection which we do not use. + config.setBoolean(CONFIG_DIFF_SECTION, null, CONFIG_KEY_RENAMES, false); + + config.save(); + } + } + + private static Repository buildjGitRepo(File repositoryDir) { + try { + return new RepositoryBuilder().setGitDir(repositoryDir).setBare().build(); + } catch (IOException e) { + throw new StorageException("failed to create a repository at: " + repositoryDir, e); + } + } + + private GitRepositoryV2(Project parent, File repoDir, Executor repositoryWorker, + @Nullable RepositoryCache cache) { + this.parent = requireNonNull(parent, "parent"); + this.repositoryDir = requireNonNull(repoDir, "repoDir"); + name = repoDir.getName(); + this.repositoryWorker = requireNonNull(repositoryWorker, "repositoryWorker"); + this.cache = cache; + + RepositoryMetadataDatabase repoMetadata; + try { + repoMetadata = new RepositoryMetadataDatabase(repoDir, false); + } catch (Throwable t) { + // The metadata doesn't exist so check if the repository exists in the form of the old version. + final Repository oldRepo = buildjGitRepo(repoDir); + final boolean oldRepoExist = exist(oldRepo); + closeJGitRepo(oldRepo); + if (!oldRepoExist) { + throw new RepositoryNotFoundException(repoDir.toString()); + } + final File primaryRepoDir = RepositoryMetadataDatabase.initialPrimaryRepoDir(repoDir); + final File tmpRepoDir = new File(repoDir.getParentFile(), UUID.randomUUID().toString()); + logger.debug("Migrating {} to {} using temp repository: {}", repoDir, primaryRepoDir, tmpRepoDir); + if (!repoDir.renameTo(tmpRepoDir)) { + throw new StorageException("failed to migrate a repository at: " + repoDir + + ", to the tmp dir: " + tmpRepoDir); + } + primaryRepoDir.mkdirs(); + if (!tmpRepoDir.renameTo(primaryRepoDir)) { + // Rename it back. + tmpRepoDir.renameTo(repoDir); + throw new StorageException("failed to migrate a repository at: " + tmpRepoDir + + ", to: " + primaryRepoDir); + } + assert !tmpRepoDir.exists(); + logger.debug("Migrating {} is done.", repoDir); + repoMetadata = new RepositoryMetadataDatabase(repoDir, true); + } + + this.repoMetadata = repoMetadata; + try { + final InternalRepository primaryRepo = + openInternalRepository(repoMetadata.primaryRepoDir(), true); + assert primaryRepo != null; + this.primaryRepo = primaryRepo; + final Commit initialCommit = currentInitialCommit(primaryRepo); + creationTimeMillis = initialCommit.when(); + author = initialCommit.author(); + secondaryRepo = openInternalRepository(repoMetadata.secondaryRepoDir(), false); + } catch (Throwable t) { + repoMetadata.close(); + closeInternalRepository(primaryRepo); + closeInternalRepository(secondaryRepo); + throw t; + } + } + + @Nullable + private InternalRepository openInternalRepository(File repoDir, boolean primary) { + final Repository jGitRepo = buildjGitRepo(repoDir); + CommitIdDatabase commitIdDatabase = null; + try { + if (!exist(jGitRepo)) { + if (primary) { + throw new RepositoryNotFoundException(repoDir.toString()); + } else { + return null; + } + } + checkGitRepositoryFormat(jGitRepo); + final Revision headRevision = uncachedHeadRevision(jGitRepo); + commitIdDatabase = new CommitIdDatabase(jGitRepo); + if (!headRevision.equals(commitIdDatabase.headRevision())) { + commitIdDatabase.rebuild(jGitRepo); + assert headRevision.equals(commitIdDatabase.headRevision()); + } + if (primary) { + this.headRevision = headRevision; + } + return new InternalRepository(parent, name, repoDir, jGitRepo, commitIdDatabase); + } catch (Throwable t) { + closeJGitRepo(jGitRepo); + if (commitIdDatabase != null) { + commitIdDatabase.close(); + } + throw t; + } + } + + private static void checkGitRepositoryFormat(Repository repository) { + final int formatVersion = repository.getConfig().getInt( + CONFIG_CORE_SECTION, null, CONFIG_KEY_REPO_FORMAT_VERSION, 0); + if (formatVersion != 1) { + throw new StorageException("unsupported repository format version: " + formatVersion); + } + } + + private static boolean exist(Repository repository) { + if (repository.getConfig() instanceof FileBasedConfig) { + return ((FileBasedConfig) repository.getConfig()).getFile().exists(); + } + return repository.getDirectory().exists(); + } + + @VisibleForTesting + void internalClose() { + close(() -> new CentralDogmaException("should never reach here")); + } + + /** + * Waits until all pending operations are complete and closes this repository. + * + * @param failureCauseSupplier the {@link Supplier} that creates a new {@link CentralDogmaException} + * which will be used to fail the operations issued after this method is called + */ + void close(Supplier failureCauseSupplier) { + requireNonNull(failureCauseSupplier, "failureCauseSupplier"); + if (closePending.compareAndSet(null, failureCauseSupplier)) { + repositoryWorker.execute(() -> { + rwLock.writeLock().lock(); + try { + closeInternalRepository(primaryRepo); + closeInternalRepository(secondaryRepo); + } finally { + rwLock.writeLock().unlock(); + commitWatchers.close(failureCauseSupplier); + closeFuture.complete(null); + } + }); + } + + closeFuture.join(); + } + + private static void closeInternalRepository(@Nullable InternalRepository internalRepo) { + if (internalRepo == null) { + return; + } + internalRepo.close(); + } + + private static void deleteCruft(File repoDir) { + try { + Util.deleteFileTree(repoDir); + } catch (IOException e) { + logger.error("Failed to delete a half-created repository at: {}", repoDir, e); + } + } + + /** + * Returns the current revision. + */ + private Revision uncachedHeadRevision(Repository jGitRepo) { + try (RevWalk revWalk = newRevWalk(jGitRepo)) { + final ObjectId headRevisionId = jGitRepo.resolve(R_HEADS_MASTER); + if (headRevisionId != null) { + final RevCommit revCommit = revWalk.parseCommit(headRevisionId); + return CommitUtil.extractRevision(revCommit.getFullMessage()); + } + } catch (CentralDogmaException e) { + throw e; + } catch (Exception e) { + throw new StorageException("failed to get the current revision", e); + } + + throw new StorageException("failed to determine the HEAD: " + parent.name() + '/' + name); + } + + @Override + public Project parent() { + return parent; + } + + @Override + public String name() { + return name; + } + + @Override + public long creationTimeMillis() { + return creationTimeMillis; + } + + @Override + public Author author() { + return author; + } + + @Override + public Revision normalizeNow(Revision revision) { + return normalizeNow(revision, headRevision.major()); + } + + private Revision normalizeNow(Revision revision, int headMajor) { + requireNonNull(revision, "revision"); + + int major = revision.major(); + + if (major >= 0) { + if (major > headMajor) { + throw new RevisionNotFoundException( + "revision: " + revision + " (expected: <= " + headMajor + ")"); + } + } else { + major = headMajor + major + 1; + if (major <= 0) { + throw new RevisionNotFoundException( + "revision: " + revision + " (expected: " + revision + " + " + headMajor + " + 1 > 0)"); + } + } + + final Revision firstRevision = primaryRepo.commitIdDatabase().firstRevision(); + assert firstRevision != null; + final int firstMajor = firstRevision.major(); + if (major < firstMajor) { + major = firstMajor; + } + + // Create a new instance only when necessary. + if (revision.major() == major) { + return revision; + } else { + return new Revision(major); + } + } + + @Override + public RevisionRange normalizeNow(Revision from, Revision to) { + final int headMajor = headRevision.major(); + return new RevisionRange(normalizeNow(from, headMajor), normalizeNow(to, headMajor)); + } + + @Override + public CompletableFuture>> find( + Revision revision, String pathPattern, Map, ?> options) { + final ServiceRequestContext ctx = context(); + return CompletableFuture.supplyAsync(() -> { + failFastIfTimedOut(this, logger, ctx, "find", revision, pathPattern, options); + return blockingFind(revision, pathPattern, options); + }, repositoryWorker); + } + + private Map> blockingFind( + Revision revision, String pathPattern, Map, ?> options) { + + requireNonNull(pathPattern, "pathPattern"); + requireNonNull(revision, "revision"); + requireNonNull(options, "options"); + + final Revision normRevision = normalizeNow(revision); + final boolean fetchContent = FindOption.FETCH_CONTENT.get(options); + final int maxEntries = FindOption.MAX_ENTRIES.get(options); + + readLock(); + final InternalRepository primaryRepo = this.primaryRepo; + try (ObjectReader reader = primaryRepo.jGitRepo().newObjectReader(); + TreeWalk treeWalk = new TreeWalk(reader); + RevWalk revWalk = newRevWalk(reader)) { + + // Query on a non-exist revision will return empty result. + final Revision headRevision = this.headRevision; + if (normRevision.compareTo(headRevision) > 0) { + return Collections.emptyMap(); + } + + if ("/".equals(pathPattern)) { + return Collections.singletonMap(pathPattern, Entry.ofDirectory(normRevision, "/")); + } + + final Map> result = new LinkedHashMap<>(); + final ObjectId commitId = primaryRepo.commitIdDatabase().get(normRevision); + final RevCommit revCommit = revWalk.parseCommit(commitId); + final PathPatternFilter filter = PathPatternFilter.of(pathPattern); + + final RevTree revTree = revCommit.getTree(); + treeWalk.addTree(revTree.getId()); + while (treeWalk.next() && result.size() < maxEntries) { + final boolean matches = filter.matches(treeWalk); + final String path = '/' + treeWalk.getPathString(); + + // Recurse into a directory if necessary. + if (treeWalk.isSubtree()) { + if (matches) { + // Add the directory itself to the result set if its path matches the pattern. + result.put(path, Entry.ofDirectory(normRevision, path)); + } + + treeWalk.enterSubtree(); + continue; + } + + if (!matches) { + continue; + } + + // Build an entry as requested. + final Entry entry; + final EntryType entryType = EntryType.guessFromPath(path); + if (fetchContent) { + final byte[] content = reader.open(treeWalk.getObjectId(0)).getBytes(); + switch (entryType) { + case JSON: + final JsonNode jsonNode = Jackson.readTree(content); + entry = Entry.ofJson(normRevision, path, jsonNode); + break; + case TEXT: + final String strVal = sanitizeText(new String(content, UTF_8)); + entry = Entry.ofText(normRevision, path, strVal); + break; + default: + throw new Error("unexpected entry type: " + entryType); + } + } else { + switch (entryType) { + case JSON: + entry = Entry.ofJson(normRevision, path, Jackson.nullNode); + break; + case TEXT: + entry = Entry.ofText(normRevision, path, ""); + break; + default: + throw new Error("unexpected entry type: " + entryType); + } + } + + result.put(path, entry); + } + + return Util.unsafeCast(result); + } catch (CentralDogmaException | IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new StorageException( + "failed to get data from '" + parent.name() + '/' + name + "' at " + pathPattern + + " for " + revision, e); + } finally { + readUnlock(); + } + } + + /** + * Get the diff between any two valid revisions. + * + * @param from revision from + * @param to revision to + * @param pathPattern target path pattern + * @return the map of changes mapped by path + * @throws StorageException if {@code from} or {@code to} does not exist. + */ + @Override + public CompletableFuture>> diff(Revision from, Revision to, String pathPattern) { + final ServiceRequestContext ctx = context(); + return CompletableFuture.supplyAsync(() -> { + requireNonNull(from, "from"); + requireNonNull(to, "to"); + requireNonNull(pathPattern, "pathPattern"); + + failFastIfTimedOut(this, logger, ctx, "diff", from, to, pathPattern); + + final RevisionRange range = normalizeNow(from, to).toAscending(); + readLock(); + final InternalRepository primaryRepo = this.primaryRepo; + final Repository jGitRepo = primaryRepo.jGitRepo(); + try (RevWalk rw = newRevWalk(jGitRepo)) { + final CommitIdDatabase commitIdDatabase = primaryRepo.commitIdDatabase(); + final RevTree treeA = rw.parseTree(commitIdDatabase.get(range.from())); + final RevTree treeB = rw.parseTree(commitIdDatabase.get(range.to())); + + // Compare the two Git trees. + // Note that we do not cache here because CachingRepository caches the final result already. + return toChangeMap(jGitRepo, + blockingCompareTreesUncached(jGitRepo, treeA, treeB, + pathPatternFilterOrTreeFilter(pathPattern))); + } catch (StorageException e) { + throw e; + } catch (Exception e) { + throw new StorageException("failed to parse two trees: range=" + range, e); + } finally { + readUnlock(); + } + }, repositoryWorker); + } + + private List blockingCompareTreesUncached(Repository jGitRepo, + @Nullable RevTree treeA, @Nullable RevTree treeB, + TreeFilter filter) { + readLock(); + try (DiffFormatter diffFormatter = new DiffFormatter(null)) { + diffFormatter.setRepository(jGitRepo); + diffFormatter.setPathFilter(filter); + return ImmutableList.copyOf(diffFormatter.scan(treeA, treeB)); + } catch (IOException e) { + throw new StorageException("failed to compare two trees: " + treeA + " vs. " + treeB, e); + } finally { + readUnlock(); + } + } + + private static TreeFilter pathPatternFilterOrTreeFilter(@Nullable String pathPattern) { + if (pathPattern == null) { + return TreeFilter.ALL; + } + + final PathPatternFilter pathPatternFilter = PathPatternFilter.of(pathPattern); + return pathPatternFilter.matchesAll() ? TreeFilter.ALL : pathPatternFilter; + } + + private Map> toChangeMap(Repository jGitRepo, List diffEntryList) { + try (ObjectReader reader = jGitRepo.newObjectReader()) { + final Map> changeMap = new LinkedHashMap<>(); + + for (DiffEntry diffEntry : diffEntryList) { + final String oldPath = '/' + diffEntry.getOldPath(); + final String newPath = '/' + diffEntry.getNewPath(); + + switch (diffEntry.getChangeType()) { + case MODIFY: + final EntryType oldEntryType = EntryType.guessFromPath(oldPath); + switch (oldEntryType) { + case JSON: + if (!oldPath.equals(newPath)) { + putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath)); + } + + final JsonNode oldJsonNode = + Jackson.readTree( + reader.open(diffEntry.getOldId().toObjectId()).getBytes()); + final JsonNode newJsonNode = + Jackson.readTree( + reader.open(diffEntry.getNewId().toObjectId()).getBytes()); + final JsonPatch patch = + JsonPatch.generate(oldJsonNode, newJsonNode, ReplaceMode.SAFE); + + if (!patch.isEmpty()) { + putChange(changeMap, newPath, + Change.ofJsonPatch(newPath, Jackson.valueToTree(patch))); + } + break; + case TEXT: + final String oldText = sanitizeText(new String( + reader.open(diffEntry.getOldId().toObjectId()).getBytes(), UTF_8)); + + final String newText = sanitizeText(new String( + reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8)); + + if (!oldPath.equals(newPath)) { + putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath)); + } + + if (!oldText.equals(newText)) { + putChange(changeMap, newPath, + Change.ofTextPatch(newPath, oldText, newText)); + } + break; + default: + throw new Error("unexpected old entry type: " + oldEntryType); + } + break; + case ADD: + final EntryType newEntryType = EntryType.guessFromPath(newPath); + switch (newEntryType) { + case JSON: { + final JsonNode jsonNode = Jackson.readTree( + reader.open(diffEntry.getNewId().toObjectId()).getBytes()); + + putChange(changeMap, newPath, Change.ofJsonUpsert(newPath, jsonNode)); + break; + } + case TEXT: { + final String text = sanitizeText(new String( + reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8)); + + putChange(changeMap, newPath, Change.ofTextUpsert(newPath, text)); + break; + } + default: + throw new Error("unexpected new entry type: " + newEntryType); + } + break; + case DELETE: + putChange(changeMap, oldPath, Change.ofRemoval(oldPath)); + break; + default: + throw new Error(); + } + } + return changeMap; + } catch (Exception e) { + throw new StorageException("failed to convert list of DiffEntry to Changes map", e); + } + } + + private static void putChange(Map> changeMap, String path, Change change) { + final Change oldChange = changeMap.put(path, change); + assert oldChange == null; + } + + /** + * Removes {@code \r} and appends {@code \n} on the last line if it does not end with {@code \n}. + */ + private static String sanitizeText(String text) { + if (text.indexOf('\r') >= 0) { + text = CR.matcher(text).replaceAll(""); + } + if (!text.isEmpty() && !text.endsWith("\n")) { + text += "\n"; + } + return text; + } + + @Override + public CompletableFuture>> previewDiff(Revision baseRevision, + Iterable> changes) { + final ServiceRequestContext ctx = context(); + return CompletableFuture.supplyAsync(() -> { + failFastIfTimedOut(this, logger, ctx, "previewDiff", baseRevision); + return blockingPreviewDiff(baseRevision, changes); + }, repositoryWorker); + } + + private Map> blockingPreviewDiff(Revision baseRevision, Iterable> changes) { + requireNonNull(baseRevision, "baseRevision"); + requireNonNull(changes, "changes"); + baseRevision = normalizeNow(baseRevision); + + readLock(); + final InternalRepository primaryRepo = this.primaryRepo; + final Repository jGitRepo = primaryRepo.jGitRepo(); + try (ObjectReader reader = jGitRepo.newObjectReader(); + RevWalk revWalk = newRevWalk(reader); + DiffFormatter diffFormatter = new DiffFormatter(null)) { + + final ObjectId baseTreeId = toTree(primaryRepo.commitIdDatabase(), revWalk, baseRevision); + final DirCache dirCache = DirCache.newInCore(); + final int numEdits = applyChanges(jGitRepo, baseRevision, baseTreeId, dirCache, changes); + if (numEdits == 0) { + return Collections.emptyMap(); + } + + final CanonicalTreeParser p = new CanonicalTreeParser(); + p.reset(reader, baseTreeId); + diffFormatter.setRepository(jGitRepo); + final List result = diffFormatter.scan(p, new DirCacheIterator(dirCache)); + return toChangeMap(jGitRepo, result); + } catch (IOException e) { + throw new StorageException("failed to perform a dry-run diff", e); + } finally { + readUnlock(); + } + } + + @Override + public CompletableFuture commit(Revision baseRevision, long commitTimeMillis, Author author, + String summary, String detail, Markup markup, + Iterable> changes, boolean directExecution) { + requireNonNull(baseRevision, "baseRevision"); + requireNonNull(author, "author"); + requireNonNull(summary, "summary"); + requireNonNull(detail, "detail"); + requireNonNull(markup, "markup"); + requireNonNull(changes, "changes"); + + final ServiceRequestContext ctx = context(); + return CompletableFuture.supplyAsync(() -> { + failFastIfTimedOut(this, logger, ctx, "commit", baseRevision, author, summary); + return blockingCommit(baseRevision, commitTimeMillis, + author, summary, detail, markup, changes, false, directExecution); + }, repositoryWorker); + } + + private CommitResult blockingCommit( + Revision baseRevision, long commitTimeMillis, Author author, String summary, + String detail, Markup markup, Iterable> changes, boolean allowEmptyCommit, + boolean directExecution) { + + requireNonNull(baseRevision, "baseRevision"); + + final RevisionAndEntries res; + final Iterable> applyingChanges; + writeLock(); + try { + final Revision normBaseRevision = normalizeNow(baseRevision); + final Revision headRevision = this.headRevision; + if (headRevision.major() != normBaseRevision.major()) { + throw new ChangeConflictException( + "invalid baseRevision: " + baseRevision + " (expected: " + headRevision + + " or equivalent)"); + } + + if (directExecution) { + applyingChanges = blockingPreviewDiff(normBaseRevision, changes).values(); + } else { + applyingChanges = changes; + } + res = commit0(primaryRepo, headRevision, headRevision.forward(1), commitTimeMillis, + author, summary, detail, markup, applyingChanges, allowEmptyCommit); + this.headRevision = res.revision; + final InternalRepository secondaryRepo = this.secondaryRepo; + if (secondaryRepo != null) { + // Push the same commit to the secondary repo. + commit0(secondaryRepo, headRevision, headRevision.forward(1), commitTimeMillis, + author, summary, detail, markup, applyingChanges, allowEmptyCommit); + } else if (isCreatingSecondaryRepo) { + laggedCommitsForSecondary.add(new LaggedCommit( + headRevision, commitTimeMillis, author, summary, detail, markup, applyingChanges, + allowEmptyCommit)); + } + } finally { + writeUnLock(); + } + + // Note that the notification is made while no lock is held to avoid the risk of a dead lock. + notifyWatchers(res.revision, res.diffEntries); + return CommitResult.of(res.revision, applyingChanges); + } + + private static RevisionAndEntries commit0(InternalRepository repo, @Nullable Revision prevRevision, + Revision nextRevision, long commitTimeMillis, Author author, + String summary, String detail, Markup markup, + Iterable> changes, boolean allowEmpty) { + + requireNonNull(author, "author"); + requireNonNull(summary, "summary"); + requireNonNull(changes, "changes"); + requireNonNull(detail, "detail"); + requireNonNull(markup, "markup"); + + assert prevRevision == null || prevRevision.major() > 0; + assert nextRevision.major() > 0; + + final Repository jGitRepo = repo.jGitRepo(); + final CommitIdDatabase commitIdDatabase = repo.commitIdDatabase(); + try (ObjectInserter inserter = jGitRepo.newObjectInserter(); + ObjectReader reader = jGitRepo.newObjectReader(); + RevWalk revWalk = newRevWalk(reader)) { + + final ObjectId prevTreeId; + if (prevRevision != null) { + prevTreeId = toTree(commitIdDatabase, revWalk, prevRevision); + } else { + prevTreeId = null; + } + + // The staging area that keeps the entries of the new tree. + // It starts with the entries of the tree at the prevRevision (or with no entries if the + // prevRevision is the initial commit), and then this method will apply the requested changes + // to build the new tree. + final DirCache dirCache = DirCache.newInCore(); + + // Apply the changes and retrieve the list of the affected files. + final int numEdits = applyChanges(jGitRepo, prevRevision, prevTreeId, dirCache, changes); + + // Reject empty commit if necessary. + final List diffEntries; + boolean isEmpty = numEdits == 0; + if (isEmpty || prevTreeId == null) { + // We do not need the diffEntries when creating a new repository which means prevTreeId is null. + diffEntries = ImmutableList.of(); + } else { + // Even if there are edits, the resulting tree might be identical with the previous tree. + final CanonicalTreeParser p = new CanonicalTreeParser(); + p.reset(reader, prevTreeId); + final DiffFormatter diffFormatter = new DiffFormatter(null); + diffFormatter.setRepository(jGitRepo); + diffEntries = diffFormatter.scan(p, new DirCacheIterator(dirCache)); + isEmpty = diffEntries.isEmpty(); + } + + if (!allowEmpty && isEmpty) { + throw new RedundantChangeException( + "changes did not change anything in " + + repo.project().name() + '/' + repo.originalRepoName() + + " at revision " + (prevRevision != null ? prevRevision.major() : 0) + ": " + changes); + } + + // flush the current index to repository and get the result tree object id. + final ObjectId nextTreeId = dirCache.writeTree(inserter); + + // build a commit object + final PersonIdent personIdent = new PersonIdent(author.name(), author.email(), + commitTimeMillis / 1000L * 1000L, 0); + + final CommitBuilder commitBuilder = new CommitBuilder(); + + commitBuilder.setAuthor(personIdent); + commitBuilder.setCommitter(personIdent); + commitBuilder.setTreeId(nextTreeId); + commitBuilder.setEncoding(UTF_8); + + // Write summary, detail and revision to commit's message as JSON format. + commitBuilder.setMessage(CommitUtil.toJsonString(summary, detail, markup, nextRevision)); + + // if the head commit exists, use it as the parent commit. + if (prevRevision != null) { + commitBuilder.setParentId(commitIdDatabase.get(prevRevision)); + } + + final ObjectId nextCommitId = inserter.insert(commitBuilder); + inserter.flush(); + + // tagging the revision object, for history lookup purpose. + commitIdDatabase.put(nextRevision, nextCommitId); + doRefUpdate(jGitRepo, revWalk, R_HEADS_MASTER, nextCommitId); + + return new RevisionAndEntries(nextRevision, diffEntries); + } catch (CentralDogmaException | IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new StorageException("failed to push at '" + + repo.project().name() + '/' + repo.originalRepoName() + '\'', e); + } + } + + private void notifyWatchers(Revision newRevision, List diffEntries) { + for (DiffEntry entry : diffEntries) { + switch (entry.getChangeType()) { + case ADD: + commitWatchers.notify(newRevision, entry.getNewPath()); + break; + case MODIFY: + case DELETE: + commitWatchers.notify(newRevision, entry.getOldPath()); + break; + default: + throw new Error(); + } + } + } + + @Override + public CompletableFuture> history(Revision from, Revision to, String pathPattern, + int maxCommits) { + final ServiceRequestContext ctx = context(); + return CompletableFuture.supplyAsync(() -> { + failFastIfTimedOut(this, logger, ctx, "history", from, to, pathPattern, maxCommits); + return blockingHistory(from, to, pathPattern, maxCommits); + }, repositoryWorker); + } + + private List blockingHistory(Revision from, Revision to, String pathPattern, int maxCommits) { + requireNonNull(pathPattern, "pathPattern"); + requireNonNull(from, "from"); + requireNonNull(to, "to"); + if (maxCommits <= 0) { + throw new IllegalArgumentException("maxCommits: " + maxCommits + " (expected: > 0)"); + } + + maxCommits = Math.min(maxCommits, MAX_MAX_COMMITS); + + final RevisionRange range = normalizeNow(from, to); + final RevisionRange descendingRange = range.toDescending(); + + // At this point, we are sure: from.major >= to.major + readLock(); + final InternalRepository primaryRepo = this.primaryRepo; + try { + return getCommits(primaryRepo, pathPattern, maxCommits, range, descendingRange); + } finally { + readUnlock(); + } + } + + @Override + public CompletableFuture findLatestRevision(Revision lastKnownRevision, String pathPattern) { + requireNonNull(lastKnownRevision, "lastKnownRevision"); + requireNonNull(pathPattern, "pathPattern"); + + final ServiceRequestContext ctx = context(); + return CompletableFuture.supplyAsync(() -> { + failFastIfTimedOut(this, logger, ctx, "findLatestRevision", lastKnownRevision, pathPattern); + return blockingFindLatestRevision(lastKnownRevision, pathPattern); + }, repositoryWorker); + } + + @Nullable + private Revision blockingFindLatestRevision(Revision lastKnownRevision, String pathPattern) { + final RevisionRange range = normalizeNow(lastKnownRevision, Revision.HEAD); + if (range.from().equals(range.to())) { + // Empty range. + return null; + } + + if (range.from().major() == 1) { + // Fast path: no need to compare because we are sure there is nothing at revision 1. + final Map> entries = + blockingFind(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT); + return !entries.isEmpty() ? range.to() : null; + } + + // Slow path: compare the two trees. + final PathPatternFilter filter = PathPatternFilter.of(pathPattern); + // Convert the revisions to Git trees. + final List diffEntries; + readLock(); + final InternalRepository primaryRepo = this.primaryRepo; + try (RevWalk revWalk = newRevWalk(primaryRepo.jGitRepo())) { + final CommitIdDatabase commitIdDatabase = primaryRepo.commitIdDatabase(); + final RevTree treeA = toTree(commitIdDatabase, revWalk, range.from()); + final RevTree treeB = toTree(commitIdDatabase, revWalk, range.to()); + diffEntries = blockingCompareTrees(primaryRepo.jGitRepo(), treeA, treeB); + } finally { + readUnlock(); + } + + // Return the latest revision if the changes between the two trees contain the file. + for (DiffEntry e : diffEntries) { + final String path; + switch (e.getChangeType()) { + case ADD: + path = e.getNewPath(); + break; + case MODIFY: + case DELETE: + path = e.getOldPath(); + break; + default: + throw new Error(); + } + + if (filter.matches(path)) { + return range.to(); + } + } + + return null; + } + + /** + * Compares the two Git trees (with caching). + */ + private List blockingCompareTrees(Repository jGitRepo, RevTree treeA, RevTree treeB) { + if (cache == null) { + return blockingCompareTreesUncached(jGitRepo, treeA, treeB, TreeFilter.ALL); + } + + final CacheableCompareTreesCall key = new CacheableCompareTreesCall(this, treeA, treeB); + CompletableFuture> existingFuture = cache.getIfPresent(key); + if (existingFuture != null) { + final List existingDiffEntries = existingFuture.getNow(null); + if (existingDiffEntries != null) { + // Cached already. + return existingDiffEntries; + } + } + + // Not cached yet. Acquire a lock so that we do not compare the same tree pairs simultaneously. + final List newDiffEntries; + final Lock lock = key.coarseGrainedLock(); + lock.lock(); + try { + existingFuture = cache.getIfPresent(key); + if (existingFuture != null) { + final List existingDiffEntries = existingFuture.getNow(null); + if (existingDiffEntries != null) { + // Other thread already put the entries to the cache before we acquire the lock. + return existingDiffEntries; + } + } + + newDiffEntries = blockingCompareTreesUncached(jGitRepo, treeA, treeB, TreeFilter.ALL); + cache.put(key, newDiffEntries); + } finally { + lock.unlock(); + } + + logger.debug("Cache miss: {}", key); + return newDiffEntries; + } + + @Override + public CompletableFuture watch(Revision lastKnownRevision, String pathPattern) { + requireNonNull(lastKnownRevision, "lastKnownRevision"); + requireNonNull(pathPattern, "pathPattern"); + + final ServiceRequestContext ctx = context(); + final Revision normLastKnownRevision = normalizeNow(lastKnownRevision); + final CompletableFuture future = new CompletableFuture<>(); + CompletableFuture.runAsync(() -> { + failFastIfTimedOut(this, logger, ctx, "watch", lastKnownRevision, pathPattern); + readLock(); + try { + // If lastKnownRevision is outdated already and the recent changes match, + // there's no need to watch. + final Revision latestRevision = blockingFindLatestRevision(normLastKnownRevision, pathPattern); + if (latestRevision != null) { + future.complete(latestRevision); + } else { + commitWatchers.add(normLastKnownRevision, pathPattern, future); + } + } finally { + readUnlock(); + } + }, repositoryWorker).exceptionally(cause -> { + future.completeExceptionally(cause); + return null; + }); + + return future; + } + + private void readLock() { + rwLock.readLock().lock(); + if (closePending.get() != null) { + rwLock.readLock().unlock(); + throw closePending.get().get(); + } + } + + private void readUnlock() { + rwLock.readLock().unlock(); + } + + private void writeLock() { + rwLock.writeLock().lock(); + if (closePending.get() != null) { + writeUnLock(); + throw closePending.get().get(); + } + } + + private void writeUnLock() { + rwLock.writeLock().unlock(); + } + + private Commit currentInitialCommit(InternalRepository primaryRepo) { + final Revision firstRevision = primaryRepo.commitIdDatabase().firstRevision(); + // This method is called when opening an existing repository so firstRevision is not null. + assert firstRevision != null; + final RevisionRange range = new RevisionRange(firstRevision, firstRevision); + return getCommits(primaryRepo, ALL_PATH, 1, range, range).get(0); + } + + private static List> toChanges(Map> entries) { + final Builder> builder = ImmutableList.builder(); + for (Entry entry : entries.values()) { + final EntryType type = entry.type(); + if (type == EntryType.DIRECTORY) { + continue; + } + if (type == EntryType.JSON) { + final JsonNode content = (JsonNode) entry.content(); + builder.add(Change.ofJsonUpsert(entry.path(), content)); + } else { + assert type == EntryType.TEXT; + final String content = (String) entry.content(); + builder.add(Change.ofTextUpsert(entry.path(), content)); + } + } + return builder.build(); + } + + /** + * This repository removes the old commits by creating the secondary repository that commits are pushed + * together with the primary repository and removing the primary repository if the secondary repository + * contains at least the number of {@code minRetentionCommits} and contains the recent + * {@code minRetentionDays} commits. + */ + @Override + public void removeOldCommits(int minRetentionCommits, int minRetentionDays) { + // TODO(minwoox): provide a way to set different minRetentionCommits and minRetentionDays + // in each repository. + if (minRetentionCommits == 0 && minRetentionDays == 0) { + // Not enabled. + return; + } + + final InternalRepository secondaryRepo = this.secondaryRepo; + if (secondaryRepo != null) { + if (exceedsMinRetention(secondaryRepo, minRetentionCommits, minRetentionDays)) { + promoteSecondaryRepo(); + } + } else if (exceedsMinRetention(primaryRepo, minRetentionCommits, minRetentionDays)) { + createSecondaryRepo(); + } + } + + private boolean exceedsMinRetention(InternalRepository repo, + int minRetentionCommits, int minRetentionDays) { + final CommitIdDatabase commitIdDatabase = repo.commitIdDatabase(); + final Revision headRevision = commitIdDatabase.headRevision(); + final Revision firstRevision = commitIdDatabase.firstRevision(); + if (headRevision == null || firstRevision == null) { + return false; + } + if (minRetentionCommits != 0) { + if (headRevision.major() - firstRevision.major() <= minRetentionCommits) { + return false; + } + } + + if (minRetentionDays != 0) { + final Instant creationTime = repo.secondCommitCreationTimeInstant(); + return creationTime != null && + Instant.now().minus(Duration.ofDays(minRetentionDays)).isBefore(creationTime); + } + return true; + } + + private void promoteSecondaryRepo() { + final InternalRepository secondaryRepo = this.secondaryRepo; + if (secondaryRepo != null) { + writeLock(); + try { + logger.info("Promoting the secondary repository in {}/{}.", parent.name(), name); + repoMetadata.setPrimaryRepoDir(secondaryRepo.jGitRepo().getDirectory()); + final InternalRepository primaryRepo = this.primaryRepo; + this.primaryRepo = secondaryRepo; + this.secondaryRepo = null; + repositoryWorker.execute(() -> { + closeInternalRepository(primaryRepo); + final File repoDir = primaryRepo.repoDir(); + try { + if (!deleteDirectory(repoDir)) { + logger.warn("Failed to delete the old primary repository: {}", repoDir); + } + } catch (Throwable t) { + logger.warn("Failed to delete the old primary repository: {}", repoDir); + } + }); + logger.info("Promotion is done for {}/{}. {} is now the primary.", + parent.name(), name, secondaryRepo.repoDir()); + } finally { + writeUnLock(); + } + } + + createSecondaryRepo(); + } + + private void createSecondaryRepo() { + try { + writeLock(); + final Revision headRevision = this.headRevision; + isCreatingSecondaryRepo = true; + writeUnLock(); + + logger.info("Creating the secondary repository in {}/{} with the head revision: {}.", + parent.name(), name, headRevision); + final Map> entries = find(headRevision, ALL_PATH, ImmutableMap.of()).join(); + final List> changes = toChanges(entries); + final File secondaryRepoDir = repoMetadata.secondaryRepoDir(); + final InternalRepository secondaryRepo = + createInternalRepo(parent, name, secondaryRepoDir, headRevision, + creationTimeMillis, author, changes); + writeLock(); + try { + if (laggedCommitsForSecondary.isEmpty()) { + // There were no commits after creating the secondary repo. + assert headRevision == this.headRevision; + this.secondaryRepo = secondaryRepo; + } else { + // We should catch up. + for (LaggedCommit c : laggedCommitsForSecondary) { + commit0(secondaryRepo, c.revision, c.revision.forward(1), c.commitTimeMillis, c.author, + c.summary, c.detail, c.markup, c.changes, c.allowEmpty); + } + laggedCommitsForSecondary.clear(); + isCreatingSecondaryRepo = false; + this.secondaryRepo = secondaryRepo; + } + } finally { + writeUnLock(); + } + logger.info("The secondary repository {} is created in {}/{}.", + secondaryRepoDir.getName(), parent.name(), name, headRevision); + } catch (Throwable t) { + logger.warn("Failed to create the secondary repository", t); + } + } + + private static final class RevisionAndEntries { + final Revision revision; + final List diffEntries; + + RevisionAndEntries(Revision revision, List diffEntries) { + this.revision = revision; + this.diffEntries = diffEntries; + } + } + + private static class LaggedCommit { + + private final Revision revision; + private final long commitTimeMillis; + private final Author author; + private final String summary; + private final String detail; + private final Markup markup; + private final Iterable> changes; + private final boolean allowEmpty; + + LaggedCommit(Revision revision, long commitTimeMillis, Author author, String summary, + String detail, Markup markup, Iterable> changes, + boolean allowEmpty) { + this.revision = revision; + this.commitTimeMillis = commitTimeMillis; + this.author = author; + this.summary = summary; + this.detail = detail; + this.markup = markup; + this.changes = changes; + this.allowEmpty = allowEmpty; + } + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java new file mode 100644 index 000000000..198c91883 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java @@ -0,0 +1,109 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.closeJGitRepo; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.getCommits; +import static com.linecorp.centraldogma.server.storage.repository.Repository.ALL_PATH; + +import java.io.File; +import java.time.Instant; + +import javax.annotation.Nullable; + +import org.eclipse.jgit.lib.Repository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.common.RevisionRange; +import com.linecorp.centraldogma.server.storage.project.Project; + +final class InternalRepository { + + private static final Logger logger = LoggerFactory.getLogger(InternalRepository.class); + + private final Project project; + private final String originalRepoName; // e.g. foo + private final File repoDir; // e.g. foo_0000000000 + private final Repository jGitRepository; + private final CommitIdDatabase commitIdDatabase; + + // Only accessed by the worker in CommitRetentionManagementPlugin. + @Nullable + private Instant secondCommitCreationTimeInstant; + + InternalRepository(Project project, String originalRepoName, File repoDir, + Repository jGitRepository, CommitIdDatabase commitIdDatabase) { + this.project = project; + this.originalRepoName = originalRepoName; + this.repoDir = repoDir; + this.jGitRepository = jGitRepository; + this.commitIdDatabase = commitIdDatabase; + } + + Project project() { + return project; + } + + String originalRepoName() { + return originalRepoName; + } + + File repoDir() { + return repoDir; + } + + Repository jGitRepo() { + return jGitRepository; + } + + CommitIdDatabase commitIdDatabase() { + return commitIdDatabase; + } + + @Nullable + Instant secondCommitCreationTimeInstant() { + if (secondCommitCreationTimeInstant == null) { + final Revision firstRevision = commitIdDatabase.firstRevision(); + if (firstRevision == null) { + return null; + } + final Revision headRevision = commitIdDatabase.headRevision(); + if (headRevision == null) { + return null; + } + if (firstRevision.equals(headRevision)) { + // The second commit is not made yet. + return null; + } + final Revision secondRevision = firstRevision.forward(1); + final RevisionRange range = new RevisionRange(secondRevision, secondRevision); + secondCommitCreationTimeInstant = + Instant.ofEpochMilli(getCommits(this, ALL_PATH, 1, range, range).get(0).when()); + } + return secondCommitCreationTimeInstant; + } + + void close() { + try { + commitIdDatabase.close(); + } catch (Throwable t) { + logger.warn("Failed to close a commitId database:", t); + } + closeJGitRepo(jGitRepository); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java new file mode 100644 index 000000000..621dc9639 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java @@ -0,0 +1,199 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import java.io.EOFException; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import javax.annotation.Nullable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cronutils.utils.VisibleForTesting; + +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.server.storage.StorageException; + +/** + * Simple file-based database that has the suffix of the primary Git repository. + */ +final class RepositoryMetadataDatabase implements AutoCloseable { + + private static final Logger logger = LoggerFactory.getLogger(RepositoryMetadataDatabase.class); + + private static final String INITIAL_PRIMARY_SUFFIX = "0000000000"; + + private static final int PRIMARY_SUFFIX_LEN = INITIAL_PRIMARY_SUFFIX.length(); + + private static final ThreadLocal threadLocalBuffer = + ThreadLocal.withInitial(() -> ByteBuffer.allocateDirect(PRIMARY_SUFFIX_LEN)); + + static File initialPrimaryRepoDir(File rootDir) { + return repoDir(rootDir, INITIAL_PRIMARY_SUFFIX); + } + + private static File repoDir(File rootDir, String suffix) { + return new File(rootDir, rootDir.getName() + '_' + suffix); + } + + private final File rootDir; + private final Path path; + private final FileChannel channel; + @Nullable + private volatile Revision headRevision; + @Nullable + private volatile Revision firstRevision; + + private String primarySuffix; + + RepositoryMetadataDatabase(File rootDir, boolean create) { + this.rootDir = rootDir; + path = new File(rootDir, "metadata.dat").toPath(); + try { + if (create) { + channel = FileChannel.open(path, + StandardOpenOption.CREATE, + StandardOpenOption.READ, + StandardOpenOption.WRITE); + } else { + channel = FileChannel.open(path, + StandardOpenOption.READ, + StandardOpenOption.WRITE); + } + } catch (IOException e) { + throw new StorageException("failed to open a repository database: " + path, e); + } + + if (create) { + writeSuffix(INITIAL_PRIMARY_SUFFIX); + primarySuffix = INITIAL_PRIMARY_SUFFIX; + } else { + boolean success = false; + try { + final long size; + try { + size = channel.size(); + } catch (IOException e) { + throw new StorageException("failed to get the file length: " + path, e); + } + + if (size != PRIMARY_SUFFIX_LEN) { + throw new StorageException("incorrect file length: " + path + " (" + size + " bytes)"); + } + + final ByteBuffer buf = threadLocalBuffer.get(); + buf.clear(); + readTo(buf); + buf.flip(); + primarySuffix = StandardCharsets.UTF_8.decode(buf).toString(); + // To check if the suffix is a correct integer value. + // noinspection ResultOfMethodCallIgnored + Integer.parseInt(primarySuffix); + success = true; + } finally { + if (!success) { + close(); + } + } + } + } + + private void writeSuffix(String suffix) { + final ByteBuffer buf = threadLocalBuffer.get(); + buf.clear(); + buf.put(suffix.getBytes()); + buf.flip(); + + long pos = 0; + try { + do { + pos += channel.write(buf, pos); + } while (buf.hasRemaining()); + channel.force(true); + } catch (IOException e) { + throw new StorageException("failed to update the suffix (" + suffix + + ") of the primary repository: " + path, e); + } + } + + private void readTo(ByteBuffer buf) { + long pos = 0; + try { + do { + final int readBytes = channel.read(buf, pos); + if (readBytes < 0) { + throw new EOFException(); + } + pos += readBytes; + } while (buf.hasRemaining()); + } catch (IOException e) { + throw new StorageException("failed to read the primary repository database: " + path, e); + } + } + + File primaryRepoDir() { + return repoDir(rootDir, primarySuffix); + } + + File secondaryRepoDir() { + return repoDir(rootDir, addOne(primarySuffix)); + } + + void setPrimaryRepoDir(File newRepoDir) { + final String newPrimarySuffix = addOne(primarySuffix); + final File secondary = new File(rootDir, rootDir.getName() + '_' + newPrimarySuffix); + assert newRepoDir.equals(secondaryRepoDir()); + primarySuffix = newPrimarySuffix; + writeSuffix(newPrimarySuffix); + } + + @Override + public void close() { + try { + channel.close(); + } catch (IOException e) { + logger.warn("Failed to close the commit ID database: {}", path, e); + } + } + + void closeAndDelete() { + try { + channel.close(); + } catch (IOException e) { + logger.warn("Failed to close the commit ID database: {}", path, e); + } + if (!path.toFile().delete()) { + logger.warn("Failed to delete the commit ID database: {}", path); + } + } + + @VisibleForTesting + static String addOne(String suffix) { + final int intSuffix = Integer.parseInt(suffix); + final String str = String.valueOf(intSuffix + 1); + if (str.length() < 10) { + return INITIAL_PRIMARY_SUFFIX.substring(0, 10 - str.length()) + str; + } + return str; + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java index 1f4951662..588ddd41e 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java @@ -30,10 +30,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.spotify.futures.CompletableFutures; @@ -82,30 +80,12 @@ public interface Repository { /** * Returns the creation time of this {@link Repository}. */ - default long creationTimeMillis() { - try { - final List history = history(Revision.INIT, Revision.INIT, ALL_PATH, 1).join(); - return history.get(0).when(); - } catch (CompletionException e) { - final Throwable cause = Throwables.getRootCause(e); - Throwables.throwIfUnchecked(cause); - throw new StorageException("failed to retrieve the initial commit", cause); - } - } + long creationTimeMillis(); /** * Returns the author who created this {@link Repository}. */ - default Author author() { - try { - final List history = history(Revision.INIT, Revision.INIT, ALL_PATH, 1).join(); - return history.get(0).author(); - } catch (CompletionException e) { - final Throwable cause = Throwables.getRootCause(e); - Throwables.throwIfUnchecked(cause); - throw new StorageException("failed to retrieve the initial commit", cause); - } - } + Author author(); /** * Returns the {@link CompletableFuture} whose value is the absolute {@link Revision} of the @@ -498,4 +478,10 @@ default CompletableFuture> mergeFiles(Revision revision, Merg return future; } + + /** + * Removes the old commits that are passed more than the {@code minRetentionDays} and are exceeded the + * number of {@code minRetentionCommits}. + */ + void removeOldCommits(int minRetentionCommits, int minRetentionDays); } diff --git a/server/src/main/resources/META-INF/services/com.linecorp.centraldogma.server.plugin.Plugin b/server/src/main/resources/META-INF/services/com.linecorp.centraldogma.server.plugin.Plugin index bb8987036..2cb1e0f21 100644 --- a/server/src/main/resources/META-INF/services/com.linecorp.centraldogma.server.plugin.Plugin +++ b/server/src/main/resources/META-INF/services/com.linecorp.centraldogma.server.plugin.Plugin @@ -1,2 +1,3 @@ com.linecorp.centraldogma.server.internal.mirror.DefaultMirroringServicePlugin com.linecorp.centraldogma.server.internal.storage.PurgeSchedulingServicePlugin +com.linecorp.centraldogma.server.internal.storage.repository.git.CommitRetentionManagementPlugin \ No newline at end of file diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepositoryTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepositoryTest.java index 68c9d2d0e..0e32fffd6 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepositoryTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepositoryTest.java @@ -28,7 +28,6 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; @@ -407,15 +406,9 @@ private Repository newCachingRepo() { } private Repository newCachingRepo(MeterRegistry meterRegistry) { - when(delegateRepo.history(INIT, INIT, Repository.ALL_PATH, 1)).thenReturn(completedFuture( - ImmutableList.of(new Commit(INIT, SYSTEM, "", "", Markup.PLAINTEXT)))); - final Repository cachingRepo = new CachingRepository( delegateRepo, new RepositoryCache("maximumSize=1000", meterRegistry)); - // Verify that CachingRepository calls delegateRepo.history() once to retrieve the initial commit. - verify(delegateRepo, times(1)).history(INIT, INIT, Repository.ALL_PATH, 1); - verifyNoMoreInteractions(delegateRepo); clearInvocations(delegateRepo); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabaseTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabaseTest.java index e5109ecb0..9a7099177 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabaseTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabaseTest.java @@ -21,7 +21,6 @@ import static org.mockito.Mockito.mock; import java.io.File; -import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.StandardOpenOption; @@ -32,6 +31,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.common.Change; @@ -65,11 +66,12 @@ void emptyDatabase() { assertThatThrownBy(() -> db.get(Revision.INIT)).isInstanceOf(IllegalStateException.class); } - @Test - void simpleAccess() { + @ValueSource(ints = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }) + @ParameterizedTest + void simpleAccess(int startRevision) { final int numCommits = 10; - final ObjectId[] expectedCommitIds = new ObjectId[numCommits + 1]; - for (int i = 1; i <= numCommits; i++) { + final ObjectId[] expectedCommitIds = new ObjectId[numCommits + startRevision]; + for (int i = startRevision; i < startRevision + numCommits; i++) { final Revision revision = new Revision(i); final ObjectId commitId = randomCommitId(); expectedCommitIds[i] = commitId; @@ -77,15 +79,20 @@ void simpleAccess() { assertThat(db.headRevision()).isEqualTo(revision); } - for (int i = 1; i <= numCommits; i++) { + for (int i = startRevision; i < startRevision + numCommits; i++) { assertThat(db.get(new Revision(i))).isEqualTo(expectedCommitIds[i]); } assertThatThrownBy(() -> db.get(Revision.HEAD)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("absolute revision"); - assertThatThrownBy(() -> db.get(new Revision(numCommits + 1))) + assertThatThrownBy(() -> db.get(new Revision(numCommits + startRevision))) .isInstanceOf(RevisionNotFoundException.class); + if (startRevision > 1) { + assertThatThrownBy(() -> db.get(new Revision(startRevision - 1))) + .isInstanceOf(RevisionNotFoundException.class); + } + assertThat(db.firstRevision()).isEqualTo(new Revision(startRevision)); } @Test @@ -95,7 +102,7 @@ void truncatedDatabase() throws Exception { // Truncate the database file. try (FileChannel f = FileChannel.open(new File(tempDir, "commit_ids.dat").toPath(), - StandardOpenOption.APPEND)) { + StandardOpenOption.APPEND)) { assertThat(f.size()).isEqualTo(24); f.truncate(23); @@ -106,50 +113,25 @@ void truncatedDatabase() throws Exception { .hasMessageContaining("incorrect file length"); } - @Test - void mismatchingRevision() throws Exception { - db.close(); - - // Append a record with incorrect revision number. - try (FileChannel f = FileChannel.open(new File(tempDir, "commit_ids.dat").toPath(), - StandardOpenOption.APPEND)) { - - final ByteBuffer buf = ByteBuffer.allocate(24); - buf.putInt(42); // Expected to be 1. - randomCommitId().copyRawTo(buf); - buf.flip(); - do { - f.write(buf); - } while (buf.hasRemaining()); - - assertThat(f.size()).isEqualTo(buf.capacity()); - } - - // Reopen the database and see if it fails to resolve the revision 1. - db = new CommitIdDatabase(tempDir); - assertThatThrownBy(() -> db.get(Revision.INIT)) - .isInstanceOf(StorageException.class) - .hasMessageContaining("incorrect revision number"); - } - - @Test - void rebuildingBadDatabase() throws Exception { + @ValueSource(ints = { 1, 3, 5, 7, 9 }) + @ParameterizedTest + void rebuildingBadDatabase(int startRevision) throws Exception { final int numCommits = 10; final File repoDir = tempDir; - final File commitIdDatabaseFile = new File(repoDir, "commit_ids.dat"); - // Create a repository which contains some commits. - GitRepository repo = new GitRepository(mock(Project.class), repoDir, commonPool(), 0, Author.SYSTEM); + GitRepositoryV2 repo = createRepoWithFirstRevision(repoDir, startRevision); Revision headRevision = null; try { - for (int i = 1; i <= numCommits; i++) { - headRevision = repo.commit(new Revision(i), 0, Author.SYSTEM, "", - Change.ofTextUpsert("/" + i + ".txt", "")).join().revision(); + // We already add 2 commits in createRepoWithFirstRevision so let's start with startRevision + 2. + for (int i = startRevision + 2; i < startRevision + numCommits; i++) { + headRevision = addCommit(repo, i); } } finally { repo.internalClose(); } + final File primaryRepoDir = repo.primaryRepo.repoDir(); + final File commitIdDatabaseFile = new File(primaryRepoDir, "commit_ids.dat"); // Wipe out the commit ID database. assertThat(commitIdDatabaseFile).exists(); try (FileChannel ch = FileChannel.open(commitIdDatabaseFile.toPath(), StandardOpenOption.WRITE)) { @@ -157,19 +139,54 @@ void rebuildingBadDatabase() throws Exception { } // Open the repository again to see if the commit ID database is regenerated automatically. - repo = new GitRepository(mock(Project.class), repoDir, commonPool(), null); + repo = GitRepositoryV2.open(mock(Project.class), repoDir, commonPool(), null); try { assertThat(repo.normalizeNow(Revision.HEAD)).isEqualTo(headRevision); - for (int i = 1; i <= numCommits; i++) { + for (int i = startRevision; i < startRevision + numCommits; i++) { assertThat(repo.find(new Revision(i + 1), "/" + i + ".txt").join()).hasSize(1); } } finally { repo.internalClose(); } + assertThat(commitIdDatabaseFile).exists(); assertThat(Files.size(commitIdDatabaseFile.toPath())).isEqualTo((numCommits + 1) * 24L); } + private GitRepositoryV2 createRepoWithFirstRevision(File repoDir, int startRevision) { + final GitRepositoryV2 repo = new GitRepositoryV2(mock(Project.class), repoDir, + commonPool(), + 0, Author.SYSTEM, null); + if (startRevision == 1) { + addCommit(repo, startRevision); + addCommit(repo, startRevision + 1); + return repo; + } + + for (int i = 1; i < startRevision; i++) { + addCommit(repo, i); + } + repo.removeOldCommits(1, 0); + // Now the first revision of secondary repository is startRevision; + final InternalRepository secondaryRepo = repo.secondaryRepo; + assertThat(secondaryRepo.commitIdDatabase().firstRevision().major()).isEqualTo(startRevision); + + addCommit(repo, startRevision); + addCommit(repo, startRevision + 1); + repo.removeOldCommits(1, 0); + // The secondary repo is promoted. + assertThat(repo.primaryRepo).isSameAs(secondaryRepo); + assertThat(repo.primaryRepo.commitIdDatabase().firstRevision().major()).isEqualTo(startRevision); + assertThat(repo.primaryRepo.commitIdDatabase().headRevision().major()).isEqualTo(startRevision + 2); + return repo; + } + + private Revision addCommit(GitRepositoryV2 repo, int i) { + return repo.commit(Revision.HEAD, 0, Author.SYSTEM, "", Change.ofTextUpsert("/" + i + ".txt", "")) + .join() + .revision(); + } + private static ObjectId randomCommitId() { final byte[] commitId = new byte[20]; ThreadLocalRandom.current().nextBytes(commitId); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManagerTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManagerTest.java index 1f99c3473..d4b86f1f9 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManagerTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManagerTest.java @@ -54,8 +54,8 @@ void setUp(@TempDir Path tempDir) { void testCreate() { final GitRepositoryManager gitRepositoryManager = newRepositoryManager(); final Repository repository = gitRepositoryManager.create(TEST_REPO, Author.SYSTEM); - assertThat(repository).isInstanceOf(GitRepository.class); - assertThat(((GitRepository) repository).cache).isNotNull(); + assertThat(repository).isInstanceOf(GitRepositoryV2.class); + assertThat(((GitRepositoryV2) repository).cache).isNotNull(); // Must disallow creating a duplicate. assertThatThrownBy(() -> gitRepositoryManager.create(TEST_REPO, Author.SYSTEM)) @@ -75,8 +75,8 @@ void testOpen() { // Create a new manager so that it loads the repository we created above. gitRepositoryManager = newRepositoryManager(); final Repository repository = gitRepositoryManager.get(TEST_REPO); - assertThat(repository).isInstanceOf(GitRepository.class); - assertThat(((GitRepository) repository).cache).isNotNull(); + assertThat(repository).isInstanceOf(GitRepositoryV2.class); + assertThat(((GitRepositoryV2) repository).cache).isNotNull(); gitRepositoryManager.remove(TEST_REPO); gitRepositoryManager.purgeMarked(); @@ -88,7 +88,7 @@ void testGet() { assertThat(gitRepositoryManager.exists(TEST_REPO)).isFalse(); final Repository repository = gitRepositoryManager.create(TEST_REPO, Author.SYSTEM); - assertThat(repository).isInstanceOf(GitRepository.class); + assertThat(repository).isInstanceOf(GitRepositoryV2.class); assertThat(gitRepositoryManager.get(TEST_REPO)).isSameAs(repository); gitRepositoryManager.remove(TEST_REPO); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java new file mode 100644 index 000000000..ef80783cd --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java @@ -0,0 +1,63 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static java.util.concurrent.ForkJoinPool.commonPool; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +import java.io.File; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Commit; +import com.linecorp.centraldogma.common.Markup; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.server.storage.project.Project; + +class GitRepositoryMigrationTest { + + @TempDir + static File repoDir; + + @Test + void migrate() { + final GitRepository oldRepo = new GitRepository(mock(Project.class), new File(repoDir, "test_repo"), + commonPool(), 0L, Author.SYSTEM); + addCommits(oldRepo); + oldRepo.internalClose(); + final GitRepositoryV2 migrated = GitRepositoryV2.open(mock(Project.class), + new File(repoDir, "test_repo"), commonPool(), + null); + final List commits = migrated.history(Revision.INIT, Revision.HEAD, "/**").join(); + assertThat(commits.get(0).summary()).contains("new repository"); + for (int i = 1; i < 10; i++) { + assertThat(commits.get(i).summary()).isEqualTo("Summary" + i); + } + } + + private static void addCommits(GitRepository oldRepo) { + for (int i = 1; i < 10; i++) { + oldRepo.commit(Revision.HEAD, 0, Author.SYSTEM, + "Summary" + i, "Detail", Markup.PLAINTEXT, + Change.ofTextUpsert("/file_" + i + ".txt", String.valueOf(i))).join(); + } + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryTest.java index 981f71e00..93ea86d75 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryTest.java @@ -24,9 +24,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.fail; import static org.awaitility.Awaitility.await; -import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import java.io.File; import java.util.ArrayList; @@ -47,11 +45,6 @@ import javax.annotation.Nullable; -import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.revwalk.RevWalk; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -75,7 +68,6 @@ import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.common.RevisionNotFoundException; import com.linecorp.centraldogma.internal.Util; -import com.linecorp.centraldogma.server.storage.StorageException; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.repository.Repository; import com.linecorp.centraldogma.testing.internal.TestUtil; @@ -89,7 +81,7 @@ class GitRepositoryTest { @TempDir static File repoDir; - private static GitRepository repo; + private static GitRepositoryV2 repo; /** * Used by {@link GitRepositoryTest#testWatchWithQueryCancellation()}. @@ -99,8 +91,8 @@ class GitRepositoryTest { @BeforeAll static void init() { - repo = new GitRepository(mock(Project.class), new File(repoDir, "test_repo"), - commonPool(), 0L, Author.SYSTEM) { + repo = new GitRepositoryV2(mock(Project.class), new File(repoDir, "test_repo"), + commonPool(), 0L, Author.SYSTEM, null) { /** * Used by {@link GitRepositoryTest#testWatchWithQueryCancellation()}. */ @@ -1266,48 +1258,12 @@ private static void ensureWatcherCleanUp() { await().untilAsserted(() -> assertThat(repo.commitWatchers.watchesMap).isEmpty()); } - @Test - void testDoUpdateRef() throws Exception { - final ObjectId commitId = mock(ObjectId.class); - - // A commit on the mainlane - testDoUpdateRef(Constants.R_TAGS + '1', commitId, false); - testDoUpdateRef(Constants.R_HEADS + Constants.MASTER, commitId, false); - } - - private static void testDoUpdateRef(String ref, ObjectId commitId, boolean tagExists) throws Exception { - final org.eclipse.jgit.lib.Repository jGitRepo = mock(org.eclipse.jgit.lib.Repository.class); - final RevWalk revWalk = mock(RevWalk.class); - final RefUpdate refUpdate = mock(RefUpdate.class); - - lenient().when(jGitRepo.exactRef(ref)).thenReturn(tagExists ? mock(Ref.class) : null); - lenient().when(jGitRepo.updateRef(ref)).thenReturn(refUpdate); - - lenient().when(refUpdate.update(revWalk)).thenReturn(RefUpdate.Result.NEW); - GitRepository.doRefUpdate(jGitRepo, revWalk, ref, commitId); - - when(refUpdate.update(revWalk)).thenReturn(RefUpdate.Result.FAST_FORWARD); - GitRepository.doRefUpdate(jGitRepo, revWalk, ref, commitId); - - when(refUpdate.update(revWalk)).thenReturn(RefUpdate.Result.LOCK_FAILURE); - assertThatThrownBy(() -> GitRepository.doRefUpdate(jGitRepo, revWalk, ref, commitId)) - .isInstanceOf(StorageException.class); - } - - @Test - void testDoUpdateRefOnExistingTag() { - final ObjectId commitId = mock(ObjectId.class); - - assertThatThrownBy(() -> testDoUpdateRef(Constants.R_TAGS + "01/1.0", commitId, true)) - .isInstanceOf(StorageException.class); - } - @Test void operationOnClosedRepository() { final CentralDogmaException expectedException = new CentralDogmaException(); - final GitRepository repo = new GitRepository(mock(Project.class), - new File(repoDir, "close_test_repo"), - commonPool(), 0L, Author.SYSTEM); + final GitRepositoryV2 repo = new GitRepositoryV2(mock(Project.class), + new File(repoDir, "close_test_repo"), + commonPool(), 0L, Author.SYSTEM, null); repo.close(() -> expectedException); assertThatThrownBy(() -> repo.find(INIT, "/**").join()).hasCause(expectedException); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtilTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtilTest.java new file mode 100644 index 000000000..2c4b253c8 --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtilTest.java @@ -0,0 +1,69 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.RefUpdate; +import org.eclipse.jgit.revwalk.RevWalk; +import org.junit.jupiter.api.Test; + +import com.linecorp.centraldogma.server.storage.StorageException; + +final class GitRepositoryUtilTest { + + @Test + void testDoUpdateRef() throws Exception { + final ObjectId commitId = mock(ObjectId.class); + + // A commit on the mainlane + testDoUpdateRef(Constants.R_TAGS + '1', commitId, false); + testDoUpdateRef(Constants.R_HEADS + Constants.MASTER, commitId, false); + } + + private static void testDoUpdateRef(String ref, ObjectId commitId, boolean tagExists) throws Exception { + final org.eclipse.jgit.lib.Repository jGitRepo = mock(org.eclipse.jgit.lib.Repository.class); + final RevWalk revWalk = mock(RevWalk.class); + final RefUpdate refUpdate = mock(RefUpdate.class); + + lenient().when(jGitRepo.exactRef(ref)).thenReturn(tagExists ? mock(Ref.class) : null); + lenient().when(jGitRepo.updateRef(ref)).thenReturn(refUpdate); + + lenient().when(refUpdate.update(revWalk)).thenReturn(RefUpdate.Result.NEW); + GitRepositoryUtil.doRefUpdate(jGitRepo, revWalk, ref, commitId); + + when(refUpdate.update(revWalk)).thenReturn(RefUpdate.Result.FAST_FORWARD); + GitRepositoryUtil.doRefUpdate(jGitRepo, revWalk, ref, commitId); + + when(refUpdate.update(revWalk)).thenReturn(RefUpdate.Result.LOCK_FAILURE); + assertThatThrownBy(() -> GitRepositoryUtil.doRefUpdate(jGitRepo, revWalk, ref, commitId)) + .isInstanceOf(StorageException.class); + } + + @Test + void testDoUpdateRefOnExistingTag() { + final ObjectId commitId = mock(ObjectId.class); + + assertThatThrownBy(() -> testDoUpdateRef(Constants.R_TAGS + "01/1.0", commitId, true)) + .isInstanceOf(StorageException.class); + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java new file mode 100644 index 000000000..1fa020293 --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java @@ -0,0 +1,103 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static java.util.concurrent.ForkJoinPool.commonPool; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.Mockito.mock; + +import java.io.File; +import java.time.Instant; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Markup; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.server.storage.project.Project; + +class GitRepositoryV2PromotionTest { + + @TempDir + static File repoDir; + + @Test + void promote() { + final GitRepositoryV2 repo = new GitRepositoryV2(mock(Project.class), + new File(repoDir, "test_repo"), commonPool(), + 0L, Author.SYSTEM, null); + final InternalRepository primaryRepo = repo.primaryRepo; + final CommitIdDatabase primaryCommitIdDatabase = primaryRepo.commitIdDatabase(); + assertThat(primaryCommitIdDatabase.firstRevision()).isEqualTo(primaryCommitIdDatabase.headRevision()); + addCommits(repo, 2, 11); + assertThat(primaryCommitIdDatabase.headRevision().major()).isEqualTo(11); + + repo.removeOldCommits(10, 0); + // Nothing happened because 10 commits are made so far. + assertThat(repo.secondaryRepo).isNull(); + + // Add one more commit. + addCommits(repo, 12, 12); + repo.removeOldCommits(10, 0); + assertThat(primaryCommitIdDatabase.headRevision().major()).isEqualTo(12); + final InternalRepository secondaryRepo = repo.secondaryRepo; + assertThat(secondaryRepo).isNotNull(); + assertThat(secondaryRepo.commitIdDatabase().firstRevision().major()).isEqualTo(12); + assertThat(secondaryRepo.commitIdDatabase().headRevision().major()).isEqualTo(12); + assertThat(secondaryRepo.secondCommitCreationTimeInstant()).isNull(); + + // Add ten more commit. + addCommits(repo, 13, 22); + // No changes. + repo.removeOldCommits(10, 0); + assertThat(primaryCommitIdDatabase.headRevision().major()).isEqualTo(22); + assertThat(secondaryRepo).isEqualTo(repo.secondaryRepo); + assertThat(repo.primaryRepo).isNotSameAs(repo.secondaryRepo); + assertThat(secondaryRepo.commitIdDatabase().firstRevision().major()).isEqualTo(12); + assertThat(secondaryRepo.commitIdDatabase().headRevision().major()).isEqualTo(22); + assertThat(secondaryRepo.secondCommitCreationTimeInstant()).isEqualTo(Instant.ofEpochMilli(13 * 1000)); + + // Add one more commit. + addCommits(repo, 23, 23); + repo.removeOldCommits(10, 0); + // The secondary repo is promoted to the primary repo. + assertThat(repo.primaryRepo).isSameAs(secondaryRepo); + assertThat(repo.primaryRepo).isNotSameAs(primaryRepo); + // The old primary repo is gone now. + await().until(() -> !primaryRepo.repoDir().exists()); + + final CommitIdDatabase newPrimaryDatabase = repo.primaryRepo.commitIdDatabase(); + assertThat(newPrimaryDatabase.firstRevision().major()).isEqualTo(12); + assertThat(newPrimaryDatabase.headRevision().major()).isEqualTo(23); + + // A new secondary repo is created. + assertThat(repo.secondaryRepo).isNotSameAs(secondaryRepo); + assertThat(repo.secondaryRepo.commitIdDatabase().firstRevision().major()).isEqualTo(23); + assertThat(repo.secondaryRepo.commitIdDatabase().headRevision().major()).isEqualTo(23); + assertThat(repo.secondaryRepo.secondCommitCreationTimeInstant()).isNull(); + } + + private static void addCommits(GitRepositoryV2 repo, int start, int end) { + for (int i = start; i <= end; i++) { + repo.commit(Revision.HEAD, i * 1000, Author.SYSTEM, + "Summary" + i, "Detail", Markup.PLAINTEXT, + Change.ofTextUpsert("/file_" + i + ".txt", String.valueOf(i))).join(); + } + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabaseTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabaseTest.java new file mode 100644 index 000000000..2e4badf62 --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabaseTest.java @@ -0,0 +1,84 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.linecorp.centraldogma.server.internal.storage.repository.git.RepositoryMetadataDatabase.addOne; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; + +import javax.annotation.Nullable; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class RepositoryMetadataDatabaseTest { + + @TempDir + File tempDir; + + @Nullable + private RepositoryMetadataDatabase db; + + @BeforeEach + void setUp() { + db = new RepositoryMetadataDatabase(tempDir, true); + } + + @AfterEach + void tearDown() { + if (db != null) { + db.close(); + } + } + + @Test + void secondaryDirIsGreaterThanPrimaryByOne() { + assertThat(db.primaryRepoDir().getName()).isEqualTo(tempDir.getName() + "_0000000000"); + assertThat(db.secondaryRepoDir().getName()).isEqualTo(tempDir.getName() + "_0000000001"); + + db.setPrimaryRepoDir(new File(tempDir, tempDir.getName() + "_0000000001")); + assertThat(db.primaryRepoDir().getName()).isEqualTo(tempDir.getName() + "_0000000001"); + assertThat(db.secondaryRepoDir().getName()).isEqualTo(tempDir.getName() + "_0000000002"); + } + + @Test + void nextPrimaryRepoShouldBeGreaterByOne() { + assertThat(db.primaryRepoDir().getName()).isEqualTo(tempDir.getName() + "_0000000000"); + assertThat(db.secondaryRepoDir().getName()).isEqualTo(tempDir.getName() + "_0000000001"); + + assertThatThrownBy(() -> db.setPrimaryRepoDir(new File(tempDir, tempDir.getName() + "_1111111111"))) + .isInstanceOf(AssertionError.class); + } + + @Test + void addOneToSuffixTest() { + String suffix = "0000000000"; + assertThat(addOne(suffix)).isEqualTo("0000000001"); + + suffix = "0000000009"; + assertThat(addOne(suffix)).isEqualTo("0000000010"); + + suffix = "0000000099"; + assertThat(addOne(suffix)).isEqualTo("0000000100"); + + suffix = "1111111111"; + assertThat(addOne(suffix)).isEqualTo("1111111112"); + } +} diff --git a/site/src/sphinx/setup-configuration.rst b/site/src/sphinx/setup-configuration.rst index ddbaaba7c..e096837a3 100644 --- a/site/src/sphinx/setup-configuration.rst +++ b/site/src/sphinx/setup-configuration.rst @@ -47,6 +47,11 @@ defaults: "numMirroringThreads": null, "maxNumFilesPerMirror": null, "maxNumBytesPerMirror": null, + "commitRetentionConfig": { + "minRetentionCommits" : 2000, + "minRetentionDays": 14, + "schedule": "0 0 * * * ?" + }, "writeQuotaPerRepository": { "requestQuota" : 5, "timeWindowSeconds": 1 @@ -217,6 +222,29 @@ Core properties - a time windows in seconds. +- ``commitRetention`` + + - the configuration for retaining commits in a repository. The repository retains at least the number of + ``minRetentionCommits`` when more than ``minRetentionCommits`` are made. The repository also retains + commits at least ``minRetentionDays``. If both conditions are not satisfied, the commits are not removed. + For example, when ``minRetentionCommits`` is set to 2000 and ``minRetentionDays`` is set to 14, + the commits that are created more than 2000 are not removed until 14 days have passed. Set 0 to retain + all commits. + + - ``minRetentionCommits`` (integer) + + - a minimum number of commits that a repository should retain. The number should be greater than or + equal to 1000. Specify 0 to disable it. + + - ``minRetentionDays`` (integer) + + - a minimum number of days of a commit that a repository should retain. Specify 0 to disable it. + + - ``schedule`` (string) + + - a `Quartz cron expression `_ + that schedules the job of removing old commits. + - ``accessLogFormat`` (string) - the format to be used for writing an access log. ``common`` and ``combined`` are pre-defined for NCSA From 9d5e250acc04d36504588a183080de062f591194 Mon Sep 17 00:00:00 2001 From: minwoox Date: Mon, 9 Aug 2021 10:36:49 +0900 Subject: [PATCH 02/14] WIP --- .../centraldogma/common/Revision.java | 8 + .../common/util/NonNullByDefault.java | 2 +- .../server/internal/api/ContentServiceV1.java | 4 +- .../storage/repository/CacheableCall.java | 2 +- .../repository/git/CommitIdDatabase.java | 15 +- .../repository/git/GitRepositoryUtil.java | 595 +-------- .../repository/git/GitRepositoryV2.java | 950 ++++---------- .../repository/git/InternalRepository.java | 1112 ++++++++++++++++- .../repository/git/GitRepositoryTest.java | 17 +- .../git/GitRepositoryV2DiffTest.java | 95 ++ .../git/GitRepositoryV2FindTest.java | 118 ++ .../git/GitRepositoryV2HistoryTest.java | 115 ++ .../git/GitRepositoryV2PromotionTest.java | 4 +- ...lTest.java => InternalRepositoryTest.java} | 9 +- 14 files changed, 1691 insertions(+), 1355 deletions(-) create mode 100644 server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2DiffTest.java create mode 100644 server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2FindTest.java create mode 100644 server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2HistoryTest.java rename server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/{GitRepositoryUtilTest.java => InternalRepositoryTest.java} (88%) diff --git a/common/src/main/java/com/linecorp/centraldogma/common/Revision.java b/common/src/main/java/com/linecorp/centraldogma/common/Revision.java index b7d9faf8d..4d293e691 100644 --- a/common/src/main/java/com/linecorp/centraldogma/common/Revision.java +++ b/common/src/main/java/com/linecorp/centraldogma/common/Revision.java @@ -119,6 +119,14 @@ public int major() { return major; } + /** + * Tells whether the {@link #major()} of this {@link Revision} is lower than the {@code other} + * {@link Revision}. + */ + public boolean isLowerThan(Revision other) { + return major < other.major(); + } + /** * Returns {@code 0}. * diff --git a/common/src/main/java/com/linecorp/centraldogma/common/util/NonNullByDefault.java b/common/src/main/java/com/linecorp/centraldogma/common/util/NonNullByDefault.java index 4838fd32a..bfa801cd9 100644 --- a/common/src/main/java/com/linecorp/centraldogma/common/util/NonNullByDefault.java +++ b/common/src/main/java/com/linecorp/centraldogma/common/util/NonNullByDefault.java @@ -34,6 +34,6 @@ @Documented @Target(ElementType.PACKAGE) @Retention(RetentionPolicy.RUNTIME) -@TypeQualifierDefault({ ElementType.METHOD, ElementType.PARAMETER, ElementType.FIELD, ElementType.TYPE_USE }) +@TypeQualifierDefault({ ElementType.METHOD, ElementType.PARAMETER, ElementType.FIELD }) public @interface NonNullByDefault { } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java index 3e1e8a658..6accadfc2 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java @@ -58,7 +58,6 @@ import com.linecorp.centraldogma.common.MergeQuery; import com.linecorp.centraldogma.common.Query; import com.linecorp.centraldogma.common.Revision; -import com.linecorp.centraldogma.common.RevisionRange; import com.linecorp.centraldogma.internal.api.v1.ChangeDto; import com.linecorp.centraldogma.internal.api.v1.CommitMessageDto; import com.linecorp.centraldogma.internal.api.v1.EntryDto; @@ -329,10 +328,9 @@ public CompletableFuture listCommits(@Param String revision, toRevision = to != null ? new Revision(to) : fromRevision; } - final RevisionRange range = repository.normalizeNow(fromRevision, toRevision).toDescending(); final int maxCommits0 = firstNonNull(maxCommits, Repository.DEFAULT_MAX_COMMITS); return repository - .history(range.from(), range.to(), normalizePath(path), maxCommits0) + .history(fromRevision, toRevision, normalizePath(path), maxCommits0) .thenApply(commits -> { final boolean toList = to != null || isNullOrEmpty(revision) || diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CacheableCall.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CacheableCall.java index 2e7c9aa1a..ae4ff7110 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CacheableCall.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CacheableCall.java @@ -38,7 +38,7 @@ public abstract class CacheableCall { } } - final Repository repo; + private final Repository repo; protected CacheableCall(Repository repo) { this.repo = requireNonNull(repo, "repo"); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java index 8c659f49d..0206fc89f 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java @@ -119,7 +119,8 @@ private CommitIdDatabase(File rootDir, boolean fsync) { final int numRecords = (int) (size / RECORD_LEN); if (numRecords > 0) { - firstRevision = retrieveFirstRevision(); + final Revision firstRevision = retrieveFirstRevision(); + this.firstRevision = firstRevision; headRevision = new Revision(numRecords + firstRevision.major() - 1); } else { firstRevision = null; @@ -169,11 +170,17 @@ Revision firstRevision() { } ObjectId get(Revision revision) { - final Revision headRevision = this.headRevision; - checkState(headRevision != null, "initial commit not available yet: %s", path); checkArgument(!revision.isRelative(), "revision: %s (expected: an absolute revision)", revision); + final Revision headRevision = this.headRevision; + final Revision firstRevision = this.firstRevision; + if (headRevision == null || firstRevision == null) { + throw new IllegalStateException("initial commit not available yet: " + path); + } + if (!(firstRevision.major() <= revision.major() && revision.major() <= headRevision.major())) { - throw new RevisionNotFoundException(revision); + throw new RevisionNotFoundException( + "revision: " + revision + + " (expected: " + firstRevision.major() + " <= revision <= " + headRevision.major() + ")"); } final ByteBuffer buf = threadLocalBuffer.get(); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtil.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtil.java index b6ac6033b..668f2c3af 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtil.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtil.java @@ -15,546 +15,20 @@ */ package com.linecorp.centraldogma.server.internal.storage.repository.git; -import static com.google.common.base.MoreObjects.firstNonNull; -import static com.linecorp.centraldogma.server.storage.repository.Repository.ALL_PATH; -import static com.linecorp.centraldogma.server.storage.repository.Repository.MAX_MAX_COMMITS; -import static java.nio.charset.StandardCharsets.UTF_8; - -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Field; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Objects; -import java.util.StringJoiner; -import java.util.regex.Pattern; -import java.util.stream.Stream; - -import javax.annotation.Nullable; - -import org.eclipse.jgit.dircache.DirCache; -import org.eclipse.jgit.dircache.DirCacheBuilder; -import org.eclipse.jgit.dircache.DirCacheEditor; -import org.eclipse.jgit.dircache.DirCacheEditor.DeletePath; -import org.eclipse.jgit.dircache.DirCacheEditor.DeleteTree; -import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit; -import org.eclipse.jgit.dircache.DirCacheEntry; -import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.FileMode; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdOwnerMap; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.lib.RefUpdate.Result; import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevTree; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.revwalk.TreeRevFilter; -import org.eclipse.jgit.revwalk.filter.RevFilter; -import org.eclipse.jgit.treewalk.filter.AndTreeFilter; -import org.eclipse.jgit.treewalk.filter.TreeFilter; +import org.eclipse.jgit.storage.file.FileBasedConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.google.common.annotations.VisibleForTesting; - -import com.linecorp.centraldogma.common.Author; -import com.linecorp.centraldogma.common.CentralDogmaException; -import com.linecorp.centraldogma.common.Change; -import com.linecorp.centraldogma.common.ChangeConflictException; -import com.linecorp.centraldogma.common.Commit; -import com.linecorp.centraldogma.common.Revision; -import com.linecorp.centraldogma.common.RevisionRange; -import com.linecorp.centraldogma.internal.Jackson; -import com.linecorp.centraldogma.internal.Util; -import com.linecorp.centraldogma.internal.jsonpatch.JsonPatch; -import com.linecorp.centraldogma.server.storage.StorageException; - -import difflib.DiffUtils; -import difflib.Patch; - final class GitRepositoryUtil { private static final Logger logger = LoggerFactory.getLogger(GitRepositoryUtil.class); - private static final byte[] EMPTY_BYTE = new byte[0]; - - private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL); - - private static final Field revWalkObjectsField; - - static { - Field field = null; - try { - field = RevWalk.class.getDeclaredField("objects"); - if (field.getType() != ObjectIdOwnerMap.class) { - throw new IllegalStateException( - RevWalk.class.getSimpleName() + ".objects is not an " + - ObjectIdOwnerMap.class.getSimpleName() + '.'); - } - field.setAccessible(true); - } catch (NoSuchFieldException e) { - throw new IllegalStateException( - RevWalk.class.getSimpleName() + ".objects does not exist."); - } - - revWalkObjectsField = field; - } - - static RevWalk newRevWalk(Repository jGitRepository) { - final RevWalk revWalk = new RevWalk(jGitRepository); - // Disable rewriteParents because otherwise `RevWalk` will load every commit into memory. - revWalk.setRewriteParents(false); - return revWalk; - } - - static RevWalk newRevWalk(ObjectReader reader) { - final RevWalk revWalk = new RevWalk(reader); - // Disable rewriteParents because otherwise `RevWalk` will load every commit into memory. - revWalk.setRewriteParents(false); - return revWalk; - } - - static RevTree toTree(CommitIdDatabase commitIdDatabase, RevWalk revWalk, Revision revision) { - final ObjectId commitId = commitIdDatabase.get(revision); - try { - return revWalk.parseCommit(commitId).getTree(); - } catch (IOException e) { - throw new StorageException("failed to parse a commit: " + commitId, e); - } - } - - static int applyChanges(Repository jGitRepo, @Nullable Revision baseRevision, - @Nullable ObjectId baseTreeId, DirCache dirCache, Iterable> changes) { - int numEdits = 0; - - try (ObjectInserter inserter = jGitRepo.newObjectInserter(); - ObjectReader reader = jGitRepo.newObjectReader()) { - - if (baseTreeId != null) { - // the DirCacheBuilder is to used for doing update operations on the given DirCache object - final DirCacheBuilder builder = dirCache.builder(); - - // Add the tree object indicated by the prevRevision to the temporary DirCache object. - builder.addTree(EMPTY_BYTE, 0, reader, baseTreeId); - builder.finish(); - } - - // loop over the specified changes. - for (Change change : changes) { - final String changePath = change.path().substring(1); // Strip the leading '/'. - final DirCacheEntry oldEntry = dirCache.getEntry(changePath); - final byte[] oldContent = oldEntry != null ? reader.open(oldEntry.getObjectId()).getBytes() - : null; - - switch (change.type()) { - case UPSERT_JSON: { - final JsonNode oldJsonNode = oldContent != null ? Jackson.readTree(oldContent) : null; - final JsonNode newJsonNode = firstNonNull((JsonNode) change.content(), - JsonNodeFactory.instance.nullNode()); - - // Upsert only when the contents are really different. - if (!Objects.equals(newJsonNode, oldJsonNode)) { - applyPathEdit(dirCache, new InsertJson(changePath, inserter, newJsonNode)); - numEdits++; - } - break; - } - case UPSERT_TEXT: { - final String sanitizedOldText; - if (oldContent != null) { - sanitizedOldText = sanitizeText(new String(oldContent, UTF_8)); - } else { - sanitizedOldText = null; - } - - final String sanitizedNewText = sanitizeText(change.contentAsText()); - - // Upsert only when the contents are really different. - if (!sanitizedNewText.equals(sanitizedOldText)) { - applyPathEdit(dirCache, new InsertText(changePath, inserter, sanitizedNewText)); - numEdits++; - } - break; - } - case REMOVE: - if (oldEntry != null) { - applyPathEdit(dirCache, new DeletePath(changePath)); - numEdits++; - break; - } - - // The path might be a directory. - if (applyDirectoryEdits(dirCache, changePath, null, change)) { - numEdits++; - } else { - // Was not a directory either; conflict. - reportNonExistentEntry(change); - break; - } - break; - case RENAME: { - final String newPath = - ((String) change.content()).substring(1); // Strip the leading '/'. - - if (dirCache.getEntry(newPath) != null) { - throw new ChangeConflictException("a file exists at the target path: " + change); - } - - if (oldEntry != null) { - if (changePath.equals(newPath)) { - // Redundant rename request - old path and new path are same. - break; - } - - final DirCacheEditor editor = dirCache.editor(); - editor.add(new DeletePath(changePath)); - editor.add(new CopyOldEntry(newPath, oldEntry)); - editor.finish(); - numEdits++; - break; - } - - // The path might be a directory. - if (applyDirectoryEdits(dirCache, changePath, newPath, change)) { - numEdits++; - } else { - // Was not a directory either; conflict. - reportNonExistentEntry(change); - } - break; - } - case APPLY_JSON_PATCH: { - final JsonNode oldJsonNode; - if (oldContent != null) { - oldJsonNode = Jackson.readTree(oldContent); - } else { - oldJsonNode = Jackson.nullNode; - } - - final JsonNode newJsonNode; - try { - newJsonNode = JsonPatch.fromJson((JsonNode) change.content()).apply(oldJsonNode); - } catch (Exception e) { - throw new ChangeConflictException("failed to apply JSON patch: " + change, e); - } - - // Apply only when the contents are really different. - if (!newJsonNode.equals(oldJsonNode)) { - applyPathEdit(dirCache, new InsertJson(changePath, inserter, newJsonNode)); - numEdits++; - } - break; - } - case APPLY_TEXT_PATCH: - final Patch patch = DiffUtils.parseUnifiedDiff( - Util.stringToLines(sanitizeText((String) change.content()))); - - final String sanitizedOldText; - final List sanitizedOldTextLines; - if (oldContent != null) { - sanitizedOldText = sanitizeText(new String(oldContent, UTF_8)); - sanitizedOldTextLines = Util.stringToLines(sanitizedOldText); - } else { - sanitizedOldText = null; - sanitizedOldTextLines = Collections.emptyList(); - } - - final String newText; - try { - final List newTextLines = DiffUtils.patch(sanitizedOldTextLines, patch); - if (newTextLines.isEmpty()) { - newText = ""; - } else { - final StringJoiner joiner = new StringJoiner("\n", "", "\n"); - for (String line : newTextLines) { - joiner.add(line); - } - newText = joiner.toString(); - } - } catch (Exception e) { - throw new ChangeConflictException("failed to apply text patch: " + change, e); - } - - // Apply only when the contents are really different. - if (!newText.equals(sanitizedOldText)) { - applyPathEdit(dirCache, new InsertText(changePath, inserter, newText)); - numEdits++; - } - break; - } - } - } catch (CentralDogmaException | IllegalArgumentException e) { - throw e; - } catch (Exception e) { - throw new StorageException("failed to apply changes on revision " + baseRevision, e); - } - return numEdits; - } - - private static void applyPathEdit(DirCache dirCache, PathEdit edit) { - final DirCacheEditor e = dirCache.editor(); - e.add(edit); - e.finish(); - } - - /** - * Removes {@code \r} and appends {@code \n} on the last line if it does not end with {@code \n}. - */ - private static String sanitizeText(String text) { - if (text.indexOf('\r') >= 0) { - text = CR.matcher(text).replaceAll(""); - } - if (!text.isEmpty() && !text.endsWith("\n")) { - text += "\n"; - } - return text; - } - - /** - * Applies recursive directory edits. - * - * @param oldDir the path to the directory to make a recursive change - * @param newDir the path to the renamed directory, or {@code null} to remove the directory. - * - * @return {@code true} if any edits were made to {@code dirCache}, {@code false} otherwise - */ - private static boolean applyDirectoryEdits(DirCache dirCache, - String oldDir, @Nullable String newDir, Change change) { - - if (!oldDir.endsWith("/")) { - oldDir += '/'; - } - if (newDir != null && !newDir.endsWith("/")) { - newDir += '/'; - } - - final byte[] rawOldDir = Constants.encode(oldDir); - final byte[] rawNewDir = newDir != null ? Constants.encode(newDir) : null; - final int numEntries = dirCache.getEntryCount(); - DirCacheEditor editor = null; - - loop: - for (int i = 0; i < numEntries; i++) { - final DirCacheEntry e = dirCache.getEntry(i); - final byte[] rawPath = e.getRawPath(); - - // Ensure that there are no entries under the newDir; we have a conflict otherwise. - if (rawNewDir != null) { - boolean conflict = true; - if (rawPath.length > rawNewDir.length) { - // Check if there is a file whose path starts with 'newDir'. - for (int j = 0; j < rawNewDir.length; j++) { - if (rawNewDir[j] != rawPath[j]) { - conflict = false; - break; - } - } - } else if (rawPath.length == rawNewDir.length - 1) { - // Check if there is a file whose path is exactly same with newDir without trailing '/'. - for (int j = 0; j < rawNewDir.length - 1; j++) { - if (rawNewDir[j] != rawPath[j]) { - conflict = false; - break; - } - } - } else { - conflict = false; - } - - if (conflict) { - throw new ChangeConflictException("target directory exists already: " + change); - } - } - - // Skip the entries that do not belong to the oldDir. - if (rawPath.length <= rawOldDir.length) { - continue; - } - for (int j = 0; j < rawOldDir.length; j++) { - if (rawOldDir[j] != rawPath[j]) { - continue loop; - } - } - - // Do not create an editor until we find an entry to rename/remove. - // We can tell if there was any matching entries or not from the nullness of editor later. - if (editor == null) { - editor = dirCache.editor(); - editor.add(new DeleteTree(oldDir)); - if (newDir == null) { - // Recursive removal - break; - } - } - - assert newDir != null; // We should get here only when it's a recursive rename. - - final String oldPath = e.getPathString(); - final String newPath = newDir + oldPath.substring(oldDir.length()); - editor.add(new CopyOldEntry(newPath, e)); - } - - if (editor != null) { - editor.finish(); - return true; - } else { - return false; - } - } - - private static void reportNonExistentEntry(Change change) { - throw new ChangeConflictException("non-existent file/directory: " + change); - } - - @VisibleForTesting - static void doRefUpdate(Repository jGitRepo, RevWalk revWalk, - String ref, ObjectId commitId) throws IOException { - if (ref.startsWith(Constants.R_TAGS)) { - final Ref oldRef = jGitRepo.exactRef(ref); - if (oldRef != null) { - throw new StorageException("tag ref exists already: " + ref); - } - } - - final RefUpdate refUpdate = jGitRepo.updateRef(ref); - refUpdate.setNewObjectId(commitId); - - final Result res = refUpdate.update(revWalk); - switch (res) { - case NEW: - case FAST_FORWARD: - // Expected - break; - default: - throw new StorageException("unexpected refUpdate state: " + res); - } - } - - static List getCommits(InternalRepository repo, String pathPattern, int maxCommits, - RevisionRange range, RevisionRange descendingRange) { - try (RevWalk revWalk = newRevWalk(repo.jGitRepo())) { - final ObjectIdOwnerMap revWalkInternalMap = - (ObjectIdOwnerMap) revWalkObjectsField.get(revWalk); - - final CommitIdDatabase commitIdDatabase = repo.commitIdDatabase(); - final ObjectId fromCommitId = commitIdDatabase.get(descendingRange.from()); - final ObjectId toCommitId = commitIdDatabase.get(descendingRange.to()); - - revWalk.markStart(revWalk.parseCommit(fromCommitId)); - revWalk.setRetainBody(false); - - // Instead of relying on RevWalk to filter the commits, - // we let RevWalk yield all commits so we can: - // - Have more control on when iteration should be stopped. - // (A single Iterator.next() doesn't take long.) - // - Clean up the internal map as early as possible. - final RevFilter filter = new TreeRevFilter(revWalk, AndTreeFilter.create( - TreeFilter.ANY_DIFF, PathPatternFilter.of(pathPattern))); - - // Search up to 1000 commits when maxCommits <= 100. - // Search up to (maxCommits * 10) commits when 100 < maxCommits <= 1000. - final int maxNumProcessedCommits = Math.max(maxCommits * 10, MAX_MAX_COMMITS); - - final List commitList = new ArrayList<>(); - int numProcessedCommits = 0; - for (RevCommit revCommit : revWalk) { - numProcessedCommits++; - - if (filter.include(revWalk, revCommit)) { - revWalk.parseBody(revCommit); - commitList.add(toCommit(revCommit)); - revCommit.disposeBody(); - } - - if (revCommit.getId().equals(toCommitId) || - commitList.size() >= maxCommits || - // Prevent from iterating for too long. - numProcessedCommits >= maxNumProcessedCommits) { - break; - } - - // Clear the internal lookup table of RevWalk to reduce the memory usage. - // This is safe because we have linear history and traverse in one direction. - if (numProcessedCommits % 16 == 0) { - revWalkInternalMap.clear(); - } - } - - // Include the initial empty commit only when the caller specified - // the initial revision (1) in the range and the pathPattern contains '/**'. - if (commitList.size() < maxCommits && - descendingRange.to().major() == 1 && - pathPattern.contains(ALL_PATH)) { - try (RevWalk tmpRevWalk = newRevWalk(repo.jGitRepo())) { - final RevCommit lastRevCommit = tmpRevWalk.parseCommit(toCommitId); - commitList.add(toCommit(lastRevCommit)); - } - } - - if (!descendingRange.equals(range)) { // from and to is swapped so reverse the list. - Collections.reverse(commitList); - } - - return commitList; - } catch (CentralDogmaException e) { - throw e; - } catch (Exception e) { - throw new StorageException( - "failed to retrieve the history: " + repo.repoDir() + - " (" + pathPattern + ", " + range.from() + ".." + range.to() + ')', e); - } - } - - private static Commit toCommit(RevCommit revCommit) { - final Author author; - final PersonIdent committerIdent = revCommit.getCommitterIdent(); - final long when; - if (committerIdent == null) { - author = Author.UNKNOWN; - when = 0; - } else { - author = new Author(committerIdent.getName(), committerIdent.getEmailAddress()); - when = committerIdent.getWhen().getTime(); - } - - try { - return CommitUtil.newCommit(author, when, revCommit.getFullMessage()); - } catch (Exception e) { - throw new StorageException("failed to create a Commit", e); - } - } - - static boolean deleteDirectory(File dir) throws IOException { - try (Stream walk = Files.walk(dir.toPath())) { - return walk.sorted(Comparator.reverseOrder()) - .map(Path::toFile) - .map(File::delete) - // Return false if it fails to delete a file. - .reduce(true, (a, b) -> a && b); - } - } - - static boolean isEmpty(File dir) throws IOException { - if (!dir.isDirectory()) { - return false; - } - if (!dir.exists()) { - return true; - } - try (Stream entries = Files.list(dir.toPath())) { - return entries.findFirst().isPresent(); + static boolean exists(Repository repository) { + if (repository.getConfig() instanceof FileBasedConfig) { + return ((FileBasedConfig) repository.getConfig()).getFile().exists(); } + return repository.getDirectory().exists(); } static void closeJGitRepo(Repository repository) { @@ -565,64 +39,5 @@ static void closeJGitRepo(Repository repository) { } } - // PathEdit implementations which is used when applying changes. - - private static final class InsertText extends PathEdit { - private final ObjectInserter inserter; - private final String text; - - InsertText(String entryPath, ObjectInserter inserter, String text) { - super(entryPath); - this.inserter = inserter; - this.text = text; - } - - @Override - public void apply(DirCacheEntry ent) { - try { - ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, text.getBytes(UTF_8))); - ent.setFileMode(FileMode.REGULAR_FILE); - } catch (IOException e) { - throw new StorageException("failed to create a new text blob", e); - } - } - } - - private static final class InsertJson extends PathEdit { - private final ObjectInserter inserter; - private final JsonNode jsonNode; - - InsertJson(String entryPath, ObjectInserter inserter, JsonNode jsonNode) { - super(entryPath); - this.inserter = inserter; - this.jsonNode = jsonNode; - } - - @Override - public void apply(DirCacheEntry ent) { - try { - ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, Jackson.writeValueAsBytes(jsonNode))); - ent.setFileMode(FileMode.REGULAR_FILE); - } catch (IOException e) { - throw new StorageException("failed to create a new JSON blob", e); - } - } - } - - private static final class CopyOldEntry extends PathEdit { - private final DirCacheEntry oldEntry; - - CopyOldEntry(String entryPath, DirCacheEntry oldEntry) { - super(entryPath); - this.oldEntry = oldEntry; - } - - @Override - public void apply(DirCacheEntry ent) { - ent.setFileMode(oldEntry.getFileMode()); - ent.setObjectId(oldEntry.getObjectId()); - } - } - private GitRepositoryUtil() {} } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java index 7377588f9..1347e9030 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java @@ -16,72 +16,39 @@ package com.linecorp.centraldogma.server.internal.storage.repository.git; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.context; import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.failFastIfTimedOut; -import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.applyChanges; import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.closeJGitRepo; -import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.deleteDirectory; -import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.doRefUpdate; -import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.getCommits; -import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.isEmpty; -import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.newRevWalk; -import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.toTree; -import static java.nio.charset.StandardCharsets.UTF_8; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.InternalRepository.buildJGitRepo; import static java.util.Objects.requireNonNull; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_COMMIT_SECTION; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DIFF_SECTION; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_ALGORITHM; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_FILEMODE; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_GPGSIGN; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_HIDEDOTFILES; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_RENAMES; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REPO_FORMAT_VERSION; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_SYMLINKS; import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedHashMap; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Supplier; -import java.util.regex.Pattern; +import java.util.stream.Stream; import javax.annotation.Nullable; import org.eclipse.jgit.diff.DiffEntry; -import org.eclipse.jgit.diff.DiffFormatter; -import org.eclipse.jgit.dircache.DirCache; -import org.eclipse.jgit.dircache.DirCacheIterator; -import org.eclipse.jgit.lib.CommitBuilder; import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.CoreConfig.HideDotFiles; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.RepositoryBuilder; -import org.eclipse.jgit.lib.StoredConfig; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevTree; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.storage.file.FileBasedConfig; -import org.eclipse.jgit.treewalk.CanonicalTreeParser; -import org.eclipse.jgit.treewalk.TreeWalk; -import org.eclipse.jgit.treewalk.filter.TreeFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -100,17 +67,13 @@ import com.linecorp.centraldogma.common.Entry; import com.linecorp.centraldogma.common.EntryType; import com.linecorp.centraldogma.common.Markup; -import com.linecorp.centraldogma.common.RedundantChangeException; import com.linecorp.centraldogma.common.RepositoryNotFoundException; import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.common.RevisionNotFoundException; import com.linecorp.centraldogma.common.RevisionRange; -import com.linecorp.centraldogma.internal.Jackson; -import com.linecorp.centraldogma.internal.Util; -import com.linecorp.centraldogma.internal.jsonpatch.JsonPatch; -import com.linecorp.centraldogma.internal.jsonpatch.ReplaceMode; import com.linecorp.centraldogma.server.command.CommitResult; import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache; +import com.linecorp.centraldogma.server.internal.storage.repository.git.InternalRepository.RevisionAndEntries; import com.linecorp.centraldogma.server.storage.StorageException; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.repository.FindOption; @@ -126,7 +89,6 @@ class GitRepositoryV2 implements com.linecorp.centraldogma.server.storage.reposi static final String R_HEADS_MASTER = Constants.R_HEADS + Constants.MASTER; private static final byte[] EMPTY_BYTE = new byte[0]; - private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL); /** * Opens an existing Git-backed repository. @@ -201,8 +163,8 @@ static GitRepositoryV2 open(Project parent, File repositoryDir, repoMetadata = new RepositoryMetadataDatabase(repositoryDir, true); final File primaryRepoDir = repoMetadata.primaryRepoDir(); try { - primaryRepo = createInternalRepo(parent, name, primaryRepoDir, Revision.INIT, - creationTimeMillis, author, ImmutableList.of()); + primaryRepo = InternalRepository.of(parent, name, primaryRepoDir, Revision.INIT, + creationTimeMillis, author, ImmutableList.of()); } catch (Throwable t) { repoMetadata.close(); throw t; @@ -212,73 +174,15 @@ static GitRepositoryV2 open(Project parent, File repositoryDir, this.author = author; } - @VisibleForTesting - static InternalRepository createInternalRepo(Project parent, String originalRepoName, File repoDir, - Revision nextRevision, long commitTimeMillis, Author author, - Iterable> changes) { - boolean success = false; - InternalRepository internalRepo = null; - try { - createEmptyJGitRepo(repoDir); - - // Re-open the repository with the updated settings and format version. - final Repository jGitRepo = new RepositoryBuilder().setGitDir(repoDir) - .build(); - internalRepo = new InternalRepository(parent, originalRepoName, repoDir, - jGitRepo, new CommitIdDatabase(jGitRepo)); - - // Initialize the master branch. - final RefUpdate head = jGitRepo.updateRef(Constants.HEAD); - head.disableRefLog(); - head.link(R_HEADS_MASTER); - - // Insert the initial commit into the master branch. - commit0(internalRepo, null, nextRevision, commitTimeMillis, author, - "Create a new repository", "", Markup.PLAINTEXT, changes, true); - success = true; - return internalRepo; - } catch (IOException e) { - throw new StorageException("failed to create a repository at: " + repoDir, e); - } finally { - if (!success) { - closeInternalRepository(internalRepo); - // Failed to create a repository. Remove any cruft so that it is not loaded on the next run. - deleteCruft(repoDir); - } + private static boolean isEmpty(File dir) throws IOException { + if (!dir.isDirectory()) { + return false; } - } - - private static void createEmptyJGitRepo(File repositoryDir) throws IOException { - try (org.eclipse.jgit.lib.Repository initRepository = buildjGitRepo(repositoryDir)) { - initRepository.create(true); - - final StoredConfig config = initRepository.getConfig(); - // Update the repository settings to upgrade to format version 1 and reftree. - config.setInt(CONFIG_CORE_SECTION, null, CONFIG_KEY_REPO_FORMAT_VERSION, 1); - - // Disable hidden files, symlinks and file modes we do not use. - config.setEnum(CONFIG_CORE_SECTION, null, CONFIG_KEY_HIDEDOTFILES, HideDotFiles.FALSE); - config.setBoolean(CONFIG_CORE_SECTION, null, CONFIG_KEY_SYMLINKS, false); - config.setBoolean(CONFIG_CORE_SECTION, null, CONFIG_KEY_FILEMODE, false); - - // Disable GPG signing. - config.setBoolean(CONFIG_COMMIT_SECTION, null, CONFIG_KEY_GPGSIGN, false); - - // Set the diff algorithm. - config.setString(CONFIG_DIFF_SECTION, null, CONFIG_KEY_ALGORITHM, "histogram"); - - // Disable rename detection which we do not use. - config.setBoolean(CONFIG_DIFF_SECTION, null, CONFIG_KEY_RENAMES, false); - - config.save(); + if (!dir.exists()) { + return true; } - } - - private static Repository buildjGitRepo(File repositoryDir) { - try { - return new RepositoryBuilder().setGitDir(repositoryDir).setBare().build(); - } catch (IOException e) { - throw new StorageException("failed to create a repository at: " + repositoryDir, e); + try (Stream entries = Files.list(dir.toPath())) { + return entries.findFirst().isPresent(); } } @@ -295,41 +199,22 @@ private GitRepositoryV2(Project parent, File repoDir, Executor repositoryWorker, repoMetadata = new RepositoryMetadataDatabase(repoDir, false); } catch (Throwable t) { // The metadata doesn't exist so check if the repository exists in the form of the old version. - final Repository oldRepo = buildjGitRepo(repoDir); - final boolean oldRepoExist = exist(oldRepo); - closeJGitRepo(oldRepo); - if (!oldRepoExist) { - throw new RepositoryNotFoundException(repoDir.toString()); - } - final File primaryRepoDir = RepositoryMetadataDatabase.initialPrimaryRepoDir(repoDir); - final File tmpRepoDir = new File(repoDir.getParentFile(), UUID.randomUUID().toString()); - logger.debug("Migrating {} to {} using temp repository: {}", repoDir, primaryRepoDir, tmpRepoDir); - if (!repoDir.renameTo(tmpRepoDir)) { - throw new StorageException("failed to migrate a repository at: " + repoDir + - ", to the tmp dir: " + tmpRepoDir); - } - primaryRepoDir.mkdirs(); - if (!tmpRepoDir.renameTo(primaryRepoDir)) { - // Rename it back. - tmpRepoDir.renameTo(repoDir); - throw new StorageException("failed to migrate a repository at: " + tmpRepoDir + - ", to: " + primaryRepoDir); - } - assert !tmpRepoDir.exists(); - logger.debug("Migrating {} is done.", repoDir); + checkRepositoryExists(repoDir); + migrateToV2(repoDir); repoMetadata = new RepositoryMetadataDatabase(repoDir, true); } this.repoMetadata = repoMetadata; try { final InternalRepository primaryRepo = - openInternalRepository(repoMetadata.primaryRepoDir(), true); + InternalRepository.open(parent, name, repoMetadata.primaryRepoDir(), true); assert primaryRepo != null; this.primaryRepo = primaryRepo; + headRevision = primaryRepo.headRevision(); final Commit initialCommit = currentInitialCommit(primaryRepo); creationTimeMillis = initialCommit.when(); author = initialCommit.author(); - secondaryRepo = openInternalRepository(repoMetadata.secondaryRepoDir(), false); + secondaryRepo = InternalRepository.open(parent, name, repoMetadata.secondaryRepoDir(), false); } catch (Throwable t) { repoMetadata.close(); closeInternalRepository(primaryRepo); @@ -338,51 +223,45 @@ private GitRepositoryV2(Project parent, File repoDir, Executor repositoryWorker, } } - @Nullable - private InternalRepository openInternalRepository(File repoDir, boolean primary) { - final Repository jGitRepo = buildjGitRepo(repoDir); - CommitIdDatabase commitIdDatabase = null; - try { - if (!exist(jGitRepo)) { - if (primary) { - throw new RepositoryNotFoundException(repoDir.toString()); - } else { - return null; - } - } - checkGitRepositoryFormat(jGitRepo); - final Revision headRevision = uncachedHeadRevision(jGitRepo); - commitIdDatabase = new CommitIdDatabase(jGitRepo); - if (!headRevision.equals(commitIdDatabase.headRevision())) { - commitIdDatabase.rebuild(jGitRepo); - assert headRevision.equals(commitIdDatabase.headRevision()); - } - if (primary) { - this.headRevision = headRevision; - } - return new InternalRepository(parent, name, repoDir, jGitRepo, commitIdDatabase); - } catch (Throwable t) { - closeJGitRepo(jGitRepo); - if (commitIdDatabase != null) { - commitIdDatabase.close(); - } - throw t; + private static void checkRepositoryExists(File repoDir) { + final Repository oldRepo = buildJGitRepo(repoDir); + final boolean oldRepoExist = GitRepositoryUtil.exists(oldRepo); + closeJGitRepo(oldRepo); + if (!oldRepoExist) { + throw new RepositoryNotFoundException(repoDir.toString()); } } - private static void checkGitRepositoryFormat(Repository repository) { - final int formatVersion = repository.getConfig().getInt( - CONFIG_CORE_SECTION, null, CONFIG_KEY_REPO_FORMAT_VERSION, 0); - if (formatVersion != 1) { - throw new StorageException("unsupported repository format version: " + formatVersion); - } - } + private static void migrateToV2(File repoDir) { + // When: + // - repoDir: /foo + // - primaryRepoDir: /foo/foo_0000000000 + // - tmpRepoDir: /bar + // + // Migration steps will be: + // - /foo becomes /bar + // - /foo/foo_0000000000 directory is created. + // - /bar becomes /foo/foo_0000000000 - private static boolean exist(Repository repository) { - if (repository.getConfig() instanceof FileBasedConfig) { - return ((FileBasedConfig) repository.getConfig()).getFile().exists(); + final File primaryRepoDir = RepositoryMetadataDatabase.initialPrimaryRepoDir(repoDir); + final File tmpRepoDir = new File(repoDir.getParentFile(), UUID.randomUUID().toString()); + logger.debug("Migrating {} to {} using temp repository: {}", repoDir, primaryRepoDir, tmpRepoDir); + if (!repoDir.renameTo(tmpRepoDir)) { + throw new StorageException("failed to migrate a repository at: " + repoDir + + ", to the tmp dir: " + tmpRepoDir); + } + if (!primaryRepoDir.mkdirs()) { + throw new StorageException("failed to create " + primaryRepoDir + " while migrating to V2."); + } + if (!tmpRepoDir.renameTo(primaryRepoDir)) { + // Rename it back. + //noinspection ResultOfMethodCallIgnored + tmpRepoDir.renameTo(repoDir); + throw new StorageException("failed to migrate a repository at: " + tmpRepoDir + + ", to: " + primaryRepoDir); } - return repository.getDirectory().exists(); + checkState(!tmpRepoDir.exists(), "%s is not renamed.", tmpRepoDir); + logger.debug("Migrating {} is done.", repoDir); } @VisibleForTesting @@ -422,33 +301,6 @@ private static void closeInternalRepository(@Nullable InternalRepository interna internalRepo.close(); } - private static void deleteCruft(File repoDir) { - try { - Util.deleteFileTree(repoDir); - } catch (IOException e) { - logger.error("Failed to delete a half-created repository at: {}", repoDir, e); - } - } - - /** - * Returns the current revision. - */ - private Revision uncachedHeadRevision(Repository jGitRepo) { - try (RevWalk revWalk = newRevWalk(jGitRepo)) { - final ObjectId headRevisionId = jGitRepo.resolve(R_HEADS_MASTER); - if (headRevisionId != null) { - final RevCommit revCommit = revWalk.parseCommit(headRevisionId); - return CommitUtil.extractRevision(revCommit.getFullMessage()); - } - } catch (CentralDogmaException e) { - throw e; - } catch (Exception e) { - throw new StorageException("failed to get the current revision", e); - } - - throw new StorageException("failed to determine the HEAD: " + parent.name() + '/' + name); - } - @Override public Project parent() { return parent; @@ -471,12 +323,21 @@ public Author author() { @Override public Revision normalizeNow(Revision revision) { - return normalizeNow(revision, headRevision.major()); + return normalizeNow(revision, true); } - private Revision normalizeNow(Revision revision, int headMajor) { - requireNonNull(revision, "revision"); + private Revision normalizeNow(Revision revision, boolean checkFirstRevision) { + return normalizeNow(revision, headRevision.major(), checkFirstRevision); + } + + @Override + public RevisionRange normalizeNow(Revision from, Revision to) { + final int headMajor = headRevision.major(); + return new RevisionRange(normalizeNow(from, headMajor, true), normalizeNow(to, headMajor, true)); + } + private Revision normalizeNow(Revision revision, int headMajor, boolean checkFirstRevision) { + requireNonNull(revision, "revision"); int major = revision.major(); if (major >= 0) { @@ -492,11 +353,17 @@ private Revision normalizeNow(Revision revision, int headMajor) { } } - final Revision firstRevision = primaryRepo.commitIdDatabase().firstRevision(); - assert firstRevision != null; - final int firstMajor = firstRevision.major(); - if (major < firstMajor) { - major = firstMajor; + if (checkFirstRevision) { + final int firstRevisionMajor = primaryRepo.firstRevision().major(); + if (major < firstRevisionMajor) { + if (major == 1) { + // We silently update the major to the first revision when it's Revision.INIT. + major = firstRevisionMajor; + } else { + throw new RevisionNotFoundException( + "revision: " + revision + " (expected: >= " + firstRevisionMajor + ")"); + } + } } // Create a new instance only when necessary. @@ -507,118 +374,31 @@ private Revision normalizeNow(Revision revision, int headMajor) { } } - @Override - public RevisionRange normalizeNow(Revision from, Revision to) { - final int headMajor = headRevision.major(); - return new RevisionRange(normalizeNow(from, headMajor), normalizeNow(to, headMajor)); - } - @Override public CompletableFuture>> find( Revision revision, String pathPattern, Map, ?> options) { - final ServiceRequestContext ctx = context(); - return CompletableFuture.supplyAsync(() -> { - failFastIfTimedOut(this, logger, ctx, "find", revision, pathPattern, options); - return blockingFind(revision, pathPattern, options); - }, repositoryWorker); - } - - private Map> blockingFind( - Revision revision, String pathPattern, Map, ?> options) { - requireNonNull(pathPattern, "pathPattern"); requireNonNull(revision, "revision"); requireNonNull(options, "options"); - - final Revision normRevision = normalizeNow(revision); - final boolean fetchContent = FindOption.FETCH_CONTENT.get(options); - final int maxEntries = FindOption.MAX_ENTRIES.get(options); - - readLock(); - final InternalRepository primaryRepo = this.primaryRepo; - try (ObjectReader reader = primaryRepo.jGitRepo().newObjectReader(); - TreeWalk treeWalk = new TreeWalk(reader); - RevWalk revWalk = newRevWalk(reader)) { - - // Query on a non-exist revision will return empty result. - final Revision headRevision = this.headRevision; - if (normRevision.compareTo(headRevision) > 0) { - return Collections.emptyMap(); - } - - if ("/".equals(pathPattern)) { - return Collections.singletonMap(pathPattern, Entry.ofDirectory(normRevision, "/")); - } - - final Map> result = new LinkedHashMap<>(); - final ObjectId commitId = primaryRepo.commitIdDatabase().get(normRevision); - final RevCommit revCommit = revWalk.parseCommit(commitId); - final PathPatternFilter filter = PathPatternFilter.of(pathPattern); - - final RevTree revTree = revCommit.getTree(); - treeWalk.addTree(revTree.getId()); - while (treeWalk.next() && result.size() < maxEntries) { - final boolean matches = filter.matches(treeWalk); - final String path = '/' + treeWalk.getPathString(); - - // Recurse into a directory if necessary. - if (treeWalk.isSubtree()) { - if (matches) { - // Add the directory itself to the result set if its path matches the pattern. - result.put(path, Entry.ofDirectory(normRevision, path)); - } - - treeWalk.enterSubtree(); - continue; - } - - if (!matches) { - continue; + final ServiceRequestContext ctx = context(); + return CompletableFuture.supplyAsync(() -> { + failFastIfTimedOut(this, logger, ctx, "find", revision, pathPattern, options); + readLock(); + try { + final Revision normalizedRevision = normalizeNow(revision); + // Query on a non-exist revision will return empty result. + final Revision headRevision = this.headRevision; + if (normalizedRevision.compareTo(headRevision) > 0) { + return Collections.emptyMap(); } - - // Build an entry as requested. - final Entry entry; - final EntryType entryType = EntryType.guessFromPath(path); - if (fetchContent) { - final byte[] content = reader.open(treeWalk.getObjectId(0)).getBytes(); - switch (entryType) { - case JSON: - final JsonNode jsonNode = Jackson.readTree(content); - entry = Entry.ofJson(normRevision, path, jsonNode); - break; - case TEXT: - final String strVal = sanitizeText(new String(content, UTF_8)); - entry = Entry.ofText(normRevision, path, strVal); - break; - default: - throw new Error("unexpected entry type: " + entryType); - } - } else { - switch (entryType) { - case JSON: - entry = Entry.ofJson(normRevision, path, Jackson.nullNode); - break; - case TEXT: - entry = Entry.ofText(normRevision, path, ""); - break; - default: - throw new Error("unexpected entry type: " + entryType); - } + if ("/".equals(pathPattern)) { + return Collections.singletonMap(pathPattern, Entry.ofDirectory(normalizedRevision, "/")); } - - result.put(path, entry); + return primaryRepo.find(normalizedRevision, pathPattern, options); + } finally { + readUnlock(); } - - return Util.unsafeCast(result); - } catch (CentralDogmaException | IllegalArgumentException e) { - throw e; - } catch (Exception e) { - throw new StorageException( - "failed to get data from '" + parent.name() + '/' + name + "' at " + pathPattern + - " for " + revision, e); - } finally { - readUnlock(); - } + }, repositoryWorker); } /** @@ -632,204 +412,42 @@ private Map> blockingFind( */ @Override public CompletableFuture>> diff(Revision from, Revision to, String pathPattern) { + requireNonNull(from, "from"); + requireNonNull(to, "to"); + requireNonNull(pathPattern, "pathPattern"); final ServiceRequestContext ctx = context(); return CompletableFuture.supplyAsync(() -> { - requireNonNull(from, "from"); - requireNonNull(to, "to"); - requireNonNull(pathPattern, "pathPattern"); - failFastIfTimedOut(this, logger, ctx, "diff", from, to, pathPattern); - - final RevisionRange range = normalizeNow(from, to).toAscending(); readLock(); - final InternalRepository primaryRepo = this.primaryRepo; - final Repository jGitRepo = primaryRepo.jGitRepo(); - try (RevWalk rw = newRevWalk(jGitRepo)) { - final CommitIdDatabase commitIdDatabase = primaryRepo.commitIdDatabase(); - final RevTree treeA = rw.parseTree(commitIdDatabase.get(range.from())); - final RevTree treeB = rw.parseTree(commitIdDatabase.get(range.to())); - - // Compare the two Git trees. - // Note that we do not cache here because CachingRepository caches the final result already. - return toChangeMap(jGitRepo, - blockingCompareTreesUncached(jGitRepo, treeA, treeB, - pathPatternFilterOrTreeFilter(pathPattern))); - } catch (StorageException e) { - throw e; - } catch (Exception e) { - throw new StorageException("failed to parse two trees: range=" + range, e); + try { + final RevisionRange range = normalizeNow(from, to).toAscending(); + if (range.from().equals(range.to())) { + // Empty range. + return ImmutableMap.of(); + } + return primaryRepo.diff(range, pathPattern); } finally { readUnlock(); } }, repositoryWorker); } - private List blockingCompareTreesUncached(Repository jGitRepo, - @Nullable RevTree treeA, @Nullable RevTree treeB, - TreeFilter filter) { - readLock(); - try (DiffFormatter diffFormatter = new DiffFormatter(null)) { - diffFormatter.setRepository(jGitRepo); - diffFormatter.setPathFilter(filter); - return ImmutableList.copyOf(diffFormatter.scan(treeA, treeB)); - } catch (IOException e) { - throw new StorageException("failed to compare two trees: " + treeA + " vs. " + treeB, e); - } finally { - readUnlock(); - } - } - - private static TreeFilter pathPatternFilterOrTreeFilter(@Nullable String pathPattern) { - if (pathPattern == null) { - return TreeFilter.ALL; - } - - final PathPatternFilter pathPatternFilter = PathPatternFilter.of(pathPattern); - return pathPatternFilter.matchesAll() ? TreeFilter.ALL : pathPatternFilter; - } - - private Map> toChangeMap(Repository jGitRepo, List diffEntryList) { - try (ObjectReader reader = jGitRepo.newObjectReader()) { - final Map> changeMap = new LinkedHashMap<>(); - - for (DiffEntry diffEntry : diffEntryList) { - final String oldPath = '/' + diffEntry.getOldPath(); - final String newPath = '/' + diffEntry.getNewPath(); - - switch (diffEntry.getChangeType()) { - case MODIFY: - final EntryType oldEntryType = EntryType.guessFromPath(oldPath); - switch (oldEntryType) { - case JSON: - if (!oldPath.equals(newPath)) { - putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath)); - } - - final JsonNode oldJsonNode = - Jackson.readTree( - reader.open(diffEntry.getOldId().toObjectId()).getBytes()); - final JsonNode newJsonNode = - Jackson.readTree( - reader.open(diffEntry.getNewId().toObjectId()).getBytes()); - final JsonPatch patch = - JsonPatch.generate(oldJsonNode, newJsonNode, ReplaceMode.SAFE); - - if (!patch.isEmpty()) { - putChange(changeMap, newPath, - Change.ofJsonPatch(newPath, Jackson.valueToTree(patch))); - } - break; - case TEXT: - final String oldText = sanitizeText(new String( - reader.open(diffEntry.getOldId().toObjectId()).getBytes(), UTF_8)); - - final String newText = sanitizeText(new String( - reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8)); - - if (!oldPath.equals(newPath)) { - putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath)); - } - - if (!oldText.equals(newText)) { - putChange(changeMap, newPath, - Change.ofTextPatch(newPath, oldText, newText)); - } - break; - default: - throw new Error("unexpected old entry type: " + oldEntryType); - } - break; - case ADD: - final EntryType newEntryType = EntryType.guessFromPath(newPath); - switch (newEntryType) { - case JSON: { - final JsonNode jsonNode = Jackson.readTree( - reader.open(diffEntry.getNewId().toObjectId()).getBytes()); - - putChange(changeMap, newPath, Change.ofJsonUpsert(newPath, jsonNode)); - break; - } - case TEXT: { - final String text = sanitizeText(new String( - reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8)); - - putChange(changeMap, newPath, Change.ofTextUpsert(newPath, text)); - break; - } - default: - throw new Error("unexpected new entry type: " + newEntryType); - } - break; - case DELETE: - putChange(changeMap, oldPath, Change.ofRemoval(oldPath)); - break; - default: - throw new Error(); - } - } - return changeMap; - } catch (Exception e) { - throw new StorageException("failed to convert list of DiffEntry to Changes map", e); - } - } - - private static void putChange(Map> changeMap, String path, Change change) { - final Change oldChange = changeMap.put(path, change); - assert oldChange == null; - } - - /** - * Removes {@code \r} and appends {@code \n} on the last line if it does not end with {@code \n}. - */ - private static String sanitizeText(String text) { - if (text.indexOf('\r') >= 0) { - text = CR.matcher(text).replaceAll(""); - } - if (!text.isEmpty() && !text.endsWith("\n")) { - text += "\n"; - } - return text; - } - @Override public CompletableFuture>> previewDiff(Revision baseRevision, Iterable> changes) { + requireNonNull(baseRevision, "baseRevision"); + requireNonNull(changes, "changes"); final ServiceRequestContext ctx = context(); return CompletableFuture.supplyAsync(() -> { failFastIfTimedOut(this, logger, ctx, "previewDiff", baseRevision); - return blockingPreviewDiff(baseRevision, changes); - }, repositoryWorker); - } - - private Map> blockingPreviewDiff(Revision baseRevision, Iterable> changes) { - requireNonNull(baseRevision, "baseRevision"); - requireNonNull(changes, "changes"); - baseRevision = normalizeNow(baseRevision); - - readLock(); - final InternalRepository primaryRepo = this.primaryRepo; - final Repository jGitRepo = primaryRepo.jGitRepo(); - try (ObjectReader reader = jGitRepo.newObjectReader(); - RevWalk revWalk = newRevWalk(reader); - DiffFormatter diffFormatter = new DiffFormatter(null)) { - - final ObjectId baseTreeId = toTree(primaryRepo.commitIdDatabase(), revWalk, baseRevision); - final DirCache dirCache = DirCache.newInCore(); - final int numEdits = applyChanges(jGitRepo, baseRevision, baseTreeId, dirCache, changes); - if (numEdits == 0) { - return Collections.emptyMap(); + readLock(); + try { + final Revision normalizedRevision = normalizeNow(baseRevision); + return primaryRepo.previewDiff(normalizedRevision, changes); + } finally { + readUnlock(); } - - final CanonicalTreeParser p = new CanonicalTreeParser(); - p.reset(reader, baseTreeId); - diffFormatter.setRepository(jGitRepo); - final List result = diffFormatter.scan(p, new DirCacheIterator(dirCache)); - return toChangeMap(jGitRepo, result); - } catch (IOException e) { - throw new StorageException("failed to perform a dry-run diff", e); - } finally { - readUnlock(); - } + }, repositoryWorker); } @Override @@ -847,46 +465,43 @@ public CompletableFuture commit(Revision baseRevision, long commit return CompletableFuture.supplyAsync(() -> { failFastIfTimedOut(this, logger, ctx, "commit", baseRevision, author, summary); return blockingCommit(baseRevision, commitTimeMillis, - author, summary, detail, markup, changes, false, directExecution); + author, summary, detail, markup, changes, directExecution); }, repositoryWorker); } private CommitResult blockingCommit( Revision baseRevision, long commitTimeMillis, Author author, String summary, - String detail, Markup markup, Iterable> changes, boolean allowEmptyCommit, - boolean directExecution) { - - requireNonNull(baseRevision, "baseRevision"); - + String detail, Markup markup, Iterable> changes, boolean directExecution) { final RevisionAndEntries res; final Iterable> applyingChanges; writeLock(); try { - final Revision normBaseRevision = normalizeNow(baseRevision); + final Revision normalizedRevision = normalizeNow(baseRevision); final Revision headRevision = this.headRevision; - if (headRevision.major() != normBaseRevision.major()) { + if (headRevision.major() != normalizedRevision.major()) { throw new ChangeConflictException( "invalid baseRevision: " + baseRevision + " (expected: " + headRevision + " or equivalent)"); } if (directExecution) { - applyingChanges = blockingPreviewDiff(normBaseRevision, changes).values(); + applyingChanges = primaryRepo.previewDiff(normalizedRevision, changes).values(); } else { applyingChanges = changes; } - res = commit0(primaryRepo, headRevision, headRevision.forward(1), commitTimeMillis, - author, summary, detail, markup, applyingChanges, allowEmptyCommit); + res = primaryRepo.commit(headRevision, headRevision.forward(1), commitTimeMillis, + author, summary, detail, markup, applyingChanges, false); this.headRevision = res.revision; - final InternalRepository secondaryRepo = this.secondaryRepo; - if (secondaryRepo != null) { - // Push the same commit to the secondary repo. - commit0(secondaryRepo, headRevision, headRevision.forward(1), commitTimeMillis, - author, summary, detail, markup, applyingChanges, allowEmptyCommit); - } else if (isCreatingSecondaryRepo) { - laggedCommitsForSecondary.add(new LaggedCommit( - headRevision, commitTimeMillis, author, summary, detail, markup, applyingChanges, - allowEmptyCommit)); + if (isCreatingSecondaryRepo) { + laggedCommitsForSecondary.add(new LaggedCommit(headRevision, commitTimeMillis, author, summary, + detail, markup, applyingChanges, false)); + } else { + final InternalRepository secondaryRepo = this.secondaryRepo; + if (secondaryRepo != null) { + // Push the same commit to the secondary repo. + secondaryRepo.commit(headRevision, headRevision.forward(1), commitTimeMillis, + author, summary, detail, markup, applyingChanges, false); + } } } finally { writeUnLock(); @@ -897,103 +512,6 @@ private CommitResult blockingCommit( return CommitResult.of(res.revision, applyingChanges); } - private static RevisionAndEntries commit0(InternalRepository repo, @Nullable Revision prevRevision, - Revision nextRevision, long commitTimeMillis, Author author, - String summary, String detail, Markup markup, - Iterable> changes, boolean allowEmpty) { - - requireNonNull(author, "author"); - requireNonNull(summary, "summary"); - requireNonNull(changes, "changes"); - requireNonNull(detail, "detail"); - requireNonNull(markup, "markup"); - - assert prevRevision == null || prevRevision.major() > 0; - assert nextRevision.major() > 0; - - final Repository jGitRepo = repo.jGitRepo(); - final CommitIdDatabase commitIdDatabase = repo.commitIdDatabase(); - try (ObjectInserter inserter = jGitRepo.newObjectInserter(); - ObjectReader reader = jGitRepo.newObjectReader(); - RevWalk revWalk = newRevWalk(reader)) { - - final ObjectId prevTreeId; - if (prevRevision != null) { - prevTreeId = toTree(commitIdDatabase, revWalk, prevRevision); - } else { - prevTreeId = null; - } - - // The staging area that keeps the entries of the new tree. - // It starts with the entries of the tree at the prevRevision (or with no entries if the - // prevRevision is the initial commit), and then this method will apply the requested changes - // to build the new tree. - final DirCache dirCache = DirCache.newInCore(); - - // Apply the changes and retrieve the list of the affected files. - final int numEdits = applyChanges(jGitRepo, prevRevision, prevTreeId, dirCache, changes); - - // Reject empty commit if necessary. - final List diffEntries; - boolean isEmpty = numEdits == 0; - if (isEmpty || prevTreeId == null) { - // We do not need the diffEntries when creating a new repository which means prevTreeId is null. - diffEntries = ImmutableList.of(); - } else { - // Even if there are edits, the resulting tree might be identical with the previous tree. - final CanonicalTreeParser p = new CanonicalTreeParser(); - p.reset(reader, prevTreeId); - final DiffFormatter diffFormatter = new DiffFormatter(null); - diffFormatter.setRepository(jGitRepo); - diffEntries = diffFormatter.scan(p, new DirCacheIterator(dirCache)); - isEmpty = diffEntries.isEmpty(); - } - - if (!allowEmpty && isEmpty) { - throw new RedundantChangeException( - "changes did not change anything in " + - repo.project().name() + '/' + repo.originalRepoName() + - " at revision " + (prevRevision != null ? prevRevision.major() : 0) + ": " + changes); - } - - // flush the current index to repository and get the result tree object id. - final ObjectId nextTreeId = dirCache.writeTree(inserter); - - // build a commit object - final PersonIdent personIdent = new PersonIdent(author.name(), author.email(), - commitTimeMillis / 1000L * 1000L, 0); - - final CommitBuilder commitBuilder = new CommitBuilder(); - - commitBuilder.setAuthor(personIdent); - commitBuilder.setCommitter(personIdent); - commitBuilder.setTreeId(nextTreeId); - commitBuilder.setEncoding(UTF_8); - - // Write summary, detail and revision to commit's message as JSON format. - commitBuilder.setMessage(CommitUtil.toJsonString(summary, detail, markup, nextRevision)); - - // if the head commit exists, use it as the parent commit. - if (prevRevision != null) { - commitBuilder.setParentId(commitIdDatabase.get(prevRevision)); - } - - final ObjectId nextCommitId = inserter.insert(commitBuilder); - inserter.flush(); - - // tagging the revision object, for history lookup purpose. - commitIdDatabase.put(nextRevision, nextCommitId); - doRefUpdate(jGitRepo, revWalk, R_HEADS_MASTER, nextCommitId); - - return new RevisionAndEntries(nextRevision, diffEntries); - } catch (CentralDogmaException | IllegalArgumentException e) { - throw e; - } catch (Exception e) { - throw new StorageException("failed to push at '" + - repo.project().name() + '/' + repo.originalRepoName() + '\'', e); - } - } - private void notifyWatchers(Revision newRevision, List diffEntries) { for (DiffEntry entry : diffEntries) { switch (entry.getChangeType()) { @@ -1013,41 +531,28 @@ private void notifyWatchers(Revision newRevision, List diffEntries) { @Override public CompletableFuture> history(Revision from, Revision to, String pathPattern, int maxCommits) { + requireNonNull(pathPattern, "pathPattern"); + requireNonNull(from, "from"); + requireNonNull(to, "to"); + checkArgument(maxCommits > 0, "maxCommits: %s (expected: > 0)", maxCommits); final ServiceRequestContext ctx = context(); return CompletableFuture.supplyAsync(() -> { failFastIfTimedOut(this, logger, ctx, "history", from, to, pathPattern, maxCommits); - return blockingHistory(from, to, pathPattern, maxCommits); + final int maxCommits0 = Math.min(maxCommits, MAX_MAX_COMMITS); + readLock(); + try { + final RevisionRange range = normalizeNow(from, to); + return primaryRepo.listCommits(pathPattern, maxCommits0, range); + } finally { + readUnlock(); + } }, repositoryWorker); } - private List blockingHistory(Revision from, Revision to, String pathPattern, int maxCommits) { - requireNonNull(pathPattern, "pathPattern"); - requireNonNull(from, "from"); - requireNonNull(to, "to"); - if (maxCommits <= 0) { - throw new IllegalArgumentException("maxCommits: " + maxCommits + " (expected: > 0)"); - } - - maxCommits = Math.min(maxCommits, MAX_MAX_COMMITS); - - final RevisionRange range = normalizeNow(from, to); - final RevisionRange descendingRange = range.toDescending(); - - // At this point, we are sure: from.major >= to.major - readLock(); - final InternalRepository primaryRepo = this.primaryRepo; - try { - return getCommits(primaryRepo, pathPattern, maxCommits, range, descendingRange); - } finally { - readUnlock(); - } - } - @Override public CompletableFuture findLatestRevision(Revision lastKnownRevision, String pathPattern) { requireNonNull(lastKnownRevision, "lastKnownRevision"); requireNonNull(pathPattern, "pathPattern"); - final ServiceRequestContext ctx = context(); return CompletableFuture.supplyAsync(() -> { failFastIfTimedOut(this, logger, ctx, "findLatestRevision", lastKnownRevision, pathPattern); @@ -1057,34 +562,49 @@ public CompletableFuture findLatestRevision(Revision lastKnownRevision @Nullable private Revision blockingFindLatestRevision(Revision lastKnownRevision, String pathPattern) { - final RevisionRange range = normalizeNow(lastKnownRevision, Revision.HEAD); - if (range.from().equals(range.to())) { - // Empty range. - return null; - } - - if (range.from().major() == 1) { + if (lastKnownRevision.major() == Revision.INIT.major()) { // Fast path: no need to compare because we are sure there is nothing at revision 1. - final Map> entries = - blockingFind(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT); - return !entries.isEmpty() ? range.to() : null; + readLock(); + try { + final Revision headRevision = this.headRevision; + if (headRevision.major() == 1) { + return null; + } + if ("/".equals(pathPattern)) { + return headRevision; + } + final Map> entries = primaryRepo.find( + headRevision, pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT); + return !entries.isEmpty() ? headRevision : null; + } finally { + readUnlock(); + } } // Slow path: compare the two trees. - final PathPatternFilter filter = PathPatternFilter.of(pathPattern); - // Convert the revisions to Git trees. final List diffEntries; + final RevisionRange range; readLock(); - final InternalRepository primaryRepo = this.primaryRepo; - try (RevWalk revWalk = newRevWalk(primaryRepo.jGitRepo())) { - final CommitIdDatabase commitIdDatabase = primaryRepo.commitIdDatabase(); - final RevTree treeA = toTree(commitIdDatabase, revWalk, range.from()); - final RevTree treeB = toTree(commitIdDatabase, revWalk, range.to()); - diffEntries = blockingCompareTrees(primaryRepo.jGitRepo(), treeA, treeB); + try { + range = new RevisionRange(normalizeNow(lastKnownRevision, false), headRevision); + if (range.from().isLowerThan(primaryRepo.firstRevision())) { + // We should return range.to() without comparing two tree. It's because: + // - the entry might be changed between the lastKnownRevision and first revision. + // - but the previous commits before the first revision is packed and committed to the + // primaryRepo at once, we really don't know if there was a change. + // - so it's safe to return the latest revision so that the client get notified when watching. + return range.to(); + } + if (range.from().equals(range.to())) { + // Empty range. + return null; + } + diffEntries = primaryRepo.diff(range, cache); } finally { readUnlock(); } + final PathPatternFilter filter = PathPatternFilter.of(pathPattern); // Return the latest revision if the changes between the two trees contain the file. for (DiffEntry e : diffEntries) { final String path; @@ -1108,48 +628,6 @@ private Revision blockingFindLatestRevision(Revision lastKnownRevision, String p return null; } - /** - * Compares the two Git trees (with caching). - */ - private List blockingCompareTrees(Repository jGitRepo, RevTree treeA, RevTree treeB) { - if (cache == null) { - return blockingCompareTreesUncached(jGitRepo, treeA, treeB, TreeFilter.ALL); - } - - final CacheableCompareTreesCall key = new CacheableCompareTreesCall(this, treeA, treeB); - CompletableFuture> existingFuture = cache.getIfPresent(key); - if (existingFuture != null) { - final List existingDiffEntries = existingFuture.getNow(null); - if (existingDiffEntries != null) { - // Cached already. - return existingDiffEntries; - } - } - - // Not cached yet. Acquire a lock so that we do not compare the same tree pairs simultaneously. - final List newDiffEntries; - final Lock lock = key.coarseGrainedLock(); - lock.lock(); - try { - existingFuture = cache.getIfPresent(key); - if (existingFuture != null) { - final List existingDiffEntries = existingFuture.getNow(null); - if (existingDiffEntries != null) { - // Other thread already put the entries to the cache before we acquire the lock. - return existingDiffEntries; - } - } - - newDiffEntries = blockingCompareTreesUncached(jGitRepo, treeA, treeB, TreeFilter.ALL); - cache.put(key, newDiffEntries); - } finally { - lock.unlock(); - } - - logger.debug("Cache miss: {}", key); - return newDiffEntries; - } - @Override public CompletableFuture watch(Revision lastKnownRevision, String pathPattern) { requireNonNull(lastKnownRevision, "lastKnownRevision"); @@ -1210,26 +688,7 @@ private Commit currentInitialCommit(InternalRepository primaryRepo) { // This method is called when opening an existing repository so firstRevision is not null. assert firstRevision != null; final RevisionRange range = new RevisionRange(firstRevision, firstRevision); - return getCommits(primaryRepo, ALL_PATH, 1, range, range).get(0); - } - - private static List> toChanges(Map> entries) { - final Builder> builder = ImmutableList.builder(); - for (Entry entry : entries.values()) { - final EntryType type = entry.type(); - if (type == EntryType.DIRECTORY) { - continue; - } - if (type == EntryType.JSON) { - final JsonNode content = (JsonNode) entry.content(); - builder.add(Change.ofJsonUpsert(entry.path(), content)); - } else { - assert type == EntryType.TEXT; - final String content = (String) entry.content(); - builder.add(Change.ofTextUpsert(entry.path(), content)); - } - } - return builder.build(); + return primaryRepo.listCommits(ALL_PATH, 1, range).get(0); } /** @@ -1272,9 +731,9 @@ private boolean exceedsMinRetention(InternalRepository repo, } if (minRetentionDays != 0) { - final Instant creationTime = repo.secondCommitCreationTimeInstant(); - return creationTime != null && - Instant.now().minus(Duration.ofDays(minRetentionDays)).isBefore(creationTime); + final Instant secondCommitCreationTime = repo.secondCommitCreationTimeInstant(); + return secondCommitCreationTime != null && + Instant.now().minus(Duration.ofDays(minRetentionDays)).isBefore(secondCommitCreationTime); } return true; } @@ -1310,6 +769,16 @@ private void promoteSecondaryRepo() { createSecondaryRepo(); } + private static boolean deleteDirectory(File dir) throws IOException { + try (Stream walk = Files.walk(dir.toPath())) { + return walk.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .map(File::delete) + // Return false if it fails to delete a file. + .reduce(true, (a, b) -> a && b); + } + } + private void createSecondaryRepo() { try { writeLock(); @@ -1323,24 +792,24 @@ private void createSecondaryRepo() { final List> changes = toChanges(entries); final File secondaryRepoDir = repoMetadata.secondaryRepoDir(); final InternalRepository secondaryRepo = - createInternalRepo(parent, name, secondaryRepoDir, headRevision, - creationTimeMillis, author, changes); + InternalRepository.of(parent, name, secondaryRepoDir, headRevision, + creationTimeMillis, author, changes); writeLock(); try { if (laggedCommitsForSecondary.isEmpty()) { - // There were no commits after creating the secondary repo. + // There were no commits after creating the secondary repo + // so the headRevision hasn't changed. assert headRevision == this.headRevision; - this.secondaryRepo = secondaryRepo; } else { // We should catch up. for (LaggedCommit c : laggedCommitsForSecondary) { - commit0(secondaryRepo, c.revision, c.revision.forward(1), c.commitTimeMillis, c.author, - c.summary, c.detail, c.markup, c.changes, c.allowEmpty); + secondaryRepo.commit(c.revision, c.revision.forward(1), c.commitTimeMillis, c.author, + c.summary, c.detail, c.markup, c.changes, c.allowEmpty); } laggedCommitsForSecondary.clear(); - isCreatingSecondaryRepo = false; - this.secondaryRepo = secondaryRepo; } + isCreatingSecondaryRepo = false; + this.secondaryRepo = secondaryRepo; } finally { writeUnLock(); } @@ -1351,14 +820,23 @@ private void createSecondaryRepo() { } } - private static final class RevisionAndEntries { - final Revision revision; - final List diffEntries; - - RevisionAndEntries(Revision revision, List diffEntries) { - this.revision = revision; - this.diffEntries = diffEntries; + private static List> toChanges(Map> entries) { + final Builder> builder = ImmutableList.builder(); + for (Entry entry : entries.values()) { + final EntryType type = entry.type(); + if (type == EntryType.DIRECTORY) { + continue; + } + if (type == EntryType.JSON) { + final JsonNode content = (JsonNode) entry.content(); + builder.add(Change.ofJsonUpsert(entry.path(), content)); + } else { + assert type == EntryType.TEXT; + final String content = (String) entry.content(); + builder.add(Change.ofTextUpsert(entry.path(), content)); + } } + return builder.build(); } private static class LaggedCommit { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java index 198c91883..8d2f59cb0 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java @@ -15,27 +15,278 @@ */ package com.linecorp.centraldogma.server.internal.storage.repository.git; +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.google.common.base.Preconditions.checkState; import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.closeJGitRepo; -import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.getCommits; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.exists; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryV2.R_HEADS_MASTER; import static com.linecorp.centraldogma.server.storage.repository.Repository.ALL_PATH; +import static com.linecorp.centraldogma.server.storage.repository.Repository.MAX_MAX_COMMITS; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_COMMIT_SECTION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DIFF_SECTION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_ALGORITHM; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_FILEMODE; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_GPGSIGN; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_HIDEDOTFILES; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_RENAMES; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REPO_FORMAT_VERSION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_SYMLINKS; import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.StringJoiner; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.locks.Lock; +import java.util.regex.Pattern; import javax.annotation.Nullable; +import org.eclipse.jgit.diff.DiffEntry; +import org.eclipse.jgit.diff.DiffFormatter; +import org.eclipse.jgit.dircache.DirCache; +import org.eclipse.jgit.dircache.DirCacheBuilder; +import org.eclipse.jgit.dircache.DirCacheEditor; +import org.eclipse.jgit.dircache.DirCacheEditor.DeletePath; +import org.eclipse.jgit.dircache.DirCacheEditor.DeleteTree; +import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit; +import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.dircache.DirCacheIterator; +import org.eclipse.jgit.lib.CommitBuilder; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.CoreConfig.HideDotFiles; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectIdOwnerMap; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.RefUpdate; +import org.eclipse.jgit.lib.RefUpdate.Result; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.RepositoryBuilder; +import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.revwalk.TreeRevFilter; +import org.eclipse.jgit.revwalk.filter.RevFilter; +import org.eclipse.jgit.treewalk.CanonicalTreeParser; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.AndTreeFilter; +import org.eclipse.jgit.treewalk.filter.TreeFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.CentralDogmaException; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.ChangeConflictException; +import com.linecorp.centraldogma.common.Commit; +import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.EntryType; +import com.linecorp.centraldogma.common.Markup; +import com.linecorp.centraldogma.common.RedundantChangeException; +import com.linecorp.centraldogma.common.RepositoryNotFoundException; import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.common.RevisionRange; +import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.internal.Util; +import com.linecorp.centraldogma.internal.jsonpatch.JsonPatch; +import com.linecorp.centraldogma.internal.jsonpatch.ReplaceMode; +import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache; +import com.linecorp.centraldogma.server.storage.StorageException; import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.repository.FindOption; + +import difflib.DiffUtils; +import difflib.Patch; final class InternalRepository { private static final Logger logger = LoggerFactory.getLogger(InternalRepository.class); + private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL); + private static final byte[] EMPTY_BYTE = new byte[0]; + + private static final Field revWalkObjectsField; + + static { + try { + revWalkObjectsField = RevWalk.class.getDeclaredField("objects"); + if (revWalkObjectsField.getType() != ObjectIdOwnerMap.class) { + throw new IllegalStateException( + RevWalk.class.getSimpleName() + ".objects is not an " + + ObjectIdOwnerMap.class.getSimpleName() + '.'); + } + revWalkObjectsField.setAccessible(true); + } catch (NoSuchFieldException e) { + throw new IllegalStateException( + RevWalk.class.getSimpleName() + ".objects does not exist."); + } + } + + static InternalRepository of(Project parent, String originalRepoName, File repoDir, + Revision nextRevision, long commitTimeMillis, + Author author, Iterable> changes) { + boolean success = false; + InternalRepository internalRepo = null; + try { + createEmptyJGitRepo(repoDir); + + // Re-open the repository with the updated settings and format version. + final Repository jGitRepo = new RepositoryBuilder().setGitDir(repoDir).build(); + internalRepo = new InternalRepository(parent, originalRepoName, repoDir, + jGitRepo, new CommitIdDatabase(jGitRepo)); + + // Initialize the master branch. + final RefUpdate head = jGitRepo.updateRef(Constants.HEAD); + head.disableRefLog(); + head.link(R_HEADS_MASTER); + + // Insert the initial commit into the master branch. + internalRepo.commit(null, nextRevision, commitTimeMillis, author, + "Create a new repository", "", Markup.PLAINTEXT, changes, true); + success = true; + return internalRepo; + } catch (IOException e) { + throw new StorageException("failed to create a repository at: " + repoDir, e); + } finally { + if (!success) { + if (internalRepo != null) { + internalRepo.close(); + } + // Failed to create a repository. Remove any cruft so that it is not loaded on the next run. + deleteCruft(repoDir); + } + } + } + + private static void deleteCruft(File repoDir) { + try { + Util.deleteFileTree(repoDir); + } catch (IOException e) { + logger.error("Failed to delete a half-created repository at: {}", repoDir, e); + } + } + + private static void createEmptyJGitRepo(File repositoryDir) throws IOException { + try (org.eclipse.jgit.lib.Repository initRepository = buildJGitRepo(repositoryDir)) { + initRepository.create(true); + + final StoredConfig config = initRepository.getConfig(); + // Update the repository settings to upgrade to format version 1 and reftree. + config.setInt(CONFIG_CORE_SECTION, null, CONFIG_KEY_REPO_FORMAT_VERSION, 1); + + // Disable hidden files, symlinks and file modes we do not use. + config.setEnum(CONFIG_CORE_SECTION, null, CONFIG_KEY_HIDEDOTFILES, HideDotFiles.FALSE); + config.setBoolean(CONFIG_CORE_SECTION, null, CONFIG_KEY_SYMLINKS, false); + config.setBoolean(CONFIG_CORE_SECTION, null, CONFIG_KEY_FILEMODE, false); + + // Disable GPG signing. + config.setBoolean(CONFIG_COMMIT_SECTION, null, CONFIG_KEY_GPGSIGN, false); + + // Set the diff algorithm. + config.setString(CONFIG_DIFF_SECTION, null, CONFIG_KEY_ALGORITHM, "histogram"); + + // Disable rename detection which we do not use. + config.setBoolean(CONFIG_DIFF_SECTION, null, CONFIG_KEY_RENAMES, false); + + config.save(); + } + } + + static Repository buildJGitRepo(File repositoryDir) { + try { + return new RepositoryBuilder().setGitDir(repositoryDir).setBare().build(); + } catch (IOException e) { + throw new StorageException("failed to create a repository at: " + repositoryDir, e); + } + } + + @Nullable + static InternalRepository open(Project parent, String name, File repoDir, boolean errorIfNotExist) { + final Repository jGitRepo = buildJGitRepo(repoDir); + CommitIdDatabase commitIdDatabase = null; + try { + if (!exists(jGitRepo)) { + if (errorIfNotExist) { + throw new RepositoryNotFoundException(repoDir.toString()); + } else { + return null; + } + } + checkGitRepositoryFormat(jGitRepo); + final Revision headRevision = uncachedHeadRevision(parent, name, jGitRepo); + commitIdDatabase = new CommitIdDatabase(jGitRepo); + if (!headRevision.equals(commitIdDatabase.headRevision())) { + commitIdDatabase.rebuild(jGitRepo); + assert headRevision.equals(commitIdDatabase.headRevision()); + } + return new InternalRepository(parent, name, repoDir, jGitRepo, commitIdDatabase); + } catch (Throwable t) { + closeJGitRepo(jGitRepo); + if (commitIdDatabase != null) { + commitIdDatabase.close(); + } + throw t; + } + } + + private static Revision uncachedHeadRevision(Project parent, String name, Repository jGitRepo) { + try (RevWalk revWalk = newRevWalk(jGitRepo)) { + final ObjectId headRevisionId = jGitRepo.resolve(R_HEADS_MASTER); + if (headRevisionId != null) { + final RevCommit revCommit = revWalk.parseCommit(headRevisionId); + return CommitUtil.extractRevision(revCommit.getFullMessage()); + } + } catch (CentralDogmaException e) { + throw e; + } catch (Exception e) { + throw new StorageException("failed to get the current revision", e); + } + + throw new StorageException("failed to determine the HEAD: " + parent.name() + '/' + name); + } + + private static RevWalk newRevWalk(Repository jGitRepository) { + final RevWalk revWalk = new RevWalk(jGitRepository); + // Disable rewriteParents because otherwise `RevWalk` will load every commit into memory. + revWalk.setRewriteParents(false); + return revWalk; + } + + private static RevWalk newRevWalk(ObjectReader reader) { + final RevWalk revWalk = new RevWalk(reader); + // Disable rewriteParents because otherwise `RevWalk` will load every commit into memory. + revWalk.setRewriteParents(false); + return revWalk; + } + + private static void checkGitRepositoryFormat(Repository repository) { + final int formatVersion = repository.getConfig().getInt( + CONFIG_CORE_SECTION, null, CONFIG_KEY_REPO_FORMAT_VERSION, 0); + if (formatVersion != 1) { + throw new StorageException("unsupported repository format version: " + formatVersion); + } + } + private final Project project; private final String originalRepoName; // e.g. foo private final File repoDir; // e.g. foo_0000000000 @@ -46,8 +297,8 @@ final class InternalRepository { @Nullable private Instant secondCommitCreationTimeInstant; - InternalRepository(Project project, String originalRepoName, File repoDir, - Repository jGitRepository, CommitIdDatabase commitIdDatabase) { + private InternalRepository(Project project, String originalRepoName, File repoDir, + Repository jGitRepository, CommitIdDatabase commitIdDatabase) { this.project = project; this.originalRepoName = originalRepoName; this.repoDir = repoDir; @@ -75,6 +326,22 @@ CommitIdDatabase commitIdDatabase() { return commitIdDatabase; } + Revision headRevision() { + final Revision headRevision = commitIdDatabase.headRevision(); + checkState(headRevision != null, "the headRevision is not set."); + return headRevision; + } + + Revision firstRevision() { + final Revision firstRevision = commitIdDatabase.firstRevision(); + checkState(firstRevision != null, "the firstRevision is not set."); + return firstRevision; + } + + /** + * Returns the {@link Instant} of the time when the second commit is created. This {@link Instant} is used + * to check if the second commit of this repository exceeds the minimum retention days or not. + */ @Nullable Instant secondCommitCreationTimeInstant() { if (secondCommitCreationTimeInstant == null) { @@ -93,11 +360,779 @@ Instant secondCommitCreationTimeInstant() { final Revision secondRevision = firstRevision.forward(1); final RevisionRange range = new RevisionRange(secondRevision, secondRevision); secondCommitCreationTimeInstant = - Instant.ofEpochMilli(getCommits(this, ALL_PATH, 1, range, range).get(0).when()); + Instant.ofEpochMilli(listCommits(ALL_PATH, 1, range).get(0).when()); } return secondCommitCreationTimeInstant; } + List listCommits(String pathPattern, int maxCommits, RevisionRange range) { + final RevisionRange descendingRange = range.toDescending(); + try (RevWalk revWalk = newRevWalk(jGitRepository)) { + final ObjectIdOwnerMap revWalkInternalMap = + (ObjectIdOwnerMap) revWalkObjectsField.get(revWalk); + final ObjectId fromCommitId = commitIdDatabase.get(descendingRange.from()); + final ObjectId toCommitId = commitIdDatabase.get(descendingRange.to()); + + revWalk.markStart(revWalk.parseCommit(fromCommitId)); + revWalk.setRetainBody(false); + + // Instead of relying on RevWalk to filter the commits, + // we let RevWalk yield all commits so we can: + // - Have more control on when iteration should be stopped. + // (A single Iterator.next() doesn't take long.) + // - Clean up the internal map as early as possible. + final RevFilter filter = new TreeRevFilter(revWalk, AndTreeFilter.create( + TreeFilter.ANY_DIFF, PathPatternFilter.of(pathPattern))); + + // Search up to 1000 commits when maxCommits <= 100. + // Search up to (maxCommits * 10) commits when 100 < maxCommits <= 1000. + final int maxNumProcessedCommits = Math.max(maxCommits * 10, MAX_MAX_COMMITS); + + final List commitList = new ArrayList<>(); + int numProcessedCommits = 0; + for (RevCommit revCommit : revWalk) { + numProcessedCommits++; + + if (filter.include(revWalk, revCommit)) { + revWalk.parseBody(revCommit); + commitList.add(toCommit(revCommit)); + revCommit.disposeBody(); + } + + if (revCommit.getId().equals(toCommitId) || + commitList.size() >= maxCommits || + // Prevent from iterating for too long. + numProcessedCommits >= maxNumProcessedCommits) { + break; + } + + // Clear the internal lookup table of RevWalk to reduce the memory usage. + // This is safe because we have linear history and traverse in one direction. + if (numProcessedCommits % 16 == 0) { + revWalkInternalMap.clear(); + } + } + + // Include the initial empty commit only when the caller specified + // the initial revision (1) in the range and the pathPattern contains '/**'. + if (commitList.size() < maxCommits && + descendingRange.to().major() == 1 && + pathPattern.contains(ALL_PATH)) { + try (RevWalk tmpRevWalk = newRevWalk(jGitRepository)) { + final RevCommit lastRevCommit = tmpRevWalk.parseCommit(toCommitId); + commitList.add(toCommit(lastRevCommit)); + } + } + + if (!descendingRange.equals(range)) { // from and to is swapped so reverse the list. + Collections.reverse(commitList); + } + + return commitList; + } catch (CentralDogmaException e) { + throw e; + } catch (Exception e) { + throw new StorageException( + "failed to retrieve the history: " + originalRepoName + + " (" + pathPattern + ", " + range.from() + ".." + range.to() + ')', e); + } + } + + private static Commit toCommit(RevCommit revCommit) { + final Author author; + final PersonIdent committerIdent = revCommit.getCommitterIdent(); + final long when; + if (committerIdent == null) { + author = Author.UNKNOWN; + when = 0; + } else { + author = new Author(committerIdent.getName(), committerIdent.getEmailAddress()); + when = committerIdent.getWhen().getTime(); + } + + try { + return CommitUtil.newCommit(author, when, revCommit.getFullMessage()); + } catch (Exception e) { + throw new StorageException("failed to create a Commit", e); + } + } + + Map> find(Revision normalizedRevision, String pathPattern, Map, ?> options) { + try (ObjectReader reader = jGitRepository.newObjectReader(); + TreeWalk treeWalk = new TreeWalk(reader); + RevWalk revWalk = newRevWalk(reader)) { + final Map> result = new LinkedHashMap<>(); + final ObjectId commitId = commitIdDatabase.get(normalizedRevision); + final RevCommit revCommit = revWalk.parseCommit(commitId); + final PathPatternFilter filter = PathPatternFilter.of(pathPattern); + + final int maxEntries = FindOption.MAX_ENTRIES.get(options); + final RevTree revTree = revCommit.getTree(); + treeWalk.addTree(revTree.getId()); + while (treeWalk.next() && result.size() < maxEntries) { + final boolean matches = filter.matches(treeWalk); + final String path = '/' + treeWalk.getPathString(); + + // Recurse into a directory if necessary. + if (treeWalk.isSubtree()) { + if (matches) { + // Add the directory itself to the result set if its path matches the pattern. + result.put(path, Entry.ofDirectory(normalizedRevision, path)); + } + + treeWalk.enterSubtree(); + continue; + } + + if (!matches) { + continue; + } + + final boolean fetchContent = FindOption.FETCH_CONTENT.get(options); + // Build an entry as requested. + final Entry entry; + final EntryType entryType = EntryType.guessFromPath(path); + if (fetchContent) { + final byte[] content = reader.open(treeWalk.getObjectId(0)).getBytes(); + switch (entryType) { + case JSON: + final JsonNode jsonNode = Jackson.readTree(content); + entry = Entry.ofJson(normalizedRevision, path, jsonNode); + break; + case TEXT: + final String strVal = sanitizeText(new String(content, UTF_8)); + entry = Entry.ofText(normalizedRevision, path, strVal); + break; + default: + throw new Error("unexpected entry type: " + entryType); + } + } else { + switch (entryType) { + case JSON: + entry = Entry.ofJson(normalizedRevision, path, Jackson.nullNode); + break; + case TEXT: + entry = Entry.ofText(normalizedRevision, path, ""); + break; + default: + throw new Error("unexpected entry type: " + entryType); + } + } + result.put(path, entry); + } + + return Util.unsafeCast(result); + } catch (CentralDogmaException | IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new StorageException( + "failed to get data from '" + project.name() + '/' + originalRepoName + "' at " + + pathPattern + " for " + normalizedRevision, e); + } + } + + /** + * Removes {@code \r} and appends {@code \n} on the last line if it does not end with {@code \n}. + */ + private static String sanitizeText(String text) { + if (text.indexOf('\r') >= 0) { + text = CR.matcher(text).replaceAll(""); + } + if (!text.isEmpty() && !text.endsWith("\n")) { + text += "\n"; + } + return text; + } + + List diff(RevisionRange range, @Nullable RepositoryCache cache) { + return diff0(range, TreeFilter.ALL, cache); + } + + Map> diff(RevisionRange range, String pathPattern) { + // Note that we do not cache here because CachingRepository caches the final result. + return toChangeMap(diff0(range, filter(pathPattern), null)); + } + + private List diff0(RevisionRange range, TreeFilter filter, @Nullable RepositoryCache cache) { + try (RevWalk rw = newRevWalk(jGitRepository)) { + final RevTree treeA = rw.parseTree(commitIdDatabase.get(range.from())); + final RevTree treeB = rw.parseTree(commitIdDatabase.get(range.to())); + return blockingCompareTrees(treeA, treeB, filter, cache); + } catch (StorageException e) { + throw e; + } catch (Exception e) { + throw new StorageException("failed to parse two trees: from= " + range.from() + + ", to= " + range.to(), e); + } + } + + private List blockingCompareTrees(@Nullable RevTree treeA, @Nullable RevTree treeB, + TreeFilter filter, @Nullable RepositoryCache cache) { + if (cache == null) { + return blockingCompareTreesUncached(treeA, treeB, filter); + } + final CacheableCompareTreesCall key = + new CacheableCompareTreesCall(project.repos().get(originalRepoName), treeA, treeB); + CompletableFuture> existingFuture = cache.getIfPresent(key); + if (existingFuture != null) { + final List existingDiffEntries = existingFuture.getNow(null); + if (existingDiffEntries != null) { + // Cached already. + return existingDiffEntries; + } + } + + // Not cached yet. Acquire a lock so that we do not compare the same tree pairs simultaneously. + final List newDiffEntries; + final Lock lock = key.coarseGrainedLock(); + lock.lock(); + try { + existingFuture = cache.getIfPresent(key); + if (existingFuture != null) { + final List existingDiffEntries = existingFuture.getNow(null); + if (existingDiffEntries != null) { + // Other thread already put the entries to the cache before we acquire the lock. + return existingDiffEntries; + } + } + + newDiffEntries = blockingCompareTreesUncached(treeA, treeB, filter); + cache.put(key, newDiffEntries); + } finally { + lock.unlock(); + } + + logger.debug("Cache miss: {}", key); + return newDiffEntries; + } + + private List blockingCompareTreesUncached(@Nullable RevTree treeA, @Nullable RevTree treeB, + TreeFilter filter) { + try (DiffFormatter diffFormatter = new DiffFormatter(null)) { + diffFormatter.setRepository(jGitRepository); + diffFormatter.setPathFilter(filter); + return ImmutableList.copyOf(diffFormatter.scan(treeA, treeB)); + } catch (IOException e) { + throw new StorageException("failed to compare two trees: " + treeA + " vs. " + treeB, e); + } + } + + private static TreeFilter filter(@Nullable String pathPattern) { + if (pathPattern == null) { + return TreeFilter.ALL; + } + + final PathPatternFilter pathPatternFilter = PathPatternFilter.of(pathPattern); + return pathPatternFilter.matchesAll() ? TreeFilter.ALL : pathPatternFilter; + } + + private Map> toChangeMap(List diffEntryList) { + try (ObjectReader reader = jGitRepository.newObjectReader()) { + final Map> changeMap = new LinkedHashMap<>(); + + for (DiffEntry diffEntry : diffEntryList) { + final String oldPath = '/' + diffEntry.getOldPath(); + final String newPath = '/' + diffEntry.getNewPath(); + + switch (diffEntry.getChangeType()) { + case MODIFY: + final EntryType oldEntryType = EntryType.guessFromPath(oldPath); + switch (oldEntryType) { + case JSON: + if (!oldPath.equals(newPath)) { + putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath)); + } + + final JsonNode oldJsonNode = + Jackson.readTree( + reader.open(diffEntry.getOldId().toObjectId()).getBytes()); + final JsonNode newJsonNode = + Jackson.readTree( + reader.open(diffEntry.getNewId().toObjectId()).getBytes()); + final JsonPatch patch = + JsonPatch.generate(oldJsonNode, newJsonNode, ReplaceMode.SAFE); + + if (!patch.isEmpty()) { + putChange(changeMap, newPath, + Change.ofJsonPatch(newPath, Jackson.valueToTree(patch))); + } + break; + case TEXT: + final String oldText = sanitizeText(new String( + reader.open(diffEntry.getOldId().toObjectId()).getBytes(), UTF_8)); + + final String newText = sanitizeText(new String( + reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8)); + + if (!oldPath.equals(newPath)) { + putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath)); + } + + if (!oldText.equals(newText)) { + putChange(changeMap, newPath, + Change.ofTextPatch(newPath, oldText, newText)); + } + break; + default: + throw new Error("unexpected old entry type: " + oldEntryType); + } + break; + case ADD: + final EntryType newEntryType = EntryType.guessFromPath(newPath); + switch (newEntryType) { + case JSON: { + final JsonNode jsonNode = Jackson.readTree( + reader.open(diffEntry.getNewId().toObjectId()).getBytes()); + + putChange(changeMap, newPath, Change.ofJsonUpsert(newPath, jsonNode)); + break; + } + case TEXT: { + final String text = sanitizeText(new String( + reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8)); + + putChange(changeMap, newPath, Change.ofTextUpsert(newPath, text)); + break; + } + default: + throw new Error("unexpected new entry type: " + newEntryType); + } + break; + case DELETE: + putChange(changeMap, oldPath, Change.ofRemoval(oldPath)); + break; + default: + throw new Error(); + } + } + return changeMap; + } catch (Exception e) { + throw new StorageException("failed to convert list of DiffEntry to Changes map", e); + } + } + + private static void putChange(Map> changeMap, String path, Change change) { + final Change oldChange = changeMap.put(path, change); + assert oldChange == null; + } + + Map> previewDiff(Revision baseRevision, Iterable> changes) { + try (ObjectReader reader = jGitRepository.newObjectReader(); + RevWalk revWalk = newRevWalk(reader); + DiffFormatter diffFormatter = new DiffFormatter(null)) { + final ObjectId baseTreeId = toTree(commitIdDatabase, revWalk, baseRevision); + final DirCache dirCache = DirCache.newInCore(); + final int numEdits = applyChanges(baseRevision, baseTreeId, dirCache, changes); + if (numEdits == 0) { + return Collections.emptyMap(); + } + + final CanonicalTreeParser p = new CanonicalTreeParser(); + p.reset(reader, baseTreeId); + diffFormatter.setRepository(jGitRepository); + final List result = diffFormatter.scan(p, new DirCacheIterator(dirCache)); + return toChangeMap(result); + } catch (IOException e) { + throw new StorageException("failed to perform a dry-run diff", e); + } + } + + private static RevTree toTree(CommitIdDatabase commitIdDatabase, RevWalk revWalk, Revision revision) { + final ObjectId commitId = commitIdDatabase.get(revision); + try { + return revWalk.parseTree(commitId); + } catch (IOException e) { + throw new StorageException("failed to parse a commit: " + commitId, e); + } + } + + private int applyChanges(@Nullable Revision baseRevision, @Nullable ObjectId baseTreeId, + DirCache dirCache, Iterable> changes) { + int numEdits = 0; + + try (ObjectInserter inserter = jGitRepository.newObjectInserter(); + ObjectReader reader = jGitRepository.newObjectReader()) { + + if (baseTreeId != null) { + // the DirCacheBuilder is to used for doing update operations on the given DirCache object + final DirCacheBuilder builder = dirCache.builder(); + + // Add the tree object indicated by the prevRevision to the temporary DirCache object. + builder.addTree(EMPTY_BYTE, 0, reader, baseTreeId); + builder.finish(); + } + + // loop over the specified changes. + for (Change change : changes) { + final String changePath = change.path().substring(1); // Strip the leading '/'. + final DirCacheEntry oldEntry = dirCache.getEntry(changePath); + final byte[] oldContent = oldEntry != null ? reader.open(oldEntry.getObjectId()).getBytes() + : null; + + switch (change.type()) { + case UPSERT_JSON: { + final JsonNode oldJsonNode = oldContent != null ? Jackson.readTree(oldContent) : null; + final JsonNode newJsonNode = firstNonNull((JsonNode) change.content(), + JsonNodeFactory.instance.nullNode()); + + // Upsert only when the contents are really different. + if (!Objects.equals(newJsonNode, oldJsonNode)) { + applyPathEdit(dirCache, new InsertJson(changePath, inserter, newJsonNode)); + numEdits++; + } + break; + } + case UPSERT_TEXT: { + final String sanitizedOldText; + if (oldContent != null) { + sanitizedOldText = sanitizeText(new String(oldContent, UTF_8)); + } else { + sanitizedOldText = null; + } + + final String sanitizedNewText = sanitizeText(change.contentAsText()); + + // Upsert only when the contents are really different. + if (!sanitizedNewText.equals(sanitizedOldText)) { + applyPathEdit(dirCache, new InsertText(changePath, inserter, sanitizedNewText)); + numEdits++; + } + break; + } + case REMOVE: + if (oldEntry != null) { + applyPathEdit(dirCache, new DeletePath(changePath)); + numEdits++; + break; + } + + // The path might be a directory. + if (applyDirectoryEdits(dirCache, changePath, null, change)) { + numEdits++; + } else { + // Was not a directory either; conflict. + reportNonExistentEntry(change); + break; + } + break; + case RENAME: { + final String newPath = + ((String) change.content()).substring(1); // Strip the leading '/'. + + if (dirCache.getEntry(newPath) != null) { + throw new ChangeConflictException("a file exists at the target path: " + change); + } + + if (oldEntry != null) { + if (changePath.equals(newPath)) { + // Redundant rename request - old path and new path are same. + break; + } + + final DirCacheEditor editor = dirCache.editor(); + editor.add(new DeletePath(changePath)); + editor.add(new CopyOldEntry(newPath, oldEntry)); + editor.finish(); + numEdits++; + break; + } + + // The path might be a directory. + if (applyDirectoryEdits(dirCache, changePath, newPath, change)) { + numEdits++; + } else { + // Was not a directory either; conflict. + reportNonExistentEntry(change); + } + break; + } + case APPLY_JSON_PATCH: { + final JsonNode oldJsonNode; + if (oldContent != null) { + oldJsonNode = Jackson.readTree(oldContent); + } else { + oldJsonNode = Jackson.nullNode; + } + + final JsonNode newJsonNode; + try { + newJsonNode = JsonPatch.fromJson((JsonNode) change.content()).apply(oldJsonNode); + } catch (Exception e) { + throw new ChangeConflictException("failed to apply JSON patch: " + change, e); + } + + // Apply only when the contents are really different. + if (!newJsonNode.equals(oldJsonNode)) { + applyPathEdit(dirCache, new InsertJson(changePath, inserter, newJsonNode)); + numEdits++; + } + break; + } + case APPLY_TEXT_PATCH: + final Patch patch = DiffUtils.parseUnifiedDiff( + Util.stringToLines(sanitizeText((String) change.content()))); + + final String sanitizedOldText; + final List sanitizedOldTextLines; + if (oldContent != null) { + sanitizedOldText = sanitizeText(new String(oldContent, UTF_8)); + sanitizedOldTextLines = Util.stringToLines(sanitizedOldText); + } else { + sanitizedOldText = null; + sanitizedOldTextLines = Collections.emptyList(); + } + + final String newText; + try { + final List newTextLines = DiffUtils.patch(sanitizedOldTextLines, patch); + if (newTextLines.isEmpty()) { + newText = ""; + } else { + final StringJoiner joiner = new StringJoiner("\n", "", "\n"); + for (String line : newTextLines) { + joiner.add(line); + } + newText = joiner.toString(); + } + } catch (Exception e) { + throw new ChangeConflictException("failed to apply text patch: " + change, e); + } + + // Apply only when the contents are really different. + if (!newText.equals(sanitizedOldText)) { + applyPathEdit(dirCache, new InsertText(changePath, inserter, newText)); + numEdits++; + } + break; + } + } + } catch (CentralDogmaException | IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new StorageException("failed to apply changes on revision " + baseRevision, e); + } + return numEdits; + } + + private static void applyPathEdit(DirCache dirCache, PathEdit edit) { + final DirCacheEditor e = dirCache.editor(); + e.add(edit); + e.finish(); + } + + /** + * Applies recursive directory edits. + * + * @param oldDir the path to the directory to make a recursive change + * @param newDir the path to the renamed directory, or {@code null} to remove the directory. + * + * @return {@code true} if any edits were made to {@code dirCache}, {@code false} otherwise + */ + private static boolean applyDirectoryEdits(DirCache dirCache, + String oldDir, @Nullable String newDir, Change change) { + + if (!oldDir.endsWith("/")) { + oldDir += '/'; + } + if (newDir != null && !newDir.endsWith("/")) { + newDir += '/'; + } + + final byte[] rawOldDir = Constants.encode(oldDir); + final byte[] rawNewDir = newDir != null ? Constants.encode(newDir) : null; + final int numEntries = dirCache.getEntryCount(); + DirCacheEditor editor = null; + + loop: + for (int i = 0; i < numEntries; i++) { + final DirCacheEntry e = dirCache.getEntry(i); + final byte[] rawPath = e.getRawPath(); + + // Ensure that there are no entries under the newDir; we have a conflict otherwise. + if (rawNewDir != null) { + boolean conflict = true; + if (rawPath.length > rawNewDir.length) { + // Check if there is a file whose path starts with 'newDir'. + for (int j = 0; j < rawNewDir.length; j++) { + if (rawNewDir[j] != rawPath[j]) { + conflict = false; + break; + } + } + } else if (rawPath.length == rawNewDir.length - 1) { + // Check if there is a file whose path is exactly same with newDir without trailing '/'. + for (int j = 0; j < rawNewDir.length - 1; j++) { + if (rawNewDir[j] != rawPath[j]) { + conflict = false; + break; + } + } + } else { + conflict = false; + } + + if (conflict) { + throw new ChangeConflictException("target directory exists already: " + change); + } + } + + // Skip the entries that do not belong to the oldDir. + if (rawPath.length <= rawOldDir.length) { + continue; + } + for (int j = 0; j < rawOldDir.length; j++) { + if (rawOldDir[j] != rawPath[j]) { + continue loop; + } + } + + // Do not create an editor until we find an entry to rename/remove. + // We can tell if there was any matching entries or not from the nullness of editor later. + if (editor == null) { + editor = dirCache.editor(); + editor.add(new DeleteTree(oldDir)); + if (newDir == null) { + // Recursive removal + break; + } + } + + assert newDir != null; // We should get here only when it's a recursive rename. + + final String oldPath = e.getPathString(); + final String newPath = newDir + oldPath.substring(oldDir.length()); + editor.add(new CopyOldEntry(newPath, e)); + } + + if (editor != null) { + editor.finish(); + return true; + } else { + return false; + } + } + + private static void reportNonExistentEntry(Change change) { + throw new ChangeConflictException("non-existent file/directory: " + change); + } + + RevisionAndEntries commit(@Nullable Revision prevRevision, Revision nextRevision, + long commitTimeMillis, Author author, String summary, String detail, + Markup markup, Iterable> changes, boolean allowEmpty) { + requireNonNull(author, "author"); + requireNonNull(summary, "summary"); + requireNonNull(changes, "changes"); + requireNonNull(detail, "detail"); + requireNonNull(markup, "markup"); + + assert prevRevision == null || prevRevision.major() > 0; + assert nextRevision.major() > 0; + + try (ObjectInserter inserter = jGitRepository.newObjectInserter(); + ObjectReader reader = jGitRepository.newObjectReader(); + RevWalk revWalk = newRevWalk(reader)) { + + final ObjectId prevTreeId; + if (prevRevision != null) { + prevTreeId = toTree(commitIdDatabase, revWalk, prevRevision); + } else { + prevTreeId = null; + } + + // The staging area that keeps the entries of the new tree. + // It starts with the entries of the tree at the prevRevision (or with no entries if the + // prevRevision is the initial commit), and then this method will apply the requested changes + // to build the new tree. + final DirCache dirCache = DirCache.newInCore(); + + // Apply the changes and retrieve the list of the affected files. + final int numEdits = applyChanges(prevRevision, prevTreeId, dirCache, changes); + + // Reject empty commit if necessary. + final List diffEntries; + boolean isEmpty = numEdits == 0; + if (isEmpty || prevTreeId == null) { + // We do not need the diffEntries when creating a new repository which means prevTreeId is null. + diffEntries = ImmutableList.of(); + } else { + // Even if there are edits, the resulting tree might be identical with the previous tree. + final CanonicalTreeParser p = new CanonicalTreeParser(); + p.reset(reader, prevTreeId); + final DiffFormatter diffFormatter = new DiffFormatter(null); + diffFormatter.setRepository(jGitRepository); + diffEntries = diffFormatter.scan(p, new DirCacheIterator(dirCache)); + isEmpty = diffEntries.isEmpty(); + } + + if (!allowEmpty && isEmpty) { + throw new RedundantChangeException( + "changes did not change anything in " + + project.name() + '/' + originalRepoName + + " at revision " + (prevRevision != null ? prevRevision.major() : 0) + ": " + changes); + } + + // flush the current index to repository and get the result tree object id. + final ObjectId nextTreeId = dirCache.writeTree(inserter); + + // build a commit object + final PersonIdent personIdent = new PersonIdent(author.name(), author.email(), + commitTimeMillis / 1000L * 1000L, 0); + + final CommitBuilder commitBuilder = new CommitBuilder(); + + commitBuilder.setAuthor(personIdent); + commitBuilder.setCommitter(personIdent); + commitBuilder.setTreeId(nextTreeId); + commitBuilder.setEncoding(UTF_8); + + // Write summary, detail and revision to commit's message as JSON format. + commitBuilder.setMessage(CommitUtil.toJsonString(summary, detail, markup, nextRevision)); + + // if the head commit exists, use it as the parent commit. + if (prevRevision != null) { + commitBuilder.setParentId(commitIdDatabase.get(prevRevision)); + } + + final ObjectId nextCommitId = inserter.insert(commitBuilder); + inserter.flush(); + + // tagging the revision object, for history lookup purpose. + commitIdDatabase.put(nextRevision, nextCommitId); + doRefUpdate(jGitRepository, revWalk, R_HEADS_MASTER, nextCommitId); + + return new RevisionAndEntries(nextRevision, diffEntries); + } catch (CentralDogmaException | IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new StorageException("failed to push at '" + project.name() + '/' + originalRepoName + '\'', + e); + } + } + + @VisibleForTesting + static void doRefUpdate(Repository jGitRepo, RevWalk revWalk, + String ref, ObjectId commitId) throws IOException { + if (ref.startsWith(Constants.R_TAGS)) { + final Ref oldRef = jGitRepo.exactRef(ref); + if (oldRef != null) { + throw new StorageException("tag ref exists already: " + ref); + } + } + + final RefUpdate refUpdate = jGitRepo.updateRef(ref); + refUpdate.setNewObjectId(commitId); + + final Result res = refUpdate.update(revWalk); + switch (res) { + case NEW: + case FAST_FORWARD: + // Expected + break; + default: + throw new StorageException("unexpected refUpdate state: " + res); + } + } + void close() { try { commitIdDatabase.close(); @@ -106,4 +1141,73 @@ void close() { } closeJGitRepo(jGitRepository); } + + static final class RevisionAndEntries { + final Revision revision; + final List diffEntries; + + RevisionAndEntries(Revision revision, List diffEntries) { + this.revision = revision; + this.diffEntries = diffEntries; + } + } + + // PathEdit implementations which is used when applying changes. + + private static final class InsertText extends PathEdit { + private final ObjectInserter inserter; + private final String text; + + InsertText(String entryPath, ObjectInserter inserter, String text) { + super(entryPath); + this.inserter = inserter; + this.text = text; + } + + @Override + public void apply(DirCacheEntry ent) { + try { + ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, text.getBytes(UTF_8))); + ent.setFileMode(FileMode.REGULAR_FILE); + } catch (IOException e) { + throw new StorageException("failed to create a new text blob", e); + } + } + } + + private static final class InsertJson extends PathEdit { + private final ObjectInserter inserter; + private final JsonNode jsonNode; + + InsertJson(String entryPath, ObjectInserter inserter, JsonNode jsonNode) { + super(entryPath); + this.inserter = inserter; + this.jsonNode = jsonNode; + } + + @Override + public void apply(DirCacheEntry ent) { + try { + ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, Jackson.writeValueAsBytes(jsonNode))); + ent.setFileMode(FileMode.REGULAR_FILE); + } catch (IOException e) { + throw new StorageException("failed to create a new JSON blob", e); + } + } + } + + private static final class CopyOldEntry extends PathEdit { + private final DirCacheEntry oldEntry; + + CopyOldEntry(String entryPath, DirCacheEntry oldEntry) { + super(entryPath); + this.oldEntry = oldEntry; + } + + @Override + public void apply(DirCacheEntry ent) { + ent.setFileMode(oldEntry.getFileMode()); + ent.setObjectId(oldEntry.getObjectId()); + } + } } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryTest.java index 93ea86d75..ef0a3a194 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryTest.java @@ -540,20 +540,17 @@ void testDiff_invalidParameters() { assertThat(repo.diff(revision1, revision2, "non_existing_path").join()).isEmpty(); assertThatThrownBy(() -> repo.diff(revision1, revision2, (String) null).join()) - .isInstanceOf(CompletionException.class) - .hasCauseInstanceOf(NullPointerException.class); + .isInstanceOf(NullPointerException.class); assertThatThrownBy(() -> repo.diff(null, revision2, path).join()) - .isInstanceOf(CompletionException.class) - .hasCauseInstanceOf(NullPointerException.class); + .isInstanceOf(NullPointerException.class); assertThatThrownBy(() -> repo.diff(revision1, new Revision(revision2.major() + 1), path).join()) - .isInstanceOf(CompletionException.class) - .hasCauseInstanceOf(RevisionNotFoundException.class); + .hasRootCauseInstanceOf(RevisionNotFoundException.class); assertThatThrownBy(() -> repo.diff(new Revision(revision2.major() + 1), revision2, path).join()) .isInstanceOf(CompletionException.class) - .hasCauseInstanceOf(RevisionNotFoundException.class); + .hasRootCauseInstanceOf(RevisionNotFoundException.class); } @Test @@ -873,12 +870,10 @@ void testHistory_parameterCheck() { .hasCauseInstanceOf(RevisionNotFoundException.class); assertThatThrownBy(() -> repo.history(null, HEAD, "non_existing_path").join()) - .isInstanceOf(CompletionException.class) - .hasCauseInstanceOf(NullPointerException.class); + .isInstanceOf(NullPointerException.class); assertThatThrownBy(() -> repo.history(HEAD, null, "non_existing_path").join()) - .isInstanceOf(CompletionException.class) - .hasCauseInstanceOf(NullPointerException.class); + .isInstanceOf(NullPointerException.class); } @Test diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2DiffTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2DiffTest.java new file mode 100644 index 000000000..f41c536f5 --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2DiffTest.java @@ -0,0 +1,95 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryV2HistoryTest.createRepository; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.stream.IntStream; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.common.RevisionNotFoundException; + +class GitRepositoryV2DiffTest { + + @TempDir + static File repoDir; + private static GitRepositoryV2 repo; + + @BeforeAll + static void setUp() { + // The repository contains commits from 10 to 20(inclusive). + repo = createRepository(repoDir, 10, 20); + } + + @AfterAll + static void tearDown() { + repo.internalClose(); + } + + @Test + void diff() { + Map> diffs = repo.diff(new Revision(18), new Revision(20), "/**").join(); + assertThat(diffs.values()).containsExactly( + Change.ofTextUpsert("/file_19.txt", "19" + System.lineSeparator()), + Change.ofTextUpsert("/file_20.txt", "20" + System.lineSeparator())); + + // It's same when the revisions are reversed. + assertThat(repo.diff(Revision.HEAD, new Revision(18), "/**").join().values()) + .containsExactlyInAnyOrderElementsOf(diffs.values()); + + diffs = repo.diff(new Revision(10), new Revision(20), "/**").join(); + assertThat(diffs).hasSize(10); // from 11 to 20. + final List> expected = + IntStream.range(11, 21) + .mapToObj(i -> Change.ofTextUpsert("/file_" + i + ".txt", i + System.lineSeparator())) + .collect(toImmutableList()); + assertThat(diffs.values()).containsExactlyInAnyOrderElementsOf(expected); + + // If the revision is INIT, the result is same as Revision(10) which is the first revision + // of the new primary repository. + assertThat(repo.diff(Revision.INIT, new Revision(20), "/**").join().values()) + .containsExactlyInAnyOrderElementsOf(diffs.values()); + + assertThat(repo.diff(Revision.INIT, Revision.INIT, "/**").join()).isEmpty(); + } + + @Test + void invalidRevisionDiff() { + assertThatThrownBy(() -> repo.diff(new Revision(9), new Revision(10), "/**").join()) + .hasRootCauseInstanceOf(RevisionNotFoundException.class) + .hasRootCauseMessage("revision: Revision(9) (expected: >= 10)"); + + assertThatThrownBy(() -> repo.diff(Revision.INIT, new Revision(9), "/**").join()) + .hasRootCauseInstanceOf(RevisionNotFoundException.class) + .hasRootCauseMessage("revision: Revision(9) (expected: >= 10)"); + + assertThatThrownBy(() -> repo.diff(Revision.HEAD, new Revision(9), "/**").join()) + .hasRootCauseInstanceOf(RevisionNotFoundException.class) + .hasRootCauseMessage("revision: Revision(9) (expected: >= 10)"); + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2FindTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2FindTest.java new file mode 100644 index 000000000..0c08d7ac7 --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2FindTest.java @@ -0,0 +1,118 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryV2HistoryTest.createRepository; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.stream.IntStream; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.google.common.collect.Iterables; + +import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.common.RevisionNotFoundException; + +class GitRepositoryV2FindTest { + + @TempDir + static File repoDir; + private static GitRepositoryV2 repo; + + @BeforeAll + static void setUp() { + // The repository contains commits from 10 to 20(inclusive). + repo = createRepository(repoDir, 10, 20); + } + + @AfterAll + static void tearDown() { + repo.internalClose(); + } + + @Test + void normalFind() { + Map> entries = repo.find(new Revision(20), "/**").join(); + assertThat(entries).hasSize(19); // 2 to 20 (inclusively) + + entries = repo.find(new Revision(15), "/file_15.txt").join(); + assertThat(entries.size()).isOne(); + Entry entry = Iterables.get(entries.values(), 0); + assertThat(entry.revision().major()).isEqualTo(15); + assertThat(entry.path()).isEqualTo("/file_15.txt"); + + // file_2.txt is committed together when the current primary repository is created. + entries = repo.find(new Revision(10), "/file_2.txt").join(); + assertThat(entries.size()).isOne(); + entry = Iterables.get(entries.values(), 0); + assertThat(entry.revision().major()).isEqualTo(10); + assertThat(entry.path()).isEqualTo("/file_2.txt"); + + entries = repo.find(new Revision(10), "/**").join(); + assertThat(entries).hasSize(9); // from 2 ~ 10 + final List expected = IntStream.range(2, 11).mapToObj(i -> "/file_" + i + ".txt") + .collect(toImmutableList()); + final List actual = entries.values().stream().map(Entry::path).collect(toImmutableList()); + assertThat(actual).containsExactlyInAnyOrderElementsOf(expected); + + // If the revision is INIT, the result is same as Revision(10) which is the first revision + // of the new primary repository. + assertThat(repo.find(Revision.INIT, "/**").join()).isEqualTo(entries); + } + + @Test + void invalidRevisionFind() { + assertThatThrownBy(() -> repo.find(new Revision(9), "/**").join()) + .hasCauseInstanceOf(RevisionNotFoundException.class) + .hasMessageContaining("revision: Revision(9) (expected: >= 10)"); + } + + @Test + void findLatestRevision() { + Revision revision = repo.findLatestRevision(new Revision(20), "/**").join(); + assertThat(revision).isNull(); // there's no commits after the revision 20 so it's null. + + revision = repo.findLatestRevision(new Revision(19), "/**").join(); + assertThat(revision.major()).isEqualTo(20); + + revision = repo.findLatestRevision(new Revision(13), "/file_15.txt").join(); + // file_15 is committed at the revision 15 which is before the revision 19. + assertThat(revision.major()).isEqualTo(20); + + revision = repo.findLatestRevision(new Revision(19), "/file_15.txt").join(); + // file_15 is committed at the revision 15 which is before the revision 19. + assertThat(revision).isNull(); + + revision = repo.findLatestRevision(new Revision(10), "/file_5.txt").join(); + assertThat(revision).isNull(); // It's null because file_5 is not changed after the revision 10. + + revision = repo.findLatestRevision(new Revision(4), "/file_5.txt").join(); + assertThat(revision.major()).isEqualTo(20); + + revision = repo.findLatestRevision(new Revision(6), "/file_5.txt").join(); + assertThat(revision.major()).isEqualTo(20); + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2HistoryTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2HistoryTest.java new file mode 100644 index 000000000..329c625e0 --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2HistoryTest.java @@ -0,0 +1,115 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryV2PromotionTest.addCommits; +import static java.util.concurrent.ForkJoinPool.commonPool; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; + +import java.io.File; +import java.util.List; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Commit; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.common.RevisionNotFoundException; +import com.linecorp.centraldogma.server.storage.project.Project; + +class GitRepositoryV2HistoryTest { + + @TempDir + static File repoDir; + private static GitRepositoryV2 repo; + + @BeforeAll + static void setUp() { + // The repository contains commits from 10 to 20(inclusive). + repo = createRepository(repoDir, 10, 20); + } + + @AfterAll + static void tearDown() { + repo.internalClose(); + } + + @Test + void invalidRevision() { + // The larger of the two revisions should be greater or equal to the firstRevision. + assertThatThrownBy(() -> repo.history(new Revision(2), new Revision(9), "/**") + .join()) + .hasCauseInstanceOf(RevisionNotFoundException.class) + .hasMessageContaining("revision: Revision(2) (expected: >= 10)"); + assertThatThrownBy(() -> repo.history(new Revision(9), new Revision(2), "/**") + .join()) + .hasCauseInstanceOf(RevisionNotFoundException.class) + .hasMessageContaining("revision: Revision(9) (expected: >= 10)"); + + // The larger of the two revisions should be lower or equal to the headRevision. + assertThatThrownBy(() -> repo.history(new Revision(2), new Revision(21), "/**") + .join()) + .hasCauseInstanceOf(RevisionNotFoundException.class) + .hasMessageContaining("revision: Revision(2) (expected: >= 10)"); + assertThatThrownBy(() -> repo.history(new Revision(21), new Revision(2), "/**") + .join()) + .hasCauseInstanceOf(RevisionNotFoundException.class) + .hasMessageContaining("revision: Revision(21) (expected: <= 20)"); + + assertThatThrownBy(() -> repo.history(new Revision(2), new Revision(10), "/**") + .join()) + .hasCauseInstanceOf(RevisionNotFoundException.class) + .hasMessageContaining("revision: Revision(2) (expected: >= 10)"); + + assertThatThrownBy(() -> repo.history(new Revision(10), new Revision(2), "/**") + .join()) + .hasCauseInstanceOf(RevisionNotFoundException.class) + .hasMessageContaining("revision: Revision(2) (expected: >= 10)"); + } + + @Test + void normalHistoryCall() { + List commits = repo.history(new Revision(10), new Revision(14), "/**") + .join(); + assertThat(commits).hasSize(5); + + commits = repo.history(Revision.INIT, Revision.INIT, "/**").join(); + System.err.println(commits); + assertThat(commits).hasSize(1); + } + + static GitRepositoryV2 createRepository(File repoDir, int from, int to) { + final GitRepositoryV2 repo = new GitRepositoryV2(mock(Project.class), + new File(repoDir, "test_repo"), commonPool(), + 0L, Author.SYSTEM, null); + addCommits(repo, 2, from); + repo.removeOldCommits(from - 2, 0); + // The headRevision of the secondary repository is now from revision. + + addCommits(repo, from + 1, to); + repo.removeOldCommits(to - from - 1, 0); + + final CommitIdDatabase commitIdDatabase = repo.primaryRepo.commitIdDatabase(); + assertThat(commitIdDatabase.firstRevision().major()).isSameAs(10); + assertThat(commitIdDatabase.headRevision().major()).isSameAs(20); + return repo; + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java index 1fa020293..0a2bdb6ea 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java @@ -91,9 +91,11 @@ void promote() { assertThat(repo.secondaryRepo.commitIdDatabase().firstRevision().major()).isEqualTo(23); assertThat(repo.secondaryRepo.commitIdDatabase().headRevision().major()).isEqualTo(23); assertThat(repo.secondaryRepo.secondCommitCreationTimeInstant()).isNull(); + + repo.internalClose(); } - private static void addCommits(GitRepositoryV2 repo, int start, int end) { + static void addCommits(GitRepositoryV2 repo, int start, int end) { for (int i = start; i <= end; i++) { repo.commit(Revision.HEAD, i * 1000, Author.SYSTEM, "Summary" + i, "Detail", Markup.PLAINTEXT, diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtilTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepositoryTest.java similarity index 88% rename from server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtilTest.java rename to server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepositoryTest.java index 2c4b253c8..a1ab1fba9 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtilTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepositoryTest.java @@ -15,6 +15,7 @@ */ package com.linecorp.centraldogma.server.internal.storage.repository.git; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.InternalRepository.doRefUpdate; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; @@ -29,7 +30,7 @@ import com.linecorp.centraldogma.server.storage.StorageException; -final class GitRepositoryUtilTest { +class InternalRepositoryTest { @Test void testDoUpdateRef() throws Exception { @@ -49,13 +50,13 @@ private static void testDoUpdateRef(String ref, ObjectId commitId, boolean tagEx lenient().when(jGitRepo.updateRef(ref)).thenReturn(refUpdate); lenient().when(refUpdate.update(revWalk)).thenReturn(RefUpdate.Result.NEW); - GitRepositoryUtil.doRefUpdate(jGitRepo, revWalk, ref, commitId); + doRefUpdate(jGitRepo, revWalk, ref, commitId); when(refUpdate.update(revWalk)).thenReturn(RefUpdate.Result.FAST_FORWARD); - GitRepositoryUtil.doRefUpdate(jGitRepo, revWalk, ref, commitId); + doRefUpdate(jGitRepo, revWalk, ref, commitId); when(refUpdate.update(revWalk)).thenReturn(RefUpdate.Result.LOCK_FAILURE); - assertThatThrownBy(() -> GitRepositoryUtil.doRefUpdate(jGitRepo, revWalk, ref, commitId)) + assertThatThrownBy(() -> doRefUpdate(jGitRepo, revWalk, ref, commitId)) .isInstanceOf(StorageException.class); } From c0bafb308aa7fdd4843a2fce246857f9fdea4b25 Mon Sep 17 00:00:00 2001 From: minwoox Date: Mon, 13 Sep 2021 18:06:25 +0900 Subject: [PATCH 03/14] Add tests --- .../server/CentralDogmaConfig.java | 3 +- .../repository/git/CommitIdDatabase.java | 7 +- .../storage/repository/git/GitRepository.java | 2 +- .../repository/git/GitRepositoryV2.java | 105 +++++++----------- .../repository/git/InternalRepository.java | 15 ++- .../git/RepositoryMetadataDatabase.java | 29 ++--- ...linecorp.centraldogma.server.plugin.Plugin | 2 +- .../git/GitRepositoryV2PromotionTest.java | 10 +- .../git/GitRepositoryV2WatchTest.java | 66 +++++++++++ 9 files changed, 142 insertions(+), 97 deletions(-) create mode 100644 server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2WatchTest.java diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaConfig.java b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaConfig.java index d7fefe36b..a816c4a3b 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaConfig.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaConfig.java @@ -169,8 +169,7 @@ public final class CentralDogmaConfig { @JsonProperty("accessLogFormat") @Nullable String accessLogFormat, @JsonProperty("authentication") @Nullable AuthConfig authConfig, @JsonProperty("writeQuotaPerRepository") @Nullable QuotaConfig writeQuotaPerRepository, - @JsonProperty("commitRetention") @Nullable - CommitRetentionConfig commitRetentionConfig) { + @JsonProperty("commitRetention") @Nullable CommitRetentionConfig commitRetentionConfig) { this.dataDir = requireNonNull(dataDir, "dataDir"); this.ports = ImmutableList.copyOf(requireNonNull(ports, "ports")); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java index 0206fc89f..ed94a8c89 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java @@ -204,6 +204,7 @@ void put(Revision revision, ObjectId commitId) { private synchronized void put(Revision revision, ObjectId commitId, boolean isNewHeadRevision) { if (isNewHeadRevision) { + final Revision headRevision = this.headRevision; if (headRevision != null) { final Revision expected = headRevision.forward(1); checkState(revision.equals(expected), "incorrect revision: %s (expected: %s)", @@ -234,10 +235,11 @@ private synchronized void put(Revision revision, ObjectId commitId, boolean isNe throw new StorageException("failed to update the commit ID database: " + path, e); } + final Revision headRevision = this.headRevision; if (isNewHeadRevision || headRevision == null || headRevision.major() < revision.major()) { - headRevision = revision; + this.headRevision = revision; } } @@ -292,8 +294,7 @@ private void rebuild(Repository gitRepo, RevWalk revWalk, Revision headRevision, currentId = revCommit.getParent(0); break; default: - throw new StorageException("found more than one parent: " + - gitRepo.getDirectory()); + throw new StorageException("found more than one parent: " + gitRepo.getDirectory()); } revCommit = revWalk.parseCommit(currentId); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java index 5225d31dc..9bc370ec0 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java @@ -1617,7 +1617,7 @@ private static void deleteCruft(File repoDir) { @Override public void removeOldCommits(int minRetentionCommits, int minRetentionDays) { - // Not supported. + throw new UnsupportedOperationException(); } @Override diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java index 1347e9030..8abb92105 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 LINE Corporation + * Copyright 2021 LINE Corporation * * LINE Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance @@ -13,7 +13,6 @@ * License for the specific language governing permissions and limitations * under the License. */ - package com.linecorp.centraldogma.server.internal.storage.repository.git; import static com.google.common.base.Preconditions.checkArgument; @@ -83,13 +82,8 @@ class GitRepositoryV2 implements com.linecorp.centraldogma.server.storage.reposi private static final Logger logger = LoggerFactory.getLogger(GitRepositoryV2.class); - private static final String PRIMARY_REPOSITORY_SUFFIX = "_primary"; - private static final String FOLLOWER_REPOSITORY_SUFFIX = "_secondary"; - static final String R_HEADS_MASTER = Constants.R_HEADS + Constants.MASTER; - private static final byte[] EMPTY_BYTE = new byte[0]; - /** * Opens an existing Git-backed repository. * @@ -105,8 +99,7 @@ static GitRepositoryV2 open(Project parent, File repositoryDir, private final ReadWriteLock rwLock = new ReentrantReadWriteLock(); private final Project parent; - private final File repositoryDir; - private final String name; + private final String originalRepoName; private final Executor repositoryWorker; private final RepositoryMetadataDatabase repoMetadata; @@ -139,10 +132,8 @@ static GitRepositoryV2 open(Project parent, File repositoryDir, GitRepositoryV2(Project parent, File repositoryDir, Executor repositoryWorker, long creationTimeMillis, Author author, @Nullable RepositoryCache cache) { - this.parent = requireNonNull(parent, "parent"); - this.repositoryDir = requireNonNull(repositoryDir, "repositoryDir"); - name = repositoryDir.getName(); + originalRepoName = requireNonNull(repositoryDir, "repositoryDir").getName(); this.repositoryWorker = requireNonNull(repositoryWorker, "repositoryWorker"); this.cache = cache; @@ -163,7 +154,7 @@ static GitRepositoryV2 open(Project parent, File repositoryDir, repoMetadata = new RepositoryMetadataDatabase(repositoryDir, true); final File primaryRepoDir = repoMetadata.primaryRepoDir(); try { - primaryRepo = InternalRepository.of(parent, name, primaryRepoDir, Revision.INIT, + primaryRepo = InternalRepository.of(parent, originalRepoName, primaryRepoDir, Revision.INIT, creationTimeMillis, author, ImmutableList.of()); } catch (Throwable t) { repoMetadata.close(); @@ -189,8 +180,7 @@ private static boolean isEmpty(File dir) throws IOException { private GitRepositoryV2(Project parent, File repoDir, Executor repositoryWorker, @Nullable RepositoryCache cache) { this.parent = requireNonNull(parent, "parent"); - this.repositoryDir = requireNonNull(repoDir, "repoDir"); - name = repoDir.getName(); + originalRepoName = requireNonNull(repoDir, "repoDir").getName(); this.repositoryWorker = requireNonNull(repositoryWorker, "repositoryWorker"); this.cache = cache; @@ -207,14 +197,15 @@ private GitRepositoryV2(Project parent, File repoDir, Executor repositoryWorker, this.repoMetadata = repoMetadata; try { final InternalRepository primaryRepo = - InternalRepository.open(parent, name, repoMetadata.primaryRepoDir(), true); + InternalRepository.open(parent, originalRepoName, repoMetadata.primaryRepoDir(), true); assert primaryRepo != null; this.primaryRepo = primaryRepo; headRevision = primaryRepo.headRevision(); final Commit initialCommit = currentInitialCommit(primaryRepo); creationTimeMillis = initialCommit.when(); author = initialCommit.author(); - secondaryRepo = InternalRepository.open(parent, name, repoMetadata.secondaryRepoDir(), false); + secondaryRepo = InternalRepository.open(parent, originalRepoName, + repoMetadata.secondaryRepoDir(), false); } catch (Throwable t) { repoMetadata.close(); closeInternalRepository(primaryRepo); @@ -283,6 +274,7 @@ void close(Supplier failureCauseSupplier) { try { closeInternalRepository(primaryRepo); closeInternalRepository(secondaryRepo); + repoMetadata.close(); } finally { rwLock.writeLock().unlock(); commitWatchers.close(failureCauseSupplier); @@ -308,7 +300,7 @@ public Project parent() { @Override public String name() { - return name; + return originalRepoName; } @Override @@ -386,11 +378,7 @@ public CompletableFuture>> find( readLock(); try { final Revision normalizedRevision = normalizeNow(revision); - // Query on a non-exist revision will return empty result. final Revision headRevision = this.headRevision; - if (normalizedRevision.compareTo(headRevision) > 0) { - return Collections.emptyMap(); - } if ("/".equals(pathPattern)) { return Collections.singletonMap(pathPattern, Entry.ofDirectory(normalizedRevision, "/")); } @@ -538,11 +526,10 @@ public CompletableFuture> history(Revision from, Revision to, Strin final ServiceRequestContext ctx = context(); return CompletableFuture.supplyAsync(() -> { failFastIfTimedOut(this, logger, ctx, "history", from, to, pathPattern, maxCommits); - final int maxCommits0 = Math.min(maxCommits, MAX_MAX_COMMITS); readLock(); try { final RevisionRange range = normalizeNow(from, to); - return primaryRepo.listCommits(pathPattern, maxCommits0, range); + return primaryRepo.listCommits(pathPattern, Math.min(maxCommits, MAX_MAX_COMMITS), range); } finally { readUnlock(); } @@ -589,7 +576,9 @@ private Revision blockingFindLatestRevision(Revision lastKnownRevision, String p range = new RevisionRange(normalizeNow(lastKnownRevision, false), headRevision); if (range.from().isLowerThan(primaryRepo.firstRevision())) { // We should return range.to() without comparing two tree. It's because: - // - the entry might be changed between the lastKnownRevision and first revision. + // - the entry might be changed between the lastKnownRevision(in the previous repository + // that is removed and superseded by the current primaryRepo) + // and first revision(in the current primaryRepo). // - but the previous commits before the first revision is packed and committed to the // primaryRepo at once, we really don't know if there was a change. // - so it's safe to return the latest revision so that the client get notified when watching. @@ -634,7 +623,7 @@ public CompletableFuture watch(Revision lastKnownRevision, String path requireNonNull(pathPattern, "pathPattern"); final ServiceRequestContext ctx = context(); - final Revision normLastKnownRevision = normalizeNow(lastKnownRevision); + final Revision normLastKnownRevision = normalizeNow(lastKnownRevision, false); final CompletableFuture future = new CompletableFuture<>(); CompletableFuture.runAsync(() -> { failFastIfTimedOut(this, logger, ctx, "watch", lastKnownRevision, pathPattern); @@ -684,9 +673,7 @@ private void writeUnLock() { } private Commit currentInitialCommit(InternalRepository primaryRepo) { - final Revision firstRevision = primaryRepo.commitIdDatabase().firstRevision(); - // This method is called when opening an existing repository so firstRevision is not null. - assert firstRevision != null; + final Revision firstRevision = primaryRepo.firstRevision(); final RevisionRange range = new RevisionRange(firstRevision, firstRevision); return primaryRepo.listCommits(ALL_PATH, 1, range).get(0); } @@ -709,7 +696,7 @@ public void removeOldCommits(int minRetentionCommits, int minRetentionDays) { final InternalRepository secondaryRepo = this.secondaryRepo; if (secondaryRepo != null) { if (exceedsMinRetention(secondaryRepo, minRetentionCommits, minRetentionDays)) { - promoteSecondaryRepo(); + promoteSecondaryRepo(secondaryRepo); } } else if (exceedsMinRetention(primaryRepo, minRetentionCommits, minRetentionDays)) { createSecondaryRepo(); @@ -719,11 +706,8 @@ public void removeOldCommits(int minRetentionCommits, int minRetentionDays) { private boolean exceedsMinRetention(InternalRepository repo, int minRetentionCommits, int minRetentionDays) { final CommitIdDatabase commitIdDatabase = repo.commitIdDatabase(); - final Revision headRevision = commitIdDatabase.headRevision(); - final Revision firstRevision = commitIdDatabase.firstRevision(); - if (headRevision == null || firstRevision == null) { - return false; - } + final Revision headRevision = repo.headRevision(); + final Revision firstRevision = repo.firstRevision(); if (minRetentionCommits != 0) { if (headRevision.major() - firstRevision.major() <= minRetentionCommits) { return false; @@ -738,32 +722,29 @@ private boolean exceedsMinRetention(InternalRepository repo, return true; } - private void promoteSecondaryRepo() { - final InternalRepository secondaryRepo = this.secondaryRepo; - if (secondaryRepo != null) { - writeLock(); - try { - logger.info("Promoting the secondary repository in {}/{}.", parent.name(), name); - repoMetadata.setPrimaryRepoDir(secondaryRepo.jGitRepo().getDirectory()); - final InternalRepository primaryRepo = this.primaryRepo; - this.primaryRepo = secondaryRepo; - this.secondaryRepo = null; - repositoryWorker.execute(() -> { - closeInternalRepository(primaryRepo); - final File repoDir = primaryRepo.repoDir(); - try { - if (!deleteDirectory(repoDir)) { - logger.warn("Failed to delete the old primary repository: {}", repoDir); - } - } catch (Throwable t) { + private void promoteSecondaryRepo(InternalRepository secondaryRepo) { + writeLock(); + try { + logger.info("Promoting the secondary repository in {}/{}.", parent.name(), originalRepoName); + repoMetadata.setPrimaryRepoDir(secondaryRepo.jGitRepo().getDirectory()); + final InternalRepository primaryRepo = this.primaryRepo; + this.primaryRepo = secondaryRepo; + this.secondaryRepo = null; + repositoryWorker.execute(() -> { + closeInternalRepository(primaryRepo); + final File repoDir = primaryRepo.repoDir(); + try { + if (!deleteDirectory(repoDir)) { logger.warn("Failed to delete the old primary repository: {}", repoDir); } - }); - logger.info("Promotion is done for {}/{}. {} is now the primary.", - parent.name(), name, secondaryRepo.repoDir()); - } finally { - writeUnLock(); - } + } catch (Throwable t) { + logger.warn("Failed to delete the old primary repository: {}", repoDir); + } + }); + logger.info("Promotion is done for {}/{}. {} is now the primary.", + parent.name(), originalRepoName, secondaryRepo.repoDir()); + } finally { + writeUnLock(); } createSecondaryRepo(); @@ -787,12 +768,12 @@ private void createSecondaryRepo() { writeUnLock(); logger.info("Creating the secondary repository in {}/{} with the head revision: {}.", - parent.name(), name, headRevision); + parent.name(), originalRepoName, headRevision); final Map> entries = find(headRevision, ALL_PATH, ImmutableMap.of()).join(); final List> changes = toChanges(entries); final File secondaryRepoDir = repoMetadata.secondaryRepoDir(); final InternalRepository secondaryRepo = - InternalRepository.of(parent, name, secondaryRepoDir, headRevision, + InternalRepository.of(parent, originalRepoName, secondaryRepoDir, headRevision, creationTimeMillis, author, changes); writeLock(); try { @@ -814,7 +795,7 @@ private void createSecondaryRepo() { writeUnLock(); } logger.info("The secondary repository {} is created in {}/{}.", - secondaryRepoDir.getName(), parent.name(), name, headRevision); + secondaryRepoDir.getName(), parent.name(), originalRepoName, headRevision); } catch (Throwable t) { logger.warn("Failed to create the secondary repository", t); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java index 8d2f59cb0..eb3196ecb 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java @@ -187,10 +187,10 @@ private static void deleteCruft(File repoDir) { } private static void createEmptyJGitRepo(File repositoryDir) throws IOException { - try (org.eclipse.jgit.lib.Repository initRepository = buildJGitRepo(repositoryDir)) { - initRepository.create(true); + try (org.eclipse.jgit.lib.Repository jGitRepository = buildJGitRepo(repositoryDir)) { + jGitRepository.create(true); - final StoredConfig config = initRepository.getConfig(); + final StoredConfig config = jGitRepository.getConfig(); // Update the repository settings to upgrade to format version 1 and reftree. config.setInt(CONFIG_CORE_SECTION, null, CONFIG_KEY_REPO_FORMAT_VERSION, 1); @@ -267,16 +267,19 @@ private static Revision uncachedHeadRevision(Project parent, String name, Reposi private static RevWalk newRevWalk(Repository jGitRepository) { final RevWalk revWalk = new RevWalk(jGitRepository); - // Disable rewriteParents because otherwise `RevWalk` will load every commit into memory. - revWalk.setRewriteParents(false); + disableRewriteParents(revWalk); return revWalk; } private static RevWalk newRevWalk(ObjectReader reader) { final RevWalk revWalk = new RevWalk(reader); + disableRewriteParents(revWalk); + return revWalk; + } + + private static void disableRewriteParents(RevWalk revWalk) { // Disable rewriteParents because otherwise `RevWalk` will load every commit into memory. revWalk.setRewriteParents(false); - return revWalk; } private static void checkGitRepositoryFormat(Repository repository) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java index 621dc9639..f42233c04 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java @@ -85,8 +85,8 @@ private static File repoDir(File rootDir, String suffix) { } if (create) { - writeSuffix(INITIAL_PRIMARY_SUFFIX); primarySuffix = INITIAL_PRIMARY_SUFFIX; + writeSuffix(primarySuffix); } else { boolean success = false; try { @@ -131,7 +131,7 @@ private void writeSuffix(String suffix) { } while (buf.hasRemaining()); channel.force(true); } catch (IOException e) { - throw new StorageException("failed to update the suffix (" + suffix + + throw new StorageException("failed to write the suffix (" + suffix + ") of the primary repository: " + path, e); } } @@ -159,10 +159,12 @@ File secondaryRepoDir() { return repoDir(rootDir, addOne(primarySuffix)); } - void setPrimaryRepoDir(File newRepoDir) { + void setPrimaryRepoDir(File newRepoDir) { // e.g. /foo_0000123457 + // e.g. primarySuffix: 0000123456, newPrimarySuffix: 0000123457 final String newPrimarySuffix = addOne(primarySuffix); + // e.g. secondary: /foo_0000123457 final File secondary = new File(rootDir, rootDir.getName() + '_' + newPrimarySuffix); - assert newRepoDir.equals(secondaryRepoDir()); + assert newRepoDir.equals(secondary); primarySuffix = newPrimarySuffix; writeSuffix(newPrimarySuffix); } @@ -176,23 +178,12 @@ public void close() { } } - void closeAndDelete() { - try { - channel.close(); - } catch (IOException e) { - logger.warn("Failed to close the commit ID database: {}", path, e); - } - if (!path.toFile().delete()) { - logger.warn("Failed to delete the commit ID database: {}", path); - } - } - @VisibleForTesting - static String addOne(String suffix) { - final int intSuffix = Integer.parseInt(suffix); - final String str = String.valueOf(intSuffix + 1); + static String addOne(String suffix) { // e.g. "0000123456" + final int intSuffix = Integer.parseInt(suffix); // e.g. 123456 + final String str = String.valueOf(intSuffix + 1); // e.g. "123457" if (str.length() < 10) { - return INITIAL_PRIMARY_SUFFIX.substring(0, 10 - str.length()) + str; + return INITIAL_PRIMARY_SUFFIX.substring(0, 10 - str.length()) + str; // e.g. "0000123457" } return str; } diff --git a/server/src/main/resources/META-INF/services/com.linecorp.centraldogma.server.plugin.Plugin b/server/src/main/resources/META-INF/services/com.linecorp.centraldogma.server.plugin.Plugin index 2cb1e0f21..1d38f906f 100644 --- a/server/src/main/resources/META-INF/services/com.linecorp.centraldogma.server.plugin.Plugin +++ b/server/src/main/resources/META-INF/services/com.linecorp.centraldogma.server.plugin.Plugin @@ -1,3 +1,3 @@ com.linecorp.centraldogma.server.internal.mirror.DefaultMirroringServicePlugin com.linecorp.centraldogma.server.internal.storage.PurgeSchedulingServicePlugin -com.linecorp.centraldogma.server.internal.storage.repository.git.CommitRetentionManagementPlugin \ No newline at end of file +com.linecorp.centraldogma.server.internal.storage.repository.git.CommitRetentionManagementPlugin diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java index 0a2bdb6ea..5fa3f7910 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java @@ -97,9 +97,13 @@ void promote() { static void addCommits(GitRepositoryV2 repo, int start, int end) { for (int i = start; i <= end; i++) { - repo.commit(Revision.HEAD, i * 1000, Author.SYSTEM, - "Summary" + i, "Detail", Markup.PLAINTEXT, - Change.ofTextUpsert("/file_" + i + ".txt", String.valueOf(i))).join(); + final Change change = Change.ofTextUpsert("/file_" + i + ".txt", String.valueOf(i)); + addCommit(repo, i, change); } } + + static void addCommit(GitRepositoryV2 repo, int index, Change change) { + repo.commit(Revision.HEAD, index * 1000, Author.SYSTEM, + "Summary" + index, "Detail", Markup.PLAINTEXT, change).join(); + } } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2WatchTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2WatchTest.java new file mode 100644 index 000000000..2f0849846 --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2WatchTest.java @@ -0,0 +1,66 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryV2HistoryTest.createRepository; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryV2PromotionTest.addCommit; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.util.concurrent.CompletableFuture; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Revision; + +class GitRepositoryV2WatchTest { + + @TempDir + static File repoDir; + private static GitRepositoryV2 repo; + + @BeforeAll + static void setUp() { + // The repository contains commits from 10 to 20(inclusive). + repo = createRepository(repoDir, 10, 20); + } + + @AfterAll + static void tearDown() { + repo.internalClose(); + } + + @Test + void testWatch() { + assertThat(repo.watch(Revision.INIT, "/file_2.txt").join()).isEqualTo(new Revision(20)); + // Return the head revision even though there was no change for "/file_2.txt" which is committed at + // the revision 2. The current primary repo does not know if there was a commit for the entry between + // the specified(5) revision and the first revision(10) of the repo. So it's safe to return the + // head revision. + assertThat(repo.watch(new Revision(5), "/file_2.txt").join()).isEqualTo(new Revision(20)); + assertThat(repo.watch(new Revision(5), "/file_6.txt").join()).isEqualTo(new Revision(20)); + + final CompletableFuture future = repo.watch(new Revision(10), "/file_2.txt"); + assertThat(future.isDone()).isFalse(); + addCommit(repo, 21, Change.ofTextUpsert("/file_2.txt", String.valueOf(2 + 2))); + + assertThat(future.join()).isEqualTo(new Revision(21)); + } +} From 2f8e1076edefb798d3d181c15033448f6741c3d0 Mon Sep 17 00:00:00 2001 From: minwoox Date: Tue, 14 Sep 2021 11:58:20 +0900 Subject: [PATCH 04/14] Fix test --- .../storage/repository/git/GitRepositoryV2.java | 9 ++++++--- .../git/GitRepositoryMigrationTest.java | 5 +++-- .../repository/git/GitRepositoryV2DiffTest.java | 16 ++++++++++------ .../repository/git/GitRepositoryV2WatchTest.java | 3 +++ 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java index 8abb92105..58bfea11b 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java @@ -21,6 +21,7 @@ import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.failFastIfTimedOut; import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.closeJGitRepo; import static com.linecorp.centraldogma.server.internal.storage.repository.git.InternalRepository.buildJGitRepo; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import static java.util.Objects.requireNonNull; import java.io.File; @@ -244,12 +245,14 @@ private static void migrateToV2(File repoDir) { if (!primaryRepoDir.mkdirs()) { throw new StorageException("failed to create " + primaryRepoDir + " while migrating to V2."); } - if (!tmpRepoDir.renameTo(primaryRepoDir)) { - // Rename it back. + try { + final Path moved = Files.move(tmpRepoDir.toPath(), primaryRepoDir.toPath(), REPLACE_EXISTING); + assert moved == primaryRepoDir.toPath(); + } catch (IOException e) { //noinspection ResultOfMethodCallIgnored tmpRepoDir.renameTo(repoDir); throw new StorageException("failed to migrate a repository at: " + tmpRepoDir + - ", to: " + primaryRepoDir); + ", to: " + primaryRepoDir, e); } checkState(!tmpRepoDir.exists(), "%s is not renamed.", tmpRepoDir); logger.debug("Migrating {} is done.", repoDir); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java index ef80783cd..0c5e96796 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java @@ -39,12 +39,13 @@ class GitRepositoryMigrationTest { @Test void migrate() { - final GitRepository oldRepo = new GitRepository(mock(Project.class), new File(repoDir, "test_repo"), + final File fooDir = new File(repoDir, "foo"); + final GitRepository oldRepo = new GitRepository(mock(Project.class), new File(fooDir, "test_repo"), commonPool(), 0L, Author.SYSTEM); addCommits(oldRepo); oldRepo.internalClose(); final GitRepositoryV2 migrated = GitRepositoryV2.open(mock(Project.class), - new File(repoDir, "test_repo"), commonPool(), + new File(fooDir, "test_repo"), commonPool(), null); final List commits = migrated.history(Revision.INIT, Revision.HEAD, "/**").join(); assertThat(commits.get(0).summary()).contains("new repository"); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2DiffTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2DiffTest.java index f41c536f5..1941f4734 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2DiffTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2DiffTest.java @@ -30,6 +30,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import com.google.common.collect.ImmutableList; + import com.linecorp.centraldogma.common.Change; import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.common.RevisionNotFoundException; @@ -54,9 +56,7 @@ static void tearDown() { @Test void diff() { Map> diffs = repo.diff(new Revision(18), new Revision(20), "/**").join(); - assertThat(diffs.values()).containsExactly( - Change.ofTextUpsert("/file_19.txt", "19" + System.lineSeparator()), - Change.ofTextUpsert("/file_20.txt", "20" + System.lineSeparator())); + assertThat(paths(diffs)).containsExactly("/file_19.txt", "/file_20.txt"); // It's same when the revisions are reversed. assertThat(repo.diff(Revision.HEAD, new Revision(18), "/**").join().values()) @@ -64,11 +64,11 @@ void diff() { diffs = repo.diff(new Revision(10), new Revision(20), "/**").join(); assertThat(diffs).hasSize(10); // from 11 to 20. - final List> expected = + final List expected = IntStream.range(11, 21) - .mapToObj(i -> Change.ofTextUpsert("/file_" + i + ".txt", i + System.lineSeparator())) + .mapToObj(i -> "/file_" + i + ".txt") .collect(toImmutableList()); - assertThat(diffs.values()).containsExactlyInAnyOrderElementsOf(expected); + assertThat(paths(diffs)).containsExactlyInAnyOrderElementsOf(expected); // If the revision is INIT, the result is same as Revision(10) which is the first revision // of the new primary repository. @@ -78,6 +78,10 @@ void diff() { assertThat(repo.diff(Revision.INIT, Revision.INIT, "/**").join()).isEmpty(); } + private ImmutableList paths(Map> diffs) { + return diffs.values().stream().map(Change::path).collect(toImmutableList()); + } + @Test void invalidRevisionDiff() { assertThatThrownBy(() -> repo.diff(new Revision(9), new Revision(10), "/**").join()) diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2WatchTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2WatchTest.java index 2f0849846..d07537dc5 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2WatchTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2WatchTest.java @@ -18,6 +18,7 @@ import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryV2HistoryTest.createRepository; import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryV2PromotionTest.addCommit; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import java.io.File; import java.util.concurrent.CompletableFuture; @@ -62,5 +63,7 @@ void testWatch() { addCommit(repo, 21, Change.ofTextUpsert("/file_2.txt", String.valueOf(2 + 2))); assertThat(future.join()).isEqualTo(new Revision(21)); + // Make sure CommitWatchers has cleared the watch. + await().untilAsserted(() -> assertThat(repo.commitWatchers.watchesMap).isEmpty()); } } From 96fd44c8a28f8ea07e3f8655625c98f5caeb0df2 Mon Sep 17 00:00:00 2001 From: minwoox Date: Tue, 14 Sep 2021 14:39:53 +0900 Subject: [PATCH 05/14] Close repo in test --- .../storage/repository/git/GitRepositoryMigrationTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java index 0c5e96796..52fb992e1 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java @@ -52,6 +52,8 @@ void migrate() { for (int i = 1; i < 10; i++) { assertThat(commits.get(i).summary()).isEqualTo("Summary" + i); } + + migrated.internalClose(); } private static void addCommits(GitRepository oldRepo) { From e0a78f8120516c49f308092029af4f8030210396 Mon Sep 17 00:00:00 2001 From: minwoox Date: Fri, 24 Sep 2021 17:21:03 +0900 Subject: [PATCH 06/14] Address comments by @ikhoon --- .../server/CommitRetentionConfig.java | 2 +- .../storage/DirectoryBasedStorageManager.java | 2 +- .../storage/repository/git/FailFastUtil.java | 8 +-- .../repository/git/GitRepositoryManager.java | 49 --------------- .../repository/git/GitRepositoryV2.java | 59 +++++++++++-------- ...pository.java => LegacyGitRepository.java} | 19 +++--- .../git/RepositoryMetadataDatabase.java | 14 +---- .../git/GitRepositoryMigrationTest.java | 7 ++- .../git/GitRepositoryV2PromotionTest.java | 4 +- .../git/RepositoryMetadataDatabaseTest.java | 10 ++-- 10 files changed, 64 insertions(+), 110 deletions(-) rename server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/{GitRepository.java => LegacyGitRepository.java} (98%) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java b/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java index 78d83fcfa..994889bda 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java @@ -36,7 +36,7 @@ public final class CommitRetentionConfig { // The minimum of minRetentionCommits - private static final int MINIMUM_MINIMUM_RETENTION_COMMITS = 1000; + private static final int MINIMUM_MINIMUM_RETENTION_COMMITS = 5000; private static final String DEFAULT_SCHEDULE = "0 0 * * * ?"; // Every day private static final CronParser cronParser = new CronParser(CronDefinitionBuilder.instanceDefinitionFor( diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/DirectoryBasedStorageManager.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/DirectoryBasedStorageManager.java index f9405568b..1d1b97003 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/DirectoryBasedStorageManager.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/DirectoryBasedStorageManager.java @@ -56,6 +56,7 @@ public abstract class DirectoryBasedStorageManager implements StorageManager< private static final Logger logger = LoggerFactory.getLogger(DirectoryBasedStorageManager.class); + public static final String SUFFIX_REMOVED = ".removed"; /** * Start with an alphanumeric character. * An alphanumeric character, minus, plus, underscore and dot are allowed in the middle. @@ -63,7 +64,6 @@ public abstract class DirectoryBasedStorageManager implements StorageManager< */ private static final Pattern CHILD_NAME = Pattern.compile("^[0-9A-Za-z](?:[-+_0-9A-Za-z.]*[0-9A-Za-z])?$"); - private static final String SUFFIX_REMOVED = ".removed"; private static final String SUFFIX_PURGED = ".purged"; private final String childTypeName; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/FailFastUtil.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/FailFastUtil.java index b55fe7c11..121a0eb2a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/FailFastUtil.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/FailFastUtil.java @@ -39,7 +39,7 @@ static ServiceRequestContext context() { } @SuppressWarnings("MethodParameterNamingConvention") - static void failFastIfTimedOut(GitRepository repo, Logger logger, @Nullable ServiceRequestContext ctx, + static void failFastIfTimedOut(LegacyGitRepository repo, Logger logger, @Nullable ServiceRequestContext ctx, String methodName, Object arg1) { if (ctx != null && ctx.isTimedOut()) { logger.info("{} Rejecting a request timed out already: repo={}/{}, method={}, args={}", @@ -49,7 +49,7 @@ static void failFastIfTimedOut(GitRepository repo, Logger logger, @Nullable Serv } @SuppressWarnings("MethodParameterNamingConvention") - static void failFastIfTimedOut(GitRepository repo, Logger logger, @Nullable ServiceRequestContext ctx, + static void failFastIfTimedOut(LegacyGitRepository repo, Logger logger, @Nullable ServiceRequestContext ctx, String methodName, Object arg1, Object arg2) { if (ctx != null && ctx.isTimedOut()) { logger.info("{} Rejecting a request timed out already: repo={}/{}, method={}, args=[{}, {}]", @@ -59,7 +59,7 @@ static void failFastIfTimedOut(GitRepository repo, Logger logger, @Nullable Serv } @SuppressWarnings("MethodParameterNamingConvention") - static void failFastIfTimedOut(GitRepository repo, Logger logger, @Nullable ServiceRequestContext ctx, + static void failFastIfTimedOut(LegacyGitRepository repo, Logger logger, @Nullable ServiceRequestContext ctx, String methodName, Object arg1, Object arg2, Object arg3) { if (ctx != null && ctx.isTimedOut()) { logger.info("{} Rejecting a request timed out already: repo={}/{}, method={}, args=[{}, {}, {}]", @@ -69,7 +69,7 @@ static void failFastIfTimedOut(GitRepository repo, Logger logger, @Nullable Serv } @SuppressWarnings("MethodParameterNamingConvention") - static void failFastIfTimedOut(GitRepository repo, Logger logger, @Nullable ServiceRequestContext ctx, + static void failFastIfTimedOut(LegacyGitRepository repo, Logger logger, @Nullable ServiceRequestContext ctx, String methodName, Object arg1, Object arg2, Object arg3, int arg4) { if (ctx != null && ctx.isTimedOut()) { logger.info( diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManager.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManager.java index 0b745ee5a..e67138159 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManager.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManager.java @@ -19,23 +19,15 @@ import static java.util.Objects.requireNonNull; import java.io.File; -import java.io.IOException; import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; -import java.util.function.BiConsumer; import java.util.function.Supplier; import javax.annotation.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.linecorp.armeria.common.util.TextFormatter; import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.common.CentralDogmaException; import com.linecorp.centraldogma.common.RepositoryExistsException; import com.linecorp.centraldogma.common.RepositoryNotFoundException; -import com.linecorp.centraldogma.internal.Util; import com.linecorp.centraldogma.server.internal.storage.DirectoryBasedStorageManager; import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache; import com.linecorp.centraldogma.server.storage.project.Project; @@ -45,8 +37,6 @@ public class GitRepositoryManager extends DirectoryBasedStorageManager implements RepositoryManager { - private static final Logger logger = LoggerFactory.getLogger(GitRepositoryManager.class); - private final Project parent; private final Executor repositoryWorker; @@ -72,12 +62,6 @@ protected Repository openChild(File childDir) throws Exception { return GitRepositoryV2.open(parent, childDir, repositoryWorker, cache); } - private static void deleteCruft(File dir) throws IOException { - logger.info("Deleting the cruft from previous migration: {}", dir); - Util.deleteFileTree(dir); - logger.info("Deleted the cruft from previous migration: {}", dir); - } - @Override protected Repository createChild(File childDir, Author author, long creationTimeMillis) throws Exception { return new GitRepositoryV2(parent, childDir, repositoryWorker, @@ -99,37 +83,4 @@ protected CentralDogmaException newStorageExistsException(String name) { protected CentralDogmaException newStorageNotFoundException(String name) { return new RepositoryNotFoundException(parent().name() + '/' + name); } - - /** - * Logs the migration progress periodically. - */ - private static class MigrationProgressLogger implements BiConsumer { - - private static final long REPORT_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(10); - - private final String name; - private final long startTimeNanos; - private long lastReportTimeNanos; - - MigrationProgressLogger(Repository repo) { - name = repo.parent().name() + '/' + repo.name(); - startTimeNanos = lastReportTimeNanos = System.nanoTime(); - } - - @Override - public void accept(Integer current, Integer total) { - final long currentTimeNanos = System.nanoTime(); - final long elapsedTimeNanos = currentTimeNanos - startTimeNanos; - if (currentTimeNanos - lastReportTimeNanos > REPORT_INTERVAL_NANOS) { - logger.info("{}: {}% ({}/{}) - took {}", - name, (int) ((double) current / total * 100), - current, total, TextFormatter.elapsed(elapsedTimeNanos)); - lastReportTimeNanos = currentTimeNanos; - } else if (current.equals(total)) { - logger.info("{}: 100% ({}/{}) - took {}", - name, current, total, - TextFormatter.elapsed(elapsedTimeNanos)); - } - } - } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java index 58bfea11b..00496f0a8 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java @@ -17,6 +17,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; +import static com.linecorp.centraldogma.server.internal.storage.DirectoryBasedStorageManager.SUFFIX_REMOVED; import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.context; import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.failFastIfTimedOut; import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.closeJGitRepo; @@ -28,10 +29,9 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Duration; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; -import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -381,9 +381,8 @@ public CompletableFuture>> find( readLock(); try { final Revision normalizedRevision = normalizeNow(revision); - final Revision headRevision = this.headRevision; if ("/".equals(pathPattern)) { - return Collections.singletonMap(pathPattern, Entry.ofDirectory(normalizedRevision, "/")); + return ImmutableMap.of(pathPattern, Entry.ofDirectory(normalizedRevision, "/")); } return primaryRepo.find(normalizedRevision, pathPattern, options); } finally { @@ -706,9 +705,8 @@ public void removeOldCommits(int minRetentionCommits, int minRetentionDays) { } } - private boolean exceedsMinRetention(InternalRepository repo, - int minRetentionCommits, int minRetentionDays) { - final CommitIdDatabase commitIdDatabase = repo.commitIdDatabase(); + private static boolean exceedsMinRetention(InternalRepository repo, + int minRetentionCommits, int minRetentionDays) { final Revision headRevision = repo.headRevision(); final Revision firstRevision = repo.firstRevision(); if (minRetentionCommits != 0) { @@ -720,7 +718,7 @@ private boolean exceedsMinRetention(InternalRepository repo, if (minRetentionDays != 0) { final Instant secondCommitCreationTime = repo.secondCommitCreationTimeInstant(); return secondCommitCreationTime != null && - Instant.now().minus(Duration.ofDays(minRetentionDays)).isBefore(secondCommitCreationTime); + secondCommitCreationTime.isBefore(Instant.now().minus(minRetentionDays, ChronoUnit.DAYS)); } return true; } @@ -736,12 +734,12 @@ private void promoteSecondaryRepo(InternalRepository secondaryRepo) { repositoryWorker.execute(() -> { closeInternalRepository(primaryRepo); final File repoDir = primaryRepo.repoDir(); + final Path path = repoDir.toPath(); + final Path newPath = path.resolveSibling(path.getFileName().toString() + SUFFIX_REMOVED); try { - if (!deleteDirectory(repoDir)) { - logger.warn("Failed to delete the old primary repository: {}", repoDir); - } - } catch (Throwable t) { - logger.warn("Failed to delete the old primary repository: {}", repoDir); + Files.move(path, newPath); + } catch (IOException e) { + logger.warn("Failed to mark the old primary repository: {} to {}", repoDir, newPath); } }); logger.info("Promotion is done for {}/{}. {} is now the primary.", @@ -753,29 +751,20 @@ private void promoteSecondaryRepo(InternalRepository secondaryRepo) { createSecondaryRepo(); } - private static boolean deleteDirectory(File dir) throws IOException { - try (Stream walk = Files.walk(dir.toPath())) { - return walk.sorted(Comparator.reverseOrder()) - .map(Path::toFile) - .map(File::delete) - // Return false if it fails to delete a file. - .reduce(true, (a, b) -> a && b); - } - } - private void createSecondaryRepo() { + InternalRepository secondaryRepo = null; try { writeLock(); final Revision headRevision = this.headRevision; isCreatingSecondaryRepo = true; writeUnLock(); - logger.info("Creating the secondary repository in {}/{} with the head revision: {}.", + logger.info("Creating the secondary repository in {}/{}. head revision: {}.", parent.name(), originalRepoName, headRevision); final Map> entries = find(headRevision, ALL_PATH, ImmutableMap.of()).join(); final List> changes = toChanges(entries); final File secondaryRepoDir = repoMetadata.secondaryRepoDir(); - final InternalRepository secondaryRepo = + secondaryRepo = InternalRepository.of(parent, originalRepoName, secondaryRepoDir, headRevision, creationTimeMillis, author, changes); writeLock(); @@ -797,10 +786,18 @@ private void createSecondaryRepo() { } finally { writeUnLock(); } - logger.info("The secondary repository {} is created in {}/{}.", + logger.info("The secondary repository {} is created in {}/{}. head revision: {}", secondaryRepoDir.getName(), parent.name(), originalRepoName, headRevision); } catch (Throwable t) { logger.warn("Failed to create the secondary repository", t); + if (secondaryRepo != null) { + closeInternalRepository(secondaryRepo); + try { + deleteDirectory(secondaryRepo.repoDir()); + } catch (IOException e) { + logger.warn("Failed to delete the directory: {}", secondaryRepo.repoDir()); + } + } } } @@ -823,6 +820,16 @@ private static List> toChanges(Map> entries) { return builder.build(); } + private static boolean deleteDirectory(File dir) throws IOException { + try (Stream walk = Files.walk(dir.toPath())) { + return walk.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .map(File::delete) + // Return false if it fails to delete a file. + .reduce(true, (a, b) -> a && b); + } + } + private static class LaggedCommit { private final Revision revision; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/LegacyGitRepository.java similarity index 98% rename from server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java rename to server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/LegacyGitRepository.java index 9bc370ec0..d1fe0f802 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/LegacyGitRepository.java @@ -131,9 +131,9 @@ /** * A {@link Repository} based on Git. */ -class GitRepository implements Repository { +class LegacyGitRepository implements Repository { - private static final Logger logger = LoggerFactory.getLogger(GitRepository.class); + private static final Logger logger = LoggerFactory.getLogger(LegacyGitRepository.class); private static final String R_HEADS_MASTER = Constants.R_HEADS + Constants.MASTER; @@ -189,8 +189,8 @@ class GitRepository implements Repository { * @throws StorageException if failed to create a new repository */ @VisibleForTesting - GitRepository(Project parent, File repoDir, Executor repositoryWorker, - long creationTimeMillis, Author author) { + LegacyGitRepository(Project parent, File repoDir, Executor repositoryWorker, + long creationTimeMillis, Author author) { this(parent, repoDir, repositoryWorker, creationTimeMillis, author, null); } @@ -204,8 +204,8 @@ class GitRepository implements Repository { * * @throws StorageException if failed to create a new repository */ - GitRepository(Project parent, File repoDir, Executor repositoryWorker, - long creationTimeMillis, Author author, @Nullable RepositoryCache cache) { + LegacyGitRepository(Project parent, File repoDir, Executor repositoryWorker, + long creationTimeMillis, Author author, @Nullable RepositoryCache cache) { this.parent = requireNonNull(parent, "parent"); name = requireNonNull(repoDir, "repoDir").getName(); @@ -283,7 +283,8 @@ class GitRepository implements Repository { * * @throws StorageException if failed to open the repository at the specified location */ - GitRepository(Project parent, File repoDir, Executor repositoryWorker, @Nullable RepositoryCache cache) { + LegacyGitRepository(Project parent, File repoDir, Executor repositoryWorker, + @Nullable RepositoryCache cache) { this.parent = requireNonNull(parent, "parent"); name = requireNonNull(repoDir, "repoDir").getName(); this.repositoryWorker = requireNonNull(repositoryWorker, "repositoryWorker"); @@ -1552,8 +1553,8 @@ public void cloneTo(File newRepoDir, BiConsumer progressListen requireNonNull(progressListener, "progressListener"); final Revision endRevision = normalizeNow(Revision.HEAD); - final GitRepository newRepo = new GitRepository(parent, newRepoDir, repositoryWorker, - creationTimeMillis(), author(), cache); + final LegacyGitRepository newRepo = new LegacyGitRepository(parent, newRepoDir, repositoryWorker, + creationTimeMillis(), author(), cache); progressListener.accept(1, endRevision.major()); boolean success = false; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java index f42233c04..80e46b10f 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java @@ -24,14 +24,11 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import javax.annotation.Nullable; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.cronutils.utils.VisibleForTesting; -import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.server.storage.StorageException; /** @@ -59,11 +56,6 @@ private static File repoDir(File rootDir, String suffix) { private final File rootDir; private final Path path; private final FileChannel channel; - @Nullable - private volatile Revision headRevision; - @Nullable - private volatile Revision firstRevision; - private String primarySuffix; RepositoryMetadataDatabase(File rootDir, boolean create) { @@ -156,12 +148,12 @@ File primaryRepoDir() { } File secondaryRepoDir() { - return repoDir(rootDir, addOne(primarySuffix)); + return repoDir(rootDir, increment(primarySuffix)); } void setPrimaryRepoDir(File newRepoDir) { // e.g. /foo_0000123457 // e.g. primarySuffix: 0000123456, newPrimarySuffix: 0000123457 - final String newPrimarySuffix = addOne(primarySuffix); + final String newPrimarySuffix = increment(primarySuffix); // e.g. secondary: /foo_0000123457 final File secondary = new File(rootDir, rootDir.getName() + '_' + newPrimarySuffix); assert newRepoDir.equals(secondary); @@ -179,7 +171,7 @@ public void close() { } @VisibleForTesting - static String addOne(String suffix) { // e.g. "0000123456" + static String increment(String suffix) { // e.g. "0000123456" final int intSuffix = Integer.parseInt(suffix); // e.g. 123456 final String str = String.valueOf(intSuffix + 1); // e.g. "123457" if (str.length() < 10) { diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java index 52fb992e1..d37d25fba 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java @@ -40,8 +40,9 @@ class GitRepositoryMigrationTest { @Test void migrate() { final File fooDir = new File(repoDir, "foo"); - final GitRepository oldRepo = new GitRepository(mock(Project.class), new File(fooDir, "test_repo"), - commonPool(), 0L, Author.SYSTEM); + final LegacyGitRepository oldRepo = + new LegacyGitRepository(mock(Project.class), new File(fooDir, "test_repo"), + commonPool(), 0L, Author.SYSTEM); addCommits(oldRepo); oldRepo.internalClose(); final GitRepositoryV2 migrated = GitRepositoryV2.open(mock(Project.class), @@ -56,7 +57,7 @@ void migrate() { migrated.internalClose(); } - private static void addCommits(GitRepository oldRepo) { + private static void addCommits(LegacyGitRepository oldRepo) { for (int i = 1; i < 10; i++) { oldRepo.commit(Revision.HEAD, 0, Author.SYSTEM, "Summary" + i, "Detail", Markup.PLAINTEXT, diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java index 5fa3f7910..8d381711d 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java @@ -15,6 +15,7 @@ */ package com.linecorp.centraldogma.server.internal.storage.repository.git; +import static com.linecorp.centraldogma.server.internal.storage.DirectoryBasedStorageManager.SUFFIX_REMOVED; import static java.util.concurrent.ForkJoinPool.commonPool; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; @@ -79,8 +80,9 @@ void promote() { // The secondary repo is promoted to the primary repo. assertThat(repo.primaryRepo).isSameAs(secondaryRepo); assertThat(repo.primaryRepo).isNotSameAs(primaryRepo); - // The old primary repo is gone now. + // The old primary repo is gone and renamed. await().until(() -> !primaryRepo.repoDir().exists()); + assertThat(new File(primaryRepo.repoDir() + SUFFIX_REMOVED).exists()).isTrue(); final CommitIdDatabase newPrimaryDatabase = repo.primaryRepo.commitIdDatabase(); assertThat(newPrimaryDatabase.firstRevision().major()).isEqualTo(12); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabaseTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabaseTest.java index 2e4badf62..28d1d8df0 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabaseTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabaseTest.java @@ -15,7 +15,7 @@ */ package com.linecorp.centraldogma.server.internal.storage.repository.git; -import static com.linecorp.centraldogma.server.internal.storage.repository.git.RepositoryMetadataDatabase.addOne; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.RepositoryMetadataDatabase.increment; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -70,15 +70,15 @@ void nextPrimaryRepoShouldBeGreaterByOne() { @Test void addOneToSuffixTest() { String suffix = "0000000000"; - assertThat(addOne(suffix)).isEqualTo("0000000001"); + assertThat(increment(suffix)).isEqualTo("0000000001"); suffix = "0000000009"; - assertThat(addOne(suffix)).isEqualTo("0000000010"); + assertThat(increment(suffix)).isEqualTo("0000000010"); suffix = "0000000099"; - assertThat(addOne(suffix)).isEqualTo("0000000100"); + assertThat(increment(suffix)).isEqualTo("0000000100"); suffix = "1111111111"; - assertThat(addOne(suffix)).isEqualTo("1111111112"); + assertThat(increment(suffix)).isEqualTo("1111111112"); } } From 1c5185a881cb2d8ddf2c36b882c265e8b38209d6 Mon Sep 17 00:00:00 2001 From: minwoox Date: Thu, 23 Dec 2021 23:59:59 +0900 Subject: [PATCH 07/14] WIP --- .../centraldogma/server/command/Command.java | 24 +++ .../server/command/CommandType.java | 1 + .../CreateRollingRepositoryCommand.java | 48 ++++++ .../command/StandaloneCommandExecutor.java | 11 ++ .../repository/git/CommitIdDatabase.java | 10 ++ .../git/CommitRetentionManagementPlugin.java | 13 +- .../repository/git/GitRepositoryV2.java | 140 +++++++++++++----- .../repository/git/InternalRepository.java | 20 ++- .../server/storage/repository/Repository.java | 7 + 9 files changed, 230 insertions(+), 44 deletions(-) create mode 100644 server/src/main/java/com/linecorp/centraldogma/server/command/CreateRollingRepositoryCommand.java diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/Command.java b/server/src/main/java/com/linecorp/centraldogma/server/command/Command.java index 5041f17a2..3ed7090be 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/Command.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/Command.java @@ -16,6 +16,7 @@ package com.linecorp.centraldogma.server.command; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import javax.annotation.Nullable; @@ -48,6 +49,7 @@ @Type(value = RemoveRepositoryCommand.class, name = "REMOVE_REPOSITORY"), @Type(value = PurgeRepositoryCommand.class, name = "PURGE_REPOSITORY"), @Type(value = UnremoveRepositoryCommand.class, name = "UNREMOVE_REPOSITORY"), + @Type(value = CreateRollingRepositoryCommand.class, name = "CREATE_ROLLING_REPOSITORY"), @Type(value = NormalizingPushCommand.class, name = "NORMALIZING_PUSH"), @Type(value = PushAsIsCommand.class, name = "PUSH"), @Type(value = CreateSessionCommand.class, name = "CREATE_SESSIONS"), @@ -246,6 +248,28 @@ static Command purgeRepository(@Nullable Long timestamp, Author author, return new PurgeRepositoryCommand(timestamp, author, projectName, repositoryName); } + /** + * Returns a new {@link Command} which is used to create a rolling repository. + * + * @param projectName the name of the project + * @param repositoryName the name of the repository which is supposed to be purged + * @param initialRevision the initial {@link Revision} of the rolling repository + * @param minRetentionCommits the configured number of recent commits that a repository should retain + * @param minRetentionDays the configured number of days for recent commits that a repository should retain + */ + static Command createRollingRepository( + String projectName, String repositoryName, Revision initialRevision, + int minRetentionCommits, int minRetentionDays) { + requireNonNull(projectName, "projectName"); + requireNonNull(repositoryName, "repositoryName"); + requireNonNull(initialRevision, "initialRevision"); + checkArgument(minRetentionCommits > 0, "minRetentionCommits: %s (expected: > 0)", + minRetentionCommits); + checkArgument(minRetentionDays > 0, "minRetentionDays: %s (expected: > 0)", minRetentionDays); + return new CreateRollingRepositoryCommand(projectName, repositoryName, initialRevision, + minRetentionCommits, minRetentionDays); + } + /** * Returns a new {@link Command} which is used to push the changes. The changes are normalized via * {@link Repository#previewDiff(Revision, Iterable)} before they are applied. diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/CommandType.java b/server/src/main/java/com/linecorp/centraldogma/server/command/CommandType.java index 7543dd4ef..17022013a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/CommandType.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/CommandType.java @@ -28,6 +28,7 @@ public enum CommandType { CREATE_REPOSITORY(Void.class), REMOVE_REPOSITORY(Void.class), UNREMOVE_REPOSITORY(Void.class), + CREATE_ROLLING_REPOSITORY(Void.class), NORMALIZING_PUSH(CommitResult.class), PUSH(Revision.class), SAVE_NAMED_QUERY(Void.class), diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/CreateRollingRepositoryCommand.java b/server/src/main/java/com/linecorp/centraldogma/server/command/CreateRollingRepositoryCommand.java new file mode 100644 index 000000000..11df5f7cc --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/CreateRollingRepositoryCommand.java @@ -0,0 +1,48 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.command; + +import static java.util.Objects.requireNonNull; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Revision; + +public final class CreateRollingRepositoryCommand extends RepositoryCommand { + + private final Revision initialRevision; + private final int minRetentionCommits; + private final int minRetentionDays; + + CreateRollingRepositoryCommand(String projectName, String repositoryName, + Revision initialRevision, int minRetentionCommits, int minRetentionDays) { + super(CommandType.CREATE_ROLLING_REPOSITORY, null, Author.SYSTEM, projectName, repositoryName); + this.initialRevision = requireNonNull(initialRevision, "initialRevision"); + this.minRetentionCommits = minRetentionCommits; + this.minRetentionDays = minRetentionDays; + } + + public Revision initialRevision() { + return initialRevision; + } + + public int minRetentionCommits() { + return minRetentionCommits; + } + + public int minRetentionDays() { + return minRetentionDays; + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java b/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java index 18784ca3d..ecde4e707 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java @@ -170,6 +170,10 @@ protected CompletableFuture doExecute(Command command) throws Exceptio return (CompletableFuture) purgeRepository((PurgeRepositoryCommand) command); } + if (command instanceof CreateRollingRepositoryCommand) { + return (CompletableFuture) createRollingRepository((CreateRollingRepositoryCommand) command); + } + if (command instanceof NormalizingPushCommand) { return (CompletableFuture) push((NormalizingPushCommand) command, true); } @@ -253,6 +257,13 @@ private CompletableFuture purgeRepository(PurgeRepositoryCommand c) { }, repositoryWorker); } + private CompletableFuture createRollingRepository(CreateRollingRepositoryCommand c) { + return CompletableFuture.supplyAsync(() -> { + repo(c).createRollingRepository(c.initialRevision(), c.minRetentionCommits(), c.minRetentionDays()); + return null; + }, repositoryWorker); + } + private CompletableFuture push(AbstractPushCommand c, boolean normalizing) { if (c.projectName().equals(INTERNAL_PROJECT_DOGMA) || c.repositoryName().equals(Project.REPO_DOGMA) || !writeQuotaEnabled()) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java index ed94a8c89..5978807ff 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java @@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.MoreObjects; import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.common.RevisionNotFoundException; @@ -327,4 +328,13 @@ public void close() { logger.warn("Failed to close the commit ID database: {}", path, e); } } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).omitNullValues() + .add("path", path) + .add("headRevision", headRevision) + .add("firstRevision", firstRevision) + .toString(); + } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java index bbadfc49f..d929767d6 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java @@ -34,8 +34,10 @@ import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.server.CentralDogmaConfig; import com.linecorp.centraldogma.server.CommitRetentionConfig; +import com.linecorp.centraldogma.server.command.Command; import com.linecorp.centraldogma.server.plugin.Plugin; import com.linecorp.centraldogma.server.plugin.PluginContext; import com.linecorp.centraldogma.server.plugin.PluginTarget; @@ -58,7 +60,7 @@ public final class CommitRetentionManagementPlugin implements Plugin { @Override public PluginTarget target() { - return PluginTarget.ALL_REPLICAS; + return PluginTarget.LEADER_ONLY; } @Override @@ -111,7 +113,14 @@ private void removeOldCommits(PluginContext context, CommitRetentionConfig confi final ProjectManager pm = context.projectManager(); for (Project project : pm.list().values()) { for (Repository repo : project.repos().list().values()) { - repo.removeOldCommits(config.minRetentionCommits(), config.minRetentionDays()); + final Revision revision = repo.shouldCreateRollingRepository(config.minRetentionCommits(), + config.minRetentionDays()); + if (revision != null) { + context.commandExecutor().execute( + Command.createRollingRepository(project.name(), repo.name(), revision, + config.minRetentionCommits(), + config.minRetentionDays())); + } } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java index 00496f0a8..966cd893c 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java @@ -65,6 +65,7 @@ import com.linecorp.centraldogma.common.ChangeConflictException; import com.linecorp.centraldogma.common.Commit; import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.EntryNotFoundException; import com.linecorp.centraldogma.common.EntryType; import com.linecorp.centraldogma.common.Markup; import com.linecorp.centraldogma.common.RepositoryNotFoundException; @@ -539,32 +540,44 @@ public CompletableFuture> history(Revision from, Revision to, Strin } @Override - public CompletableFuture findLatestRevision(Revision lastKnownRevision, String pathPattern) { + public CompletableFuture findLatestRevision(Revision lastKnownRevision, String pathPattern, + boolean errorOnEntryNotFound) { requireNonNull(lastKnownRevision, "lastKnownRevision"); requireNonNull(pathPattern, "pathPattern"); final ServiceRequestContext ctx = context(); return CompletableFuture.supplyAsync(() -> { failFastIfTimedOut(this, logger, ctx, "findLatestRevision", lastKnownRevision, pathPattern); - return blockingFindLatestRevision(lastKnownRevision, pathPattern); + return blockingFindLatestRevision(lastKnownRevision, pathPattern, errorOnEntryNotFound); }, repositoryWorker); } @Nullable - private Revision blockingFindLatestRevision(Revision lastKnownRevision, String pathPattern) { + private Revision blockingFindLatestRevision(Revision lastKnownRevision, String pathPattern, + boolean errorOnEntryNotFound) { if (lastKnownRevision.major() == Revision.INIT.major()) { // Fast path: no need to compare because we are sure there is nothing at revision 1. + final Revision headRevision = this.headRevision; + if (headRevision.major() == 1) { + if (errorOnEntryNotFound) { + throw new EntryNotFoundException(lastKnownRevision, pathPattern); + } + + return null; + } + if ("/".equals(pathPattern)) { + return headRevision; + } readLock(); try { - final Revision headRevision = this.headRevision; - if (headRevision.major() == 1) { - return null; - } - if ("/".equals(pathPattern)) { - return headRevision; - } final Map> entries = primaryRepo.find( headRevision, pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT); - return !entries.isEmpty() ? headRevision : null; + if (!entries.isEmpty()) { + return headRevision; + } + if (errorOnEntryNotFound) { + throw new EntryNotFoundException(lastKnownRevision, pathPattern); + } + return null; } finally { readUnlock(); } @@ -588,7 +601,17 @@ private Revision blockingFindLatestRevision(Revision lastKnownRevision, String p } if (range.from().equals(range.to())) { // Empty range. - return null; + if (!errorOnEntryNotFound) { + return null; + } + // We have to check if we have the entry. + final Map> entries = + primaryRepo.find(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT); + if (!entries.isEmpty()) { + // We have the entry so just return null because there's no change. + return null; + } + throw new EntryNotFoundException(lastKnownRevision, pathPattern); } diffEntries = primaryRepo.diff(range, cache); } finally { @@ -616,11 +639,20 @@ private Revision blockingFindLatestRevision(Revision lastKnownRevision, String p } } - return null; + if (!errorOnEntryNotFound) { + return null; + } + if (!primaryRepo.find(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT).isEmpty()) { + // We have to make sure that the entry does not exist because the size of diffEntries can be 0 + // when the contents of range.from() and range.to() are identical. (e.g. add, remove and add again) + return null; + } + throw new EntryNotFoundException(lastKnownRevision, pathPattern); } @Override - public CompletableFuture watch(Revision lastKnownRevision, String pathPattern) { + public CompletableFuture watch(Revision lastKnownRevision, String pathPattern, + boolean errorOnEntryNotFound) { requireNonNull(lastKnownRevision, "lastKnownRevision"); requireNonNull(pathPattern, "pathPattern"); @@ -633,7 +665,8 @@ public CompletableFuture watch(Revision lastKnownRevision, String path try { // If lastKnownRevision is outdated already and the recent changes match, // there's no need to watch. - final Revision latestRevision = blockingFindLatestRevision(normLastKnownRevision, pathPattern); + final Revision latestRevision = blockingFindLatestRevision(normLastKnownRevision, pathPattern, + errorOnEntryNotFound); if (latestRevision != null) { future.complete(latestRevision); } else { @@ -674,7 +707,7 @@ private void writeUnLock() { rwLock.writeLock().unlock(); } - private Commit currentInitialCommit(InternalRepository primaryRepo) { + private static Commit currentInitialCommit(InternalRepository primaryRepo) { final Revision firstRevision = primaryRepo.firstRevision(); final RevisionRange range = new RevisionRange(firstRevision, firstRevision); return primaryRepo.listCommits(ALL_PATH, 1, range).get(0); @@ -697,17 +730,40 @@ public void removeOldCommits(int minRetentionCommits, int minRetentionDays) { final InternalRepository secondaryRepo = this.secondaryRepo; if (secondaryRepo != null) { - if (exceedsMinRetention(secondaryRepo, minRetentionCommits, minRetentionDays)) { - promoteSecondaryRepo(secondaryRepo); + if (exceedsMinRetention(secondaryRepo, headRevision, minRetentionCommits, minRetentionDays)) { + promoteSecondaryRepo(secondaryRepo, rollingRepositoryInitialRevision); } - } else if (exceedsMinRetention(primaryRepo, minRetentionCommits, minRetentionDays)) { - createSecondaryRepo(); + } else if (exceedsMinRetention(primaryRepo, headRevision, minRetentionCommits, minRetentionDays)) { + createSecondaryRepo(rollingRepositoryInitialRevision); + } + } + + @Override + public Revision shouldCreateRollingRepository(int minRetentionCommits, int minRetentionDays) { + final InternalRepository repo = secondaryRepo != null ? secondaryRepo : primaryRepo; + return shouldCreateRollingRepository(repo.headRevision(), minRetentionCommits, minRetentionDays); + } + + @Nullable + private Revision shouldCreateRollingRepository(Revision headRevision, int minRetentionCommits, + int minRetentionDays) { + // TODO(minwoox): provide a way to set different minRetentionCommits and minRetentionDays + // in each repository. + if ((minRetentionCommits == 0 && minRetentionDays == 0) || + (minRetentionCommits == Integer.MAX_VALUE && minRetentionDays == Integer.MAX_VALUE)) { + // Not enabled. + return null; } + + final InternalRepository repo = secondaryRepo != null ? secondaryRepo : primaryRepo; + if (exceedsMinRetention(repo, headRevision, minRetentionCommits, minRetentionDays)) { + return headRevision; + } + return null; } - private static boolean exceedsMinRetention(InternalRepository repo, + private static boolean exceedsMinRetention(InternalRepository repo, Revision headRevision, int minRetentionCommits, int minRetentionDays) { - final Revision headRevision = repo.headRevision(); final Revision firstRevision = repo.firstRevision(); if (minRetentionCommits != 0) { if (headRevision.major() - firstRevision.major() <= minRetentionCommits) { @@ -723,7 +779,23 @@ private static boolean exceedsMinRetention(InternalRepository repo, return true; } - private void promoteSecondaryRepo(InternalRepository secondaryRepo) { + @Override + public void createRollingRepository(Revision rollingRepositoryInitialRevision, + int minRetentionCommits, int minRetentionDays) { + checkState(shouldCreateRollingRepository(rollingRepositoryInitialRevision, + minRetentionCommits, minRetentionDays) == + rollingRepositoryInitialRevision, "aaa"); + + final InternalRepository secondaryRepo = this.secondaryRepo; + if (secondaryRepo != null) { + promoteSecondaryRepo(secondaryRepo, rollingRepositoryInitialRevision); + } else { + createSecondaryRepo(rollingRepositoryInitialRevision); + } + } + + private void promoteSecondaryRepo(InternalRepository secondaryRepo, + Revision rollingRepositoryInitialRevision) { writeLock(); try { logger.info("Promoting the secondary repository in {}/{}.", parent.name(), originalRepoName); @@ -748,31 +820,31 @@ private void promoteSecondaryRepo(InternalRepository secondaryRepo) { writeUnLock(); } - createSecondaryRepo(); + createSecondaryRepo(rollingRepositoryInitialRevision); } - private void createSecondaryRepo() { + private void createSecondaryRepo(Revision rollingRepositoryInitialRevision) { InternalRepository secondaryRepo = null; try { writeLock(); - final Revision headRevision = this.headRevision; isCreatingSecondaryRepo = true; writeUnLock(); logger.info("Creating the secondary repository in {}/{}. head revision: {}.", - parent.name(), originalRepoName, headRevision); - final Map> entries = find(headRevision, ALL_PATH, ImmutableMap.of()).join(); + parent.name(), originalRepoName, rollingRepositoryInitialRevision); + final Map> entries = find(rollingRepositoryInitialRevision, ALL_PATH, + ImmutableMap.of()).join(); final List> changes = toChanges(entries); final File secondaryRepoDir = repoMetadata.secondaryRepoDir(); - secondaryRepo = - InternalRepository.of(parent, originalRepoName, secondaryRepoDir, headRevision, - creationTimeMillis, author, changes); + secondaryRepo = InternalRepository.of(parent, originalRepoName, secondaryRepoDir, + rollingRepositoryInitialRevision, creationTimeMillis, + author, changes); writeLock(); try { if (laggedCommitsForSecondary.isEmpty()) { // There were no commits after creating the secondary repo - // so the headRevision hasn't changed. - assert headRevision == this.headRevision; + // so the rollingRepositoryInitialRevision hasn't changed. + assert rollingRepositoryInitialRevision == this.headRevision; } else { // We should catch up. for (LaggedCommit c : laggedCommitsForSecondary) { @@ -787,7 +859,7 @@ private void createSecondaryRepo() { writeUnLock(); } logger.info("The secondary repository {} is created in {}/{}. head revision: {}", - secondaryRepoDir.getName(), parent.name(), originalRepoName, headRevision); + secondaryRepoDir.getName(), parent.name(), originalRepoName, rollingRepositoryInitialRevision); } catch (Throwable t) { logger.warn("Failed to create the secondary repository", t); if (secondaryRepo != null) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java index eb3196ecb..ef94aea33 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java @@ -92,6 +92,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import com.linecorp.centraldogma.common.Author; @@ -309,14 +310,6 @@ private InternalRepository(Project project, String originalRepoName, File repoDi this.commitIdDatabase = commitIdDatabase; } - Project project() { - return project; - } - - String originalRepoName() { - return originalRepoName; - } - File repoDir() { return repoDir; } @@ -1145,6 +1138,17 @@ void close() { closeJGitRepo(jGitRepository); } + @Override + public String toString() { + return MoreObjects.toStringHelper(this).omitNullValues() + .add("project", project) + .add("originalRepoName", originalRepoName) + .add("repoDir", repoDir) + .add("commitIdDatabase", commitIdDatabase) + .add("jGitRepository", jGitRepository) + .toString(); + } + static final class RevisionAndEntries { final Revision revision; final List diffEntries; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java index fdd5978c5..4bd398322 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java @@ -31,6 +31,8 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; +import javax.annotation.Nullable; + import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -526,4 +528,9 @@ default CompletableFuture> mergeFiles(Revision revision, Merg * number of {@code minRetentionCommits}. */ void removeOldCommits(int minRetentionCommits, int minRetentionDays); + + @Nullable + Revision shouldCreateRollingRepository(int minRetentionCommits, int minRetentionDays); + + void createRollingRepository(Revision initialRevision, int minRetentionCommits, int minRetentionDays); } From 7b21b1aec0e5f2ae5e95ff79a0572a214d17f030 Mon Sep 17 00:00:00 2001 From: minwoox Date: Mon, 3 Jan 2022 16:03:52 +0900 Subject: [PATCH 08/14] Introduce CreateRollingRepository command to create the rolling repository by the leader --- .../server/CommitRetentionConfig.java | 6 +- .../CreateRollingRepositoryCommand.java | 20 +++ .../storage/repository/RepositoryWrapper.java | 13 +- .../repository/cache/CachingRepository.java | 10 +- .../git/CommitRetentionManagementPlugin.java | 4 +- .../repository/git/GitRepositoryV2.java | 117 +++++------------- .../repository/git/InternalRepository.java | 8 +- .../repository/git/LegacyGitRepository.java | 8 +- .../server/storage/repository/Repository.java | 16 ++- .../repository/git/CommitIdDatabaseTest.java | 11 +- .../git/GitRepositoryV2DiffTest.java | 2 +- .../git/GitRepositoryV2FindTest.java | 2 +- .../git/GitRepositoryV2HistoryTest.java | 16 ++- .../git/GitRepositoryV2PromotionTest.java | 45 +++++-- .../git/GitRepositoryV2WatchTest.java | 2 +- 15 files changed, 157 insertions(+), 123 deletions(-) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java b/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java index 994889bda..05df8d8d9 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java @@ -65,8 +65,8 @@ public CommitRetentionConfig(@JsonProperty("minRetentionCommits") int minRetenti /** * Returns the minimum number of commits that a {@link Repository} should retain. 0 means that - * the number of commits are not taken into account when {@link Repository#removeOldCommits(int, int)} - * is called. + * the number of commits are not taken into account when + * {@link Repository#shouldCreateRollingRepository(int, int)} is called. */ public int minRetentionCommits() { return minRetentionCommits; @@ -75,7 +75,7 @@ public int minRetentionCommits() { /** * Returns the minimum number of days of a commit that a {@link Repository} should retain. 0 means that * the number of retention days of commits are not taken into account when - * {@link Repository#removeOldCommits(int, int)} is called. + * {@link Repository#shouldCreateRollingRepository(int, int)} is called. */ public int minRetentionDays() { return minRetentionDays; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/CreateRollingRepositoryCommand.java b/server/src/main/java/com/linecorp/centraldogma/server/command/CreateRollingRepositoryCommand.java index 11df5f7cc..99cafdd8f 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/CreateRollingRepositoryCommand.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/CreateRollingRepositoryCommand.java @@ -19,7 +19,14 @@ import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.server.CommitRetentionConfig; +import com.linecorp.centraldogma.server.storage.repository.Repository; +/** + * A {@link Command} which is used for creating a new rolling repository. + * + * @see CommitRetentionConfig + */ public final class CreateRollingRepositoryCommand extends RepositoryCommand { private final Revision initialRevision; @@ -34,14 +41,27 @@ public final class CreateRollingRepositoryCommand extends RepositoryCommand CompletableFuture> mergeFiles(Revision revision, Merge return unwrap().mergeFiles(revision, query); } + @Nullable + @Override + public Revision shouldCreateRollingRepository(int minRetentionCommits, int minRetentionDays) { + return unwrap().shouldCreateRollingRepository(minRetentionCommits, minRetentionDays); + } + @Override - public void removeOldCommits(int minRetentionCommits, int minRetentionDays) { - unwrap().removeOldCommits(minRetentionCommits, minRetentionDays); + public void createRollingRepository(Revision initialRevision, int minRetentionCommits, + int minRetentionDays) { + unwrap().createRollingRepository(initialRevision, minRetentionCommits, minRetentionDays); } @Override diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java index 8f12a35d4..7f1534cbc 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java @@ -308,8 +308,14 @@ public CompletableFuture commit(Revision baseRevision, long commit } @Override - public void removeOldCommits(int minRetentionCommits, int minRetentionDays) { - repo.removeOldCommits(minRetentionCommits, minRetentionDays); + public Revision shouldCreateRollingRepository(int minRetentionCommits, int minRetentionDays) { + return repo.shouldCreateRollingRepository(minRetentionCommits, minRetentionDays); + } + + @Override + public void createRollingRepository(Revision initialRevision, int minRetentionCommits, + int minRetentionDays) { + repo.createRollingRepository(initialRevision, minRetentionCommits, minRetentionDays); } @Override diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java index d929767d6..3542234c2 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java @@ -89,7 +89,7 @@ private void scheduleRemovingOldCommits(PluginContext context, CommitRetentionCo .timeToNextExecution(ZonedDateTime.now()); final ListeningScheduledExecutorService worker = this.worker; assert worker != null; - scheduledFuture = worker.schedule(() -> removeOldCommits(context, config), nextExecution); + scheduledFuture = worker.schedule(() -> createRollingRepository(context, config), nextExecution); Futures.addCallback(scheduledFuture, new FutureCallback() { @Override @@ -105,7 +105,7 @@ public void onFailure(Throwable cause) { }, worker); } - private void removeOldCommits(PluginContext context, CommitRetentionConfig config) { + private void createRollingRepository(PluginContext context, CommitRetentionConfig config) { if (stopping) { return; } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java index 966cd893c..20e1b6875 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java @@ -31,7 +31,6 @@ import java.nio.file.Path; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -123,10 +122,6 @@ static GitRepositoryV2 open(Project parent, File repositoryDir, private final long creationTimeMillis; private final Author author; - // Guarded by the write lock. - private boolean isCreatingSecondaryRepo; - private final List laggedCommitsForSecondary = new ArrayList<>(); - /** * The current head revision. Initialized by the constructor and updated by commit(). */ @@ -483,16 +478,11 @@ private CommitResult blockingCommit( res = primaryRepo.commit(headRevision, headRevision.forward(1), commitTimeMillis, author, summary, detail, markup, applyingChanges, false); this.headRevision = res.revision; - if (isCreatingSecondaryRepo) { - laggedCommitsForSecondary.add(new LaggedCommit(headRevision, commitTimeMillis, author, summary, - detail, markup, applyingChanges, false)); - } else { - final InternalRepository secondaryRepo = this.secondaryRepo; - if (secondaryRepo != null) { - // Push the same commit to the secondary repo. - secondaryRepo.commit(headRevision, headRevision.forward(1), commitTimeMillis, - author, summary, detail, markup, applyingChanges, false); - } + final InternalRepository secondaryRepo = this.secondaryRepo; + if (secondaryRepo != null) { + // Push the same commit to the secondary repo. + secondaryRepo.commit(headRevision, headRevision.forward(1), commitTimeMillis, + author, summary, detail, markup, applyingChanges, false); } } finally { writeUnLock(); @@ -657,12 +647,12 @@ public CompletableFuture watch(Revision lastKnownRevision, String path requireNonNull(pathPattern, "pathPattern"); final ServiceRequestContext ctx = context(); - final Revision normLastKnownRevision = normalizeNow(lastKnownRevision, false); final CompletableFuture future = new CompletableFuture<>(); CompletableFuture.runAsync(() -> { failFastIfTimedOut(this, logger, ctx, "watch", lastKnownRevision, pathPattern); readLock(); try { + final Revision normLastKnownRevision = normalizeNow(lastKnownRevision, false); // If lastKnownRevision is outdated already and the recent changes match, // there's no need to watch. final Revision latestRevision = blockingFindLatestRevision(normLastKnownRevision, pathPattern, @@ -713,31 +703,6 @@ private static Commit currentInitialCommit(InternalRepository primaryRepo) { return primaryRepo.listCommits(ALL_PATH, 1, range).get(0); } - /** - * This repository removes the old commits by creating the secondary repository that commits are pushed - * together with the primary repository and removing the primary repository if the secondary repository - * contains at least the number of {@code minRetentionCommits} and contains the recent - * {@code minRetentionDays} commits. - */ - @Override - public void removeOldCommits(int minRetentionCommits, int minRetentionDays) { - // TODO(minwoox): provide a way to set different minRetentionCommits and minRetentionDays - // in each repository. - if (minRetentionCommits == 0 && minRetentionDays == 0) { - // Not enabled. - return; - } - - final InternalRepository secondaryRepo = this.secondaryRepo; - if (secondaryRepo != null) { - if (exceedsMinRetention(secondaryRepo, headRevision, minRetentionCommits, minRetentionDays)) { - promoteSecondaryRepo(secondaryRepo, rollingRepositoryInitialRevision); - } - } else if (exceedsMinRetention(primaryRepo, headRevision, minRetentionCommits, minRetentionDays)) { - createSecondaryRepo(rollingRepositoryInitialRevision); - } - } - @Override public Revision shouldCreateRollingRepository(int minRetentionCommits, int minRetentionDays) { final InternalRepository repo = secondaryRepo != null ? secondaryRepo : primaryRepo; @@ -782,6 +747,7 @@ private static boolean exceedsMinRetention(InternalRepository repo, Revision hea @Override public void createRollingRepository(Revision rollingRepositoryInitialRevision, int minRetentionCommits, int minRetentionDays) { + requireNonNull(rollingRepositoryInitialRevision, "rollingRepositoryInitialRevision"); checkState(shouldCreateRollingRepository(rollingRepositoryInitialRevision, minRetentionCommits, minRetentionDays) == rollingRepositoryInitialRevision, "aaa"); @@ -826,40 +792,42 @@ private void promoteSecondaryRepo(InternalRepository secondaryRepo, private void createSecondaryRepo(Revision rollingRepositoryInitialRevision) { InternalRepository secondaryRepo = null; try { - writeLock(); - isCreatingSecondaryRepo = true; - writeUnLock(); - logger.info("Creating the secondary repository in {}/{}. head revision: {}.", parent.name(), originalRepoName, rollingRepositoryInitialRevision); - final Map> entries = find(rollingRepositoryInitialRevision, ALL_PATH, - ImmutableMap.of()).join(); - final List> changes = toChanges(entries); + final List> changes = changes(rollingRepositoryInitialRevision); final File secondaryRepoDir = repoMetadata.secondaryRepoDir(); secondaryRepo = InternalRepository.of(parent, originalRepoName, secondaryRepoDir, rollingRepositoryInitialRevision, creationTimeMillis, author, changes); writeLock(); try { - if (laggedCommitsForSecondary.isEmpty()) { - // There were no commits after creating the secondary repo - // so the rollingRepositoryInitialRevision hasn't changed. - assert rollingRepositoryInitialRevision == this.headRevision; - } else { - // We should catch up. - for (LaggedCommit c : laggedCommitsForSecondary) { - secondaryRepo.commit(c.revision, c.revision.forward(1), c.commitTimeMillis, c.author, - c.summary, c.detail, c.markup, c.changes, c.allowEmpty); + final Revision headRevision = this.headRevision; + if (!rollingRepositoryInitialRevision.equals(headRevision)) { + assert rollingRepositoryInitialRevision.major() < headRevision.major(); + // There were commits after the createRollingRepositoryCommand is created + // so we should catch up. + final RevisionRange revisionRange = new RevisionRange( + rollingRepositoryInitialRevision.forward(1), headRevision); + final List commits = primaryRepo.listCommits(ALL_PATH, MAX_MAX_COMMITS, + revisionRange); + Revision fromRevision = rollingRepositoryInitialRevision; + for (Commit commit : commits) { + final Revision toRevision = commit.revision(); + final Map> diffs = primaryRepo.diff( + new RevisionRange(fromRevision, toRevision), ALL_PATH); + secondaryRepo.commit(fromRevision, toRevision, commit.when(), commit.author(), + commit.summary(), commit.detail(), commit.markup(), diffs.values(), + false); + fromRevision = toRevision; } - laggedCommitsForSecondary.clear(); } - isCreatingSecondaryRepo = false; this.secondaryRepo = secondaryRepo; } finally { writeUnLock(); } logger.info("The secondary repository {} is created in {}/{}. head revision: {}", - secondaryRepoDir.getName(), parent.name(), originalRepoName, rollingRepositoryInitialRevision); + secondaryRepoDir.getName(), parent.name(), originalRepoName, + rollingRepositoryInitialRevision); } catch (Throwable t) { logger.warn("Failed to create the secondary repository", t); if (secondaryRepo != null) { @@ -873,7 +841,9 @@ private void createSecondaryRepo(Revision rollingRepositoryInitialRevision) { } } - private static List> toChanges(Map> entries) { + private List> changes(Revision rollingRepositoryInitialRevision) { + final Map> entries = primaryRepo.find( + rollingRepositoryInitialRevision, ALL_PATH, ImmutableMap.of()); final Builder> builder = ImmutableList.builder(); for (Entry entry : entries.values()) { final EntryType type = entry.type(); @@ -901,29 +871,4 @@ private static boolean deleteDirectory(File dir) throws IOException { .reduce(true, (a, b) -> a && b); } } - - private static class LaggedCommit { - - private final Revision revision; - private final long commitTimeMillis; - private final Author author; - private final String summary; - private final String detail; - private final Markup markup; - private final Iterable> changes; - private final boolean allowEmpty; - - LaggedCommit(Revision revision, long commitTimeMillis, Author author, String summary, - String detail, Markup markup, Iterable> changes, - boolean allowEmpty) { - this.revision = revision; - this.commitTimeMillis = commitTimeMillis; - this.author = author; - this.summary = summary; - this.detail = detail; - this.markup = markup; - this.changes = changes; - this.allowEmpty = allowEmpty; - } - } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java index ef94aea33..8bd733cc7 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java @@ -454,17 +454,19 @@ private static Commit toCommit(RevCommit revCommit) { } Map> find(Revision normalizedRevision, String pathPattern, Map, ?> options) { + final PathPatternFilter filter = PathPatternFilter.of(pathPattern); + final int maxEntries = FindOption.MAX_ENTRIES.get(options); + try (ObjectReader reader = jGitRepository.newObjectReader(); TreeWalk treeWalk = new TreeWalk(reader); RevWalk revWalk = newRevWalk(reader)) { + final Map> result = new LinkedHashMap<>(); final ObjectId commitId = commitIdDatabase.get(normalizedRevision); final RevCommit revCommit = revWalk.parseCommit(commitId); - final PathPatternFilter filter = PathPatternFilter.of(pathPattern); - - final int maxEntries = FindOption.MAX_ENTRIES.get(options); final RevTree revTree = revCommit.getTree(); treeWalk.addTree(revTree.getId()); + while (treeWalk.next() && result.size() < maxEntries) { final boolean matches = filter.matches(treeWalk); final String path = '/' + treeWalk.getPathString(); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/LegacyGitRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/LegacyGitRepository.java index dd05d22ee..41a9bfcdd 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/LegacyGitRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/LegacyGitRepository.java @@ -1647,7 +1647,13 @@ private static void deleteCruft(File repoDir) { } @Override - public void removeOldCommits(int minRetentionCommits, int minRetentionDays) { + public Revision shouldCreateRollingRepository(int minRetentionCommits, int minRetentionDays) { + throw new UnsupportedOperationException(); + } + + @Override + public void createRollingRepository(Revision initialRevision, int minRetentionCommits, + int minRetentionDays) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java index 4bd398322..dfeb49524 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java @@ -524,13 +524,21 @@ default CompletableFuture> mergeFiles(Revision revision, Merg } /** - * Removes the old commits that are passed more than the {@code minRetentionDays} and are exceeded the - * number of {@code minRetentionCommits}. + * Returns the head {@link Revision} of this repository if this repository needs to create a + * rolling repository. It should meet following conditions: + * - The last created rolling repository should have more than {@code minRetentionCommits}. + * - The last created rolling repository should have commits that are older than {@code minRetentionDays}. */ - void removeOldCommits(int minRetentionCommits, int minRetentionDays); - @Nullable Revision shouldCreateRollingRepository(int minRetentionCommits, int minRetentionDays); + /** + * Creates the rolling repository. The specified {@code initialRevision} of the current repository will be + * the initial revision of the created rolling repository. + * + * @throws IllegalStateException if {@link #shouldCreateRollingRepository(int, int)} with the specified + * {@code minRetentionCommits} and {@code minRetentionDays} does not return + * a valid {@link Revision}. + */ void createRollingRepository(Revision initialRevision, int minRetentionCommits, int minRetentionDays); } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabaseTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabaseTest.java index 9a7099177..acfd7d77a 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabaseTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabaseTest.java @@ -153,7 +153,7 @@ void rebuildingBadDatabase(int startRevision) throws Exception { assertThat(Files.size(commitIdDatabaseFile.toPath())).isEqualTo((numCommits + 1) * 24L); } - private GitRepositoryV2 createRepoWithFirstRevision(File repoDir, int startRevision) { + private static GitRepositoryV2 createRepoWithFirstRevision(File repoDir, int startRevision) { final GitRepositoryV2 repo = new GitRepositoryV2(mock(Project.class), repoDir, commonPool(), 0, Author.SYSTEM, null); @@ -166,14 +166,17 @@ private GitRepositoryV2 createRepoWithFirstRevision(File repoDir, int startRevis for (int i = 1; i < startRevision; i++) { addCommit(repo, i); } - repo.removeOldCommits(1, 0); + assertThat(repo.shouldCreateRollingRepository(1, 0).major()).isEqualTo(startRevision); + repo.createRollingRepository(new Revision(startRevision), 1, 0); // Now the first revision of secondary repository is startRevision; final InternalRepository secondaryRepo = repo.secondaryRepo; assertThat(secondaryRepo.commitIdDatabase().firstRevision().major()).isEqualTo(startRevision); addCommit(repo, startRevision); addCommit(repo, startRevision + 1); - repo.removeOldCommits(1, 0); + assertThat(repo.shouldCreateRollingRepository(1, 0).major()).isEqualTo(startRevision + 2); + repo.createRollingRepository(new Revision(startRevision + 2), 1, 0); + // The secondary repo is promoted. assertThat(repo.primaryRepo).isSameAs(secondaryRepo); assertThat(repo.primaryRepo.commitIdDatabase().firstRevision().major()).isEqualTo(startRevision); @@ -181,7 +184,7 @@ private GitRepositoryV2 createRepoWithFirstRevision(File repoDir, int startRevis return repo; } - private Revision addCommit(GitRepositoryV2 repo, int i) { + private static Revision addCommit(GitRepositoryV2 repo, int i) { return repo.commit(Revision.HEAD, 0, Author.SYSTEM, "", Change.ofTextUpsert("/" + i + ".txt", "")) .join() .revision(); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2DiffTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2DiffTest.java index 1941f4734..74197ce8b 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2DiffTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2DiffTest.java @@ -45,7 +45,7 @@ class GitRepositoryV2DiffTest { @BeforeAll static void setUp() { // The repository contains commits from 10 to 20(inclusive). - repo = createRepository(repoDir, 10, 20); + repo = createRepository(repoDir); } @AfterAll diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2FindTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2FindTest.java index 0c08d7ac7..b4aa0d3ad 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2FindTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2FindTest.java @@ -45,7 +45,7 @@ class GitRepositoryV2FindTest { @BeforeAll static void setUp() { // The repository contains commits from 10 to 20(inclusive). - repo = createRepository(repoDir, 10, 20); + repo = createRepository(repoDir); } @AfterAll diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2HistoryTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2HistoryTest.java index 329c625e0..fc0ed2ac6 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2HistoryTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2HistoryTest.java @@ -44,7 +44,7 @@ class GitRepositoryV2HistoryTest { @BeforeAll static void setUp() { // The repository contains commits from 10 to 20(inclusive). - repo = createRepository(repoDir, 10, 20); + repo = createRepository(repoDir); } @AfterAll @@ -96,16 +96,20 @@ void normalHistoryCall() { assertThat(commits).hasSize(1); } - static GitRepositoryV2 createRepository(File repoDir, int from, int to) { + static GitRepositoryV2 createRepository(File repoDir) { final GitRepositoryV2 repo = new GitRepositoryV2(mock(Project.class), new File(repoDir, "test_repo"), commonPool(), 0L, Author.SYSTEM, null); - addCommits(repo, 2, from); - repo.removeOldCommits(from - 2, 0); + addCommits(repo, 2, 10); + int minRetentionCommits = 10 - 2; + assertThat(repo.shouldCreateRollingRepository(minRetentionCommits, 0).major()).isEqualTo(10); + repo.createRollingRepository(new Revision(10), minRetentionCommits, 0); // The headRevision of the secondary repository is now from revision. - addCommits(repo, from + 1, to); - repo.removeOldCommits(to - from - 1, 0); + addCommits(repo, 11, 20); + minRetentionCommits = 9; // 20 - 10 - 1 + assertThat(repo.shouldCreateRollingRepository(minRetentionCommits, 0).major()).isEqualTo(20); + repo.createRollingRepository(new Revision(20), minRetentionCommits, 0); final CommitIdDatabase commitIdDatabase = repo.primaryRepo.commitIdDatabase(); assertThat(commitIdDatabase.firstRevision().major()).isSameAs(10); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java index 8d381711d..885c65507 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java @@ -16,6 +16,7 @@ package com.linecorp.centraldogma.server.internal.storage.repository.git; import static com.linecorp.centraldogma.server.internal.storage.DirectoryBasedStorageManager.SUFFIX_REMOVED; +import static com.linecorp.centraldogma.server.storage.repository.Repository.ALL_PATH; import static java.util.concurrent.ForkJoinPool.commonPool; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; @@ -23,12 +24,16 @@ import java.io.File; import java.time.Instant; +import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import com.google.common.collect.ImmutableMap; + import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Entry; import com.linecorp.centraldogma.common.Markup; import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.server.storage.project.Project; @@ -49,13 +54,15 @@ void promote() { addCommits(repo, 2, 11); assertThat(primaryCommitIdDatabase.headRevision().major()).isEqualTo(11); - repo.removeOldCommits(10, 0); + assertThat(repo.shouldCreateRollingRepository(10, 0)).isNull(); // Nothing happened because 10 commits are made so far. assertThat(repo.secondaryRepo).isNull(); - // Add one more commit. + // Add one more commit to create a rolling repository. addCommits(repo, 12, 12); - repo.removeOldCommits(10, 0); + assertThat(repo.shouldCreateRollingRepository(10, 0).major()).isEqualTo(12); + repo.createRollingRepository(new Revision(12), 1, 0); + assertThat(primaryCommitIdDatabase.headRevision().major()).isEqualTo(12); final InternalRepository secondaryRepo = repo.secondaryRepo; assertThat(secondaryRepo).isNotNull(); @@ -66,7 +73,7 @@ void promote() { // Add ten more commit. addCommits(repo, 13, 22); // No changes. - repo.removeOldCommits(10, 0); + assertThat(repo.shouldCreateRollingRepository(10, 0)).isNull(); assertThat(primaryCommitIdDatabase.headRevision().major()).isEqualTo(22); assertThat(secondaryRepo).isEqualTo(repo.secondaryRepo); assertThat(repo.primaryRepo).isNotSameAs(repo.secondaryRepo); @@ -74,9 +81,10 @@ void promote() { assertThat(secondaryRepo.commitIdDatabase().headRevision().major()).isEqualTo(22); assertThat(secondaryRepo.secondCommitCreationTimeInstant()).isEqualTo(Instant.ofEpochMilli(13 * 1000)); - // Add one more commit. + // Add one more commit to create a rolling repository. addCommits(repo, 23, 23); - repo.removeOldCommits(10, 0); + assertThat(repo.shouldCreateRollingRepository(10, 0).major()).isEqualTo(23); + repo.createRollingRepository(new Revision(23), 1, 0); // The secondary repo is promoted to the primary repo. assertThat(repo.primaryRepo).isSameAs(secondaryRepo); assertThat(repo.primaryRepo).isNotSameAs(primaryRepo); @@ -94,6 +102,25 @@ void promote() { assertThat(repo.secondaryRepo.commitIdDatabase().headRevision().major()).isEqualTo(23); assertThat(repo.secondaryRepo.secondCommitCreationTimeInstant()).isNull(); + // Add 11 commits so that we are ready to create a rolling repository. + addCommits(repo, 24, 35); + assertThat(repo.shouldCreateRollingRepository(10, 0).major()).isEqualTo(35); + + // Add 5 more commits before creating a rolling repository to check if there are missing commits. + addCommits(repo, 36, 40); + repo.createRollingRepository(new Revision(35), 1, 0); + + assertThat(repo.primaryRepo.commitIdDatabase().firstRevision().major()).isEqualTo(23); + assertThat(repo.primaryRepo.commitIdDatabase().headRevision().major()).isEqualTo(40); + + assertThat(repo.secondaryRepo.commitIdDatabase().firstRevision().major()).isEqualTo(35); + assertThat(repo.secondaryRepo.commitIdDatabase().headRevision().major()).isEqualTo(40); + + for (int i = 36; i < 40; i++) { + assertThat(entries(repo.primaryRepo, i)).containsExactlyInAnyOrderEntriesOf( + entries(repo.primaryRepo, i)); + } + repo.internalClose(); } @@ -105,7 +132,11 @@ static void addCommits(GitRepositoryV2 repo, int start, int end) { } static void addCommit(GitRepositoryV2 repo, int index, Change change) { - repo.commit(Revision.HEAD, index * 1000, Author.SYSTEM, + repo.commit(Revision.HEAD, index * 1000L, Author.SYSTEM, "Summary" + index, "Detail", Markup.PLAINTEXT, change).join(); } + + private static Map> entries(InternalRepository repo, int revision) { + return repo.find(new Revision(revision), ALL_PATH, ImmutableMap.of()); + } } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2WatchTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2WatchTest.java index d07537dc5..895da0aaf 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2WatchTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2WatchTest.java @@ -40,7 +40,7 @@ class GitRepositoryV2WatchTest { @BeforeAll static void setUp() { // The repository contains commits from 10 to 20(inclusive). - repo = createRepository(repoDir, 10, 20); + repo = createRepository(repoDir); } @AfterAll From 958e7a96f029cc9602439982af84da720c8ec6f5 Mon Sep 17 00:00:00 2001 From: minwoox Date: Tue, 4 Jan 2022 10:52:05 +0900 Subject: [PATCH 09/14] Fix NonNullByDefault --- .../com/linecorp/centraldogma/common/util/NonNullByDefault.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/com/linecorp/centraldogma/common/util/NonNullByDefault.java b/common/src/main/java/com/linecorp/centraldogma/common/util/NonNullByDefault.java index bfa801cd9..4838fd32a 100644 --- a/common/src/main/java/com/linecorp/centraldogma/common/util/NonNullByDefault.java +++ b/common/src/main/java/com/linecorp/centraldogma/common/util/NonNullByDefault.java @@ -34,6 +34,6 @@ @Documented @Target(ElementType.PACKAGE) @Retention(RetentionPolicy.RUNTIME) -@TypeQualifierDefault({ ElementType.METHOD, ElementType.PARAMETER, ElementType.FIELD }) +@TypeQualifierDefault({ ElementType.METHOD, ElementType.PARAMETER, ElementType.FIELD, ElementType.TYPE_USE }) public @interface NonNullByDefault { } From cd622e42d53060d1e9907aa06311ae8696aa8875 Mon Sep 17 00:00:00 2001 From: minwoox Date: Mon, 21 Feb 2022 17:25:19 +0900 Subject: [PATCH 10/14] WIP --- .../repository/git/CommitIdDatabase.java | 4 ++-- .../repository/git/GitRepositoryV2.java | 23 ++++++++++--------- .../git/RepositoryMetadataDatabase.java | 3 ++- .../git/RepositoryMetadataDatabaseTest.java | 6 +++++ 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java index 5978807ff..6f6e15298 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java @@ -181,7 +181,7 @@ ObjectId get(Revision revision) { if (!(firstRevision.major() <= revision.major() && revision.major() <= headRevision.major())) { throw new RevisionNotFoundException( "revision: " + revision + - " (expected: " + firstRevision.major() + " <= revision <= " + headRevision.major() + ")"); + " (expected: " + firstRevision.major() + " <= revision <= " + headRevision.major() + ')'); } final ByteBuffer buf = threadLocalBuffer.get(); @@ -222,7 +222,7 @@ private synchronized void put(Revision revision, ObjectId commitId, boolean isNe commitId.copyRawTo(buf); buf.flip(); - // Append a record to the file. + // Append or overwrite a record in the file. long pos = (long) (revision.major() - firstRevision.major()) * RECORD_LEN; try { do { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java index 20e1b6875..8e57a3d52 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java @@ -480,6 +480,8 @@ private CommitResult blockingCommit( this.headRevision = res.revision; final InternalRepository secondaryRepo = this.secondaryRepo; if (secondaryRepo != null) { + assert headRevision.equals(secondaryRepo.headRevision()); + // Push the same commit to the secondary repo. secondaryRepo.commit(headRevision, headRevision.forward(1), commitTimeMillis, author, summary, detail, markup, applyingChanges, false); @@ -748,22 +750,23 @@ private static boolean exceedsMinRetention(InternalRepository repo, Revision hea public void createRollingRepository(Revision rollingRepositoryInitialRevision, int minRetentionCommits, int minRetentionDays) { requireNonNull(rollingRepositoryInitialRevision, "rollingRepositoryInitialRevision"); - checkState(shouldCreateRollingRepository(rollingRepositoryInitialRevision, - minRetentionCommits, minRetentionDays) == - rollingRepositoryInitialRevision, "aaa"); + final Revision rollingRepositoryRevision = shouldCreateRollingRepository(rollingRepositoryInitialRevision, + minRetentionCommits, minRetentionDays); + checkState(rollingRepositoryRevision == rollingRepositoryInitialRevision, + "shouldCreateRollingRepository() returns %s. (expected: %s)", rollingRepositoryRevision, + rollingRepositoryInitialRevision); - final InternalRepository secondaryRepo = this.secondaryRepo; if (secondaryRepo != null) { - promoteSecondaryRepo(secondaryRepo, rollingRepositoryInitialRevision); - } else { - createSecondaryRepo(rollingRepositoryInitialRevision); + promoteSecondaryRepo(); } + createSecondaryRepo(rollingRepositoryInitialRevision); } - private void promoteSecondaryRepo(InternalRepository secondaryRepo, - Revision rollingRepositoryInitialRevision) { + private void promoteSecondaryRepo() { writeLock(); try { + checkState(primaryRepo.headRevision().equals(secondaryRepo.headRevision()), ""); + logger.info("Promoting the secondary repository in {}/{}.", parent.name(), originalRepoName); repoMetadata.setPrimaryRepoDir(secondaryRepo.jGitRepo().getDirectory()); final InternalRepository primaryRepo = this.primaryRepo; @@ -785,8 +788,6 @@ private void promoteSecondaryRepo(InternalRepository secondaryRepo, } finally { writeUnLock(); } - - createSecondaryRepo(rollingRepositoryInitialRevision); } private void createSecondaryRepo(Revision rollingRepositoryInitialRevision) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java index 80e46b10f..be9cad4da 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java @@ -53,7 +53,8 @@ private static File repoDir(File rootDir, String suffix) { return new File(rootDir, rootDir.getName() + '_' + suffix); } - private final File rootDir; + @VisibleForTesting + final File rootDir; private final Path path; private final FileChannel channel; private String primarySuffix; diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabaseTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabaseTest.java index 28d1d8df0..476817936 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabaseTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabaseTest.java @@ -48,6 +48,12 @@ void tearDown() { } } + @Test + void writeAndRead() throws Exception { + final RepositoryMetadataDatabase other = new RepositoryMetadataDatabase(db.rootDir, false); + assertThat(other.primaryRepoDir().getName()).isEqualTo(db.primaryRepoDir().getName()); + } + @Test void secondaryDirIsGreaterThanPrimaryByOne() { assertThat(db.primaryRepoDir().getName()).isEqualTo(tempDir.getName() + "_0000000000"); From 9fef35d605123042841a5d2cd4703ea043ec7f40 Mon Sep 17 00:00:00 2001 From: minwoox Date: Wed, 23 Feb 2022 16:45:44 +0900 Subject: [PATCH 11/14] Address the comment by @jrhee17 and @ikhoon --- .../server/CentralDogmaBuilder.java | 13 ++-- .../repository/git/CommitIdDatabase.java | 17 ++--- .../git/CommitRetentionManagementPlugin.java | 17 +++-- .../repository/git/GitRepositoryV2.java | 71 ++++++++++--------- .../repository/git/InternalRepository.java | 6 +- .../server/metadata/MetadataService.java | 1 + site/src/sphinx/setup-configuration.rst | 6 +- 7 files changed, 75 insertions(+), 56 deletions(-) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java index cd7c4a5d6..a6907116e 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java @@ -525,13 +525,14 @@ public CentralDogmaBuilder writeQuotaPerRepository(int writeQuota, int timeWindo /** * Sets the configuration for retaining commits in a {@link Repository}. - * The {@link Repository} retains at least the number of {@code minRetentionCommits} when more than - * {@code minRetentionCommits} are made. - * The {@link Repository} also retains commits at least {@code minRetentionDays}. If both conditions are - * not satisfied, the commits are not removed. + * The {@link Repository} retains at least the number of {@code minRetentionCommits} when more than + * {@code minRetentionCommits} are made. The {@link Repository} also retains commits at least + * {@code minRetentionDays}. If both conditions are not satisfied, the commits are not removed. * For example, when {@code minRetentionCommits} is set to 2000 and {@code minRetentionDays} is set to 14, - * the commits that are created more than 2000 are not removed until 14 days have passed. Set 0 to retain - * all commits. + * the commits that are created more than 2000 will be removed after 14 days have passed. + * If {@code minRetentionCommits} is set to 2000 and {@code minRetentionDays} is set to 0, + * the commits that are created more than 2000 will be removed no matter how young the commits are. + * Set both 0 to retain all commits. */ public CentralDogmaBuilder commitRetention(int minRetentionCommits, int minRetentionDays, @Nullable String schedule) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java index 6f6e15298..8c5a8fd55 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java @@ -269,8 +269,8 @@ void rebuild(Repository gitRepo) { // NB: We did not store the last commit ID until all commit IDs are stored, // so that the partially built database always has mismatching head revision. - rebuild(gitRepo, revWalk, headRevision, revCommit, true); - rebuild(gitRepo, revWalk, headRevision, revCommit, false); + findFirstRevisionOrRebuild(gitRepo, revWalk, headRevision, revCommit, true); + findFirstRevisionOrRebuild(gitRepo, revWalk, headRevision, revCommit, false); // All commit IDs except the head have been stored. Store the head finally. put(headRevision, headCommitId); @@ -282,8 +282,9 @@ void rebuild(Repository gitRepo) { firstRevision, headRevision); } - private void rebuild(Repository gitRepo, RevWalk revWalk, Revision headRevision, - RevCommit revCommit, boolean findingFirstRevision) throws IOException { + private void findFirstRevisionOrRebuild(Repository gitRepo, RevWalk revWalk, + Revision headRevision, RevCommit revCommit, + boolean findingFirstRevision) throws IOException { ObjectId currentId; Revision previousRevision = headRevision; loop: for (;;) { @@ -300,20 +301,20 @@ private void rebuild(Repository gitRepo, RevWalk revWalk, Revision headRevision, revCommit = revWalk.parseCommit(currentId); + final Revision currentRevision; if (findingFirstRevision) { - final Revision currentRevision = CommitUtil.extractRevision(revCommit.getFullMessage()); + currentRevision = CommitUtil.extractRevision(revCommit.getFullMessage()); final Revision expectedRevision = previousRevision.backward(1); if (!currentRevision.equals(expectedRevision)) { throw new StorageException("mismatching revision: " + gitRepo.getDirectory() + " (actual: " + currentRevision.major() + ", expected: " + expectedRevision.major() + ')'); } - previousRevision = currentRevision; } else { - final Revision currentRevision = previousRevision.backward(1); + currentRevision = previousRevision.backward(1); put(currentRevision, currentId, false); - previousRevision = currentRevision; } + previousRevision = currentRevision; } if (findingFirstRevision) { firstRevision = previousRevision; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java index 3542234c2..98f5ebb57 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java @@ -113,13 +113,22 @@ private void createRollingRepository(PluginContext context, CommitRetentionConfi final ProjectManager pm = context.projectManager(); for (Project project : pm.list().values()) { for (Repository repo : project.repos().list().values()) { + if (stopping) { + return; + } final Revision revision = repo.shouldCreateRollingRepository(config.minRetentionCommits(), config.minRetentionDays()); if (revision != null) { - context.commandExecutor().execute( - Command.createRollingRepository(project.name(), repo.name(), revision, - config.minRetentionCommits(), - config.minRetentionDays())); + try { + context.commandExecutor().execute( + Command.createRollingRepository(project.name(), repo.name(), revision, + config.minRetentionCommits(), + config.minRetentionDays())) + .get(100, TimeUnit.SECONDS); + } catch (Throwable t) { + logger.warn("Failed to create a rolling repository for {}/{} with revision: {}", + project.name(), repo.name(), revision, t); + } } } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java index 8e57a3d52..c7cb9ab67 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java @@ -127,6 +127,9 @@ static GitRepositoryV2 open(Project parent, File repositoryDir, */ private volatile Revision headRevision; + /** + * Create a new Git repository. + */ GitRepositoryV2(Project parent, File repositoryDir, Executor repositoryWorker, long creationTimeMillis, Author author, @Nullable RepositoryCache cache) { this.parent = requireNonNull(parent, "parent"); @@ -151,8 +154,8 @@ static GitRepositoryV2 open(Project parent, File repositoryDir, repoMetadata = new RepositoryMetadataDatabase(repositoryDir, true); final File primaryRepoDir = repoMetadata.primaryRepoDir(); try { - primaryRepo = InternalRepository.of(parent, originalRepoName, primaryRepoDir, Revision.INIT, - creationTimeMillis, author, ImmutableList.of()); + primaryRepo = InternalRepository.create(parent, originalRepoName, primaryRepoDir, Revision.INIT, + creationTimeMillis, author, ImmutableList.of()); } catch (Throwable t) { repoMetadata.close(); throw t; @@ -162,18 +165,9 @@ static GitRepositoryV2 open(Project parent, File repositoryDir, this.author = author; } - private static boolean isEmpty(File dir) throws IOException { - if (!dir.isDirectory()) { - return false; - } - if (!dir.exists()) { - return true; - } - try (Stream entries = Files.list(dir.toPath())) { - return entries.findFirst().isPresent(); - } - } - + /** + * Opens the existing Git repository. + */ private GitRepositoryV2(Project parent, File repoDir, Executor repositoryWorker, @Nullable RepositoryCache cache) { this.parent = requireNonNull(parent, "parent"); @@ -211,6 +205,18 @@ private GitRepositoryV2(Project parent, File repoDir, Executor repositoryWorker, } } + private static boolean isEmpty(File dir) throws IOException { + if (!dir.isDirectory()) { + return false; + } + if (!dir.exists()) { + return true; + } + try (Stream entries = Files.list(dir.toPath())) { + return entries.findFirst().isPresent(); + } + } + private static void checkRepositoryExists(File repoDir) { final Repository oldRepo = buildJGitRepo(repoDir); final boolean oldRepoExist = GitRepositoryUtil.exists(oldRepo); @@ -318,23 +324,27 @@ public Revision normalizeNow(Revision revision) { } private Revision normalizeNow(Revision revision, boolean checkFirstRevision) { - return normalizeNow(revision, headRevision.major(), checkFirstRevision); + return normalizeNow(revision, headRevision, checkFirstRevision); } @Override public RevisionRange normalizeNow(Revision from, Revision to) { - final int headMajor = headRevision.major(); - return new RevisionRange(normalizeNow(from, headMajor, true), normalizeNow(to, headMajor, true)); + final Revision headRevision = this.headRevision; + return new RevisionRange(normalizeNow(from, headRevision, true), normalizeNow(to, headRevision, true)); } - private Revision normalizeNow(Revision revision, int headMajor, boolean checkFirstRevision) { + private Revision normalizeNow(Revision revision, Revision headRevision, boolean checkFirstRevision) { requireNonNull(revision, "revision"); int major = revision.major(); + final int headMajor = headRevision.major(); + if (major == -1 || major == headMajor) { + return headRevision; + } if (major >= 0) { if (major > headMajor) { throw new RevisionNotFoundException( - "revision: " + revision + " (expected: <= " + headMajor + ")"); + "revision: " + revision + " (expected: <= " + headMajor + ')'); } } else { major = headMajor + major + 1; @@ -352,17 +362,12 @@ private Revision normalizeNow(Revision revision, int headMajor, boolean checkFir major = firstRevisionMajor; } else { throw new RevisionNotFoundException( - "revision: " + revision + " (expected: >= " + firstRevisionMajor + ")"); + "revision: " + revision + " (expected: >= " + firstRevisionMajor + ')'); } } } - // Create a new instance only when necessary. - if (revision.major() == major) { - return revision; - } else { - return new Revision(major); - } + return new Revision(major); } @Override @@ -750,8 +755,8 @@ private static boolean exceedsMinRetention(InternalRepository repo, Revision hea public void createRollingRepository(Revision rollingRepositoryInitialRevision, int minRetentionCommits, int minRetentionDays) { requireNonNull(rollingRepositoryInitialRevision, "rollingRepositoryInitialRevision"); - final Revision rollingRepositoryRevision = shouldCreateRollingRepository(rollingRepositoryInitialRevision, - minRetentionCommits, minRetentionDays); + final Revision rollingRepositoryRevision = shouldCreateRollingRepository( + rollingRepositoryInitialRevision, minRetentionCommits, minRetentionDays); checkState(rollingRepositoryRevision == rollingRepositoryInitialRevision, "shouldCreateRollingRepository() returns %s. (expected: %s)", rollingRepositoryRevision, rollingRepositoryInitialRevision); @@ -771,7 +776,7 @@ private void promoteSecondaryRepo() { repoMetadata.setPrimaryRepoDir(secondaryRepo.jGitRepo().getDirectory()); final InternalRepository primaryRepo = this.primaryRepo; this.primaryRepo = secondaryRepo; - this.secondaryRepo = null; + secondaryRepo = null; repositoryWorker.execute(() -> { closeInternalRepository(primaryRepo); final File repoDir = primaryRepo.repoDir(); @@ -784,7 +789,7 @@ private void promoteSecondaryRepo() { } }); logger.info("Promotion is done for {}/{}. {} is now the primary.", - parent.name(), originalRepoName, secondaryRepo.repoDir()); + parent.name(), originalRepoName, primaryRepo.repoDir()); } finally { writeUnLock(); } @@ -797,9 +802,9 @@ private void createSecondaryRepo(Revision rollingRepositoryInitialRevision) { parent.name(), originalRepoName, rollingRepositoryInitialRevision); final List> changes = changes(rollingRepositoryInitialRevision); final File secondaryRepoDir = repoMetadata.secondaryRepoDir(); - secondaryRepo = InternalRepository.of(parent, originalRepoName, secondaryRepoDir, - rollingRepositoryInitialRevision, creationTimeMillis, - author, changes); + secondaryRepo = InternalRepository.create(parent, originalRepoName, secondaryRepoDir, + rollingRepositoryInitialRevision, creationTimeMillis, + author, changes); writeLock(); try { final Revision headRevision = this.headRevision; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java index 8bd733cc7..cab2b1d88 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java @@ -143,9 +143,9 @@ final class InternalRepository { } } - static InternalRepository of(Project parent, String originalRepoName, File repoDir, - Revision nextRevision, long commitTimeMillis, - Author author, Iterable> changes) { + static InternalRepository create(Project parent, String originalRepoName, File repoDir, + Revision nextRevision, long commitTimeMillis, + Author author, Iterable> changes) { boolean success = false; InternalRepository internalRepo = null; try { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java index 3117dcd08..13c937e0a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java @@ -58,6 +58,7 @@ import com.linecorp.centraldogma.server.internal.storage.project.SafeProjectManager; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.project.ProjectManager; +import com.linecorp.centraldogma.server.storage.repository.Repository; /** * A service class for metadata management. diff --git a/site/src/sphinx/setup-configuration.rst b/site/src/sphinx/setup-configuration.rst index e096837a3..0ad539758 100644 --- a/site/src/sphinx/setup-configuration.rst +++ b/site/src/sphinx/setup-configuration.rst @@ -228,8 +228,10 @@ Core properties ``minRetentionCommits`` when more than ``minRetentionCommits`` are made. The repository also retains commits at least ``minRetentionDays``. If both conditions are not satisfied, the commits are not removed. For example, when ``minRetentionCommits`` is set to 2000 and ``minRetentionDays`` is set to 14, - the commits that are created more than 2000 are not removed until 14 days have passed. Set 0 to retain - all commits. + the commits that are created more than 2000 will be removed after 14 days have passed. + If ``minRetentionCommits`` is set to 2000 and ``minRetentionDays`` is set to 0, + the commits that are created more than 2000 will be removed no matter how young the commits are. + Set both 0 to retain all commits. - ``minRetentionCommits`` (integer) From a40a22decdf0ef77e4e788b353a2c924dd6fdefb Mon Sep 17 00:00:00 2001 From: minwoox Date: Wed, 23 Feb 2022 17:04:40 +0900 Subject: [PATCH 12/14] Remove unused import --- .../linecorp/centraldogma/server/metadata/MetadataService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java index 13c937e0a..3117dcd08 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java @@ -58,7 +58,6 @@ import com.linecorp.centraldogma.server.internal.storage.project.SafeProjectManager; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.project.ProjectManager; -import com.linecorp.centraldogma.server.storage.repository.Repository; /** * A service class for metadata management. From ec5de1d251a63abfd26bdbb7a58dcb5b9a10d28e Mon Sep 17 00:00:00 2001 From: minwoox Date: Wed, 23 Feb 2022 23:43:24 +0900 Subject: [PATCH 13/14] Fix test --- .../centraldogma/client/armeria/CentralDogmaRepositoryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/java-armeria/src/test/java/com/linecorp/centraldogma/client/armeria/CentralDogmaRepositoryTest.java b/client/java-armeria/src/test/java/com/linecorp/centraldogma/client/armeria/CentralDogmaRepositoryTest.java index 746c49bd6..c54c9a007 100644 --- a/client/java-armeria/src/test/java/com/linecorp/centraldogma/client/armeria/CentralDogmaRepositoryTest.java +++ b/client/java-armeria/src/test/java/com/linecorp/centraldogma/client/armeria/CentralDogmaRepositoryTest.java @@ -98,7 +98,7 @@ void historyAndDiff() { final List commits = centralDogmaRepo.history().get(new Revision(2), Revision.HEAD).join(); assertThat(commits.stream() .map(Commit::summary) - .collect(toImmutableList())).containsExactly("commit3", "commit2"); + .collect(toImmutableList())).containsExactly("commit2", "commit3"); assertThat(centralDogmaRepo.diff("/foo.json") .get(Revision.INIT, Revision.HEAD) .join()) From 433dcb8678bddea20c5668a55f224f9bdde9b466 Mon Sep 17 00:00:00 2001 From: minwoox Date: Fri, 25 Aug 2023 17:40:53 +0900 Subject: [PATCH 14/14] Refine --- .../centraldogma/common/Revision.java | 1 + .../common/RolledRevisionAccessException.java | 33 +++++ .../server/CentralDogmaBuilder.java | 11 +- .../server/CommitRetentionConfig.java | 10 +- .../server/command/AbstractCommand.java | 1 + .../CreateRollingRepositoryCommand.java | 30 ++++- .../repository/git/CommitIdDatabase.java | 13 +- .../git/CommitRetentionManagementPlugin.java | 26 ++-- .../storage/repository/git/FailFastUtil.java | 50 +------ .../repository/git/GitRepositoryV2.java | 122 +++++++++++------- .../repository/git/LegacyGitRepository.java | 1 + .../git/RepositoryMetadataDatabase.java | 63 +++++---- .../server/storage/repository/Repository.java | 10 +- site/src/sphinx/setup-configuration.rst | 17 +-- 14 files changed, 212 insertions(+), 176 deletions(-) create mode 100644 common/src/main/java/com/linecorp/centraldogma/common/RolledRevisionAccessException.java diff --git a/common/src/main/java/com/linecorp/centraldogma/common/Revision.java b/common/src/main/java/com/linecorp/centraldogma/common/Revision.java index 4d293e691..bdf8739ba 100644 --- a/common/src/main/java/com/linecorp/centraldogma/common/Revision.java +++ b/common/src/main/java/com/linecorp/centraldogma/common/Revision.java @@ -124,6 +124,7 @@ public int major() { * {@link Revision}. */ public boolean isLowerThan(Revision other) { + requireNonNull(other, "other"); return major < other.major(); } diff --git a/common/src/main/java/com/linecorp/centraldogma/common/RolledRevisionAccessException.java b/common/src/main/java/com/linecorp/centraldogma/common/RolledRevisionAccessException.java new file mode 100644 index 000000000..11c9a2721 --- /dev/null +++ b/common/src/main/java/com/linecorp/centraldogma/common/RolledRevisionAccessException.java @@ -0,0 +1,33 @@ +/* + * Copyright 2018 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.common; + +/** + * A {@link RevisionNotFoundException} that is raised when attempted to access a removed revision + * by rolling repository. + */ +public class RolledRevisionAccessException extends RevisionNotFoundException { + + private static final long serialVersionUID = -8291795114909224145L; + + /** + * Creates a new instance. + */ + public RolledRevisionAccessException(String message) { + super(message); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java index 803030851..e23266bd9 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java @@ -532,14 +532,9 @@ public CentralDogmaBuilder writeQuotaPerRepository(int writeQuota, int timeWindo /** * Sets the configuration for retaining commits in a {@link Repository}. - * The {@link Repository} retains at least the number of {@code minRetentionCommits} when more than - * {@code minRetentionCommits} are made. The {@link Repository} also retains commits at least - * {@code minRetentionDays}. If both conditions are not satisfied, the commits are not removed. - * For example, when {@code minRetentionCommits} is set to 2000 and {@code minRetentionDays} is set to 14, - * the commits that are created more than 2000 will be removed after 14 days have passed. - * If {@code minRetentionCommits} is set to 2000 and {@code minRetentionDays} is set to 0, - * the commits that are created more than 2000 will be removed no matter how young the commits are. - * Set both 0 to retain all commits. + * Commits are retained for at least {@code minRetentionDays}. If the number of commits is less than + * {@code minRetentionCommits}, commits are not removed even after {@code minRetentionDays} have passed. + * Specify {@code minRetentionCommits} to {@code 0} to retain all commits. */ public CentralDogmaBuilder commitRetention(int minRetentionCommits, int minRetentionDays, @Nullable String schedule) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java b/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java index 05df8d8d9..c50d22a76 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java @@ -35,6 +35,7 @@ */ public final class CommitRetentionConfig { + // TODO(minwoox): Make this configurable. // The minimum of minRetentionCommits private static final int MINIMUM_MINIMUM_RETENTION_COMMITS = 5000; @@ -64,18 +65,15 @@ public CommitRetentionConfig(@JsonProperty("minRetentionCommits") int minRetenti } /** - * Returns the minimum number of commits that a {@link Repository} should retain. 0 means that - * the number of commits are not taken into account when - * {@link Repository#shouldCreateRollingRepository(int, int)} is called. + * Returns the minimum number of commits that a {@link Repository} should retain. {@code 0} means that + * commits are not removed. */ public int minRetentionCommits() { return minRetentionCommits; } /** - * Returns the minimum number of days of a commit that a {@link Repository} should retain. 0 means that - * the number of retention days of commits are not taken into account when - * {@link Repository#shouldCreateRollingRepository(int, int)} is called. + * Returns the minimum number of days of a commit that a {@link Repository} should retain. */ public int minRetentionDays() { return minRetentionDays; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractCommand.java b/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractCommand.java index 135a4444e..e089443b1 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractCommand.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractCommand.java @@ -78,6 +78,7 @@ public final String toString() { return toStringHelper().toString(); } + // TODO(minwoox): Do not expose ToStringHelper to public. MoreObjects.ToStringHelper toStringHelper() { return MoreObjects.toStringHelper(this) .add("type", type) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/CreateRollingRepositoryCommand.java b/server/src/main/java/com/linecorp/centraldogma/server/command/CreateRollingRepositoryCommand.java index 99cafdd8f..d8ff0c2b7 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/CreateRollingRepositoryCommand.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/CreateRollingRepositoryCommand.java @@ -17,6 +17,8 @@ import static java.util.Objects.requireNonNull; +import com.google.common.base.Objects; + import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.server.CommitRetentionConfig; @@ -49,7 +51,7 @@ public Revision initialRevision() { } /** - * Returns the minimum number of commits that a {@link Repository} should retain. 0 means that + * Returns the minimum number of commits that a {@link Repository} should retain. {@code 0} means that * the number of commits are not taken into account when * {@link Repository#shouldCreateRollingRepository(int, int)} is called. */ @@ -58,11 +60,31 @@ public int minRetentionCommits() { } /** - * Returns the minimum number of days of a commit that a {@link Repository} should retain. 0 means that - * the number of retention days of commits are not taken into account when - * {@link Repository#shouldCreateRollingRepository(int, int)} is called. + * Returns the minimum number of days of a commit that a {@link Repository} should retain. */ public int minRetentionDays() { return minRetentionDays; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CreateRollingRepositoryCommand)) { + return false; + } + final CreateRollingRepositoryCommand that = (CreateRollingRepositoryCommand) o; + return super.equals(o) && + minRetentionCommits == that.minRetentionCommits && + minRetentionDays == that.minRetentionDays && + Objects.equal(initialRevision, that.initialRevision); + } + + @Override + public int hashCode() { + return Objects.hashCode(super.hashCode(), initialRevision, minRetentionCommits, minRetentionDays); + } + + //TODO(minwoox): Add toString() after removing ToStringHelper from public API } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java index 8c5a8fd55..0fc9b2c84 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java @@ -142,7 +142,6 @@ private Revision retrieveFirstRevision() { readTo(buf, 0); buf.flip(); return new Revision(buf.getInt()); - // Do not have to call buf.clear() because clear method is called before the buffer is used. } private void readTo(ByteBuffer buf, long startPosition) { @@ -203,8 +202,9 @@ void put(Revision revision, ObjectId commitId) { put(revision, commitId, true); } - private synchronized void put(Revision revision, ObjectId commitId, boolean isNewHeadRevision) { - if (isNewHeadRevision) { + // TODO(minwoox) Use lock instead of synchronized. + private synchronized void put(Revision revision, ObjectId commitId, boolean newHead) { + if (newHead) { final Revision headRevision = this.headRevision; if (headRevision != null) { final Revision expected = headRevision.forward(1); @@ -229,7 +229,7 @@ private synchronized void put(Revision revision, ObjectId commitId, boolean isNe pos += channel.write(buf, pos); } while (buf.hasRemaining()); - if (isNewHeadRevision && fsync) { + if (newHead && fsync) { channel.force(true); } } catch (IOException e) { @@ -237,7 +237,7 @@ private synchronized void put(Revision revision, ObjectId commitId, boolean isNe } final Revision headRevision = this.headRevision; - if (isNewHeadRevision || + if (newHead || headRevision == null || headRevision.major() < revision.major()) { this.headRevision = revision; @@ -269,7 +269,10 @@ void rebuild(Repository gitRepo) { // NB: We did not store the last commit ID until all commit IDs are stored, // so that the partially built database always has mismatching head revision. + // Find firstRevision while validating whether the commitIds are stored correctly in the file. + // This won't change the file but update the firstRevision field. findFirstRevisionOrRebuild(gitRepo, revWalk, headRevision, revCommit, true); + // Now we know it's validated so rebuild the database. findFirstRevisionOrRebuild(gitRepo, revWalk, headRevision, revCommit, false); // All commit IDs except the head have been stored. Store the head finally. diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java index 4e1985e3f..189717f12 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java @@ -35,6 +35,7 @@ import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import com.linecorp.armeria.common.util.UnmodifiableFuture; import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.server.CentralDogmaConfig; import com.linecorp.centraldogma.server.CommitRetentionConfig; @@ -73,12 +74,20 @@ public boolean isEnabled(CentralDogmaConfig config) { public CompletionStage start(PluginContext context) { final CommitRetentionConfig commitRetentionConfig = context.config().commitRetentionConfig(); assert commitRetentionConfig != null; + final int minRetentionCommits = commitRetentionConfig.minRetentionCommits(); + final int minRetentionDays = commitRetentionConfig.minRetentionDays(); + if (minRetentionCommits == 0 || + minRetentionCommits == Integer.MAX_VALUE || minRetentionDays == Integer.MAX_VALUE) { + // Disabled. + return UnmodifiableFuture.completedFuture(null); + } + worker = MoreExecutors.listeningDecorator( Executors.newSingleThreadScheduledExecutor( new DefaultThreadFactory("commit-retention-worker", true))); scheduleRemovingOldCommits(context, commitRetentionConfig); - return CompletableFuture.completedFuture(null); + return UnmodifiableFuture.completedFuture(null); } private void scheduleRemovingOldCommits(PluginContext context, CommitRetentionConfig config) { @@ -105,13 +114,13 @@ public void onSuccess(@Nullable Object result) {} @Override public void onFailure(Throwable cause) { if (!stopping) { - logger.warn("Commit retention scheduler stopped due to an unexpected exception:", - cause); + logger.warn("Commit retention scheduler stopped due to an unexpected exception:", cause); } } }, worker); } + // TODO(minwoox): Add metrics. private void createRollingRepository(PluginContext context, CommitRetentionConfig config) { if (stopping) { return; @@ -123,15 +132,16 @@ private void createRollingRepository(PluginContext context, CommitRetentionConfi if (stopping) { return; } - final Revision revision = repo.shouldCreateRollingRepository(config.minRetentionCommits(), - config.minRetentionDays()); + final int minRetentionCommits = config.minRetentionCommits(); + final int minRetentionDays = config.minRetentionDays(); + final Revision revision = + repo.shouldCreateRollingRepository(minRetentionCommits, minRetentionDays); if (revision != null) { try { context.commandExecutor().execute( Command.createRollingRepository(project.name(), repo.name(), revision, - config.minRetentionCommits(), - config.minRetentionDays())) - .get(100, TimeUnit.SECONDS); + minRetentionCommits, minRetentionDays)) + .get(10, TimeUnit.MINUTES); } catch (Throwable t) { logger.warn("Failed to create a rolling repository for {}/{} with revision: {}", project.name(), repo.name(), revision, t); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/FailFastUtil.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/FailFastUtil.java index 121a0eb2a..d92079a34 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/FailFastUtil.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/FailFastUtil.java @@ -23,6 +23,7 @@ import com.linecorp.armeria.common.util.Exceptions; import com.linecorp.armeria.server.ServiceRequestContext; import com.linecorp.centraldogma.server.internal.storage.RequestAlreadyTimedOutException; +import com.linecorp.centraldogma.server.storage.repository.Repository; final class FailFastUtil { @@ -39,7 +40,7 @@ static ServiceRequestContext context() { } @SuppressWarnings("MethodParameterNamingConvention") - static void failFastIfTimedOut(LegacyGitRepository repo, Logger logger, @Nullable ServiceRequestContext ctx, + static void failFastIfTimedOut(Repository repo, Logger logger, @Nullable ServiceRequestContext ctx, String methodName, Object arg1) { if (ctx != null && ctx.isTimedOut()) { logger.info("{} Rejecting a request timed out already: repo={}/{}, method={}, args={}", @@ -49,7 +50,7 @@ static void failFastIfTimedOut(LegacyGitRepository repo, Logger logger, @Nullabl } @SuppressWarnings("MethodParameterNamingConvention") - static void failFastIfTimedOut(LegacyGitRepository repo, Logger logger, @Nullable ServiceRequestContext ctx, + static void failFastIfTimedOut(Repository repo, Logger logger, @Nullable ServiceRequestContext ctx, String methodName, Object arg1, Object arg2) { if (ctx != null && ctx.isTimedOut()) { logger.info("{} Rejecting a request timed out already: repo={}/{}, method={}, args=[{}, {}]", @@ -59,7 +60,7 @@ static void failFastIfTimedOut(LegacyGitRepository repo, Logger logger, @Nullabl } @SuppressWarnings("MethodParameterNamingConvention") - static void failFastIfTimedOut(LegacyGitRepository repo, Logger logger, @Nullable ServiceRequestContext ctx, + static void failFastIfTimedOut(Repository repo, Logger logger, @Nullable ServiceRequestContext ctx, String methodName, Object arg1, Object arg2, Object arg3) { if (ctx != null && ctx.isTimedOut()) { logger.info("{} Rejecting a request timed out already: repo={}/{}, method={}, args=[{}, {}, {}]", @@ -69,48 +70,7 @@ static void failFastIfTimedOut(LegacyGitRepository repo, Logger logger, @Nullabl } @SuppressWarnings("MethodParameterNamingConvention") - static void failFastIfTimedOut(LegacyGitRepository repo, Logger logger, @Nullable ServiceRequestContext ctx, - String methodName, Object arg1, Object arg2, Object arg3, int arg4) { - if (ctx != null && ctx.isTimedOut()) { - logger.info( - "{} Rejecting a request timed out already: repo={}/{}, method={}, args=[{}, {}, {}, {}]", - ctx, repo.parent().name(), repo.name(), methodName, arg1, arg2, arg3, arg4); - throw REQUEST_ALREADY_TIMED_OUT; - } - } - - @SuppressWarnings("MethodParameterNamingConvention") - static void failFastIfTimedOut(GitRepositoryV2 repo, Logger logger, @Nullable ServiceRequestContext ctx, - String methodName, Object arg1) { - if (ctx != null && ctx.isTimedOut()) { - logger.info("{} Rejecting a request timed out already: repo={}/{}, method={}, args={}", - ctx, repo.parent().name(), repo.name(), methodName, arg1); - throw REQUEST_ALREADY_TIMED_OUT; - } - } - - @SuppressWarnings("MethodParameterNamingConvention") - static void failFastIfTimedOut(GitRepositoryV2 repo, Logger logger, @Nullable ServiceRequestContext ctx, - String methodName, Object arg1, Object arg2) { - if (ctx != null && ctx.isTimedOut()) { - logger.info("{} Rejecting a request timed out already: repo={}/{}, method={}, args=[{}, {}]", - ctx, repo.parent().name(), repo.name(), methodName, arg1, arg2); - throw REQUEST_ALREADY_TIMED_OUT; - } - } - - @SuppressWarnings("MethodParameterNamingConvention") - static void failFastIfTimedOut(GitRepositoryV2 repo, Logger logger, @Nullable ServiceRequestContext ctx, - String methodName, Object arg1, Object arg2, Object arg3) { - if (ctx != null && ctx.isTimedOut()) { - logger.info("{} Rejecting a request timed out already: repo={}/{}, method={}, args=[{}, {}, {}]", - ctx, repo.parent().name(), repo.name(), methodName, arg1, arg2, arg3); - throw REQUEST_ALREADY_TIMED_OUT; - } - } - - @SuppressWarnings("MethodParameterNamingConvention") - static void failFastIfTimedOut(GitRepositoryV2 repo, Logger logger, @Nullable ServiceRequestContext ctx, + static void failFastIfTimedOut(Repository repo, Logger logger, @Nullable ServiceRequestContext ctx, String methodName, Object arg1, Object arg2, Object arg3, int arg4) { if (ctx != null && ctx.isTimedOut()) { logger.info( diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java index c7cb9ab67..4a0ec7c9e 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java @@ -71,6 +71,7 @@ import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.common.RevisionNotFoundException; import com.linecorp.centraldogma.common.RevisionRange; +import com.linecorp.centraldogma.common.RolledRevisionAccessException; import com.linecorp.centraldogma.server.command.CommitResult; import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache; import com.linecorp.centraldogma.server.internal.storage.repository.git.InternalRepository.RevisionAndEntries; @@ -112,9 +113,14 @@ static GitRepositoryV2 open(Project parent, File repositoryDir, private final AtomicReference> closePending = new AtomicReference<>(); private final CompletableFuture closeFuture = new CompletableFuture<>(); + // The primary internal repository where read and write operations are performed. + // The primary repository is replaced by the secondary repository when: + // - the secondary repository has more than minRetentionCommits + // - the oldest commit of the secondary repository has exceeded the minRetentionDays @VisibleForTesting InternalRepository primaryRepo; + // The secondary internal repository where only write operations are performed. @Nullable @VisibleForTesting InternalRepository secondaryRepo; @@ -128,16 +134,16 @@ static GitRepositoryV2 open(Project parent, File repositoryDir, private volatile Revision headRevision; /** - * Create a new Git repository. + * Creates a Git repository to the specified {@code repositoryDir}. */ GitRepositoryV2(Project parent, File repositoryDir, Executor repositoryWorker, long creationTimeMillis, Author author, @Nullable RepositoryCache cache) { this.parent = requireNonNull(parent, "parent"); originalRepoName = requireNonNull(repositoryDir, "repositoryDir").getName(); this.repositoryWorker = requireNonNull(repositoryWorker, "repositoryWorker"); + requireNonNull(author, "author"); this.cache = cache; - requireNonNull(author, "author"); try { if (repositoryDir.exists()) { if (!isEmpty(repositoryDir)) { @@ -145,7 +151,7 @@ static GitRepositoryV2 open(Project parent, File repositoryDir, " (exists already)"); } } else if (!repositoryDir.mkdir()) { - throw new StorageException("failed to create a repository at: " + repositoryDir); + throw new StorageException("failed to create a directory for Git at: " + repositoryDir); } } catch (IOException e) { throw new StorageException("failed to create a repository at: " + repositoryDir, e); @@ -192,9 +198,9 @@ private GitRepositoryV2(Project parent, File repoDir, Executor repositoryWorker, assert primaryRepo != null; this.primaryRepo = primaryRepo; headRevision = primaryRepo.headRevision(); - final Commit initialCommit = currentInitialCommit(primaryRepo); - creationTimeMillis = initialCommit.when(); - author = initialCommit.author(); + final Commit firstCommit = firstCommit(primaryRepo); + creationTimeMillis = firstCommit.when(); + author = firstCommit.author(); secondaryRepo = InternalRepository.open(parent, originalRepoName, repoMetadata.secondaryRepoDir(), false); } catch (Throwable t) { @@ -230,7 +236,7 @@ private static void migrateToV2(File repoDir) { // When: // - repoDir: /foo // - primaryRepoDir: /foo/foo_0000000000 - // - tmpRepoDir: /bar + // - tmpRepoDir: /bar (random UUID) // // Migration steps will be: // - /foo becomes /bar @@ -239,7 +245,7 @@ private static void migrateToV2(File repoDir) { final File primaryRepoDir = RepositoryMetadataDatabase.initialPrimaryRepoDir(repoDir); final File tmpRepoDir = new File(repoDir.getParentFile(), UUID.randomUUID().toString()); - logger.debug("Migrating {} to {} using temp repository: {}", repoDir, primaryRepoDir, tmpRepoDir); + logger.info("Migrating {} to {} using temp repository: {}", repoDir, primaryRepoDir, tmpRepoDir); if (!repoDir.renameTo(tmpRepoDir)) { throw new StorageException("failed to migrate a repository at: " + repoDir + ", to the tmp dir: " + tmpRepoDir); @@ -249,7 +255,7 @@ private static void migrateToV2(File repoDir) { } try { final Path moved = Files.move(tmpRepoDir.toPath(), primaryRepoDir.toPath(), REPLACE_EXISTING); - assert moved == primaryRepoDir.toPath(); + assert moved == primaryRepoDir.toPath() : moved + " != " + primaryRepoDir.toPath(); } catch (IOException e) { //noinspection ResultOfMethodCallIgnored tmpRepoDir.renameTo(repoDir); @@ -257,7 +263,7 @@ private static void migrateToV2(File repoDir) { ", to: " + primaryRepoDir, e); } checkState(!tmpRepoDir.exists(), "%s is not renamed.", tmpRepoDir); - logger.debug("Migrating {} is done.", repoDir); + logger.info("Migrating {} is done.", repoDir); } @VisibleForTesting @@ -361,7 +367,7 @@ private Revision normalizeNow(Revision revision, Revision headRevision, boolean // We silently update the major to the first revision when it's Revision.INIT. major = firstRevisionMajor; } else { - throw new RevisionNotFoundException( + throw new RolledRevisionAccessException( "revision: " + revision + " (expected: >= " + firstRevisionMajor + ')'); } } @@ -379,8 +385,8 @@ public CompletableFuture>> find( final ServiceRequestContext ctx = context(); return CompletableFuture.supplyAsync(() -> { failFastIfTimedOut(this, logger, ctx, "find", revision, pathPattern, options); - readLock(); try { + readLock(); final Revision normalizedRevision = normalizeNow(revision); if ("/".equals(pathPattern)) { return ImmutableMap.of(pathPattern, Entry.ofDirectory(normalizedRevision, "/")); @@ -409,8 +415,8 @@ public CompletableFuture>> diff(Revision from, Revision to final ServiceRequestContext ctx = context(); return CompletableFuture.supplyAsync(() -> { failFastIfTimedOut(this, logger, ctx, "diff", from, to, pathPattern); - readLock(); try { + readLock(); final RevisionRange range = normalizeNow(from, to).toAscending(); if (range.from().equals(range.to())) { // Empty range. @@ -431,8 +437,8 @@ public CompletableFuture>> previewDiff(Revision baseRevisi final ServiceRequestContext ctx = context(); return CompletableFuture.supplyAsync(() -> { failFastIfTimedOut(this, logger, ctx, "previewDiff", baseRevision); - readLock(); try { + readLock(); final Revision normalizedRevision = normalizeNow(baseRevision); return primaryRepo.previewDiff(normalizedRevision, changes); } finally { @@ -465,8 +471,8 @@ private CommitResult blockingCommit( String detail, Markup markup, Iterable> changes, boolean directExecution) { final RevisionAndEntries res; final Iterable> applyingChanges; - writeLock(); try { + writeLock(); final Revision normalizedRevision = normalizeNow(baseRevision); final Revision headRevision = this.headRevision; if (headRevision.major() != normalizedRevision.major()) { @@ -526,8 +532,8 @@ public CompletableFuture> history(Revision from, Revision to, Strin final ServiceRequestContext ctx = context(); return CompletableFuture.supplyAsync(() -> { failFastIfTimedOut(this, logger, ctx, "history", from, to, pathPattern, maxCommits); - readLock(); try { + readLock(); final RevisionRange range = normalizeNow(from, to); return primaryRepo.listCommits(pathPattern, Math.min(maxCommits, MAX_MAX_COMMITS), range); } finally { @@ -564,8 +570,8 @@ private Revision blockingFindLatestRevision(Revision lastKnownRevision, String p if ("/".equals(pathPattern)) { return headRevision; } - readLock(); try { + readLock(); final Map> entries = primaryRepo.find( headRevision, pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT); if (!entries.isEmpty()) { @@ -583,8 +589,8 @@ private Revision blockingFindLatestRevision(Revision lastKnownRevision, String p // Slow path: compare the two trees. final List diffEntries; final RevisionRange range; - readLock(); try { + readLock(); range = new RevisionRange(normalizeNow(lastKnownRevision, false), headRevision); if (range.from().isLowerThan(primaryRepo.firstRevision())) { // We should return range.to() without comparing two tree. It's because: @@ -657,8 +663,8 @@ public CompletableFuture watch(Revision lastKnownRevision, String path final CompletableFuture future = new CompletableFuture<>(); CompletableFuture.runAsync(() -> { failFastIfTimedOut(this, logger, ctx, "watch", lastKnownRevision, pathPattern); - readLock(); try { + readLock(); final Revision normLastKnownRevision = normalizeNow(lastKnownRevision, false); // If lastKnownRevision is outdated already and the recent changes match, // there's no need to watch. @@ -682,9 +688,9 @@ public CompletableFuture watch(Revision lastKnownRevision, String path private void readLock() { rwLock.readLock().lock(); - if (closePending.get() != null) { - rwLock.readLock().unlock(); - throw closePending.get().get(); + final Supplier exceptionSupplier = closePending.get(); + if (exceptionSupplier != null) { + throw exceptionSupplier.get(); } } @@ -694,9 +700,9 @@ private void readUnlock() { private void writeLock() { rwLock.writeLock().lock(); - if (closePending.get() != null) { - writeUnLock(); - throw closePending.get().get(); + final Supplier exceptionSupplier = closePending.get(); + if (exceptionSupplier != null) { + throw exceptionSupplier.get(); } } @@ -704,7 +710,7 @@ private void writeUnLock() { rwLock.writeLock().unlock(); } - private static Commit currentInitialCommit(InternalRepository primaryRepo) { + private static Commit firstCommit(InternalRepository primaryRepo) { final Revision firstRevision = primaryRepo.firstRevision(); final RevisionRange range = new RevisionRange(firstRevision, firstRevision); return primaryRepo.listCommits(ALL_PATH, 1, range).get(0); @@ -716,13 +722,17 @@ public Revision shouldCreateRollingRepository(int minRetentionCommits, int minRe return shouldCreateRollingRepository(repo.headRevision(), minRetentionCommits, minRetentionDays); } + /** + * Returns the specified {@code headRevision} if the repository should be rolled. Otherwise, returns + * {@code null}. + */ @Nullable private Revision shouldCreateRollingRepository(Revision headRevision, int minRetentionCommits, int minRetentionDays) { // TODO(minwoox): provide a way to set different minRetentionCommits and minRetentionDays // in each repository. - if ((minRetentionCommits == 0 && minRetentionDays == 0) || - (minRetentionCommits == Integer.MAX_VALUE && minRetentionDays == Integer.MAX_VALUE)) { + if (minRetentionCommits == 0 || + minRetentionCommits == Integer.MAX_VALUE || minRetentionDays == Integer.MAX_VALUE) { // Not enabled. return null; } @@ -739,17 +749,19 @@ private static boolean exceedsMinRetention(InternalRepository repo, Revision hea final Revision firstRevision = repo.firstRevision(); if (minRetentionCommits != 0) { if (headRevision.major() - firstRevision.major() <= minRetentionCommits) { + // Not enough commits. return false; } } - if (minRetentionDays != 0) { - final Instant secondCommitCreationTime = repo.secondCommitCreationTimeInstant(); - return secondCommitCreationTime != null && - secondCommitCreationTime.isBefore(Instant.now().minus(minRetentionDays, ChronoUnit.DAYS)); + if (minRetentionDays == 0) { + return true; } - return true; - } + + final Instant secondCommitCreationTime = repo.secondCommitCreationTimeInstant(); + return secondCommitCreationTime != null && + secondCommitCreationTime.isBefore(Instant.now().minus(minRetentionDays, ChronoUnit.DAYS)); +} @Override public void createRollingRepository(Revision rollingRepositoryInitialRevision, @@ -768,9 +780,12 @@ public void createRollingRepository(Revision rollingRepositoryInitialRevision, } private void promoteSecondaryRepo() { - writeLock(); try { - checkState(primaryRepo.headRevision().equals(secondaryRepo.headRevision()), ""); + writeLock(); + assert secondaryRepo != null; + checkState(primaryRepo.headRevision().equals(secondaryRepo.headRevision()), + "primaryRepo.headRevision() %s does not equal to secondaryRepo.headRevision() %s.", + primaryRepo.headRevision(), secondaryRepo.headRevision()); logger.info("Promoting the secondary repository in {}/{}.", parent.name(), originalRepoName); repoMetadata.setPrimaryRepoDir(secondaryRepo.jGitRepo().getDirectory()); @@ -795,28 +810,36 @@ private void promoteSecondaryRepo() { } } - private void createSecondaryRepo(Revision rollingRepositoryInitialRevision) { + private void createSecondaryRepo(Revision secondaryRepositoryInitialRevision) { InternalRepository secondaryRepo = null; try { logger.info("Creating the secondary repository in {}/{}. head revision: {}.", - parent.name(), originalRepoName, rollingRepositoryInitialRevision); - final List> changes = changes(rollingRepositoryInitialRevision); + parent.name(), originalRepoName, secondaryRepositoryInitialRevision); + final List> changes = allContents(secondaryRepositoryInitialRevision); final File secondaryRepoDir = repoMetadata.secondaryRepoDir(); secondaryRepo = InternalRepository.create(parent, originalRepoName, secondaryRepoDir, - rollingRepositoryInitialRevision, creationTimeMillis, + secondaryRepositoryInitialRevision, creationTimeMillis, author, changes); - writeLock(); try { + writeLock(); final Revision headRevision = this.headRevision; - if (!rollingRepositoryInitialRevision.equals(headRevision)) { - assert rollingRepositoryInitialRevision.major() < headRevision.major(); - // There were commits after the createRollingRepositoryCommand is created + // There's not so much chances that commits are pushed before we acquire the write lock but + // we have to check it anyway. + if (!secondaryRepositoryInitialRevision.equals(headRevision)) { + assert secondaryRepositoryInitialRevision.major() < headRevision.major(); + // There were commits after the createRollingRepositoryCommand is created, // so we should catch up. + logger.info("Catching up the interposed commits in {}/{}. from: {} to : {}.", + parent.name(), originalRepoName, + secondaryRepositoryInitialRevision, headRevision); + final RevisionRange revisionRange = new RevisionRange( - rollingRepositoryInitialRevision.forward(1), headRevision); + secondaryRepositoryInitialRevision.forward(1), headRevision); + // The number of interposed commits would be very small. + assert revisionRange.to().major() - revisionRange.from().major() < MAX_MAX_COMMITS; final List commits = primaryRepo.listCommits(ALL_PATH, MAX_MAX_COMMITS, revisionRange); - Revision fromRevision = rollingRepositoryInitialRevision; + Revision fromRevision = secondaryRepositoryInitialRevision; for (Commit commit : commits) { final Revision toRevision = commit.revision(); final Map> diffs = primaryRepo.diff( @@ -833,7 +856,7 @@ private void createSecondaryRepo(Revision rollingRepositoryInitialRevision) { } logger.info("The secondary repository {} is created in {}/{}. head revision: {}", secondaryRepoDir.getName(), parent.name(), originalRepoName, - rollingRepositoryInitialRevision); + secondaryRepositoryInitialRevision); } catch (Throwable t) { logger.warn("Failed to create the secondary repository", t); if (secondaryRepo != null) { @@ -844,12 +867,13 @@ private void createSecondaryRepo(Revision rollingRepositoryInitialRevision) { logger.warn("Failed to delete the directory: {}", secondaryRepo.repoDir()); } } + throw t; } } - private List> changes(Revision rollingRepositoryInitialRevision) { + private List> allContents(Revision secondaryRepositoryInitialRevision) { final Map> entries = primaryRepo.find( - rollingRepositoryInitialRevision, ALL_PATH, ImmutableMap.of()); + secondaryRepositoryInitialRevision, ALL_PATH, ImmutableMap.of()); final Builder> builder = ImmutableList.builder(); for (Entry entry : entries.values()) { final EntryType type = entry.type(); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/LegacyGitRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/LegacyGitRepository.java index 71157df7a..7c941a868 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/LegacyGitRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/LegacyGitRepository.java @@ -126,6 +126,7 @@ /** * A {@link Repository} based on Git. + * This class will be removed after all migration to GitRepositoryV2 is done. */ class LegacyGitRepository implements Repository { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java index be9cad4da..3adab6f1b 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java @@ -55,26 +55,27 @@ private static File repoDir(File rootDir, String suffix) { @VisibleForTesting final File rootDir; - private final Path path; - private final FileChannel channel; + private final Path metadataPath; + private final FileChannel metadataFileChannel; private String primarySuffix; RepositoryMetadataDatabase(File rootDir, boolean create) { this.rootDir = rootDir; - path = new File(rootDir, "metadata.dat").toPath(); + metadataPath = new File(rootDir, "metadata.dat").toPath(); try { if (create) { - channel = FileChannel.open(path, - StandardOpenOption.CREATE, - StandardOpenOption.READ, - StandardOpenOption.WRITE); + metadataFileChannel = FileChannel.open(metadataPath, + StandardOpenOption.CREATE, + StandardOpenOption.READ, + StandardOpenOption.WRITE); } else { - channel = FileChannel.open(path, - StandardOpenOption.READ, - StandardOpenOption.WRITE); + metadataFileChannel = FileChannel.open(metadataPath, + StandardOpenOption.READ, + StandardOpenOption.WRITE); } } catch (IOException e) { - throw new StorageException("failed to open a repository database: " + path, e); + throw new StorageException("failed to " + (create ? "create" : "open") + + " a repository database: " + metadataPath, e); } if (create) { @@ -83,20 +84,13 @@ private static File repoDir(File rootDir, String suffix) { } else { boolean success = false; try { - final long size; - try { - size = channel.size(); - } catch (IOException e) { - throw new StorageException("failed to get the file length: " + path, e); - } - - if (size != PRIMARY_SUFFIX_LEN) { - throw new StorageException("incorrect file length: " + path + " (" + size + " bytes)"); - } - final ByteBuffer buf = threadLocalBuffer.get(); buf.clear(); - readTo(buf); + final long read = readTo(buf); + if (read != PRIMARY_SUFFIX_LEN) { + throw new StorageException("incorrect prefix length for " + metadataPath + + ", length: " + read + " (expected: " + PRIMARY_SUFFIX_LEN + ')'); + } buf.flip(); primarySuffix = StandardCharsets.UTF_8.decode(buf).toString(); // To check if the suffix is a correct integer value. @@ -120,27 +114,30 @@ private void writeSuffix(String suffix) { long pos = 0; try { do { - pos += channel.write(buf, pos); + pos += metadataFileChannel.write(buf, pos); } while (buf.hasRemaining()); - channel.force(true); + metadataFileChannel.force(true); } catch (IOException e) { - throw new StorageException("failed to write the suffix (" + suffix + - ") of the primary repository: " + path, e); + close(); + throw new StorageException("failed to write the suffix (" + suffix + + ") of the primary repository: " + metadataPath, e); } } - private void readTo(ByteBuffer buf) { + private long readTo(ByteBuffer buf) { long pos = 0; try { do { - final int readBytes = channel.read(buf, pos); + final int readBytes = metadataFileChannel.read(buf, pos); if (readBytes < 0) { throw new EOFException(); } pos += readBytes; } while (buf.hasRemaining()); + return pos; } catch (IOException e) { - throw new StorageException("failed to read the primary repository database: " + path, e); + throw new StorageException("failed to read the suffix of the primary repository database: " + + metadataPath, e); } } @@ -157,7 +154,7 @@ void setPrimaryRepoDir(File newRepoDir) { // e.g. /foo_0000123457 final String newPrimarySuffix = increment(primarySuffix); // e.g. secondary: /foo_0000123457 final File secondary = new File(rootDir, rootDir.getName() + '_' + newPrimarySuffix); - assert newRepoDir.equals(secondary); + assert newRepoDir.equals(secondary) : newRepoDir + " != " + secondary; primarySuffix = newPrimarySuffix; writeSuffix(newPrimarySuffix); } @@ -165,9 +162,9 @@ void setPrimaryRepoDir(File newRepoDir) { // e.g. /foo_0000123457 @Override public void close() { try { - channel.close(); + metadataFileChannel.close(); } catch (IOException e) { - logger.warn("Failed to close the commit ID database: {}", path, e); + logger.warn("Failed to close the commit ID database: {}", metadataPath, e); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java index 70838fdf6..b0086bf4a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java @@ -526,9 +526,9 @@ default CompletableFuture> mergeFiles(Revision revision, Merg /** * Returns the head {@link Revision} of this repository if this repository needs to create a - * rolling repository. It should meet following conditions: - * - The last created rolling repository should have more than {@code minRetentionCommits}. - * - The last created rolling repository should have commits that are older than {@code minRetentionDays}. + * rolling repository. The head {@link Revision} is returned when following conditions are met: + * - The last created rolling repository has more than {@code minRetentionCommits}. + * - The last created rolling repository has commits that are older than {@code minRetentionDays}. */ @Nullable Revision shouldCreateRollingRepository(int minRetentionCommits, int minRetentionDays); @@ -536,10 +536,6 @@ default CompletableFuture> mergeFiles(Revision revision, Merg /** * Creates the rolling repository. The specified {@code initialRevision} of the current repository will be * the initial revision of the created rolling repository. - * - * @throws IllegalStateException if {@link #shouldCreateRollingRepository(int, int)} with the specified - * {@code minRetentionCommits} and {@code minRetentionDays} does not return - * a valid {@link Revision}. */ void createRollingRepository(Revision initialRevision, int minRetentionCommits, int minRetentionDays); } diff --git a/site/src/sphinx/setup-configuration.rst b/site/src/sphinx/setup-configuration.rst index d085bfdec..b91b5c532 100644 --- a/site/src/sphinx/setup-configuration.rst +++ b/site/src/sphinx/setup-configuration.rst @@ -48,7 +48,7 @@ defaults: "maxNumFilesPerMirror": null, "maxNumBytesPerMirror": null, "commitRetentionConfig": { - "minRetentionCommits" : 2000, + "minRetentionCommits" : 5000, "minRetentionDays": 14, "schedule": "0 0 * * * ?" }, @@ -225,23 +225,18 @@ Core properties - ``commitRetention`` - - the configuration for retaining commits in a repository. The repository retains at least the number of - ``minRetentionCommits`` when more than ``minRetentionCommits`` are made. The repository also retains - commits at least ``minRetentionDays``. If both conditions are not satisfied, the commits are not removed. - For example, when ``minRetentionCommits`` is set to 2000 and ``minRetentionDays`` is set to 14, - the commits that are created more than 2000 will be removed after 14 days have passed. - If ``minRetentionCommits`` is set to 2000 and ``minRetentionDays`` is set to 0, - the commits that are created more than 2000 will be removed no matter how young the commits are. - Set both 0 to retain all commits. + - the configuration for retaining commits in a repository. Commits are retained for at least + ``minRetentionDays``. If the number of commits is less than ``minRetentionCommits``, commits are + not removed even after ``minRetentionDays`` have passed. - ``minRetentionCommits`` (integer) - a minimum number of commits that a repository should retain. The number should be greater than or - equal to 1000. Specify 0 to disable it. + equal to 5000. Specify ``0`` to retain all commits. - ``minRetentionDays`` (integer) - - a minimum number of days of a commit that a repository should retain. Specify 0 to disable it. + - a minimum number of days of a commit that a repository should retain. - ``schedule`` (string)