diff --git a/client/java-armeria/src/test/java/com/linecorp/centraldogma/client/armeria/CentralDogmaRepositoryTest.java b/client/java-armeria/src/test/java/com/linecorp/centraldogma/client/armeria/CentralDogmaRepositoryTest.java index 746c49bd6..c54c9a007 100644 --- a/client/java-armeria/src/test/java/com/linecorp/centraldogma/client/armeria/CentralDogmaRepositoryTest.java +++ b/client/java-armeria/src/test/java/com/linecorp/centraldogma/client/armeria/CentralDogmaRepositoryTest.java @@ -98,7 +98,7 @@ void historyAndDiff() { final List commits = centralDogmaRepo.history().get(new Revision(2), Revision.HEAD).join(); assertThat(commits.stream() .map(Commit::summary) - .collect(toImmutableList())).containsExactly("commit3", "commit2"); + .collect(toImmutableList())).containsExactly("commit2", "commit3"); assertThat(centralDogmaRepo.diff("/foo.json") .get(Revision.INIT, Revision.HEAD) .join()) diff --git a/common/src/main/java/com/linecorp/centraldogma/common/Revision.java b/common/src/main/java/com/linecorp/centraldogma/common/Revision.java index b7d9faf8d..bdf8739ba 100644 --- a/common/src/main/java/com/linecorp/centraldogma/common/Revision.java +++ b/common/src/main/java/com/linecorp/centraldogma/common/Revision.java @@ -119,6 +119,15 @@ public int major() { return major; } + /** + * Tells whether the {@link #major()} of this {@link Revision} is lower than the {@code other} + * {@link Revision}. + */ + public boolean isLowerThan(Revision other) { + requireNonNull(other, "other"); + return major < other.major(); + } + /** * Returns {@code 0}. * diff --git a/common/src/main/java/com/linecorp/centraldogma/common/RolledRevisionAccessException.java b/common/src/main/java/com/linecorp/centraldogma/common/RolledRevisionAccessException.java new file mode 100644 index 000000000..11c9a2721 --- /dev/null +++ b/common/src/main/java/com/linecorp/centraldogma/common/RolledRevisionAccessException.java @@ -0,0 +1,33 @@ +/* + * Copyright 2018 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.common; + +/** + * A {@link RevisionNotFoundException} that is raised when attempted to access a removed revision + * by rolling repository. + */ +public class RolledRevisionAccessException extends RevisionNotFoundException { + + private static final long serialVersionUID = -8291795114909224145L; + + /** + * Creates a new instance. + */ + public RolledRevisionAccessException(String message) { + super(message); + } +} diff --git a/server/src/jmh/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryBenchmark.java b/server/src/jmh/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryBenchmark.java index fa1b7520f..8e6b615e7 100644 --- a/server/src/jmh/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryBenchmark.java +++ b/server/src/jmh/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryBenchmark.java @@ -47,14 +47,14 @@ public class GitRepositoryBenchmark { private int previousCommits; private File repoDir; - private GitRepository repo; + private GitRepositoryV2 repo; private int currentRevision; @Setup public void init() throws Exception { repoDir = Files.createTempDirectory("jmh-gitrepository.").toFile(); - repo = new GitRepository(mock(Project.class), repoDir, ForkJoinPool.commonPool(), - System.currentTimeMillis(), AUTHOR, null); + repo = new GitRepositoryV2(mock(Project.class), repoDir, ForkJoinPool.commonPool(), + System.currentTimeMillis(), AUTHOR, null); currentRevision = 1; for (int i = 0; i < previousCommits; i++) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java index b8bf6db0f..e23266bd9 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java @@ -129,6 +129,9 @@ public final class CentralDogmaBuilder { @Nullable private CorsConfig corsConfig; + @Nullable + private CommitRetentionConfig commitRetentionConfig; + /** * Creates a new builder with the specified data directory. */ @@ -527,6 +530,18 @@ public CentralDogmaBuilder writeQuotaPerRepository(int writeQuota, int timeWindo return this; } + /** + * Sets the configuration for retaining commits in a {@link Repository}. + * Commits are retained for at least {@code minRetentionDays}. If the number of commits is less than + * {@code minRetentionCommits}, commits are not removed even after {@code minRetentionDays} have passed. + * Specify {@code minRetentionCommits} to {@code 0} to retain all commits. + */ + public CentralDogmaBuilder commitRetention(int minRetentionCommits, int minRetentionDays, + @Nullable String schedule) { + commitRetentionConfig = new CommitRetentionConfig(minRetentionCommits, minRetentionDays, schedule); + return this; + } + /** * Sets the {@link MeterRegistry} used to collect metrics. */ @@ -577,6 +592,6 @@ private CentralDogmaConfig buildConfig() { webAppEnabled, webAppTitle, mirroringEnabled, numMirroringThreads, maxNumFilesPerMirror, maxNumBytesPerMirror, replicationConfig, null, accessLogFormat, authCfg, quotaConfig, - corsConfig); + commitRetentionConfig, corsConfig); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaConfig.java b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaConfig.java index 646a41611..2f9cd9264 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaConfig.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaConfig.java @@ -138,6 +138,9 @@ public final class CentralDogmaConfig { @Nullable private final QuotaConfig writeQuotaPerRepository; + @Nullable + private final CommitRetentionConfig commitRetentionConfig; + @Nullable private final CorsConfig corsConfig; @@ -169,11 +172,10 @@ public final class CentralDogmaConfig { @JsonProperty("accessLogFormat") @Nullable String accessLogFormat, @JsonProperty("authentication") @Nullable AuthConfig authConfig, @JsonProperty("writeQuotaPerRepository") @Nullable QuotaConfig writeQuotaPerRepository, + @JsonProperty("commitRetention") @Nullable CommitRetentionConfig commitRetentionConfig, @JsonProperty("cors") @Nullable CorsConfig corsConfig) { - this.dataDir = requireNonNull(dataDir, "dataDir"); this.ports = ImmutableList.copyOf(requireNonNull(ports, "ports")); - this.corsConfig = corsConfig; checkArgument(!ports.isEmpty(), "ports must have at least one port."); this.tls = tls; this.trustedProxyAddresses = trustedProxyAddresses; @@ -224,6 +226,8 @@ public final class CentralDogmaConfig { ports.stream().anyMatch(ServerPort::hasProxyProtocol)); this.writeQuotaPerRepository = writeQuotaPerRepository; + this.commitRetentionConfig = commitRetentionConfig; + this.corsConfig = corsConfig; } /** @@ -456,11 +460,20 @@ public AuthConfig authConfig() { * Returns the maximum allowed write quota per {@link Repository}. */ @Nullable - @JsonProperty("writeQuotaPerRepository") + @JsonProperty public QuotaConfig writeQuotaPerRepository() { return writeQuotaPerRepository; } + /** + * Returns the {@link CommitRetentionConfig}. + */ + @Nullable + @JsonProperty("commitRetention") + public CommitRetentionConfig commitRetentionConfig() { + return commitRetentionConfig; + } + /** * Returns the {@link CorsConfig}. */ diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java b/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java new file mode 100644 index 000000000..c50d22a76 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/CommitRetentionConfig.java @@ -0,0 +1,96 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; + +import javax.annotation.Nullable; + +import com.cronutils.model.Cron; +import com.cronutils.model.CronType; +import com.cronutils.model.definition.CronDefinitionBuilder; +import com.cronutils.parser.CronParser; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import com.linecorp.centraldogma.server.storage.repository.Repository; + +/** + * A configuration for retaining commits in a {@link Repository}. + */ +public final class CommitRetentionConfig { + + // TODO(minwoox): Make this configurable. + // The minimum of minRetentionCommits + private static final int MINIMUM_MINIMUM_RETENTION_COMMITS = 5000; + + private static final String DEFAULT_SCHEDULE = "0 0 * * * ?"; // Every day + private static final CronParser cronParser = new CronParser(CronDefinitionBuilder.instanceDefinitionFor( + CronType.QUARTZ)); + + private final int minRetentionCommits; + private final int minRetentionDays; + private final Cron schedule; + + /** + * Creates a new instance. + */ + @JsonCreator + public CommitRetentionConfig(@JsonProperty("minRetentionCommits") int minRetentionCommits, + @JsonProperty("minRetentionDays") int minRetentionDays, + @JsonProperty("schedule") @Nullable String schedule) { + checkArgument(minRetentionCommits == 0 || minRetentionCommits >= MINIMUM_MINIMUM_RETENTION_COMMITS, + "minRetentionCommits: %s (expected: 0 || >= %s)", + minRetentionCommits, MINIMUM_MINIMUM_RETENTION_COMMITS); + checkArgument(minRetentionDays >= 0, + "minRetentionDays: %s (expected: >= 0)", minRetentionDays); + this.minRetentionCommits = minRetentionCommits; + this.minRetentionDays = minRetentionDays; + this.schedule = cronParser.parse(firstNonNull(schedule, DEFAULT_SCHEDULE)); + } + + /** + * Returns the minimum number of commits that a {@link Repository} should retain. {@code 0} means that + * commits are not removed. + */ + public int minRetentionCommits() { + return minRetentionCommits; + } + + /** + * Returns the minimum number of days of a commit that a {@link Repository} should retain. + */ + public int minRetentionDays() { + return minRetentionDays; + } + + /** + * Returns the schedule of the job that removes old commits. + */ + public Cron schedule() { + return schedule; + } + + @Override + public String toString() { + return toStringHelper(this).add("minRetentionCommits", minRetentionCommits) + .add("minRetentionDays", minRetentionDays) + .add("schedule", schedule) + .toString(); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractCommand.java b/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractCommand.java index 135a4444e..e089443b1 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractCommand.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractCommand.java @@ -78,6 +78,7 @@ public final String toString() { return toStringHelper().toString(); } + // TODO(minwoox): Do not expose ToStringHelper to public. MoreObjects.ToStringHelper toStringHelper() { return MoreObjects.toStringHelper(this) .add("type", type) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/Command.java b/server/src/main/java/com/linecorp/centraldogma/server/command/Command.java index 5041f17a2..3ed7090be 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/Command.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/Command.java @@ -16,6 +16,7 @@ package com.linecorp.centraldogma.server.command; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import javax.annotation.Nullable; @@ -48,6 +49,7 @@ @Type(value = RemoveRepositoryCommand.class, name = "REMOVE_REPOSITORY"), @Type(value = PurgeRepositoryCommand.class, name = "PURGE_REPOSITORY"), @Type(value = UnremoveRepositoryCommand.class, name = "UNREMOVE_REPOSITORY"), + @Type(value = CreateRollingRepositoryCommand.class, name = "CREATE_ROLLING_REPOSITORY"), @Type(value = NormalizingPushCommand.class, name = "NORMALIZING_PUSH"), @Type(value = PushAsIsCommand.class, name = "PUSH"), @Type(value = CreateSessionCommand.class, name = "CREATE_SESSIONS"), @@ -246,6 +248,28 @@ static Command purgeRepository(@Nullable Long timestamp, Author author, return new PurgeRepositoryCommand(timestamp, author, projectName, repositoryName); } + /** + * Returns a new {@link Command} which is used to create a rolling repository. + * + * @param projectName the name of the project + * @param repositoryName the name of the repository which is supposed to be purged + * @param initialRevision the initial {@link Revision} of the rolling repository + * @param minRetentionCommits the configured number of recent commits that a repository should retain + * @param minRetentionDays the configured number of days for recent commits that a repository should retain + */ + static Command createRollingRepository( + String projectName, String repositoryName, Revision initialRevision, + int minRetentionCommits, int minRetentionDays) { + requireNonNull(projectName, "projectName"); + requireNonNull(repositoryName, "repositoryName"); + requireNonNull(initialRevision, "initialRevision"); + checkArgument(minRetentionCommits > 0, "minRetentionCommits: %s (expected: > 0)", + minRetentionCommits); + checkArgument(minRetentionDays > 0, "minRetentionDays: %s (expected: > 0)", minRetentionDays); + return new CreateRollingRepositoryCommand(projectName, repositoryName, initialRevision, + minRetentionCommits, minRetentionDays); + } + /** * Returns a new {@link Command} which is used to push the changes. The changes are normalized via * {@link Repository#previewDiff(Revision, Iterable)} before they are applied. diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/CommandType.java b/server/src/main/java/com/linecorp/centraldogma/server/command/CommandType.java index 7543dd4ef..17022013a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/CommandType.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/CommandType.java @@ -28,6 +28,7 @@ public enum CommandType { CREATE_REPOSITORY(Void.class), REMOVE_REPOSITORY(Void.class), UNREMOVE_REPOSITORY(Void.class), + CREATE_ROLLING_REPOSITORY(Void.class), NORMALIZING_PUSH(CommitResult.class), PUSH(Revision.class), SAVE_NAMED_QUERY(Void.class), diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/CreateRollingRepositoryCommand.java b/server/src/main/java/com/linecorp/centraldogma/server/command/CreateRollingRepositoryCommand.java new file mode 100644 index 000000000..d8ff0c2b7 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/CreateRollingRepositoryCommand.java @@ -0,0 +1,90 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.command; + +import static java.util.Objects.requireNonNull; + +import com.google.common.base.Objects; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.server.CommitRetentionConfig; +import com.linecorp.centraldogma.server.storage.repository.Repository; + +/** + * A {@link Command} which is used for creating a new rolling repository. + * + * @see CommitRetentionConfig + */ +public final class CreateRollingRepositoryCommand extends RepositoryCommand { + + private final Revision initialRevision; + private final int minRetentionCommits; + private final int minRetentionDays; + + CreateRollingRepositoryCommand(String projectName, String repositoryName, + Revision initialRevision, int minRetentionCommits, int minRetentionDays) { + super(CommandType.CREATE_ROLLING_REPOSITORY, null, Author.SYSTEM, projectName, repositoryName); + this.initialRevision = requireNonNull(initialRevision, "initialRevision"); + this.minRetentionCommits = minRetentionCommits; + this.minRetentionDays = minRetentionDays; + } + + /** + * Returns a {@link Revision} that will be the initial revision of the rolling repository. + */ + public Revision initialRevision() { + return initialRevision; + } + + /** + * Returns the minimum number of commits that a {@link Repository} should retain. {@code 0} means that + * the number of commits are not taken into account when + * {@link Repository#shouldCreateRollingRepository(int, int)} is called. + */ + public int minRetentionCommits() { + return minRetentionCommits; + } + + /** + * Returns the minimum number of days of a commit that a {@link Repository} should retain. + */ + public int minRetentionDays() { + return minRetentionDays; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CreateRollingRepositoryCommand)) { + return false; + } + final CreateRollingRepositoryCommand that = (CreateRollingRepositoryCommand) o; + return super.equals(o) && + minRetentionCommits == that.minRetentionCommits && + minRetentionDays == that.minRetentionDays && + Objects.equal(initialRevision, that.initialRevision); + } + + @Override + public int hashCode() { + return Objects.hashCode(super.hashCode(), initialRevision, minRetentionCommits, minRetentionDays); + } + + //TODO(minwoox): Add toString() after removing ToStringHelper from public API +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java b/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java index 18784ca3d..ecde4e707 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java @@ -170,6 +170,10 @@ protected CompletableFuture doExecute(Command command) throws Exceptio return (CompletableFuture) purgeRepository((PurgeRepositoryCommand) command); } + if (command instanceof CreateRollingRepositoryCommand) { + return (CompletableFuture) createRollingRepository((CreateRollingRepositoryCommand) command); + } + if (command instanceof NormalizingPushCommand) { return (CompletableFuture) push((NormalizingPushCommand) command, true); } @@ -253,6 +257,13 @@ private CompletableFuture purgeRepository(PurgeRepositoryCommand c) { }, repositoryWorker); } + private CompletableFuture createRollingRepository(CreateRollingRepositoryCommand c) { + return CompletableFuture.supplyAsync(() -> { + repo(c).createRollingRepository(c.initialRevision(), c.minRetentionCommits(), c.minRetentionDays()); + return null; + }, repositoryWorker); + } + private CompletableFuture push(AbstractPushCommand c, boolean normalizing) { if (c.projectName().equals(INTERNAL_PROJECT_DOGMA) || c.repositoryName().equals(Project.REPO_DOGMA) || !writeQuotaEnabled()) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java index 757194d00..ae2bb20c3 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java @@ -64,7 +64,6 @@ import com.linecorp.centraldogma.common.MergeQuery; import com.linecorp.centraldogma.common.Query; import com.linecorp.centraldogma.common.Revision; -import com.linecorp.centraldogma.common.RevisionRange; import com.linecorp.centraldogma.internal.api.v1.ChangeDto; import com.linecorp.centraldogma.internal.api.v1.CommitMessageDto; import com.linecorp.centraldogma.internal.api.v1.EntryDto; @@ -345,10 +344,9 @@ public CompletableFuture listCommits(@Param String revision, toRevision = to != null ? new Revision(to) : fromRevision; } - final RevisionRange range = repository.normalizeNow(fromRevision, toRevision).toDescending(); final int maxCommits0 = firstNonNull(maxCommits, Repository.DEFAULT_MAX_COMMITS); return repository - .history(range.from(), range.to(), normalizePath(path), maxCommits0) + .history(fromRevision, toRevision, normalizePath(path), maxCommits0) .thenApply(commits -> { final boolean toList = to != null || isNullOrEmpty(revision) || diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/DirectoryBasedStorageManager.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/DirectoryBasedStorageManager.java index f9405568b..1d1b97003 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/DirectoryBasedStorageManager.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/DirectoryBasedStorageManager.java @@ -56,6 +56,7 @@ public abstract class DirectoryBasedStorageManager implements StorageManager< private static final Logger logger = LoggerFactory.getLogger(DirectoryBasedStorageManager.class); + public static final String SUFFIX_REMOVED = ".removed"; /** * Start with an alphanumeric character. * An alphanumeric character, minus, plus, underscore and dot are allowed in the middle. @@ -63,7 +64,6 @@ public abstract class DirectoryBasedStorageManager implements StorageManager< */ private static final Pattern CHILD_NAME = Pattern.compile("^[0-9A-Za-z](?:[-+_0-9A-Za-z.]*[0-9A-Za-z])?$"); - private static final String SUFFIX_REMOVED = ".removed"; private static final String SUFFIX_PURGED = ".purged"; private final String childTypeName; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CacheableCall.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CacheableCall.java index 2e7c9aa1a..ae4ff7110 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CacheableCall.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CacheableCall.java @@ -38,7 +38,7 @@ public abstract class CacheableCall { } } - final Repository repo; + private final Repository repo; protected CacheableCall(Repository repo) { this.repo = requireNonNull(repo, "repo"); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java index cab82a7d3..69d916b93 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java @@ -22,6 +22,8 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; +import javax.annotation.Nullable; + import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.common.Change; import com.linecorp.centraldogma.common.Commit; @@ -199,6 +201,18 @@ public CompletableFuture> mergeFiles(Revision revision, Merge return unwrap().mergeFiles(revision, query); } + @Nullable + @Override + public Revision shouldCreateRollingRepository(int minRetentionCommits, int minRetentionDays) { + return unwrap().shouldCreateRollingRepository(minRetentionCommits, minRetentionDays); + } + + @Override + public void createRollingRepository(Revision initialRevision, int minRetentionCommits, + int minRetentionDays) { + unwrap().createRollingRepository(initialRevision, minRetentionCommits, minRetentionDays); + } + @Override public String toString() { return Util.simpleTypeName(this) + '(' + unwrap() + ')'; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java index 953543eac..7f1534cbc 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java @@ -307,6 +307,17 @@ public CompletableFuture commit(Revision baseRevision, long commit normalizing); } + @Override + public Revision shouldCreateRollingRepository(int minRetentionCommits, int minRetentionDays) { + return repo.shouldCreateRollingRepository(minRetentionCommits, minRetentionDays); + } + + @Override + public void createRollingRepository(Revision initialRevision, int minRetentionCommits, + int minRetentionDays) { + repo.createRollingRepository(initialRevision, minRetentionCommits, minRetentionDays); + } + @Override public String toString() { return toStringHelper(this) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CacheableCompareTreesCall.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CacheableCompareTreesCall.java index 3e068c28f..c1a86c731 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CacheableCompareTreesCall.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CacheableCompareTreesCall.java @@ -78,7 +78,7 @@ protected int weigh(List value) { } /** - * Never invoked because {@link GitRepository} produces the value of this call. + * Never invoked because {@link GitRepositoryV2} produces the value of this call. */ @Override public CompletableFuture> execute() { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java index 0b18d2681..0fc9b2c84 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabase.java @@ -17,7 +17,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; -import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepository.R_HEADS_MASTER; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryV2.R_HEADS_MASTER; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; import java.io.EOFException; @@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.MoreObjects; import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.common.RevisionNotFoundException; @@ -75,7 +76,10 @@ final class CommitIdDatabase implements AutoCloseable { private final Path path; private final FileChannel channel; private final boolean fsync; + @Nullable private volatile Revision headRevision; + @Nullable + private volatile Revision firstRevision; CommitIdDatabase(Repository repo) { // NB: We enable fsync only when our Git repository has been configured so, @@ -115,7 +119,14 @@ private CommitIdDatabase(File rootDir, boolean fsync) { } final int numRecords = (int) (size / RECORD_LEN); - headRevision = numRecords > 0 ? new Revision(numRecords) : null; + if (numRecords > 0) { + final Revision firstRevision = retrieveFirstRevision(); + this.firstRevision = firstRevision; + headRevision = new Revision(numRecords + firstRevision.major() - 1); + } else { + firstRevision = null; + headRevision = null; + } success = true; } finally { if (!success) { @@ -124,21 +135,17 @@ private CommitIdDatabase(File rootDir, boolean fsync) { } } - @Nullable Revision headRevision() { - return headRevision; - } - - ObjectId get(Revision revision) { - final Revision headRevision = this.headRevision; - checkState(headRevision != null, "initial commit not available yet: %s", path); - checkArgument(!revision.isRelative(), "revision: %s (expected: an absolute revision)", revision); - if (revision.major() > headRevision.major()) { - throw new RevisionNotFoundException(revision); - } - + private Revision retrieveFirstRevision() { final ByteBuffer buf = threadLocalBuffer.get(); buf.clear(); - long pos = (long) (revision.major() - 1) * RECORD_LEN; + buf.limit(4); + readTo(buf, 0); + buf.flip(); + return new Revision(buf.getInt()); + } + + private void readTo(ByteBuffer buf, long startPosition) { + long pos = startPosition; try { do { final int readBytes = channel.read(buf, pos); @@ -150,6 +157,35 @@ ObjectId get(Revision revision) { } catch (IOException e) { throw new StorageException("failed to read the commit ID database: " + path, e); } + } + + @Nullable + Revision headRevision() { + return headRevision; + } + + @Nullable + Revision firstRevision() { + return firstRevision; + } + + ObjectId get(Revision revision) { + checkArgument(!revision.isRelative(), "revision: %s (expected: an absolute revision)", revision); + final Revision headRevision = this.headRevision; + final Revision firstRevision = this.firstRevision; + if (headRevision == null || firstRevision == null) { + throw new IllegalStateException("initial commit not available yet: " + path); + } + + if (!(firstRevision.major() <= revision.major() && revision.major() <= headRevision.major())) { + throw new RevisionNotFoundException( + "revision: " + revision + + " (expected: " + firstRevision.major() + " <= revision <= " + headRevision.major() + ')'); + } + + final ByteBuffer buf = threadLocalBuffer.get(); + buf.clear(); + readTo(buf, (long) (revision.major() - firstRevision.major()) * RECORD_LEN); buf.flip(); @@ -166,15 +202,17 @@ void put(Revision revision, ObjectId commitId) { put(revision, commitId, true); } - private synchronized void put(Revision revision, ObjectId commitId, boolean safeMode) { - if (safeMode) { - final Revision expected; - if (headRevision == null) { - expected = Revision.INIT; + // TODO(minwoox) Use lock instead of synchronized. + private synchronized void put(Revision revision, ObjectId commitId, boolean newHead) { + if (newHead) { + final Revision headRevision = this.headRevision; + if (headRevision != null) { + final Revision expected = headRevision.forward(1); + checkState(revision.equals(expected), "incorrect revision: %s (expected: %s)", + revision, expected); } else { - expected = headRevision.forward(1); + firstRevision = revision; } - checkState(revision.equals(expected), "incorrect revision: %s (expected: %s)", revision, expected); } // Build a record. @@ -184,24 +222,25 @@ private synchronized void put(Revision revision, ObjectId commitId, boolean safe commitId.copyRawTo(buf); buf.flip(); - // Append a record to the file. - long pos = (long) (revision.major() - 1) * RECORD_LEN; + // Append or overwrite a record in the file. + long pos = (long) (revision.major() - firstRevision.major()) * RECORD_LEN; try { do { pos += channel.write(buf, pos); } while (buf.hasRemaining()); - if (safeMode && fsync) { + if (newHead && fsync) { channel.force(true); } } catch (IOException e) { throw new StorageException("failed to update the commit ID database: " + path, e); } - if (safeMode || + final Revision headRevision = this.headRevision; + if (newHead || headRevision == null || headRevision.major() < revision.major()) { - headRevision = revision; + this.headRevision = revision; } } @@ -224,48 +263,65 @@ void rebuild(Repository gitRepo) { throw new StorageException("failed to determine the HEAD: " + gitRepo.getDirectory()); } - RevCommit revCommit = revWalk.parseCommit(headCommitId); + final RevCommit revCommit = revWalk.parseCommit(headCommitId); headRevision = CommitUtil.extractRevision(revCommit.getFullMessage()); // NB: We did not store the last commit ID until all commit IDs are stored, // so that the partially built database always has mismatching head revision. - ObjectId currentId; - Revision previousRevision = headRevision; - loop: for (;;) { - switch (revCommit.getParentCount()) { - case 0: - // End of the history - break loop; - case 1: - currentId = revCommit.getParent(0); - break; - default: - throw new StorageException("found more than one parent: " + - gitRepo.getDirectory()); - } + // Find firstRevision while validating whether the commitIds are stored correctly in the file. + // This won't change the file but update the firstRevision field. + findFirstRevisionOrRebuild(gitRepo, revWalk, headRevision, revCommit, true); + // Now we know it's validated so rebuild the database. + findFirstRevisionOrRebuild(gitRepo, revWalk, headRevision, revCommit, false); + + // All commit IDs except the head have been stored. Store the head finally. + put(headRevision, headCommitId); + } catch (Exception e) { + throw new StorageException("failed to rebuild the commit ID database", e); + } + + logger.info("Rebuilt the commit ID database. firstRevision: {}, headRevision: {}", + firstRevision, headRevision); + } + + private void findFirstRevisionOrRebuild(Repository gitRepo, RevWalk revWalk, + Revision headRevision, RevCommit revCommit, + boolean findingFirstRevision) throws IOException { + ObjectId currentId; + Revision previousRevision = headRevision; + loop: for (;;) { + switch (revCommit.getParentCount()) { + case 0: + // End of the history + break loop; + case 1: + currentId = revCommit.getParent(0); + break; + default: + throw new StorageException("found more than one parent: " + gitRepo.getDirectory()); + } - revCommit = revWalk.parseCommit(currentId); + revCommit = revWalk.parseCommit(currentId); - final Revision currentRevision = CommitUtil.extractRevision(revCommit.getFullMessage()); + final Revision currentRevision; + if (findingFirstRevision) { + currentRevision = CommitUtil.extractRevision(revCommit.getFullMessage()); final Revision expectedRevision = previousRevision.backward(1); if (!currentRevision.equals(expectedRevision)) { throw new StorageException("mismatching revision: " + gitRepo.getDirectory() + " (actual: " + currentRevision.major() + ", expected: " + expectedRevision.major() + ')'); } - + } else { + currentRevision = previousRevision.backward(1); put(currentRevision, currentId, false); - previousRevision = currentRevision; } - - // All commit IDs except the head have been stored. Store the head finally. - put(headRevision, headCommitId); - } catch (Exception e) { - throw new StorageException("failed to rebuild the commit ID database", e); + previousRevision = currentRevision; + } + if (findingFirstRevision) { + firstRevision = previousRevision; } - - logger.info("Rebuilt the commit ID database."); } @Override @@ -276,4 +332,13 @@ public void close() { logger.warn("Failed to close the commit ID database: {}", path, e); } } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).omitNullValues() + .add("path", path) + .add("headRevision", headRevision) + .add("firstRevision", firstRevision) + .toString(); + } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java new file mode 100644 index 000000000..189717f12 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitRetentionManagementPlugin.java @@ -0,0 +1,187 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import java.time.Duration; +import java.time.ZonedDateTime; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import javax.annotation.Nullable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cronutils.model.time.ExecutionTime; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableScheduledFuture; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.common.util.concurrent.MoreExecutors; + +import com.linecorp.armeria.common.util.UnmodifiableFuture; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.server.CentralDogmaConfig; +import com.linecorp.centraldogma.server.CommitRetentionConfig; +import com.linecorp.centraldogma.server.command.Command; +import com.linecorp.centraldogma.server.plugin.Plugin; +import com.linecorp.centraldogma.server.plugin.PluginContext; +import com.linecorp.centraldogma.server.plugin.PluginTarget; +import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.project.ProjectManager; +import com.linecorp.centraldogma.server.storage.repository.Repository; + +import io.netty.util.concurrent.DefaultThreadFactory; + +public final class CommitRetentionManagementPlugin implements Plugin { + + private static final Logger logger = LoggerFactory.getLogger(CommitRetentionManagementPlugin.class); + + @Nullable + private ListeningScheduledExecutorService worker; + + private volatile boolean stopping; + @Nullable + private ListenableScheduledFuture scheduledFuture; + + @Override + public PluginTarget target() { + return PluginTarget.LEADER_ONLY; + } + + @Override + public boolean isEnabled(CentralDogmaConfig config) { + return config.commitRetentionConfig() != null; + } + + @Override + public CompletionStage start(PluginContext context) { + final CommitRetentionConfig commitRetentionConfig = context.config().commitRetentionConfig(); + assert commitRetentionConfig != null; + final int minRetentionCommits = commitRetentionConfig.minRetentionCommits(); + final int minRetentionDays = commitRetentionConfig.minRetentionDays(); + if (minRetentionCommits == 0 || + minRetentionCommits == Integer.MAX_VALUE || minRetentionDays == Integer.MAX_VALUE) { + // Disabled. + return UnmodifiableFuture.completedFuture(null); + } + + worker = MoreExecutors.listeningDecorator( + Executors.newSingleThreadScheduledExecutor( + new DefaultThreadFactory("commit-retention-worker", true))); + scheduleRemovingOldCommits(context, commitRetentionConfig); + + return UnmodifiableFuture.completedFuture(null); + } + + private void scheduleRemovingOldCommits(PluginContext context, CommitRetentionConfig config) { + if (stopping) { + return; + } + + final ZonedDateTime now = ZonedDateTime.now(); + final Optional duration = ExecutionTime.forCron(config.schedule()) + .timeToNextExecution(now); + if (!duration.isPresent()) { + logger.warn("Failed to calculate the next execution time of the commit retention scheduler. " + + " config: {}, now: {}", config, now); + return; + } + final ListeningScheduledExecutorService worker = this.worker; + assert worker != null; + scheduledFuture = worker.schedule(() -> createRollingRepository(context, config), duration.get()); + + Futures.addCallback(scheduledFuture, new FutureCallback() { + @Override + public void onSuccess(@Nullable Object result) {} + + @Override + public void onFailure(Throwable cause) { + if (!stopping) { + logger.warn("Commit retention scheduler stopped due to an unexpected exception:", cause); + } + } + }, worker); + } + + // TODO(minwoox): Add metrics. + private void createRollingRepository(PluginContext context, CommitRetentionConfig config) { + if (stopping) { + return; + } + + final ProjectManager pm = context.projectManager(); + for (Project project : pm.list().values()) { + for (Repository repo : project.repos().list().values()) { + if (stopping) { + return; + } + final int minRetentionCommits = config.minRetentionCommits(); + final int minRetentionDays = config.minRetentionDays(); + final Revision revision = + repo.shouldCreateRollingRepository(minRetentionCommits, minRetentionDays); + if (revision != null) { + try { + context.commandExecutor().execute( + Command.createRollingRepository(project.name(), repo.name(), revision, + minRetentionCommits, minRetentionDays)) + .get(10, TimeUnit.MINUTES); + } catch (Throwable t) { + logger.warn("Failed to create a rolling repository for {}/{} with revision: {}", + project.name(), repo.name(), revision, t); + } + } + } + } + + scheduleRemovingOldCommits(context, config); + } + + @Override + public CompletionStage stop(PluginContext context) { + stopping = true; + if (scheduledFuture != null) { + scheduledFuture.cancel(false); + } + + try { + if (worker != null && !worker.isTerminated()) { + logger.info("Stopping the commit retention worker .."); + boolean interruptLater = false; + while (!worker.isTerminated()) { + worker.shutdownNow(); + try { + worker.awaitTermination(1, TimeUnit.SECONDS); + } catch (InterruptedException e) { + // Interrupt later. + interruptLater = true; + } + } + logger.info("Stopped the commit retention worker."); + + if (interruptLater) { + Thread.currentThread().interrupt(); + } + } + } catch (Throwable t) { + logger.warn("Failed to stop the commit retention worker:", t); + } + return CompletableFuture.completedFuture(null); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/FailFastUtil.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/FailFastUtil.java index d4274534a..d92079a34 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/FailFastUtil.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/FailFastUtil.java @@ -23,6 +23,7 @@ import com.linecorp.armeria.common.util.Exceptions; import com.linecorp.armeria.server.ServiceRequestContext; import com.linecorp.centraldogma.server.internal.storage.RequestAlreadyTimedOutException; +import com.linecorp.centraldogma.server.storage.repository.Repository; final class FailFastUtil { @@ -39,7 +40,7 @@ static ServiceRequestContext context() { } @SuppressWarnings("MethodParameterNamingConvention") - static void failFastIfTimedOut(GitRepository repo, Logger logger, @Nullable ServiceRequestContext ctx, + static void failFastIfTimedOut(Repository repo, Logger logger, @Nullable ServiceRequestContext ctx, String methodName, Object arg1) { if (ctx != null && ctx.isTimedOut()) { logger.info("{} Rejecting a request timed out already: repo={}/{}, method={}, args={}", @@ -49,7 +50,7 @@ static void failFastIfTimedOut(GitRepository repo, Logger logger, @Nullable Serv } @SuppressWarnings("MethodParameterNamingConvention") - static void failFastIfTimedOut(GitRepository repo, Logger logger, @Nullable ServiceRequestContext ctx, + static void failFastIfTimedOut(Repository repo, Logger logger, @Nullable ServiceRequestContext ctx, String methodName, Object arg1, Object arg2) { if (ctx != null && ctx.isTimedOut()) { logger.info("{} Rejecting a request timed out already: repo={}/{}, method={}, args=[{}, {}]", @@ -59,7 +60,7 @@ static void failFastIfTimedOut(GitRepository repo, Logger logger, @Nullable Serv } @SuppressWarnings("MethodParameterNamingConvention") - static void failFastIfTimedOut(GitRepository repo, Logger logger, @Nullable ServiceRequestContext ctx, + static void failFastIfTimedOut(Repository repo, Logger logger, @Nullable ServiceRequestContext ctx, String methodName, Object arg1, Object arg2, Object arg3) { if (ctx != null && ctx.isTimedOut()) { logger.info("{} Rejecting a request timed out already: repo={}/{}, method={}, args=[{}, {}, {}]", @@ -69,7 +70,7 @@ static void failFastIfTimedOut(GitRepository repo, Logger logger, @Nullable Serv } @SuppressWarnings("MethodParameterNamingConvention") - static void failFastIfTimedOut(GitRepository repo, Logger logger, @Nullable ServiceRequestContext ctx, + static void failFastIfTimedOut(Repository repo, Logger logger, @Nullable ServiceRequestContext ctx, String methodName, Object arg1, Object arg2, Object arg3, int arg4) { if (ctx != null && ctx.isTimedOut()) { logger.info( diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManager.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManager.java index 551acda55..e67138159 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManager.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManager.java @@ -19,23 +19,15 @@ import static java.util.Objects.requireNonNull; import java.io.File; -import java.io.IOException; import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; -import java.util.function.BiConsumer; import java.util.function.Supplier; import javax.annotation.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.linecorp.armeria.common.util.TextFormatter; import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.common.CentralDogmaException; import com.linecorp.centraldogma.common.RepositoryExistsException; import com.linecorp.centraldogma.common.RepositoryNotFoundException; -import com.linecorp.centraldogma.internal.Util; import com.linecorp.centraldogma.server.internal.storage.DirectoryBasedStorageManager; import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache; import com.linecorp.centraldogma.server.storage.project.Project; @@ -45,8 +37,6 @@ public class GitRepositoryManager extends DirectoryBasedStorageManager implements RepositoryManager { - private static final Logger logger = LoggerFactory.getLogger(GitRepositoryManager.class); - private final Project parent; private final Executor repositoryWorker; @@ -69,25 +59,19 @@ public Project parent() { @Override protected Repository openChild(File childDir) throws Exception { - return new GitRepository(parent, childDir, repositoryWorker, cache); - } - - private static void deleteCruft(File dir) throws IOException { - logger.info("Deleting the cruft from previous migration: {}", dir); - Util.deleteFileTree(dir); - logger.info("Deleted the cruft from previous migration: {}", dir); + return GitRepositoryV2.open(parent, childDir, repositoryWorker, cache); } @Override protected Repository createChild(File childDir, Author author, long creationTimeMillis) throws Exception { - return new GitRepository(parent, childDir, repositoryWorker, - creationTimeMillis, author, cache); + return new GitRepositoryV2(parent, childDir, repositoryWorker, + creationTimeMillis, author, cache); } @Override protected void closeChild(File childDir, Repository child, Supplier failureCauseSupplier) { - ((GitRepository) child).close(failureCauseSupplier); + ((GitRepositoryV2) child).close(failureCauseSupplier); } @Override @@ -99,37 +83,4 @@ protected CentralDogmaException newStorageExistsException(String name) { protected CentralDogmaException newStorageNotFoundException(String name) { return new RepositoryNotFoundException(parent().name() + '/' + name); } - - /** - * Logs the migration progress periodically. - */ - private static class MigrationProgressLogger implements BiConsumer { - - private static final long REPORT_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(10); - - private final String name; - private final long startTimeNanos; - private long lastReportTimeNanos; - - MigrationProgressLogger(Repository repo) { - name = repo.parent().name() + '/' + repo.name(); - startTimeNanos = lastReportTimeNanos = System.nanoTime(); - } - - @Override - public void accept(Integer current, Integer total) { - final long currentTimeNanos = System.nanoTime(); - final long elapsedTimeNanos = currentTimeNanos - startTimeNanos; - if (currentTimeNanos - lastReportTimeNanos > REPORT_INTERVAL_NANOS) { - logger.info("{}: {}% ({}/{}) - took {}", - name, (int) ((double) current / total * 100), - current, total, TextFormatter.elapsed(elapsedTimeNanos)); - lastReportTimeNanos = currentTimeNanos; - } else if (current.equals(total)) { - logger.info("{}: 100% ({}/{}) - took {}", - name, current, total, - TextFormatter.elapsed(elapsedTimeNanos)); - } - } - } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtil.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtil.java new file mode 100644 index 000000000..668f2c3af --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryUtil.java @@ -0,0 +1,43 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class GitRepositoryUtil { + + private static final Logger logger = LoggerFactory.getLogger(GitRepositoryUtil.class); + + static boolean exists(Repository repository) { + if (repository.getConfig() instanceof FileBasedConfig) { + return ((FileBasedConfig) repository.getConfig()).getFile().exists(); + } + return repository.getDirectory().exists(); + } + + static void closeJGitRepo(Repository repository) { + try { + repository.close(); + } catch (Throwable t) { + logger.warn("Failed to close a Git repository: {}", repository.getDirectory(), t); + } + } + + private GitRepositoryUtil() {} +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java new file mode 100644 index 000000000..4a0ec7c9e --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2.java @@ -0,0 +1,904 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.linecorp.centraldogma.server.internal.storage.DirectoryBasedStorageManager.SUFFIX_REMOVED; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.context; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.failFastIfTimedOut; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.closeJGitRepo; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.InternalRepository.buildJGitRepo; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static java.util.Objects.requireNonNull; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Supplier; +import java.util.stream.Stream; + +import javax.annotation.Nullable; + +import org.eclipse.jgit.diff.DiffEntry; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.Repository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.ImmutableMap; + +import com.linecorp.armeria.server.ServiceRequestContext; +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.CentralDogmaException; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.ChangeConflictException; +import com.linecorp.centraldogma.common.Commit; +import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.EntryNotFoundException; +import com.linecorp.centraldogma.common.EntryType; +import com.linecorp.centraldogma.common.Markup; +import com.linecorp.centraldogma.common.RepositoryNotFoundException; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.common.RevisionNotFoundException; +import com.linecorp.centraldogma.common.RevisionRange; +import com.linecorp.centraldogma.common.RolledRevisionAccessException; +import com.linecorp.centraldogma.server.command.CommitResult; +import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache; +import com.linecorp.centraldogma.server.internal.storage.repository.git.InternalRepository.RevisionAndEntries; +import com.linecorp.centraldogma.server.storage.StorageException; +import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.repository.FindOption; +import com.linecorp.centraldogma.server.storage.repository.FindOptions; + +class GitRepositoryV2 implements com.linecorp.centraldogma.server.storage.repository.Repository { + + private static final Logger logger = LoggerFactory.getLogger(GitRepositoryV2.class); + + static final String R_HEADS_MASTER = Constants.R_HEADS + Constants.MASTER; + + /** + * Opens an existing Git-backed repository. + * + * @param repositoryDir the location of this repository + * @param repositoryWorker the {@link Executor} which will perform the blocking repository operations + * + * @throws StorageException if failed to open the repository at the specified location + */ + static GitRepositoryV2 open(Project parent, File repositoryDir, + Executor repositoryWorker, @Nullable RepositoryCache cache) { + return new GitRepositoryV2(parent, repositoryDir, repositoryWorker, cache); + } + + private final ReadWriteLock rwLock = new ReentrantReadWriteLock(); + private final Project parent; + private final String originalRepoName; + private final Executor repositoryWorker; + + private final RepositoryMetadataDatabase repoMetadata; + @Nullable + @VisibleForTesting + final RepositoryCache cache; + @VisibleForTesting + final CommitWatchers commitWatchers = new CommitWatchers(); + private final AtomicReference> closePending = new AtomicReference<>(); + private final CompletableFuture closeFuture = new CompletableFuture<>(); + + // The primary internal repository where read and write operations are performed. + // The primary repository is replaced by the secondary repository when: + // - the secondary repository has more than minRetentionCommits + // - the oldest commit of the secondary repository has exceeded the minRetentionDays + @VisibleForTesting + InternalRepository primaryRepo; + + // The secondary internal repository where only write operations are performed. + @Nullable + @VisibleForTesting + InternalRepository secondaryRepo; + + private final long creationTimeMillis; + private final Author author; + + /** + * The current head revision. Initialized by the constructor and updated by commit(). + */ + private volatile Revision headRevision; + + /** + * Creates a Git repository to the specified {@code repositoryDir}. + */ + GitRepositoryV2(Project parent, File repositoryDir, Executor repositoryWorker, + long creationTimeMillis, Author author, @Nullable RepositoryCache cache) { + this.parent = requireNonNull(parent, "parent"); + originalRepoName = requireNonNull(repositoryDir, "repositoryDir").getName(); + this.repositoryWorker = requireNonNull(repositoryWorker, "repositoryWorker"); + requireNonNull(author, "author"); + this.cache = cache; + + try { + if (repositoryDir.exists()) { + if (!isEmpty(repositoryDir)) { + throw new StorageException("failed to create a repository at: " + repositoryDir + + " (exists already)"); + } + } else if (!repositoryDir.mkdir()) { + throw new StorageException("failed to create a directory for Git at: " + repositoryDir); + } + } catch (IOException e) { + throw new StorageException("failed to create a repository at: " + repositoryDir, e); + } + + repoMetadata = new RepositoryMetadataDatabase(repositoryDir, true); + final File primaryRepoDir = repoMetadata.primaryRepoDir(); + try { + primaryRepo = InternalRepository.create(parent, originalRepoName, primaryRepoDir, Revision.INIT, + creationTimeMillis, author, ImmutableList.of()); + } catch (Throwable t) { + repoMetadata.close(); + throw t; + } + headRevision = Revision.INIT; + this.creationTimeMillis = creationTimeMillis; + this.author = author; + } + + /** + * Opens the existing Git repository. + */ + private GitRepositoryV2(Project parent, File repoDir, Executor repositoryWorker, + @Nullable RepositoryCache cache) { + this.parent = requireNonNull(parent, "parent"); + originalRepoName = requireNonNull(repoDir, "repoDir").getName(); + this.repositoryWorker = requireNonNull(repositoryWorker, "repositoryWorker"); + this.cache = cache; + + RepositoryMetadataDatabase repoMetadata; + try { + repoMetadata = new RepositoryMetadataDatabase(repoDir, false); + } catch (Throwable t) { + // The metadata doesn't exist so check if the repository exists in the form of the old version. + checkRepositoryExists(repoDir); + migrateToV2(repoDir); + repoMetadata = new RepositoryMetadataDatabase(repoDir, true); + } + + this.repoMetadata = repoMetadata; + try { + final InternalRepository primaryRepo = + InternalRepository.open(parent, originalRepoName, repoMetadata.primaryRepoDir(), true); + assert primaryRepo != null; + this.primaryRepo = primaryRepo; + headRevision = primaryRepo.headRevision(); + final Commit firstCommit = firstCommit(primaryRepo); + creationTimeMillis = firstCommit.when(); + author = firstCommit.author(); + secondaryRepo = InternalRepository.open(parent, originalRepoName, + repoMetadata.secondaryRepoDir(), false); + } catch (Throwable t) { + repoMetadata.close(); + closeInternalRepository(primaryRepo); + closeInternalRepository(secondaryRepo); + throw t; + } + } + + private static boolean isEmpty(File dir) throws IOException { + if (!dir.isDirectory()) { + return false; + } + if (!dir.exists()) { + return true; + } + try (Stream entries = Files.list(dir.toPath())) { + return entries.findFirst().isPresent(); + } + } + + private static void checkRepositoryExists(File repoDir) { + final Repository oldRepo = buildJGitRepo(repoDir); + final boolean oldRepoExist = GitRepositoryUtil.exists(oldRepo); + closeJGitRepo(oldRepo); + if (!oldRepoExist) { + throw new RepositoryNotFoundException(repoDir.toString()); + } + } + + private static void migrateToV2(File repoDir) { + // When: + // - repoDir: /foo + // - primaryRepoDir: /foo/foo_0000000000 + // - tmpRepoDir: /bar (random UUID) + // + // Migration steps will be: + // - /foo becomes /bar + // - /foo/foo_0000000000 directory is created. + // - /bar becomes /foo/foo_0000000000 + + final File primaryRepoDir = RepositoryMetadataDatabase.initialPrimaryRepoDir(repoDir); + final File tmpRepoDir = new File(repoDir.getParentFile(), UUID.randomUUID().toString()); + logger.info("Migrating {} to {} using temp repository: {}", repoDir, primaryRepoDir, tmpRepoDir); + if (!repoDir.renameTo(tmpRepoDir)) { + throw new StorageException("failed to migrate a repository at: " + repoDir + + ", to the tmp dir: " + tmpRepoDir); + } + if (!primaryRepoDir.mkdirs()) { + throw new StorageException("failed to create " + primaryRepoDir + " while migrating to V2."); + } + try { + final Path moved = Files.move(tmpRepoDir.toPath(), primaryRepoDir.toPath(), REPLACE_EXISTING); + assert moved == primaryRepoDir.toPath() : moved + " != " + primaryRepoDir.toPath(); + } catch (IOException e) { + //noinspection ResultOfMethodCallIgnored + tmpRepoDir.renameTo(repoDir); + throw new StorageException("failed to migrate a repository at: " + tmpRepoDir + + ", to: " + primaryRepoDir, e); + } + checkState(!tmpRepoDir.exists(), "%s is not renamed.", tmpRepoDir); + logger.info("Migrating {} is done.", repoDir); + } + + @VisibleForTesting + void internalClose() { + close(() -> new CentralDogmaException("should never reach here")); + } + + /** + * Waits until all pending operations are complete and closes this repository. + * + * @param failureCauseSupplier the {@link Supplier} that creates a new {@link CentralDogmaException} + * which will be used to fail the operations issued after this method is called + */ + void close(Supplier failureCauseSupplier) { + requireNonNull(failureCauseSupplier, "failureCauseSupplier"); + if (closePending.compareAndSet(null, failureCauseSupplier)) { + repositoryWorker.execute(() -> { + rwLock.writeLock().lock(); + try { + closeInternalRepository(primaryRepo); + closeInternalRepository(secondaryRepo); + repoMetadata.close(); + } finally { + rwLock.writeLock().unlock(); + commitWatchers.close(failureCauseSupplier); + closeFuture.complete(null); + } + }); + } + + closeFuture.join(); + } + + private static void closeInternalRepository(@Nullable InternalRepository internalRepo) { + if (internalRepo == null) { + return; + } + internalRepo.close(); + } + + @Override + public Project parent() { + return parent; + } + + @Override + public String name() { + return originalRepoName; + } + + @Override + public long creationTimeMillis() { + return creationTimeMillis; + } + + @Override + public Author author() { + return author; + } + + @Override + public Revision normalizeNow(Revision revision) { + return normalizeNow(revision, true); + } + + private Revision normalizeNow(Revision revision, boolean checkFirstRevision) { + return normalizeNow(revision, headRevision, checkFirstRevision); + } + + @Override + public RevisionRange normalizeNow(Revision from, Revision to) { + final Revision headRevision = this.headRevision; + return new RevisionRange(normalizeNow(from, headRevision, true), normalizeNow(to, headRevision, true)); + } + + private Revision normalizeNow(Revision revision, Revision headRevision, boolean checkFirstRevision) { + requireNonNull(revision, "revision"); + int major = revision.major(); + final int headMajor = headRevision.major(); + if (major == -1 || major == headMajor) { + return headRevision; + } + + if (major >= 0) { + if (major > headMajor) { + throw new RevisionNotFoundException( + "revision: " + revision + " (expected: <= " + headMajor + ')'); + } + } else { + major = headMajor + major + 1; + if (major <= 0) { + throw new RevisionNotFoundException( + "revision: " + revision + " (expected: " + revision + " + " + headMajor + " + 1 > 0)"); + } + } + + if (checkFirstRevision) { + final int firstRevisionMajor = primaryRepo.firstRevision().major(); + if (major < firstRevisionMajor) { + if (major == 1) { + // We silently update the major to the first revision when it's Revision.INIT. + major = firstRevisionMajor; + } else { + throw new RolledRevisionAccessException( + "revision: " + revision + " (expected: >= " + firstRevisionMajor + ')'); + } + } + } + + return new Revision(major); + } + + @Override + public CompletableFuture>> find( + Revision revision, String pathPattern, Map, ?> options) { + requireNonNull(pathPattern, "pathPattern"); + requireNonNull(revision, "revision"); + requireNonNull(options, "options"); + final ServiceRequestContext ctx = context(); + return CompletableFuture.supplyAsync(() -> { + failFastIfTimedOut(this, logger, ctx, "find", revision, pathPattern, options); + try { + readLock(); + final Revision normalizedRevision = normalizeNow(revision); + if ("/".equals(pathPattern)) { + return ImmutableMap.of(pathPattern, Entry.ofDirectory(normalizedRevision, "/")); + } + return primaryRepo.find(normalizedRevision, pathPattern, options); + } finally { + readUnlock(); + } + }, repositoryWorker); + } + + /** + * Get the diff between any two valid revisions. + * + * @param from revision from + * @param to revision to + * @param pathPattern target path pattern + * @return the map of changes mapped by path + * @throws StorageException if {@code from} or {@code to} does not exist. + */ + @Override + public CompletableFuture>> diff(Revision from, Revision to, String pathPattern) { + requireNonNull(from, "from"); + requireNonNull(to, "to"); + requireNonNull(pathPattern, "pathPattern"); + final ServiceRequestContext ctx = context(); + return CompletableFuture.supplyAsync(() -> { + failFastIfTimedOut(this, logger, ctx, "diff", from, to, pathPattern); + try { + readLock(); + final RevisionRange range = normalizeNow(from, to).toAscending(); + if (range.from().equals(range.to())) { + // Empty range. + return ImmutableMap.of(); + } + return primaryRepo.diff(range, pathPattern); + } finally { + readUnlock(); + } + }, repositoryWorker); + } + + @Override + public CompletableFuture>> previewDiff(Revision baseRevision, + Iterable> changes) { + requireNonNull(baseRevision, "baseRevision"); + requireNonNull(changes, "changes"); + final ServiceRequestContext ctx = context(); + return CompletableFuture.supplyAsync(() -> { + failFastIfTimedOut(this, logger, ctx, "previewDiff", baseRevision); + try { + readLock(); + final Revision normalizedRevision = normalizeNow(baseRevision); + return primaryRepo.previewDiff(normalizedRevision, changes); + } finally { + readUnlock(); + } + }, repositoryWorker); + } + + @Override + public CompletableFuture commit(Revision baseRevision, long commitTimeMillis, Author author, + String summary, String detail, Markup markup, + Iterable> changes, boolean directExecution) { + requireNonNull(baseRevision, "baseRevision"); + requireNonNull(author, "author"); + requireNonNull(summary, "summary"); + requireNonNull(detail, "detail"); + requireNonNull(markup, "markup"); + requireNonNull(changes, "changes"); + + final ServiceRequestContext ctx = context(); + return CompletableFuture.supplyAsync(() -> { + failFastIfTimedOut(this, logger, ctx, "commit", baseRevision, author, summary); + return blockingCommit(baseRevision, commitTimeMillis, + author, summary, detail, markup, changes, directExecution); + }, repositoryWorker); + } + + private CommitResult blockingCommit( + Revision baseRevision, long commitTimeMillis, Author author, String summary, + String detail, Markup markup, Iterable> changes, boolean directExecution) { + final RevisionAndEntries res; + final Iterable> applyingChanges; + try { + writeLock(); + final Revision normalizedRevision = normalizeNow(baseRevision); + final Revision headRevision = this.headRevision; + if (headRevision.major() != normalizedRevision.major()) { + throw new ChangeConflictException( + "invalid baseRevision: " + baseRevision + " (expected: " + headRevision + + " or equivalent)"); + } + + if (directExecution) { + applyingChanges = primaryRepo.previewDiff(normalizedRevision, changes).values(); + } else { + applyingChanges = changes; + } + res = primaryRepo.commit(headRevision, headRevision.forward(1), commitTimeMillis, + author, summary, detail, markup, applyingChanges, false); + this.headRevision = res.revision; + final InternalRepository secondaryRepo = this.secondaryRepo; + if (secondaryRepo != null) { + assert headRevision.equals(secondaryRepo.headRevision()); + + // Push the same commit to the secondary repo. + secondaryRepo.commit(headRevision, headRevision.forward(1), commitTimeMillis, + author, summary, detail, markup, applyingChanges, false); + } + } finally { + writeUnLock(); + } + + // Note that the notification is made while no lock is held to avoid the risk of a dead lock. + notifyWatchers(res.revision, res.diffEntries); + return CommitResult.of(res.revision, applyingChanges); + } + + private void notifyWatchers(Revision newRevision, List diffEntries) { + for (DiffEntry entry : diffEntries) { + switch (entry.getChangeType()) { + case ADD: + commitWatchers.notify(newRevision, entry.getNewPath()); + break; + case MODIFY: + case DELETE: + commitWatchers.notify(newRevision, entry.getOldPath()); + break; + default: + throw new Error(); + } + } + } + + @Override + public CompletableFuture> history(Revision from, Revision to, String pathPattern, + int maxCommits) { + requireNonNull(pathPattern, "pathPattern"); + requireNonNull(from, "from"); + requireNonNull(to, "to"); + checkArgument(maxCommits > 0, "maxCommits: %s (expected: > 0)", maxCommits); + final ServiceRequestContext ctx = context(); + return CompletableFuture.supplyAsync(() -> { + failFastIfTimedOut(this, logger, ctx, "history", from, to, pathPattern, maxCommits); + try { + readLock(); + final RevisionRange range = normalizeNow(from, to); + return primaryRepo.listCommits(pathPattern, Math.min(maxCommits, MAX_MAX_COMMITS), range); + } finally { + readUnlock(); + } + }, repositoryWorker); + } + + @Override + public CompletableFuture findLatestRevision(Revision lastKnownRevision, String pathPattern, + boolean errorOnEntryNotFound) { + requireNonNull(lastKnownRevision, "lastKnownRevision"); + requireNonNull(pathPattern, "pathPattern"); + final ServiceRequestContext ctx = context(); + return CompletableFuture.supplyAsync(() -> { + failFastIfTimedOut(this, logger, ctx, "findLatestRevision", lastKnownRevision, pathPattern); + return blockingFindLatestRevision(lastKnownRevision, pathPattern, errorOnEntryNotFound); + }, repositoryWorker); + } + + @Nullable + private Revision blockingFindLatestRevision(Revision lastKnownRevision, String pathPattern, + boolean errorOnEntryNotFound) { + if (lastKnownRevision.major() == Revision.INIT.major()) { + // Fast path: no need to compare because we are sure there is nothing at revision 1. + final Revision headRevision = this.headRevision; + if (headRevision.major() == 1) { + if (errorOnEntryNotFound) { + throw new EntryNotFoundException(lastKnownRevision, pathPattern); + } + + return null; + } + if ("/".equals(pathPattern)) { + return headRevision; + } + try { + readLock(); + final Map> entries = primaryRepo.find( + headRevision, pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT); + if (!entries.isEmpty()) { + return headRevision; + } + if (errorOnEntryNotFound) { + throw new EntryNotFoundException(lastKnownRevision, pathPattern); + } + return null; + } finally { + readUnlock(); + } + } + + // Slow path: compare the two trees. + final List diffEntries; + final RevisionRange range; + try { + readLock(); + range = new RevisionRange(normalizeNow(lastKnownRevision, false), headRevision); + if (range.from().isLowerThan(primaryRepo.firstRevision())) { + // We should return range.to() without comparing two tree. It's because: + // - the entry might be changed between the lastKnownRevision(in the previous repository + // that is removed and superseded by the current primaryRepo) + // and first revision(in the current primaryRepo). + // - but the previous commits before the first revision is packed and committed to the + // primaryRepo at once, we really don't know if there was a change. + // - so it's safe to return the latest revision so that the client get notified when watching. + return range.to(); + } + if (range.from().equals(range.to())) { + // Empty range. + if (!errorOnEntryNotFound) { + return null; + } + // We have to check if we have the entry. + final Map> entries = + primaryRepo.find(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT); + if (!entries.isEmpty()) { + // We have the entry so just return null because there's no change. + return null; + } + throw new EntryNotFoundException(lastKnownRevision, pathPattern); + } + diffEntries = primaryRepo.diff(range, cache); + } finally { + readUnlock(); + } + + final PathPatternFilter filter = PathPatternFilter.of(pathPattern); + // Return the latest revision if the changes between the two trees contain the file. + for (DiffEntry e : diffEntries) { + final String path; + switch (e.getChangeType()) { + case ADD: + path = e.getNewPath(); + break; + case MODIFY: + case DELETE: + path = e.getOldPath(); + break; + default: + throw new Error(); + } + + if (filter.matches(path)) { + return range.to(); + } + } + + if (!errorOnEntryNotFound) { + return null; + } + if (!primaryRepo.find(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT).isEmpty()) { + // We have to make sure that the entry does not exist because the size of diffEntries can be 0 + // when the contents of range.from() and range.to() are identical. (e.g. add, remove and add again) + return null; + } + throw new EntryNotFoundException(lastKnownRevision, pathPattern); + } + + @Override + public CompletableFuture watch(Revision lastKnownRevision, String pathPattern, + boolean errorOnEntryNotFound) { + requireNonNull(lastKnownRevision, "lastKnownRevision"); + requireNonNull(pathPattern, "pathPattern"); + + final ServiceRequestContext ctx = context(); + final CompletableFuture future = new CompletableFuture<>(); + CompletableFuture.runAsync(() -> { + failFastIfTimedOut(this, logger, ctx, "watch", lastKnownRevision, pathPattern); + try { + readLock(); + final Revision normLastKnownRevision = normalizeNow(lastKnownRevision, false); + // If lastKnownRevision is outdated already and the recent changes match, + // there's no need to watch. + final Revision latestRevision = blockingFindLatestRevision(normLastKnownRevision, pathPattern, + errorOnEntryNotFound); + if (latestRevision != null) { + future.complete(latestRevision); + } else { + commitWatchers.add(normLastKnownRevision, pathPattern, future); + } + } finally { + readUnlock(); + } + }, repositoryWorker).exceptionally(cause -> { + future.completeExceptionally(cause); + return null; + }); + + return future; + } + + private void readLock() { + rwLock.readLock().lock(); + final Supplier exceptionSupplier = closePending.get(); + if (exceptionSupplier != null) { + throw exceptionSupplier.get(); + } + } + + private void readUnlock() { + rwLock.readLock().unlock(); + } + + private void writeLock() { + rwLock.writeLock().lock(); + final Supplier exceptionSupplier = closePending.get(); + if (exceptionSupplier != null) { + throw exceptionSupplier.get(); + } + } + + private void writeUnLock() { + rwLock.writeLock().unlock(); + } + + private static Commit firstCommit(InternalRepository primaryRepo) { + final Revision firstRevision = primaryRepo.firstRevision(); + final RevisionRange range = new RevisionRange(firstRevision, firstRevision); + return primaryRepo.listCommits(ALL_PATH, 1, range).get(0); + } + + @Override + public Revision shouldCreateRollingRepository(int minRetentionCommits, int minRetentionDays) { + final InternalRepository repo = secondaryRepo != null ? secondaryRepo : primaryRepo; + return shouldCreateRollingRepository(repo.headRevision(), minRetentionCommits, minRetentionDays); + } + + /** + * Returns the specified {@code headRevision} if the repository should be rolled. Otherwise, returns + * {@code null}. + */ + @Nullable + private Revision shouldCreateRollingRepository(Revision headRevision, int minRetentionCommits, + int minRetentionDays) { + // TODO(minwoox): provide a way to set different minRetentionCommits and minRetentionDays + // in each repository. + if (minRetentionCommits == 0 || + minRetentionCommits == Integer.MAX_VALUE || minRetentionDays == Integer.MAX_VALUE) { + // Not enabled. + return null; + } + + final InternalRepository repo = secondaryRepo != null ? secondaryRepo : primaryRepo; + if (exceedsMinRetention(repo, headRevision, minRetentionCommits, minRetentionDays)) { + return headRevision; + } + return null; + } + + private static boolean exceedsMinRetention(InternalRepository repo, Revision headRevision, + int minRetentionCommits, int minRetentionDays) { + final Revision firstRevision = repo.firstRevision(); + if (minRetentionCommits != 0) { + if (headRevision.major() - firstRevision.major() <= minRetentionCommits) { + // Not enough commits. + return false; + } + } + + if (minRetentionDays == 0) { + return true; + } + + final Instant secondCommitCreationTime = repo.secondCommitCreationTimeInstant(); + return secondCommitCreationTime != null && + secondCommitCreationTime.isBefore(Instant.now().minus(minRetentionDays, ChronoUnit.DAYS)); +} + + @Override + public void createRollingRepository(Revision rollingRepositoryInitialRevision, + int minRetentionCommits, int minRetentionDays) { + requireNonNull(rollingRepositoryInitialRevision, "rollingRepositoryInitialRevision"); + final Revision rollingRepositoryRevision = shouldCreateRollingRepository( + rollingRepositoryInitialRevision, minRetentionCommits, minRetentionDays); + checkState(rollingRepositoryRevision == rollingRepositoryInitialRevision, + "shouldCreateRollingRepository() returns %s. (expected: %s)", rollingRepositoryRevision, + rollingRepositoryInitialRevision); + + if (secondaryRepo != null) { + promoteSecondaryRepo(); + } + createSecondaryRepo(rollingRepositoryInitialRevision); + } + + private void promoteSecondaryRepo() { + try { + writeLock(); + assert secondaryRepo != null; + checkState(primaryRepo.headRevision().equals(secondaryRepo.headRevision()), + "primaryRepo.headRevision() %s does not equal to secondaryRepo.headRevision() %s.", + primaryRepo.headRevision(), secondaryRepo.headRevision()); + + logger.info("Promoting the secondary repository in {}/{}.", parent.name(), originalRepoName); + repoMetadata.setPrimaryRepoDir(secondaryRepo.jGitRepo().getDirectory()); + final InternalRepository primaryRepo = this.primaryRepo; + this.primaryRepo = secondaryRepo; + secondaryRepo = null; + repositoryWorker.execute(() -> { + closeInternalRepository(primaryRepo); + final File repoDir = primaryRepo.repoDir(); + final Path path = repoDir.toPath(); + final Path newPath = path.resolveSibling(path.getFileName().toString() + SUFFIX_REMOVED); + try { + Files.move(path, newPath); + } catch (IOException e) { + logger.warn("Failed to mark the old primary repository: {} to {}", repoDir, newPath); + } + }); + logger.info("Promotion is done for {}/{}. {} is now the primary.", + parent.name(), originalRepoName, primaryRepo.repoDir()); + } finally { + writeUnLock(); + } + } + + private void createSecondaryRepo(Revision secondaryRepositoryInitialRevision) { + InternalRepository secondaryRepo = null; + try { + logger.info("Creating the secondary repository in {}/{}. head revision: {}.", + parent.name(), originalRepoName, secondaryRepositoryInitialRevision); + final List> changes = allContents(secondaryRepositoryInitialRevision); + final File secondaryRepoDir = repoMetadata.secondaryRepoDir(); + secondaryRepo = InternalRepository.create(parent, originalRepoName, secondaryRepoDir, + secondaryRepositoryInitialRevision, creationTimeMillis, + author, changes); + try { + writeLock(); + final Revision headRevision = this.headRevision; + // There's not so much chances that commits are pushed before we acquire the write lock but + // we have to check it anyway. + if (!secondaryRepositoryInitialRevision.equals(headRevision)) { + assert secondaryRepositoryInitialRevision.major() < headRevision.major(); + // There were commits after the createRollingRepositoryCommand is created, + // so we should catch up. + logger.info("Catching up the interposed commits in {}/{}. from: {} to : {}.", + parent.name(), originalRepoName, + secondaryRepositoryInitialRevision, headRevision); + + final RevisionRange revisionRange = new RevisionRange( + secondaryRepositoryInitialRevision.forward(1), headRevision); + // The number of interposed commits would be very small. + assert revisionRange.to().major() - revisionRange.from().major() < MAX_MAX_COMMITS; + final List commits = primaryRepo.listCommits(ALL_PATH, MAX_MAX_COMMITS, + revisionRange); + Revision fromRevision = secondaryRepositoryInitialRevision; + for (Commit commit : commits) { + final Revision toRevision = commit.revision(); + final Map> diffs = primaryRepo.diff( + new RevisionRange(fromRevision, toRevision), ALL_PATH); + secondaryRepo.commit(fromRevision, toRevision, commit.when(), commit.author(), + commit.summary(), commit.detail(), commit.markup(), diffs.values(), + false); + fromRevision = toRevision; + } + } + this.secondaryRepo = secondaryRepo; + } finally { + writeUnLock(); + } + logger.info("The secondary repository {} is created in {}/{}. head revision: {}", + secondaryRepoDir.getName(), parent.name(), originalRepoName, + secondaryRepositoryInitialRevision); + } catch (Throwable t) { + logger.warn("Failed to create the secondary repository", t); + if (secondaryRepo != null) { + closeInternalRepository(secondaryRepo); + try { + deleteDirectory(secondaryRepo.repoDir()); + } catch (IOException e) { + logger.warn("Failed to delete the directory: {}", secondaryRepo.repoDir()); + } + } + throw t; + } + } + + private List> allContents(Revision secondaryRepositoryInitialRevision) { + final Map> entries = primaryRepo.find( + secondaryRepositoryInitialRevision, ALL_PATH, ImmutableMap.of()); + final Builder> builder = ImmutableList.builder(); + for (Entry entry : entries.values()) { + final EntryType type = entry.type(); + if (type == EntryType.DIRECTORY) { + continue; + } + if (type == EntryType.JSON) { + final JsonNode content = (JsonNode) entry.content(); + builder.add(Change.ofJsonUpsert(entry.path(), content)); + } else { + assert type == EntryType.TEXT; + final String content = (String) entry.content(); + builder.add(Change.ofTextUpsert(entry.path(), content)); + } + } + return builder.build(); + } + + private static boolean deleteDirectory(File dir) throws IOException { + try (Stream walk = Files.walk(dir.toPath())) { + return walk.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .map(File::delete) + // Return false if it fails to delete a file. + .reduce(true, (a, b) -> a && b); + } + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java new file mode 100644 index 000000000..a2a725105 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepository.java @@ -0,0 +1,1204 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.google.common.base.Preconditions.checkState; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.closeJGitRepo; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryUtil.exists; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryV2.R_HEADS_MASTER; +import static com.linecorp.centraldogma.server.storage.repository.Repository.ALL_PATH; +import static com.linecorp.centraldogma.server.storage.repository.Repository.MAX_MAX_COMMITS; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REPO_FORMAT_VERSION; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.StringJoiner; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.locks.Lock; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; + +import org.eclipse.jgit.diff.DiffEntry; +import org.eclipse.jgit.diff.DiffFormatter; +import org.eclipse.jgit.dircache.DirCache; +import org.eclipse.jgit.dircache.DirCacheBuilder; +import org.eclipse.jgit.dircache.DirCacheEditor; +import org.eclipse.jgit.dircache.DirCacheEditor.DeletePath; +import org.eclipse.jgit.dircache.DirCacheEditor.DeleteTree; +import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit; +import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.dircache.DirCacheIterator; +import org.eclipse.jgit.lib.CommitBuilder; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectIdOwnerMap; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.RefUpdate; +import org.eclipse.jgit.lib.RefUpdate.Result; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.RepositoryBuilder; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.revwalk.TreeRevFilter; +import org.eclipse.jgit.revwalk.filter.RevFilter; +import org.eclipse.jgit.treewalk.CanonicalTreeParser; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.AndTreeFilter; +import org.eclipse.jgit.treewalk.filter.TreeFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.CentralDogmaException; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.ChangeConflictException; +import com.linecorp.centraldogma.common.Commit; +import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.EntryType; +import com.linecorp.centraldogma.common.Markup; +import com.linecorp.centraldogma.common.RedundantChangeException; +import com.linecorp.centraldogma.common.RepositoryNotFoundException; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.common.RevisionRange; +import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.internal.Util; +import com.linecorp.centraldogma.internal.jsonpatch.JsonPatch; +import com.linecorp.centraldogma.internal.jsonpatch.ReplaceMode; +import com.linecorp.centraldogma.server.internal.IsolatedSystemReader; +import com.linecorp.centraldogma.server.internal.JGitUtil; +import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache; +import com.linecorp.centraldogma.server.storage.StorageException; +import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.repository.FindOption; + +import difflib.DiffUtils; +import difflib.Patch; + +final class InternalRepository { + + private static final Logger logger = LoggerFactory.getLogger(InternalRepository.class); + + private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL); + private static final byte[] EMPTY_BYTE = new byte[0]; + + private static final Field revWalkObjectsField; + + static { + IsolatedSystemReader.install(); + + try { + revWalkObjectsField = RevWalk.class.getDeclaredField("objects"); + if (revWalkObjectsField.getType() != ObjectIdOwnerMap.class) { + throw new IllegalStateException( + RevWalk.class.getSimpleName() + ".objects is not an " + + ObjectIdOwnerMap.class.getSimpleName() + '.'); + } + revWalkObjectsField.setAccessible(true); + } catch (NoSuchFieldException e) { + throw new IllegalStateException( + RevWalk.class.getSimpleName() + ".objects does not exist."); + } + } + + static InternalRepository create(Project parent, String originalRepoName, File repoDir, + Revision nextRevision, long commitTimeMillis, + Author author, Iterable> changes) { + boolean success = false; + InternalRepository internalRepo = null; + try { + createEmptyJGitRepo(repoDir); + + // Re-open the repository with the updated settings and format version. + final Repository jGitRepo = new RepositoryBuilder().setGitDir(repoDir).build(); + internalRepo = new InternalRepository(parent, originalRepoName, repoDir, + jGitRepo, new CommitIdDatabase(jGitRepo)); + + // Initialize the master branch. + final RefUpdate head = jGitRepo.updateRef(Constants.HEAD); + head.disableRefLog(); + head.link(R_HEADS_MASTER); + + // Insert the initial commit into the master branch. + internalRepo.commit(null, nextRevision, commitTimeMillis, author, + "Create a new repository", "", Markup.PLAINTEXT, changes, true); + success = true; + return internalRepo; + } catch (IOException e) { + throw new StorageException("failed to create a repository at: " + repoDir, e); + } finally { + if (!success) { + if (internalRepo != null) { + internalRepo.close(); + } + // Failed to create a repository. Remove any cruft so that it is not loaded on the next run. + deleteCruft(repoDir); + } + } + } + + private static void deleteCruft(File repoDir) { + try { + Util.deleteFileTree(repoDir); + } catch (IOException e) { + logger.error("Failed to delete a half-created repository at: {}", repoDir, e); + } + } + + private static void createEmptyJGitRepo(File repositoryDir) throws IOException { + try (Repository jGitRepository = buildJGitRepo(repositoryDir)) { + jGitRepository.create(true); + + // Save the initial default settings. + JGitUtil.applyDefaultsAndSave(jGitRepository.getConfig()); + } + } + + static Repository buildJGitRepo(File repositoryDir) { + try { + return new RepositoryBuilder().setGitDir(repositoryDir).setBare().build(); + } catch (IOException e) { + throw new StorageException("failed to create a repository at: " + repositoryDir, e); + } + } + + @Nullable + static InternalRepository open(Project parent, String name, File repoDir, boolean errorIfNotExist) { + final Repository jGitRepo = buildJGitRepo(repoDir); + CommitIdDatabase commitIdDatabase = null; + try { + if (!exists(jGitRepo)) { + if (errorIfNotExist) { + throw new RepositoryNotFoundException(repoDir.toString()); + } else { + return null; + } + } + checkGitRepositoryFormat(jGitRepo); + // Update the default settings if necessary. + JGitUtil.applyDefaultsAndSave(jGitRepo.getConfig()); + final Revision headRevision = uncachedHeadRevision(parent, name, jGitRepo); + commitIdDatabase = new CommitIdDatabase(jGitRepo); + if (!headRevision.equals(commitIdDatabase.headRevision())) { + commitIdDatabase.rebuild(jGitRepo); + assert headRevision.equals(commitIdDatabase.headRevision()); + } + return new InternalRepository(parent, name, repoDir, jGitRepo, commitIdDatabase); + } catch (IOException e) { + throw new StorageException("failed to open a repository at: " + repoDir, e); + } catch (Throwable t) { + closeJGitRepo(jGitRepo); + if (commitIdDatabase != null) { + commitIdDatabase.close(); + } + throw t; + } + } + + private static Revision uncachedHeadRevision(Project parent, String name, Repository jGitRepo) { + try (RevWalk revWalk = newRevWalk(jGitRepo)) { + final ObjectId headRevisionId = jGitRepo.resolve(R_HEADS_MASTER); + if (headRevisionId != null) { + final RevCommit revCommit = revWalk.parseCommit(headRevisionId); + return CommitUtil.extractRevision(revCommit.getFullMessage()); + } + } catch (CentralDogmaException e) { + throw e; + } catch (Exception e) { + throw new StorageException("failed to get the current revision", e); + } + + throw new StorageException("failed to determine the HEAD: " + parent.name() + '/' + name); + } + + private static RevWalk newRevWalk(Repository jGitRepository) { + final RevWalk revWalk = new RevWalk(jGitRepository); + disableRewriteParents(revWalk); + return revWalk; + } + + private static RevWalk newRevWalk(ObjectReader reader) { + final RevWalk revWalk = new RevWalk(reader); + disableRewriteParents(revWalk); + return revWalk; + } + + private static void disableRewriteParents(RevWalk revWalk) { + // Disable rewriteParents because otherwise `RevWalk` will load every commit into memory. + revWalk.setRewriteParents(false); + } + + private static void checkGitRepositoryFormat(Repository repository) { + final int formatVersion = repository.getConfig().getInt( + CONFIG_CORE_SECTION, null, CONFIG_KEY_REPO_FORMAT_VERSION, 0); + if (formatVersion != JGitUtil.REPO_FORMAT_VERSION) { + throw new StorageException("unsupported repository format version: " + formatVersion + + " (expected: " + JGitUtil.REPO_FORMAT_VERSION + ')'); + } + } + + private final Project project; + private final String originalRepoName; // e.g. foo + private final File repoDir; // e.g. foo_0000000000 + private final Repository jGitRepository; + private final CommitIdDatabase commitIdDatabase; + + // Only accessed by the worker in CommitRetentionManagementPlugin. + @Nullable + private Instant secondCommitCreationTimeInstant; + + private InternalRepository(Project project, String originalRepoName, File repoDir, + Repository jGitRepository, CommitIdDatabase commitIdDatabase) { + this.project = project; + this.originalRepoName = originalRepoName; + this.repoDir = repoDir; + this.jGitRepository = jGitRepository; + this.commitIdDatabase = commitIdDatabase; + } + + File repoDir() { + return repoDir; + } + + Repository jGitRepo() { + return jGitRepository; + } + + CommitIdDatabase commitIdDatabase() { + return commitIdDatabase; + } + + Revision headRevision() { + final Revision headRevision = commitIdDatabase.headRevision(); + checkState(headRevision != null, "the headRevision is not set."); + return headRevision; + } + + Revision firstRevision() { + final Revision firstRevision = commitIdDatabase.firstRevision(); + checkState(firstRevision != null, "the firstRevision is not set."); + return firstRevision; + } + + /** + * Returns the {@link Instant} of the time when the second commit is created. This {@link Instant} is used + * to check if the second commit of this repository exceeds the minimum retention days or not. + */ + @Nullable + Instant secondCommitCreationTimeInstant() { + if (secondCommitCreationTimeInstant == null) { + final Revision firstRevision = commitIdDatabase.firstRevision(); + if (firstRevision == null) { + return null; + } + final Revision headRevision = commitIdDatabase.headRevision(); + if (headRevision == null) { + return null; + } + if (firstRevision.equals(headRevision)) { + // The second commit is not made yet. + return null; + } + final Revision secondRevision = firstRevision.forward(1); + final RevisionRange range = new RevisionRange(secondRevision, secondRevision); + secondCommitCreationTimeInstant = + Instant.ofEpochMilli(listCommits(ALL_PATH, 1, range).get(0).when()); + } + return secondCommitCreationTimeInstant; + } + + List listCommits(String pathPattern, int maxCommits, RevisionRange range) { + final RevisionRange descendingRange = range.toDescending(); + try (RevWalk revWalk = newRevWalk(jGitRepository)) { + final ObjectIdOwnerMap revWalkInternalMap = + (ObjectIdOwnerMap) revWalkObjectsField.get(revWalk); + final ObjectId fromCommitId = commitIdDatabase.get(descendingRange.from()); + final ObjectId toCommitId = commitIdDatabase.get(descendingRange.to()); + + revWalk.markStart(revWalk.parseCommit(fromCommitId)); + revWalk.setRetainBody(false); + + // Instead of relying on RevWalk to filter the commits, + // we let RevWalk yield all commits so we can: + // - Have more control on when iteration should be stopped. + // (A single Iterator.next() doesn't take long.) + // - Clean up the internal map as early as possible. + final RevFilter filter = new TreeRevFilter(revWalk, AndTreeFilter.create( + TreeFilter.ANY_DIFF, PathPatternFilter.of(pathPattern))); + + // Search up to 1000 commits when maxCommits <= 100. + // Search up to (maxCommits * 10) commits when 100 < maxCommits <= 1000. + final int maxNumProcessedCommits = Math.max(maxCommits * 10, MAX_MAX_COMMITS); + + final List commitList = new ArrayList<>(); + int numProcessedCommits = 0; + for (RevCommit revCommit : revWalk) { + numProcessedCommits++; + + if (filter.include(revWalk, revCommit)) { + revWalk.parseBody(revCommit); + commitList.add(toCommit(revCommit)); + revCommit.disposeBody(); + } + + if (revCommit.getId().equals(toCommitId) || + commitList.size() >= maxCommits || + // Prevent from iterating for too long. + numProcessedCommits >= maxNumProcessedCommits) { + break; + } + + // Clear the internal lookup table of RevWalk to reduce the memory usage. + // This is safe because we have linear history and traverse in one direction. + if (numProcessedCommits % 16 == 0) { + revWalkInternalMap.clear(); + } + } + + // Include the initial empty commit only when the caller specified + // the initial revision (1) in the range and the pathPattern contains '/**'. + if (commitList.size() < maxCommits && + descendingRange.to().major() == 1 && + pathPattern.contains(ALL_PATH)) { + try (RevWalk tmpRevWalk = newRevWalk(jGitRepository)) { + final RevCommit lastRevCommit = tmpRevWalk.parseCommit(toCommitId); + commitList.add(toCommit(lastRevCommit)); + } + } + + if (!descendingRange.equals(range)) { // from and to is swapped so reverse the list. + Collections.reverse(commitList); + } + + return commitList; + } catch (CentralDogmaException e) { + throw e; + } catch (Exception e) { + throw new StorageException( + "failed to retrieve the history: " + originalRepoName + + " (" + pathPattern + ", " + range.from() + ".." + range.to() + ')', e); + } + } + + private static Commit toCommit(RevCommit revCommit) { + final Author author; + final PersonIdent committerIdent = revCommit.getCommitterIdent(); + final long when; + if (committerIdent == null) { + author = Author.UNKNOWN; + when = 0; + } else { + author = new Author(committerIdent.getName(), committerIdent.getEmailAddress()); + when = committerIdent.getWhen().getTime(); + } + + try { + return CommitUtil.newCommit(author, when, revCommit.getFullMessage()); + } catch (Exception e) { + throw new StorageException("failed to create a Commit", e); + } + } + + Map> find(Revision normalizedRevision, String pathPattern, Map, ?> options) { + final PathPatternFilter filter = PathPatternFilter.of(pathPattern); + final int maxEntries = FindOption.MAX_ENTRIES.get(options); + + try (ObjectReader reader = jGitRepository.newObjectReader(); + TreeWalk treeWalk = new TreeWalk(reader); + RevWalk revWalk = newRevWalk(reader)) { + + final Map> result = new LinkedHashMap<>(); + final ObjectId commitId = commitIdDatabase.get(normalizedRevision); + final RevCommit revCommit = revWalk.parseCommit(commitId); + final RevTree revTree = revCommit.getTree(); + treeWalk.addTree(revTree.getId()); + + while (treeWalk.next() && result.size() < maxEntries) { + final boolean matches = filter.matches(treeWalk); + final String path = '/' + treeWalk.getPathString(); + + // Recurse into a directory if necessary. + if (treeWalk.isSubtree()) { + if (matches) { + // Add the directory itself to the result set if its path matches the pattern. + result.put(path, Entry.ofDirectory(normalizedRevision, path)); + } + + treeWalk.enterSubtree(); + continue; + } + + if (!matches) { + continue; + } + + final boolean fetchContent = FindOption.FETCH_CONTENT.get(options); + // Build an entry as requested. + final Entry entry; + final EntryType entryType = EntryType.guessFromPath(path); + if (fetchContent) { + final byte[] content = reader.open(treeWalk.getObjectId(0)).getBytes(); + switch (entryType) { + case JSON: + final JsonNode jsonNode = Jackson.readTree(content); + entry = Entry.ofJson(normalizedRevision, path, jsonNode); + break; + case TEXT: + final String strVal = sanitizeText(new String(content, UTF_8)); + entry = Entry.ofText(normalizedRevision, path, strVal); + break; + default: + throw new Error("unexpected entry type: " + entryType); + } + } else { + switch (entryType) { + case JSON: + entry = Entry.ofJson(normalizedRevision, path, Jackson.nullNode); + break; + case TEXT: + entry = Entry.ofText(normalizedRevision, path, ""); + break; + default: + throw new Error("unexpected entry type: " + entryType); + } + } + result.put(path, entry); + } + + return Util.unsafeCast(result); + } catch (CentralDogmaException | IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new StorageException( + "failed to get data from '" + project.name() + '/' + originalRepoName + "' at " + + pathPattern + " for " + normalizedRevision, e); + } + } + + /** + * Removes {@code \r} and appends {@code \n} on the last line if it does not end with {@code \n}. + */ + private static String sanitizeText(String text) { + if (text.indexOf('\r') >= 0) { + text = CR.matcher(text).replaceAll(""); + } + if (!text.isEmpty() && !text.endsWith("\n")) { + text += "\n"; + } + return text; + } + + List diff(RevisionRange range, @Nullable RepositoryCache cache) { + return diff0(range, TreeFilter.ALL, cache); + } + + Map> diff(RevisionRange range, String pathPattern) { + // Note that we do not cache here because CachingRepository caches the final result. + return toChangeMap(diff0(range, filter(pathPattern), null)); + } + + private List diff0(RevisionRange range, TreeFilter filter, @Nullable RepositoryCache cache) { + try (RevWalk rw = newRevWalk(jGitRepository)) { + final RevTree treeA = rw.parseTree(commitIdDatabase.get(range.from())); + final RevTree treeB = rw.parseTree(commitIdDatabase.get(range.to())); + return blockingCompareTrees(treeA, treeB, filter, cache); + } catch (StorageException e) { + throw e; + } catch (Exception e) { + throw new StorageException("failed to parse two trees: from= " + range.from() + + ", to= " + range.to(), e); + } + } + + private List blockingCompareTrees(@Nullable RevTree treeA, @Nullable RevTree treeB, + TreeFilter filter, @Nullable RepositoryCache cache) { + if (cache == null) { + return blockingCompareTreesUncached(treeA, treeB, filter); + } + final CacheableCompareTreesCall key = + new CacheableCompareTreesCall(project.repos().get(originalRepoName), treeA, treeB); + CompletableFuture> existingFuture = cache.getIfPresent(key); + if (existingFuture != null) { + final List existingDiffEntries = existingFuture.getNow(null); + if (existingDiffEntries != null) { + // Cached already. + return existingDiffEntries; + } + } + + // Not cached yet. Acquire a lock so that we do not compare the same tree pairs simultaneously. + final List newDiffEntries; + final Lock lock = key.coarseGrainedLock(); + lock.lock(); + try { + existingFuture = cache.getIfPresent(key); + if (existingFuture != null) { + final List existingDiffEntries = existingFuture.getNow(null); + if (existingDiffEntries != null) { + // Other thread already put the entries to the cache before we acquire the lock. + return existingDiffEntries; + } + } + + newDiffEntries = blockingCompareTreesUncached(treeA, treeB, filter); + cache.put(key, newDiffEntries); + } finally { + lock.unlock(); + } + + logger.debug("Cache miss: {}", key); + return newDiffEntries; + } + + private List blockingCompareTreesUncached(@Nullable RevTree treeA, @Nullable RevTree treeB, + TreeFilter filter) { + try (DiffFormatter diffFormatter = new DiffFormatter(null)) { + diffFormatter.setRepository(jGitRepository); + diffFormatter.setPathFilter(filter); + return ImmutableList.copyOf(diffFormatter.scan(treeA, treeB)); + } catch (IOException e) { + throw new StorageException("failed to compare two trees: " + treeA + " vs. " + treeB, e); + } + } + + private static TreeFilter filter(@Nullable String pathPattern) { + if (pathPattern == null) { + return TreeFilter.ALL; + } + + final PathPatternFilter pathPatternFilter = PathPatternFilter.of(pathPattern); + return pathPatternFilter.matchesAll() ? TreeFilter.ALL : pathPatternFilter; + } + + private Map> toChangeMap(List diffEntryList) { + try (ObjectReader reader = jGitRepository.newObjectReader()) { + final Map> changeMap = new LinkedHashMap<>(); + + for (DiffEntry diffEntry : diffEntryList) { + final String oldPath = '/' + diffEntry.getOldPath(); + final String newPath = '/' + diffEntry.getNewPath(); + + switch (diffEntry.getChangeType()) { + case MODIFY: + final EntryType oldEntryType = EntryType.guessFromPath(oldPath); + switch (oldEntryType) { + case JSON: + if (!oldPath.equals(newPath)) { + putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath)); + } + + final JsonNode oldJsonNode = + Jackson.readTree( + reader.open(diffEntry.getOldId().toObjectId()).getBytes()); + final JsonNode newJsonNode = + Jackson.readTree( + reader.open(diffEntry.getNewId().toObjectId()).getBytes()); + final JsonPatch patch = + JsonPatch.generate(oldJsonNode, newJsonNode, ReplaceMode.SAFE); + + if (!patch.isEmpty()) { + putChange(changeMap, newPath, + Change.ofJsonPatch(newPath, Jackson.valueToTree(patch))); + } + break; + case TEXT: + final String oldText = sanitizeText(new String( + reader.open(diffEntry.getOldId().toObjectId()).getBytes(), UTF_8)); + + final String newText = sanitizeText(new String( + reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8)); + + if (!oldPath.equals(newPath)) { + putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath)); + } + + if (!oldText.equals(newText)) { + putChange(changeMap, newPath, + Change.ofTextPatch(newPath, oldText, newText)); + } + break; + default: + throw new Error("unexpected old entry type: " + oldEntryType); + } + break; + case ADD: + final EntryType newEntryType = EntryType.guessFromPath(newPath); + switch (newEntryType) { + case JSON: { + final JsonNode jsonNode = Jackson.readTree( + reader.open(diffEntry.getNewId().toObjectId()).getBytes()); + + putChange(changeMap, newPath, Change.ofJsonUpsert(newPath, jsonNode)); + break; + } + case TEXT: { + final String text = sanitizeText(new String( + reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8)); + + putChange(changeMap, newPath, Change.ofTextUpsert(newPath, text)); + break; + } + default: + throw new Error("unexpected new entry type: " + newEntryType); + } + break; + case DELETE: + putChange(changeMap, oldPath, Change.ofRemoval(oldPath)); + break; + default: + throw new Error(); + } + } + return changeMap; + } catch (Exception e) { + throw new StorageException("failed to convert list of DiffEntry to Changes map", e); + } + } + + private static void putChange(Map> changeMap, String path, Change change) { + final Change oldChange = changeMap.put(path, change); + assert oldChange == null; + } + + Map> previewDiff(Revision baseRevision, Iterable> changes) { + try (ObjectReader reader = jGitRepository.newObjectReader(); + RevWalk revWalk = newRevWalk(reader); + DiffFormatter diffFormatter = new DiffFormatter(null)) { + final ObjectId baseTreeId = toTree(commitIdDatabase, revWalk, baseRevision); + final DirCache dirCache = DirCache.newInCore(); + final int numEdits = applyChanges(baseRevision, baseTreeId, dirCache, changes); + if (numEdits == 0) { + return Collections.emptyMap(); + } + + final CanonicalTreeParser p = new CanonicalTreeParser(); + p.reset(reader, baseTreeId); + diffFormatter.setRepository(jGitRepository); + final List result = diffFormatter.scan(p, new DirCacheIterator(dirCache)); + return toChangeMap(result); + } catch (IOException e) { + throw new StorageException("failed to perform a dry-run diff", e); + } + } + + private static RevTree toTree(CommitIdDatabase commitIdDatabase, RevWalk revWalk, Revision revision) { + final ObjectId commitId = commitIdDatabase.get(revision); + try { + return revWalk.parseTree(commitId); + } catch (IOException e) { + throw new StorageException("failed to parse a commit: " + commitId, e); + } + } + + private int applyChanges(@Nullable Revision baseRevision, @Nullable ObjectId baseTreeId, + DirCache dirCache, Iterable> changes) { + int numEdits = 0; + + try (ObjectInserter inserter = jGitRepository.newObjectInserter(); + ObjectReader reader = jGitRepository.newObjectReader()) { + + if (baseTreeId != null) { + // the DirCacheBuilder is to used for doing update operations on the given DirCache object + final DirCacheBuilder builder = dirCache.builder(); + + // Add the tree object indicated by the prevRevision to the temporary DirCache object. + builder.addTree(EMPTY_BYTE, 0, reader, baseTreeId); + builder.finish(); + } + + // loop over the specified changes. + for (Change change : changes) { + final String changePath = change.path().substring(1); // Strip the leading '/'. + final DirCacheEntry oldEntry = dirCache.getEntry(changePath); + final byte[] oldContent = oldEntry != null ? reader.open(oldEntry.getObjectId()).getBytes() + : null; + + switch (change.type()) { + case UPSERT_JSON: { + final JsonNode oldJsonNode = oldContent != null ? Jackson.readTree(oldContent) : null; + final JsonNode newJsonNode = firstNonNull((JsonNode) change.content(), + JsonNodeFactory.instance.nullNode()); + + // Upsert only when the contents are really different. + if (!Objects.equals(newJsonNode, oldJsonNode)) { + applyPathEdit(dirCache, new InsertJson(changePath, inserter, newJsonNode)); + numEdits++; + } + break; + } + case UPSERT_TEXT: { + final String sanitizedOldText; + if (oldContent != null) { + sanitizedOldText = sanitizeText(new String(oldContent, UTF_8)); + } else { + sanitizedOldText = null; + } + + final String sanitizedNewText = sanitizeText(change.contentAsText()); + + // Upsert only when the contents are really different. + if (!sanitizedNewText.equals(sanitizedOldText)) { + applyPathEdit(dirCache, new InsertText(changePath, inserter, sanitizedNewText)); + numEdits++; + } + break; + } + case REMOVE: + if (oldEntry != null) { + applyPathEdit(dirCache, new DeletePath(changePath)); + numEdits++; + break; + } + + // The path might be a directory. + if (applyDirectoryEdits(dirCache, changePath, null, change)) { + numEdits++; + } else { + // Was not a directory either; conflict. + reportNonExistentEntry(change); + break; + } + break; + case RENAME: { + final String newPath = + ((String) change.content()).substring(1); // Strip the leading '/'. + + if (dirCache.getEntry(newPath) != null) { + throw new ChangeConflictException("a file exists at the target path: " + change); + } + + if (oldEntry != null) { + if (changePath.equals(newPath)) { + // Redundant rename request - old path and new path are same. + break; + } + + final DirCacheEditor editor = dirCache.editor(); + editor.add(new DeletePath(changePath)); + editor.add(new CopyOldEntry(newPath, oldEntry)); + editor.finish(); + numEdits++; + break; + } + + // The path might be a directory. + if (applyDirectoryEdits(dirCache, changePath, newPath, change)) { + numEdits++; + } else { + // Was not a directory either; conflict. + reportNonExistentEntry(change); + } + break; + } + case APPLY_JSON_PATCH: { + final JsonNode oldJsonNode; + if (oldContent != null) { + oldJsonNode = Jackson.readTree(oldContent); + } else { + oldJsonNode = Jackson.nullNode; + } + + final JsonNode newJsonNode; + try { + newJsonNode = JsonPatch.fromJson((JsonNode) change.content()).apply(oldJsonNode); + } catch (Exception e) { + throw new ChangeConflictException("failed to apply JSON patch: " + change, e); + } + + // Apply only when the contents are really different. + if (!newJsonNode.equals(oldJsonNode)) { + applyPathEdit(dirCache, new InsertJson(changePath, inserter, newJsonNode)); + numEdits++; + } + break; + } + case APPLY_TEXT_PATCH: + final Patch patch = DiffUtils.parseUnifiedDiff( + Util.stringToLines(sanitizeText((String) change.content()))); + + final String sanitizedOldText; + final List sanitizedOldTextLines; + if (oldContent != null) { + sanitizedOldText = sanitizeText(new String(oldContent, UTF_8)); + sanitizedOldTextLines = Util.stringToLines(sanitizedOldText); + } else { + sanitizedOldText = null; + sanitizedOldTextLines = Collections.emptyList(); + } + + final String newText; + try { + final List newTextLines = DiffUtils.patch(sanitizedOldTextLines, patch); + if (newTextLines.isEmpty()) { + newText = ""; + } else { + final StringJoiner joiner = new StringJoiner("\n", "", "\n"); + for (String line : newTextLines) { + joiner.add(line); + } + newText = joiner.toString(); + } + } catch (Exception e) { + throw new ChangeConflictException("failed to apply text patch: " + change, e); + } + + // Apply only when the contents are really different. + if (!newText.equals(sanitizedOldText)) { + applyPathEdit(dirCache, new InsertText(changePath, inserter, newText)); + numEdits++; + } + break; + } + } + } catch (CentralDogmaException | IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new StorageException("failed to apply changes on revision " + baseRevision, e); + } + return numEdits; + } + + private static void applyPathEdit(DirCache dirCache, PathEdit edit) { + final DirCacheEditor e = dirCache.editor(); + e.add(edit); + e.finish(); + } + + /** + * Applies recursive directory edits. + * + * @param oldDir the path to the directory to make a recursive change + * @param newDir the path to the renamed directory, or {@code null} to remove the directory. + * + * @return {@code true} if any edits were made to {@code dirCache}, {@code false} otherwise + */ + private static boolean applyDirectoryEdits(DirCache dirCache, + String oldDir, @Nullable String newDir, Change change) { + + if (!oldDir.endsWith("/")) { + oldDir += '/'; + } + if (newDir != null && !newDir.endsWith("/")) { + newDir += '/'; + } + + final byte[] rawOldDir = Constants.encode(oldDir); + final byte[] rawNewDir = newDir != null ? Constants.encode(newDir) : null; + final int numEntries = dirCache.getEntryCount(); + DirCacheEditor editor = null; + + loop: + for (int i = 0; i < numEntries; i++) { + final DirCacheEntry e = dirCache.getEntry(i); + final byte[] rawPath = e.getRawPath(); + + // Ensure that there are no entries under the newDir; we have a conflict otherwise. + if (rawNewDir != null) { + boolean conflict = true; + if (rawPath.length > rawNewDir.length) { + // Check if there is a file whose path starts with 'newDir'. + for (int j = 0; j < rawNewDir.length; j++) { + if (rawNewDir[j] != rawPath[j]) { + conflict = false; + break; + } + } + } else if (rawPath.length == rawNewDir.length - 1) { + // Check if there is a file whose path is exactly same with newDir without trailing '/'. + for (int j = 0; j < rawNewDir.length - 1; j++) { + if (rawNewDir[j] != rawPath[j]) { + conflict = false; + break; + } + } + } else { + conflict = false; + } + + if (conflict) { + throw new ChangeConflictException("target directory exists already: " + change); + } + } + + // Skip the entries that do not belong to the oldDir. + if (rawPath.length <= rawOldDir.length) { + continue; + } + for (int j = 0; j < rawOldDir.length; j++) { + if (rawOldDir[j] != rawPath[j]) { + continue loop; + } + } + + // Do not create an editor until we find an entry to rename/remove. + // We can tell if there was any matching entries or not from the nullness of editor later. + if (editor == null) { + editor = dirCache.editor(); + editor.add(new DeleteTree(oldDir)); + if (newDir == null) { + // Recursive removal + break; + } + } + + assert newDir != null; // We should get here only when it's a recursive rename. + + final String oldPath = e.getPathString(); + final String newPath = newDir + oldPath.substring(oldDir.length()); + editor.add(new CopyOldEntry(newPath, e)); + } + + if (editor != null) { + editor.finish(); + return true; + } else { + return false; + } + } + + private static void reportNonExistentEntry(Change change) { + throw new ChangeConflictException("non-existent file/directory: " + change); + } + + RevisionAndEntries commit(@Nullable Revision prevRevision, Revision nextRevision, + long commitTimeMillis, Author author, String summary, String detail, + Markup markup, Iterable> changes, boolean allowEmpty) { + requireNonNull(author, "author"); + requireNonNull(summary, "summary"); + requireNonNull(changes, "changes"); + requireNonNull(detail, "detail"); + requireNonNull(markup, "markup"); + + assert prevRevision == null || prevRevision.major() > 0; + assert nextRevision.major() > 0; + + try (ObjectInserter inserter = jGitRepository.newObjectInserter(); + ObjectReader reader = jGitRepository.newObjectReader(); + RevWalk revWalk = newRevWalk(reader)) { + + final ObjectId prevTreeId; + if (prevRevision != null) { + prevTreeId = toTree(commitIdDatabase, revWalk, prevRevision); + } else { + prevTreeId = null; + } + + // The staging area that keeps the entries of the new tree. + // It starts with the entries of the tree at the prevRevision (or with no entries if the + // prevRevision is the initial commit), and then this method will apply the requested changes + // to build the new tree. + final DirCache dirCache = DirCache.newInCore(); + + // Apply the changes and retrieve the list of the affected files. + final int numEdits = applyChanges(prevRevision, prevTreeId, dirCache, changes); + + // Reject empty commit if necessary. + final List diffEntries; + boolean isEmpty = numEdits == 0; + if (isEmpty || prevTreeId == null) { + // We do not need the diffEntries when creating a new repository which means prevTreeId is null. + diffEntries = ImmutableList.of(); + } else { + // Even if there are edits, the resulting tree might be identical with the previous tree. + final CanonicalTreeParser p = new CanonicalTreeParser(); + p.reset(reader, prevTreeId); + final DiffFormatter diffFormatter = new DiffFormatter(null); + diffFormatter.setRepository(jGitRepository); + diffEntries = diffFormatter.scan(p, new DirCacheIterator(dirCache)); + isEmpty = diffEntries.isEmpty(); + } + + if (!allowEmpty && isEmpty) { + throw new RedundantChangeException( + "changes did not change anything in " + + project.name() + '/' + originalRepoName + + " at revision " + (prevRevision != null ? prevRevision.major() : 0) + ": " + changes); + } + + // flush the current index to repository and get the result tree object id. + final ObjectId nextTreeId = dirCache.writeTree(inserter); + + // build a commit object + final PersonIdent personIdent = new PersonIdent(author.name(), author.email(), + commitTimeMillis / 1000L * 1000L, 0); + + final CommitBuilder commitBuilder = new CommitBuilder(); + + commitBuilder.setAuthor(personIdent); + commitBuilder.setCommitter(personIdent); + commitBuilder.setTreeId(nextTreeId); + commitBuilder.setEncoding(UTF_8); + + // Write summary, detail and revision to commit's message as JSON format. + commitBuilder.setMessage(CommitUtil.toJsonString(summary, detail, markup, nextRevision)); + + // if the head commit exists, use it as the parent commit. + if (prevRevision != null) { + commitBuilder.setParentId(commitIdDatabase.get(prevRevision)); + } + + final ObjectId nextCommitId = inserter.insert(commitBuilder); + inserter.flush(); + + // tagging the revision object, for history lookup purpose. + commitIdDatabase.put(nextRevision, nextCommitId); + doRefUpdate(jGitRepository, revWalk, R_HEADS_MASTER, nextCommitId); + + return new RevisionAndEntries(nextRevision, diffEntries); + } catch (CentralDogmaException | IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new StorageException("failed to push at '" + project.name() + '/' + originalRepoName + '\'', + e); + } + } + + @VisibleForTesting + static void doRefUpdate(Repository jGitRepo, RevWalk revWalk, + String ref, ObjectId commitId) throws IOException { + if (ref.startsWith(Constants.R_TAGS)) { + final Ref oldRef = jGitRepo.exactRef(ref); + if (oldRef != null) { + throw new StorageException("tag ref exists already: " + ref); + } + } + + final RefUpdate refUpdate = jGitRepo.updateRef(ref); + refUpdate.setNewObjectId(commitId); + + final Result res = refUpdate.update(revWalk); + switch (res) { + case NEW: + case FAST_FORWARD: + // Expected + break; + default: + throw new StorageException("unexpected refUpdate state: " + res); + } + } + + void close() { + try { + commitIdDatabase.close(); + } catch (Throwable t) { + logger.warn("Failed to close a commitId database:", t); + } + closeJGitRepo(jGitRepository); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).omitNullValues() + .add("project", project) + .add("originalRepoName", originalRepoName) + .add("repoDir", repoDir) + .add("commitIdDatabase", commitIdDatabase) + .add("jGitRepository", jGitRepository) + .toString(); + } + + static final class RevisionAndEntries { + final Revision revision; + final List diffEntries; + + RevisionAndEntries(Revision revision, List diffEntries) { + this.revision = revision; + this.diffEntries = diffEntries; + } + } + + // PathEdit implementations which is used when applying changes. + + private static final class InsertText extends PathEdit { + private final ObjectInserter inserter; + private final String text; + + InsertText(String entryPath, ObjectInserter inserter, String text) { + super(entryPath); + this.inserter = inserter; + this.text = text; + } + + @Override + public void apply(DirCacheEntry ent) { + try { + ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, text.getBytes(UTF_8))); + ent.setFileMode(FileMode.REGULAR_FILE); + } catch (IOException e) { + throw new StorageException("failed to create a new text blob", e); + } + } + } + + private static final class InsertJson extends PathEdit { + private final ObjectInserter inserter; + private final JsonNode jsonNode; + + InsertJson(String entryPath, ObjectInserter inserter, JsonNode jsonNode) { + super(entryPath); + this.inserter = inserter; + this.jsonNode = jsonNode; + } + + @Override + public void apply(DirCacheEntry ent) { + try { + ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, Jackson.writeValueAsBytes(jsonNode))); + ent.setFileMode(FileMode.REGULAR_FILE); + } catch (IOException e) { + throw new StorageException("failed to create a new JSON blob", e); + } + } + } + + private static final class CopyOldEntry extends PathEdit { + private final DirCacheEntry oldEntry; + + CopyOldEntry(String entryPath, DirCacheEntry oldEntry) { + super(entryPath); + this.oldEntry = oldEntry; + } + + @Override + public void apply(DirCacheEntry ent) { + ent.setFileMode(oldEntry.getFileMode()); + ent.setObjectId(oldEntry.getObjectId()); + } + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/LegacyGitRepository.java similarity index 98% rename from server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java rename to server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/LegacyGitRepository.java index d44b5bf09..7c941a868 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/LegacyGitRepository.java @@ -126,12 +126,13 @@ /** * A {@link Repository} based on Git. + * This class will be removed after all migration to GitRepositoryV2 is done. */ -class GitRepository implements Repository { +class LegacyGitRepository implements Repository { - private static final Logger logger = LoggerFactory.getLogger(GitRepository.class); + private static final Logger logger = LoggerFactory.getLogger(LegacyGitRepository.class); - static final String R_HEADS_MASTER = Constants.R_HEADS + Constants.MASTER; + private static final String R_HEADS_MASTER = Constants.R_HEADS + Constants.MASTER; private static final byte[] EMPTY_BYTE = new byte[0]; private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL); @@ -202,8 +203,8 @@ class GitRepository implements Repository { * @throws StorageException if failed to create a new repository */ @VisibleForTesting - GitRepository(Project parent, File repoDir, Executor repositoryWorker, - long creationTimeMillis, Author author) { + LegacyGitRepository(Project parent, File repoDir, Executor repositoryWorker, + long creationTimeMillis, Author author) { this(parent, repoDir, repositoryWorker, creationTimeMillis, author, null); } @@ -217,8 +218,8 @@ class GitRepository implements Repository { * * @throws StorageException if failed to create a new repository */ - GitRepository(Project parent, File repoDir, Executor repositoryWorker, - long creationTimeMillis, Author author, @Nullable RepositoryCache cache) { + LegacyGitRepository(Project parent, File repoDir, Executor repositoryWorker, + long creationTimeMillis, Author author, @Nullable RepositoryCache cache) { this.parent = requireNonNull(parent, "parent"); name = requireNonNull(repoDir, "repoDir").getName(); @@ -279,7 +280,8 @@ class GitRepository implements Repository { * * @throws StorageException if failed to open the repository at the specified location */ - GitRepository(Project parent, File repoDir, Executor repositoryWorker, @Nullable RepositoryCache cache) { + LegacyGitRepository(Project parent, File repoDir, Executor repositoryWorker, + @Nullable RepositoryCache cache) { this.parent = requireNonNull(parent, "parent"); name = requireNonNull(repoDir, "repoDir").getName(); this.repositoryWorker = requireNonNull(repositoryWorker, "repositoryWorker"); @@ -1584,8 +1586,8 @@ public void cloneTo(File newRepoDir, BiConsumer progressListen requireNonNull(progressListener, "progressListener"); final Revision endRevision = normalizeNow(Revision.HEAD); - final GitRepository newRepo = new GitRepository(parent, newRepoDir, repositoryWorker, - creationTimeMillis(), author(), cache); + final LegacyGitRepository newRepo = new LegacyGitRepository(parent, newRepoDir, repositoryWorker, + creationTimeMillis(), author(), cache); progressListener.accept(1, endRevision.major()); boolean success = false; @@ -1647,6 +1649,17 @@ private static void deleteCruft(File repoDir) { } } + @Override + public Revision shouldCreateRollingRepository(int minRetentionCommits, int minRetentionDays) { + throw new UnsupportedOperationException(); + } + + @Override + public void createRollingRepository(Revision initialRevision, int minRetentionCommits, + int minRetentionDays) { + throw new UnsupportedOperationException(); + } + @Override public String toString() { return MoreObjects.toStringHelper(this) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java new file mode 100644 index 000000000..3adab6f1b --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabase.java @@ -0,0 +1,180 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import java.io.EOFException; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cronutils.utils.VisibleForTesting; + +import com.linecorp.centraldogma.server.storage.StorageException; + +/** + * Simple file-based database that has the suffix of the primary Git repository. + */ +final class RepositoryMetadataDatabase implements AutoCloseable { + + private static final Logger logger = LoggerFactory.getLogger(RepositoryMetadataDatabase.class); + + private static final String INITIAL_PRIMARY_SUFFIX = "0000000000"; + + private static final int PRIMARY_SUFFIX_LEN = INITIAL_PRIMARY_SUFFIX.length(); + + private static final ThreadLocal threadLocalBuffer = + ThreadLocal.withInitial(() -> ByteBuffer.allocateDirect(PRIMARY_SUFFIX_LEN)); + + static File initialPrimaryRepoDir(File rootDir) { + return repoDir(rootDir, INITIAL_PRIMARY_SUFFIX); + } + + private static File repoDir(File rootDir, String suffix) { + return new File(rootDir, rootDir.getName() + '_' + suffix); + } + + @VisibleForTesting + final File rootDir; + private final Path metadataPath; + private final FileChannel metadataFileChannel; + private String primarySuffix; + + RepositoryMetadataDatabase(File rootDir, boolean create) { + this.rootDir = rootDir; + metadataPath = new File(rootDir, "metadata.dat").toPath(); + try { + if (create) { + metadataFileChannel = FileChannel.open(metadataPath, + StandardOpenOption.CREATE, + StandardOpenOption.READ, + StandardOpenOption.WRITE); + } else { + metadataFileChannel = FileChannel.open(metadataPath, + StandardOpenOption.READ, + StandardOpenOption.WRITE); + } + } catch (IOException e) { + throw new StorageException("failed to " + (create ? "create" : "open") + + " a repository database: " + metadataPath, e); + } + + if (create) { + primarySuffix = INITIAL_PRIMARY_SUFFIX; + writeSuffix(primarySuffix); + } else { + boolean success = false; + try { + final ByteBuffer buf = threadLocalBuffer.get(); + buf.clear(); + final long read = readTo(buf); + if (read != PRIMARY_SUFFIX_LEN) { + throw new StorageException("incorrect prefix length for " + metadataPath + + ", length: " + read + " (expected: " + PRIMARY_SUFFIX_LEN + ')'); + } + buf.flip(); + primarySuffix = StandardCharsets.UTF_8.decode(buf).toString(); + // To check if the suffix is a correct integer value. + // noinspection ResultOfMethodCallIgnored + Integer.parseInt(primarySuffix); + success = true; + } finally { + if (!success) { + close(); + } + } + } + } + + private void writeSuffix(String suffix) { + final ByteBuffer buf = threadLocalBuffer.get(); + buf.clear(); + buf.put(suffix.getBytes()); + buf.flip(); + + long pos = 0; + try { + do { + pos += metadataFileChannel.write(buf, pos); + } while (buf.hasRemaining()); + metadataFileChannel.force(true); + } catch (IOException e) { + close(); + throw new StorageException("failed to write the suffix (" + suffix + + ") of the primary repository: " + metadataPath, e); + } + } + + private long readTo(ByteBuffer buf) { + long pos = 0; + try { + do { + final int readBytes = metadataFileChannel.read(buf, pos); + if (readBytes < 0) { + throw new EOFException(); + } + pos += readBytes; + } while (buf.hasRemaining()); + return pos; + } catch (IOException e) { + throw new StorageException("failed to read the suffix of the primary repository database: " + + metadataPath, e); + } + } + + File primaryRepoDir() { + return repoDir(rootDir, primarySuffix); + } + + File secondaryRepoDir() { + return repoDir(rootDir, increment(primarySuffix)); + } + + void setPrimaryRepoDir(File newRepoDir) { // e.g. /foo_0000123457 + // e.g. primarySuffix: 0000123456, newPrimarySuffix: 0000123457 + final String newPrimarySuffix = increment(primarySuffix); + // e.g. secondary: /foo_0000123457 + final File secondary = new File(rootDir, rootDir.getName() + '_' + newPrimarySuffix); + assert newRepoDir.equals(secondary) : newRepoDir + " != " + secondary; + primarySuffix = newPrimarySuffix; + writeSuffix(newPrimarySuffix); + } + + @Override + public void close() { + try { + metadataFileChannel.close(); + } catch (IOException e) { + logger.warn("Failed to close the commit ID database: {}", metadataPath, e); + } + } + + @VisibleForTesting + static String increment(String suffix) { // e.g. "0000123456" + final int intSuffix = Integer.parseInt(suffix); // e.g. 123456 + final String str = String.valueOf(intSuffix + 1); // e.g. "123457" + if (str.length() < 10) { + return INITIAL_PRIMARY_SUFFIX.substring(0, 10 - str.length()) + str; // e.g. "0000123457" + } + return str; + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java index 8e6f11266..b0086bf4a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java @@ -31,6 +31,8 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; +import javax.annotation.Nullable; + import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -521,4 +523,19 @@ default CompletableFuture> mergeFiles(Revision revision, Merg return future; } + + /** + * Returns the head {@link Revision} of this repository if this repository needs to create a + * rolling repository. The head {@link Revision} is returned when following conditions are met: + * - The last created rolling repository has more than {@code minRetentionCommits}. + * - The last created rolling repository has commits that are older than {@code minRetentionDays}. + */ + @Nullable + Revision shouldCreateRollingRepository(int minRetentionCommits, int minRetentionDays); + + /** + * Creates the rolling repository. The specified {@code initialRevision} of the current repository will be + * the initial revision of the created rolling repository. + */ + void createRollingRepository(Revision initialRevision, int minRetentionCommits, int minRetentionDays); } diff --git a/server/src/main/resources/META-INF/services/com.linecorp.centraldogma.server.plugin.Plugin b/server/src/main/resources/META-INF/services/com.linecorp.centraldogma.server.plugin.Plugin index bb8987036..1d38f906f 100644 --- a/server/src/main/resources/META-INF/services/com.linecorp.centraldogma.server.plugin.Plugin +++ b/server/src/main/resources/META-INF/services/com.linecorp.centraldogma.server.plugin.Plugin @@ -1,2 +1,3 @@ com.linecorp.centraldogma.server.internal.mirror.DefaultMirroringServicePlugin com.linecorp.centraldogma.server.internal.storage.PurgeSchedulingServicePlugin +com.linecorp.centraldogma.server.internal.storage.repository.git.CommitRetentionManagementPlugin diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabaseTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabaseTest.java index f0a0587c5..050d02fd0 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabaseTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitIdDatabaseTest.java @@ -21,7 +21,6 @@ import static org.mockito.Mockito.mock; import java.io.File; -import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.StandardOpenOption; @@ -32,6 +31,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.common.Change; @@ -65,11 +66,12 @@ void emptyDatabase() { assertThatThrownBy(() -> db.get(Revision.INIT)).isInstanceOf(IllegalStateException.class); } - @Test - void simpleAccess() { + @ValueSource(ints = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }) + @ParameterizedTest + void simpleAccess(int startRevision) { final int numCommits = 10; - final ObjectId[] expectedCommitIds = new ObjectId[numCommits + 1]; - for (int i = 1; i <= numCommits; i++) { + final ObjectId[] expectedCommitIds = new ObjectId[numCommits + startRevision]; + for (int i = startRevision; i < startRevision + numCommits; i++) { final Revision revision = new Revision(i); final ObjectId commitId = randomCommitId(); expectedCommitIds[i] = commitId; @@ -77,15 +79,20 @@ void simpleAccess() { assertThat(db.headRevision()).isEqualTo(revision); } - for (int i = 1; i <= numCommits; i++) { + for (int i = startRevision; i < startRevision + numCommits; i++) { assertThat(db.get(new Revision(i))).isEqualTo(expectedCommitIds[i]); } assertThatThrownBy(() -> db.get(Revision.HEAD)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("absolute revision"); - assertThatThrownBy(() -> db.get(new Revision(numCommits + 1))) + assertThatThrownBy(() -> db.get(new Revision(numCommits + startRevision))) .isInstanceOf(RevisionNotFoundException.class); + if (startRevision > 1) { + assertThatThrownBy(() -> db.get(new Revision(startRevision - 1))) + .isInstanceOf(RevisionNotFoundException.class); + } + assertThat(db.firstRevision()).isEqualTo(new Revision(startRevision)); } @Test @@ -95,7 +102,7 @@ void truncatedDatabase() throws Exception { // Truncate the database file. try (FileChannel f = FileChannel.open(new File(tempDir, "commit_ids.dat").toPath(), - StandardOpenOption.APPEND)) { + StandardOpenOption.APPEND)) { assertThat(f.size()).isEqualTo(24); f.truncate(23); @@ -106,50 +113,25 @@ void truncatedDatabase() throws Exception { .hasMessageContaining("incorrect file length"); } - @Test - void mismatchingRevision() throws Exception { - db.close(); - - // Append a record with incorrect revision number. - try (FileChannel f = FileChannel.open(new File(tempDir, "commit_ids.dat").toPath(), - StandardOpenOption.APPEND)) { - - final ByteBuffer buf = ByteBuffer.allocate(24); - buf.putInt(42); // Expected to be 1. - randomCommitId().copyRawTo(buf); - buf.flip(); - do { - f.write(buf); - } while (buf.hasRemaining()); - - assertThat(f.size()).isEqualTo(buf.capacity()); - } - - // Reopen the database and see if it fails to resolve the revision 1. - db = new CommitIdDatabase(tempDir); - assertThatThrownBy(() -> db.get(Revision.INIT)) - .isInstanceOf(StorageException.class) - .hasMessageContaining("incorrect revision number"); - } - - @Test - void rebuildingBadDatabase() throws Exception { + @ValueSource(ints = { 1, 3, 5, 7, 9 }) + @ParameterizedTest + void rebuildingBadDatabase(int startRevision) throws Exception { final int numCommits = 10; final File repoDir = tempDir; - final File commitIdDatabaseFile = new File(repoDir, "commit_ids.dat"); - // Create a repository which contains some commits. - GitRepository repo = new GitRepository(mock(Project.class), repoDir, commonPool(), 1000, Author.SYSTEM); + GitRepositoryV2 repo = createRepoWithFirstRevision(repoDir, startRevision); Revision headRevision = null; try { - for (int i = 1; i <= numCommits; i++) { - headRevision = repo.commit(new Revision(i), 0, Author.DEFAULT, "", - Change.ofTextUpsert("/" + i + ".txt", "")).join().revision(); + // We already add 2 commits in createRepoWithFirstRevision so let's start with startRevision + 2. + for (int i = startRevision + 2; i < startRevision + numCommits; i++) { + headRevision = addCommit(repo, i); } } finally { repo.internalClose(); } + final File primaryRepoDir = repo.primaryRepo.repoDir(); + final File commitIdDatabaseFile = new File(primaryRepoDir, "commit_ids.dat"); // Wipe out the commit ID database. assertThat(commitIdDatabaseFile).exists(); try (FileChannel ch = FileChannel.open(commitIdDatabaseFile.toPath(), StandardOpenOption.WRITE)) { @@ -157,21 +139,59 @@ void rebuildingBadDatabase() throws Exception { } // Open the repository again to see if the commit ID database is regenerated automatically. - repo = new GitRepository(mock(Project.class), repoDir, commonPool(), null); + repo = GitRepositoryV2.open(mock(Project.class), repoDir, commonPool(), null); try { assertThat(repo.normalizeNow(Revision.HEAD)).isEqualTo(headRevision); - for (int i = 1; i <= numCommits; i++) { + for (int i = startRevision; i < startRevision + numCommits; i++) { assertThat(repo.find(new Revision(i + 1), "/" + i + ".txt").join()).hasSize(1); } } finally { repo.internalClose(); } + assertThat(commitIdDatabaseFile).exists(); assertThat(repo.creationTimeMillis()).isEqualTo(1000); assertThat(repo.author()).isEqualTo(Author.SYSTEM); assertThat(Files.size(commitIdDatabaseFile.toPath())).isEqualTo((numCommits + 1) * 24L); } + private static GitRepositoryV2 createRepoWithFirstRevision(File repoDir, int startRevision) { + final GitRepositoryV2 repo = new GitRepositoryV2(mock(Project.class), repoDir, + commonPool(), + 1000, Author.SYSTEM, null); + if (startRevision == 1) { + addCommit(repo, startRevision); + addCommit(repo, startRevision + 1); + return repo; + } + + for (int i = 1; i < startRevision; i++) { + addCommit(repo, i); + } + assertThat(repo.shouldCreateRollingRepository(1, 0).major()).isEqualTo(startRevision); + repo.createRollingRepository(new Revision(startRevision), 1, 0); + // Now the first revision of secondary repository is startRevision; + final InternalRepository secondaryRepo = repo.secondaryRepo; + assertThat(secondaryRepo.commitIdDatabase().firstRevision().major()).isEqualTo(startRevision); + + addCommit(repo, startRevision); + addCommit(repo, startRevision + 1); + assertThat(repo.shouldCreateRollingRepository(1, 0).major()).isEqualTo(startRevision + 2); + repo.createRollingRepository(new Revision(startRevision + 2), 1, 0); + + // The secondary repo is promoted. + assertThat(repo.primaryRepo).isSameAs(secondaryRepo); + assertThat(repo.primaryRepo.commitIdDatabase().firstRevision().major()).isEqualTo(startRevision); + assertThat(repo.primaryRepo.commitIdDatabase().headRevision().major()).isEqualTo(startRevision + 2); + return repo; + } + + private static Revision addCommit(GitRepositoryV2 repo, int i) { + return repo.commit(Revision.HEAD, 0, Author.SYSTEM, "", Change.ofTextUpsert("/" + i + ".txt", "")) + .join() + .revision(); + } + private static ObjectId randomCommitId() { final byte[] commitId = new byte[20]; ThreadLocalRandom.current().nextBytes(commitId); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManagerTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManagerTest.java index 1f99c3473..d4b86f1f9 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManagerTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManagerTest.java @@ -54,8 +54,8 @@ void setUp(@TempDir Path tempDir) { void testCreate() { final GitRepositoryManager gitRepositoryManager = newRepositoryManager(); final Repository repository = gitRepositoryManager.create(TEST_REPO, Author.SYSTEM); - assertThat(repository).isInstanceOf(GitRepository.class); - assertThat(((GitRepository) repository).cache).isNotNull(); + assertThat(repository).isInstanceOf(GitRepositoryV2.class); + assertThat(((GitRepositoryV2) repository).cache).isNotNull(); // Must disallow creating a duplicate. assertThatThrownBy(() -> gitRepositoryManager.create(TEST_REPO, Author.SYSTEM)) @@ -75,8 +75,8 @@ void testOpen() { // Create a new manager so that it loads the repository we created above. gitRepositoryManager = newRepositoryManager(); final Repository repository = gitRepositoryManager.get(TEST_REPO); - assertThat(repository).isInstanceOf(GitRepository.class); - assertThat(((GitRepository) repository).cache).isNotNull(); + assertThat(repository).isInstanceOf(GitRepositoryV2.class); + assertThat(((GitRepositoryV2) repository).cache).isNotNull(); gitRepositoryManager.remove(TEST_REPO); gitRepositoryManager.purgeMarked(); @@ -88,7 +88,7 @@ void testGet() { assertThat(gitRepositoryManager.exists(TEST_REPO)).isFalse(); final Repository repository = gitRepositoryManager.create(TEST_REPO, Author.SYSTEM); - assertThat(repository).isInstanceOf(GitRepository.class); + assertThat(repository).isInstanceOf(GitRepositoryV2.class); assertThat(gitRepositoryManager.get(TEST_REPO)).isSameAs(repository); gitRepositoryManager.remove(TEST_REPO); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java new file mode 100644 index 000000000..d37d25fba --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryMigrationTest.java @@ -0,0 +1,67 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static java.util.concurrent.ForkJoinPool.commonPool; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +import java.io.File; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Commit; +import com.linecorp.centraldogma.common.Markup; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.server.storage.project.Project; + +class GitRepositoryMigrationTest { + + @TempDir + static File repoDir; + + @Test + void migrate() { + final File fooDir = new File(repoDir, "foo"); + final LegacyGitRepository oldRepo = + new LegacyGitRepository(mock(Project.class), new File(fooDir, "test_repo"), + commonPool(), 0L, Author.SYSTEM); + addCommits(oldRepo); + oldRepo.internalClose(); + final GitRepositoryV2 migrated = GitRepositoryV2.open(mock(Project.class), + new File(fooDir, "test_repo"), commonPool(), + null); + final List commits = migrated.history(Revision.INIT, Revision.HEAD, "/**").join(); + assertThat(commits.get(0).summary()).contains("new repository"); + for (int i = 1; i < 10; i++) { + assertThat(commits.get(i).summary()).isEqualTo("Summary" + i); + } + + migrated.internalClose(); + } + + private static void addCommits(LegacyGitRepository oldRepo) { + for (int i = 1; i < 10; i++) { + oldRepo.commit(Revision.HEAD, 0, Author.SYSTEM, + "Summary" + i, "Detail", Markup.PLAINTEXT, + Change.ofTextUpsert("/file_" + i + ".txt", String.valueOf(i))).join(); + } + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryTest.java index 557e63bbe..6a5cdda15 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryTest.java @@ -18,15 +18,14 @@ import static com.linecorp.centraldogma.common.Revision.HEAD; import static com.linecorp.centraldogma.common.Revision.INIT; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.RepositoryMetadataDatabase.initialPrimaryRepoDir; import static java.util.concurrent.ForkJoinPool.commonPool; import static net.javacrumbs.jsonunit.fluent.JsonFluentAssert.assertThatJson; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.fail; import static org.awaitility.Awaitility.await; -import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import java.io.File; import java.util.ArrayList; @@ -47,11 +46,6 @@ import javax.annotation.Nullable; -import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.util.FS; import org.junit.jupiter.api.AfterAll; @@ -78,7 +72,6 @@ import com.linecorp.centraldogma.common.RevisionNotFoundException; import com.linecorp.centraldogma.internal.Util; import com.linecorp.centraldogma.server.internal.JGitUtil; -import com.linecorp.centraldogma.server.storage.StorageException; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.repository.Repository; import com.linecorp.centraldogma.testing.internal.TestUtil; @@ -92,7 +85,7 @@ class GitRepositoryTest { @TempDir static File repoDir; - private static GitRepository repo; + private static GitRepositoryV2 repo; /** * Used by {@link GitRepositoryTest#testWatchWithQueryCancellation()}. @@ -102,8 +95,8 @@ class GitRepositoryTest { @BeforeAll static void init() { - repo = new GitRepository(mock(Project.class), new File(repoDir, "test_repo"), - commonPool(), 0L, Author.SYSTEM) { + repo = new GitRepositoryV2(mock(Project.class), new File(repoDir, "test_repo"), + commonPool(), 0L, Author.SYSTEM, null) { /** * Used by {@link GitRepositoryTest#testWatchWithQueryCancellation()}. */ @@ -167,7 +160,9 @@ void setUp(TestInfo testInfo) { @Test void defaultSettings() throws Exception { // Make sure the Git config file has been created. - final File configFile = repoDir.toPath().resolve("test_repo").resolve("config").toFile(); + final File configFile = initialPrimaryRepoDir(repoDir.toPath().resolve("test_repo").toFile()) + .toPath() + .resolve("config").toFile(); assertThat(configFile).exists(); // Load the Git config file. @@ -574,20 +569,17 @@ void testDiff_invalidParameters() { assertThat(repo.diff(revision1, revision2, "non_existing_path").join()).isEmpty(); assertThatThrownBy(() -> repo.diff(revision1, revision2, (String) null).join()) - .isInstanceOf(CompletionException.class) - .hasCauseInstanceOf(NullPointerException.class); + .isInstanceOf(NullPointerException.class); assertThatThrownBy(() -> repo.diff(null, revision2, path).join()) - .isInstanceOf(CompletionException.class) - .hasCauseInstanceOf(NullPointerException.class); + .isInstanceOf(NullPointerException.class); assertThatThrownBy(() -> repo.diff(revision1, new Revision(revision2.major() + 1), path).join()) - .isInstanceOf(CompletionException.class) - .hasCauseInstanceOf(RevisionNotFoundException.class); + .hasRootCauseInstanceOf(RevisionNotFoundException.class); assertThatThrownBy(() -> repo.diff(new Revision(revision2.major() + 1), revision2, path).join()) .isInstanceOf(CompletionException.class) - .hasCauseInstanceOf(RevisionNotFoundException.class); + .hasRootCauseInstanceOf(RevisionNotFoundException.class); } @Test @@ -907,12 +899,10 @@ void testHistory_parameterCheck() { .hasCauseInstanceOf(RevisionNotFoundException.class); assertThatThrownBy(() -> repo.history(null, HEAD, "non_existing_path").join()) - .isInstanceOf(CompletionException.class) - .hasCauseInstanceOf(NullPointerException.class); + .isInstanceOf(NullPointerException.class); assertThatThrownBy(() -> repo.history(HEAD, null, "non_existing_path").join()) - .isInstanceOf(CompletionException.class) - .hasCauseInstanceOf(NullPointerException.class); + .isInstanceOf(NullPointerException.class); } @Test @@ -1292,48 +1282,12 @@ private static void ensureWatcherCleanUp() { await().untilAsserted(() -> assertThat(repo.commitWatchers.watchesMap).isEmpty()); } - @Test - void testDoUpdateRef() throws Exception { - final ObjectId commitId = mock(ObjectId.class); - - // A commit on the mainlane - testDoUpdateRef(Constants.R_TAGS + '1', commitId, false); - testDoUpdateRef(Constants.R_HEADS + Constants.MASTER, commitId, false); - } - - private static void testDoUpdateRef(String ref, ObjectId commitId, boolean tagExists) throws Exception { - final org.eclipse.jgit.lib.Repository jGitRepo = mock(org.eclipse.jgit.lib.Repository.class); - final RevWalk revWalk = mock(RevWalk.class); - final RefUpdate refUpdate = mock(RefUpdate.class); - - lenient().when(jGitRepo.exactRef(ref)).thenReturn(tagExists ? mock(Ref.class) : null); - lenient().when(jGitRepo.updateRef(ref)).thenReturn(refUpdate); - - lenient().when(refUpdate.update(revWalk)).thenReturn(RefUpdate.Result.NEW); - GitRepository.doRefUpdate(jGitRepo, revWalk, ref, commitId); - - when(refUpdate.update(revWalk)).thenReturn(RefUpdate.Result.FAST_FORWARD); - GitRepository.doRefUpdate(jGitRepo, revWalk, ref, commitId); - - when(refUpdate.update(revWalk)).thenReturn(RefUpdate.Result.LOCK_FAILURE); - assertThatThrownBy(() -> GitRepository.doRefUpdate(jGitRepo, revWalk, ref, commitId)) - .isInstanceOf(StorageException.class); - } - - @Test - void testDoUpdateRefOnExistingTag() { - final ObjectId commitId = mock(ObjectId.class); - - assertThatThrownBy(() -> testDoUpdateRef(Constants.R_TAGS + "01/1.0", commitId, true)) - .isInstanceOf(StorageException.class); - } - @Test void operationOnClosedRepository() { final CentralDogmaException expectedException = new CentralDogmaException(); - final GitRepository repo = new GitRepository(mock(Project.class), - new File(repoDir, "close_test_repo"), - commonPool(), 0L, Author.SYSTEM); + final GitRepositoryV2 repo = new GitRepositoryV2(mock(Project.class), + new File(repoDir, "close_test_repo"), + commonPool(), 0L, Author.SYSTEM, null); repo.close(() -> expectedException); assertThatThrownBy(() -> repo.find(INIT, "/**").join()).hasCause(expectedException); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2DiffTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2DiffTest.java new file mode 100644 index 000000000..74197ce8b --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2DiffTest.java @@ -0,0 +1,99 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryV2HistoryTest.createRepository; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.stream.IntStream; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.google.common.collect.ImmutableList; + +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.common.RevisionNotFoundException; + +class GitRepositoryV2DiffTest { + + @TempDir + static File repoDir; + private static GitRepositoryV2 repo; + + @BeforeAll + static void setUp() { + // The repository contains commits from 10 to 20(inclusive). + repo = createRepository(repoDir); + } + + @AfterAll + static void tearDown() { + repo.internalClose(); + } + + @Test + void diff() { + Map> diffs = repo.diff(new Revision(18), new Revision(20), "/**").join(); + assertThat(paths(diffs)).containsExactly("/file_19.txt", "/file_20.txt"); + + // It's same when the revisions are reversed. + assertThat(repo.diff(Revision.HEAD, new Revision(18), "/**").join().values()) + .containsExactlyInAnyOrderElementsOf(diffs.values()); + + diffs = repo.diff(new Revision(10), new Revision(20), "/**").join(); + assertThat(diffs).hasSize(10); // from 11 to 20. + final List expected = + IntStream.range(11, 21) + .mapToObj(i -> "/file_" + i + ".txt") + .collect(toImmutableList()); + assertThat(paths(diffs)).containsExactlyInAnyOrderElementsOf(expected); + + // If the revision is INIT, the result is same as Revision(10) which is the first revision + // of the new primary repository. + assertThat(repo.diff(Revision.INIT, new Revision(20), "/**").join().values()) + .containsExactlyInAnyOrderElementsOf(diffs.values()); + + assertThat(repo.diff(Revision.INIT, Revision.INIT, "/**").join()).isEmpty(); + } + + private ImmutableList paths(Map> diffs) { + return diffs.values().stream().map(Change::path).collect(toImmutableList()); + } + + @Test + void invalidRevisionDiff() { + assertThatThrownBy(() -> repo.diff(new Revision(9), new Revision(10), "/**").join()) + .hasRootCauseInstanceOf(RevisionNotFoundException.class) + .hasRootCauseMessage("revision: Revision(9) (expected: >= 10)"); + + assertThatThrownBy(() -> repo.diff(Revision.INIT, new Revision(9), "/**").join()) + .hasRootCauseInstanceOf(RevisionNotFoundException.class) + .hasRootCauseMessage("revision: Revision(9) (expected: >= 10)"); + + assertThatThrownBy(() -> repo.diff(Revision.HEAD, new Revision(9), "/**").join()) + .hasRootCauseInstanceOf(RevisionNotFoundException.class) + .hasRootCauseMessage("revision: Revision(9) (expected: >= 10)"); + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2FindTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2FindTest.java new file mode 100644 index 000000000..b4aa0d3ad --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2FindTest.java @@ -0,0 +1,118 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryV2HistoryTest.createRepository; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.stream.IntStream; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.google.common.collect.Iterables; + +import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.common.RevisionNotFoundException; + +class GitRepositoryV2FindTest { + + @TempDir + static File repoDir; + private static GitRepositoryV2 repo; + + @BeforeAll + static void setUp() { + // The repository contains commits from 10 to 20(inclusive). + repo = createRepository(repoDir); + } + + @AfterAll + static void tearDown() { + repo.internalClose(); + } + + @Test + void normalFind() { + Map> entries = repo.find(new Revision(20), "/**").join(); + assertThat(entries).hasSize(19); // 2 to 20 (inclusively) + + entries = repo.find(new Revision(15), "/file_15.txt").join(); + assertThat(entries.size()).isOne(); + Entry entry = Iterables.get(entries.values(), 0); + assertThat(entry.revision().major()).isEqualTo(15); + assertThat(entry.path()).isEqualTo("/file_15.txt"); + + // file_2.txt is committed together when the current primary repository is created. + entries = repo.find(new Revision(10), "/file_2.txt").join(); + assertThat(entries.size()).isOne(); + entry = Iterables.get(entries.values(), 0); + assertThat(entry.revision().major()).isEqualTo(10); + assertThat(entry.path()).isEqualTo("/file_2.txt"); + + entries = repo.find(new Revision(10), "/**").join(); + assertThat(entries).hasSize(9); // from 2 ~ 10 + final List expected = IntStream.range(2, 11).mapToObj(i -> "/file_" + i + ".txt") + .collect(toImmutableList()); + final List actual = entries.values().stream().map(Entry::path).collect(toImmutableList()); + assertThat(actual).containsExactlyInAnyOrderElementsOf(expected); + + // If the revision is INIT, the result is same as Revision(10) which is the first revision + // of the new primary repository. + assertThat(repo.find(Revision.INIT, "/**").join()).isEqualTo(entries); + } + + @Test + void invalidRevisionFind() { + assertThatThrownBy(() -> repo.find(new Revision(9), "/**").join()) + .hasCauseInstanceOf(RevisionNotFoundException.class) + .hasMessageContaining("revision: Revision(9) (expected: >= 10)"); + } + + @Test + void findLatestRevision() { + Revision revision = repo.findLatestRevision(new Revision(20), "/**").join(); + assertThat(revision).isNull(); // there's no commits after the revision 20 so it's null. + + revision = repo.findLatestRevision(new Revision(19), "/**").join(); + assertThat(revision.major()).isEqualTo(20); + + revision = repo.findLatestRevision(new Revision(13), "/file_15.txt").join(); + // file_15 is committed at the revision 15 which is before the revision 19. + assertThat(revision.major()).isEqualTo(20); + + revision = repo.findLatestRevision(new Revision(19), "/file_15.txt").join(); + // file_15 is committed at the revision 15 which is before the revision 19. + assertThat(revision).isNull(); + + revision = repo.findLatestRevision(new Revision(10), "/file_5.txt").join(); + assertThat(revision).isNull(); // It's null because file_5 is not changed after the revision 10. + + revision = repo.findLatestRevision(new Revision(4), "/file_5.txt").join(); + assertThat(revision.major()).isEqualTo(20); + + revision = repo.findLatestRevision(new Revision(6), "/file_5.txt").join(); + assertThat(revision.major()).isEqualTo(20); + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2HistoryTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2HistoryTest.java new file mode 100644 index 000000000..fc0ed2ac6 --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2HistoryTest.java @@ -0,0 +1,119 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryV2PromotionTest.addCommits; +import static java.util.concurrent.ForkJoinPool.commonPool; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; + +import java.io.File; +import java.util.List; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Commit; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.common.RevisionNotFoundException; +import com.linecorp.centraldogma.server.storage.project.Project; + +class GitRepositoryV2HistoryTest { + + @TempDir + static File repoDir; + private static GitRepositoryV2 repo; + + @BeforeAll + static void setUp() { + // The repository contains commits from 10 to 20(inclusive). + repo = createRepository(repoDir); + } + + @AfterAll + static void tearDown() { + repo.internalClose(); + } + + @Test + void invalidRevision() { + // The larger of the two revisions should be greater or equal to the firstRevision. + assertThatThrownBy(() -> repo.history(new Revision(2), new Revision(9), "/**") + .join()) + .hasCauseInstanceOf(RevisionNotFoundException.class) + .hasMessageContaining("revision: Revision(2) (expected: >= 10)"); + assertThatThrownBy(() -> repo.history(new Revision(9), new Revision(2), "/**") + .join()) + .hasCauseInstanceOf(RevisionNotFoundException.class) + .hasMessageContaining("revision: Revision(9) (expected: >= 10)"); + + // The larger of the two revisions should be lower or equal to the headRevision. + assertThatThrownBy(() -> repo.history(new Revision(2), new Revision(21), "/**") + .join()) + .hasCauseInstanceOf(RevisionNotFoundException.class) + .hasMessageContaining("revision: Revision(2) (expected: >= 10)"); + assertThatThrownBy(() -> repo.history(new Revision(21), new Revision(2), "/**") + .join()) + .hasCauseInstanceOf(RevisionNotFoundException.class) + .hasMessageContaining("revision: Revision(21) (expected: <= 20)"); + + assertThatThrownBy(() -> repo.history(new Revision(2), new Revision(10), "/**") + .join()) + .hasCauseInstanceOf(RevisionNotFoundException.class) + .hasMessageContaining("revision: Revision(2) (expected: >= 10)"); + + assertThatThrownBy(() -> repo.history(new Revision(10), new Revision(2), "/**") + .join()) + .hasCauseInstanceOf(RevisionNotFoundException.class) + .hasMessageContaining("revision: Revision(2) (expected: >= 10)"); + } + + @Test + void normalHistoryCall() { + List commits = repo.history(new Revision(10), new Revision(14), "/**") + .join(); + assertThat(commits).hasSize(5); + + commits = repo.history(Revision.INIT, Revision.INIT, "/**").join(); + System.err.println(commits); + assertThat(commits).hasSize(1); + } + + static GitRepositoryV2 createRepository(File repoDir) { + final GitRepositoryV2 repo = new GitRepositoryV2(mock(Project.class), + new File(repoDir, "test_repo"), commonPool(), + 0L, Author.SYSTEM, null); + addCommits(repo, 2, 10); + int minRetentionCommits = 10 - 2; + assertThat(repo.shouldCreateRollingRepository(minRetentionCommits, 0).major()).isEqualTo(10); + repo.createRollingRepository(new Revision(10), minRetentionCommits, 0); + // The headRevision of the secondary repository is now from revision. + + addCommits(repo, 11, 20); + minRetentionCommits = 9; // 20 - 10 - 1 + assertThat(repo.shouldCreateRollingRepository(minRetentionCommits, 0).major()).isEqualTo(20); + repo.createRollingRepository(new Revision(20), minRetentionCommits, 0); + + final CommitIdDatabase commitIdDatabase = repo.primaryRepo.commitIdDatabase(); + assertThat(commitIdDatabase.firstRevision().major()).isSameAs(10); + assertThat(commitIdDatabase.headRevision().major()).isSameAs(20); + return repo; + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java new file mode 100644 index 000000000..885c65507 --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2PromotionTest.java @@ -0,0 +1,142 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.linecorp.centraldogma.server.internal.storage.DirectoryBasedStorageManager.SUFFIX_REMOVED; +import static com.linecorp.centraldogma.server.storage.repository.Repository.ALL_PATH; +import static java.util.concurrent.ForkJoinPool.commonPool; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.Mockito.mock; + +import java.io.File; +import java.time.Instant; +import java.util.Map; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.google.common.collect.ImmutableMap; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.Markup; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.server.storage.project.Project; + +class GitRepositoryV2PromotionTest { + + @TempDir + static File repoDir; + + @Test + void promote() { + final GitRepositoryV2 repo = new GitRepositoryV2(mock(Project.class), + new File(repoDir, "test_repo"), commonPool(), + 0L, Author.SYSTEM, null); + final InternalRepository primaryRepo = repo.primaryRepo; + final CommitIdDatabase primaryCommitIdDatabase = primaryRepo.commitIdDatabase(); + assertThat(primaryCommitIdDatabase.firstRevision()).isEqualTo(primaryCommitIdDatabase.headRevision()); + addCommits(repo, 2, 11); + assertThat(primaryCommitIdDatabase.headRevision().major()).isEqualTo(11); + + assertThat(repo.shouldCreateRollingRepository(10, 0)).isNull(); + // Nothing happened because 10 commits are made so far. + assertThat(repo.secondaryRepo).isNull(); + + // Add one more commit to create a rolling repository. + addCommits(repo, 12, 12); + assertThat(repo.shouldCreateRollingRepository(10, 0).major()).isEqualTo(12); + repo.createRollingRepository(new Revision(12), 1, 0); + + assertThat(primaryCommitIdDatabase.headRevision().major()).isEqualTo(12); + final InternalRepository secondaryRepo = repo.secondaryRepo; + assertThat(secondaryRepo).isNotNull(); + assertThat(secondaryRepo.commitIdDatabase().firstRevision().major()).isEqualTo(12); + assertThat(secondaryRepo.commitIdDatabase().headRevision().major()).isEqualTo(12); + assertThat(secondaryRepo.secondCommitCreationTimeInstant()).isNull(); + + // Add ten more commit. + addCommits(repo, 13, 22); + // No changes. + assertThat(repo.shouldCreateRollingRepository(10, 0)).isNull(); + assertThat(primaryCommitIdDatabase.headRevision().major()).isEqualTo(22); + assertThat(secondaryRepo).isEqualTo(repo.secondaryRepo); + assertThat(repo.primaryRepo).isNotSameAs(repo.secondaryRepo); + assertThat(secondaryRepo.commitIdDatabase().firstRevision().major()).isEqualTo(12); + assertThat(secondaryRepo.commitIdDatabase().headRevision().major()).isEqualTo(22); + assertThat(secondaryRepo.secondCommitCreationTimeInstant()).isEqualTo(Instant.ofEpochMilli(13 * 1000)); + + // Add one more commit to create a rolling repository. + addCommits(repo, 23, 23); + assertThat(repo.shouldCreateRollingRepository(10, 0).major()).isEqualTo(23); + repo.createRollingRepository(new Revision(23), 1, 0); + // The secondary repo is promoted to the primary repo. + assertThat(repo.primaryRepo).isSameAs(secondaryRepo); + assertThat(repo.primaryRepo).isNotSameAs(primaryRepo); + // The old primary repo is gone and renamed. + await().until(() -> !primaryRepo.repoDir().exists()); + assertThat(new File(primaryRepo.repoDir() + SUFFIX_REMOVED).exists()).isTrue(); + + final CommitIdDatabase newPrimaryDatabase = repo.primaryRepo.commitIdDatabase(); + assertThat(newPrimaryDatabase.firstRevision().major()).isEqualTo(12); + assertThat(newPrimaryDatabase.headRevision().major()).isEqualTo(23); + + // A new secondary repo is created. + assertThat(repo.secondaryRepo).isNotSameAs(secondaryRepo); + assertThat(repo.secondaryRepo.commitIdDatabase().firstRevision().major()).isEqualTo(23); + assertThat(repo.secondaryRepo.commitIdDatabase().headRevision().major()).isEqualTo(23); + assertThat(repo.secondaryRepo.secondCommitCreationTimeInstant()).isNull(); + + // Add 11 commits so that we are ready to create a rolling repository. + addCommits(repo, 24, 35); + assertThat(repo.shouldCreateRollingRepository(10, 0).major()).isEqualTo(35); + + // Add 5 more commits before creating a rolling repository to check if there are missing commits. + addCommits(repo, 36, 40); + repo.createRollingRepository(new Revision(35), 1, 0); + + assertThat(repo.primaryRepo.commitIdDatabase().firstRevision().major()).isEqualTo(23); + assertThat(repo.primaryRepo.commitIdDatabase().headRevision().major()).isEqualTo(40); + + assertThat(repo.secondaryRepo.commitIdDatabase().firstRevision().major()).isEqualTo(35); + assertThat(repo.secondaryRepo.commitIdDatabase().headRevision().major()).isEqualTo(40); + + for (int i = 36; i < 40; i++) { + assertThat(entries(repo.primaryRepo, i)).containsExactlyInAnyOrderEntriesOf( + entries(repo.primaryRepo, i)); + } + + repo.internalClose(); + } + + static void addCommits(GitRepositoryV2 repo, int start, int end) { + for (int i = start; i <= end; i++) { + final Change change = Change.ofTextUpsert("/file_" + i + ".txt", String.valueOf(i)); + addCommit(repo, i, change); + } + } + + static void addCommit(GitRepositoryV2 repo, int index, Change change) { + repo.commit(Revision.HEAD, index * 1000L, Author.SYSTEM, + "Summary" + index, "Detail", Markup.PLAINTEXT, change).join(); + } + + private static Map> entries(InternalRepository repo, int revision) { + return repo.find(new Revision(revision), ALL_PATH, ImmutableMap.of()); + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2WatchTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2WatchTest.java new file mode 100644 index 000000000..895da0aaf --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryV2WatchTest.java @@ -0,0 +1,69 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryV2HistoryTest.createRepository; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepositoryV2PromotionTest.addCommit; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import java.io.File; +import java.util.concurrent.CompletableFuture; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Revision; + +class GitRepositoryV2WatchTest { + + @TempDir + static File repoDir; + private static GitRepositoryV2 repo; + + @BeforeAll + static void setUp() { + // The repository contains commits from 10 to 20(inclusive). + repo = createRepository(repoDir); + } + + @AfterAll + static void tearDown() { + repo.internalClose(); + } + + @Test + void testWatch() { + assertThat(repo.watch(Revision.INIT, "/file_2.txt").join()).isEqualTo(new Revision(20)); + // Return the head revision even though there was no change for "/file_2.txt" which is committed at + // the revision 2. The current primary repo does not know if there was a commit for the entry between + // the specified(5) revision and the first revision(10) of the repo. So it's safe to return the + // head revision. + assertThat(repo.watch(new Revision(5), "/file_2.txt").join()).isEqualTo(new Revision(20)); + assertThat(repo.watch(new Revision(5), "/file_6.txt").join()).isEqualTo(new Revision(20)); + + final CompletableFuture future = repo.watch(new Revision(10), "/file_2.txt"); + assertThat(future.isDone()).isFalse(); + addCommit(repo, 21, Change.ofTextUpsert("/file_2.txt", String.valueOf(2 + 2))); + + assertThat(future.join()).isEqualTo(new Revision(21)); + // Make sure CommitWatchers has cleared the watch. + await().untilAsserted(() -> assertThat(repo.commitWatchers.watchesMap).isEmpty()); + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepositoryTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepositoryTest.java new file mode 100644 index 000000000..a1ab1fba9 --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/InternalRepositoryTest.java @@ -0,0 +1,70 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.linecorp.centraldogma.server.internal.storage.repository.git.InternalRepository.doRefUpdate; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.RefUpdate; +import org.eclipse.jgit.revwalk.RevWalk; +import org.junit.jupiter.api.Test; + +import com.linecorp.centraldogma.server.storage.StorageException; + +class InternalRepositoryTest { + + @Test + void testDoUpdateRef() throws Exception { + final ObjectId commitId = mock(ObjectId.class); + + // A commit on the mainlane + testDoUpdateRef(Constants.R_TAGS + '1', commitId, false); + testDoUpdateRef(Constants.R_HEADS + Constants.MASTER, commitId, false); + } + + private static void testDoUpdateRef(String ref, ObjectId commitId, boolean tagExists) throws Exception { + final org.eclipse.jgit.lib.Repository jGitRepo = mock(org.eclipse.jgit.lib.Repository.class); + final RevWalk revWalk = mock(RevWalk.class); + final RefUpdate refUpdate = mock(RefUpdate.class); + + lenient().when(jGitRepo.exactRef(ref)).thenReturn(tagExists ? mock(Ref.class) : null); + lenient().when(jGitRepo.updateRef(ref)).thenReturn(refUpdate); + + lenient().when(refUpdate.update(revWalk)).thenReturn(RefUpdate.Result.NEW); + doRefUpdate(jGitRepo, revWalk, ref, commitId); + + when(refUpdate.update(revWalk)).thenReturn(RefUpdate.Result.FAST_FORWARD); + doRefUpdate(jGitRepo, revWalk, ref, commitId); + + when(refUpdate.update(revWalk)).thenReturn(RefUpdate.Result.LOCK_FAILURE); + assertThatThrownBy(() -> doRefUpdate(jGitRepo, revWalk, ref, commitId)) + .isInstanceOf(StorageException.class); + } + + @Test + void testDoUpdateRefOnExistingTag() { + final ObjectId commitId = mock(ObjectId.class); + + assertThatThrownBy(() -> testDoUpdateRef(Constants.R_TAGS + "01/1.0", commitId, true)) + .isInstanceOf(StorageException.class); + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabaseTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabaseTest.java new file mode 100644 index 000000000..476817936 --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RepositoryMetadataDatabaseTest.java @@ -0,0 +1,90 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.linecorp.centraldogma.server.internal.storage.repository.git.RepositoryMetadataDatabase.increment; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; + +import javax.annotation.Nullable; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class RepositoryMetadataDatabaseTest { + + @TempDir + File tempDir; + + @Nullable + private RepositoryMetadataDatabase db; + + @BeforeEach + void setUp() { + db = new RepositoryMetadataDatabase(tempDir, true); + } + + @AfterEach + void tearDown() { + if (db != null) { + db.close(); + } + } + + @Test + void writeAndRead() throws Exception { + final RepositoryMetadataDatabase other = new RepositoryMetadataDatabase(db.rootDir, false); + assertThat(other.primaryRepoDir().getName()).isEqualTo(db.primaryRepoDir().getName()); + } + + @Test + void secondaryDirIsGreaterThanPrimaryByOne() { + assertThat(db.primaryRepoDir().getName()).isEqualTo(tempDir.getName() + "_0000000000"); + assertThat(db.secondaryRepoDir().getName()).isEqualTo(tempDir.getName() + "_0000000001"); + + db.setPrimaryRepoDir(new File(tempDir, tempDir.getName() + "_0000000001")); + assertThat(db.primaryRepoDir().getName()).isEqualTo(tempDir.getName() + "_0000000001"); + assertThat(db.secondaryRepoDir().getName()).isEqualTo(tempDir.getName() + "_0000000002"); + } + + @Test + void nextPrimaryRepoShouldBeGreaterByOne() { + assertThat(db.primaryRepoDir().getName()).isEqualTo(tempDir.getName() + "_0000000000"); + assertThat(db.secondaryRepoDir().getName()).isEqualTo(tempDir.getName() + "_0000000001"); + + assertThatThrownBy(() -> db.setPrimaryRepoDir(new File(tempDir, tempDir.getName() + "_1111111111"))) + .isInstanceOf(AssertionError.class); + } + + @Test + void addOneToSuffixTest() { + String suffix = "0000000000"; + assertThat(increment(suffix)).isEqualTo("0000000001"); + + suffix = "0000000009"; + assertThat(increment(suffix)).isEqualTo("0000000010"); + + suffix = "0000000099"; + assertThat(increment(suffix)).isEqualTo("0000000100"); + + suffix = "1111111111"; + assertThat(increment(suffix)).isEqualTo("1111111112"); + } +} diff --git a/site/src/sphinx/setup-configuration.rst b/site/src/sphinx/setup-configuration.rst index 346c87489..b91b5c532 100644 --- a/site/src/sphinx/setup-configuration.rst +++ b/site/src/sphinx/setup-configuration.rst @@ -47,6 +47,11 @@ defaults: "numMirroringThreads": null, "maxNumFilesPerMirror": null, "maxNumBytesPerMirror": null, + "commitRetentionConfig": { + "minRetentionCommits" : 5000, + "minRetentionDays": 14, + "schedule": "0 0 * * * ?" + }, "writeQuotaPerRepository": { "requestQuota": 5, "timeWindowSeconds": 1 @@ -218,6 +223,26 @@ Core properties - a time windows in seconds. +- ``commitRetention`` + + - the configuration for retaining commits in a repository. Commits are retained for at least + ``minRetentionDays``. If the number of commits is less than ``minRetentionCommits``, commits are + not removed even after ``minRetentionDays`` have passed. + + - ``minRetentionCommits`` (integer) + + - a minimum number of commits that a repository should retain. The number should be greater than or + equal to 5000. Specify ``0`` to retain all commits. + + - ``minRetentionDays`` (integer) + + - a minimum number of days of a commit that a repository should retain. + + - ``schedule`` (string) + + - a `Quartz cron expression `_ + that schedules the job of removing old commits. + - ``accessLogFormat`` (string) - the format to be used for writing an access log. ``common`` and ``combined`` are pre-defined for NCSA