diff --git a/.github/workflows/build-sonar.yml b/.github/workflows/build-sonar.yml index 7db810c201b3..47040c0bd3fa 100644 --- a/.github/workflows/build-sonar.yml +++ b/.github/workflows/build-sonar.yml @@ -21,6 +21,9 @@ on: branches: [ develop, release/** ] types: [opened, synchronize, reopened, labeled] +permissions: + checks: write + jobs: build-sonar: runs-on: k8s-runner-build diff --git a/.gitmodules b/.gitmodules index c12e27d3376b..d7b160cd4d19 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,4 +16,4 @@ [submodule "cdap-ui"] path = cdap-ui url = ../cdap-ui - branch = develop + branch = release/6.11 diff --git a/cdap-api-common/pom.xml b/cdap-api-common/pom.xml index ada8e8f44227..9c6b99ba5949 100644 --- a/cdap-api-common/pom.xml +++ b/cdap-api-common/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-api-common diff --git a/cdap-api-common/src/main/java/io/cdap/cdap/api/exception/ErrorCategory.java b/cdap-api-common/src/main/java/io/cdap/cdap/api/exception/ErrorCategory.java index ce6c85983851..60c4014a01b1 100644 --- a/cdap-api-common/src/main/java/io/cdap/cdap/api/exception/ErrorCategory.java +++ b/cdap-api-common/src/main/java/io/cdap/cdap/api/exception/ErrorCategory.java @@ -16,6 +16,9 @@ package io.cdap.cdap.api.exception; +import java.util.Objects; +import javax.annotation.Nullable; + /** * Class representing the category of an error. * @@ -25,6 +28,7 @@ */ public class ErrorCategory { private final ErrorCategoryEnum errorCategory; + @Nullable private final String subCategory; /** @@ -33,8 +37,7 @@ public class ErrorCategory { * @param errorCategory The category of the error. */ public ErrorCategory(ErrorCategoryEnum errorCategory) { - this.errorCategory = errorCategory; - this.subCategory = null; + this(errorCategory, null); } /** @@ -43,7 +46,7 @@ public ErrorCategory(ErrorCategoryEnum errorCategory) { * @param errorCategory The category of the error. * @param subCategory The sub-category of the error. */ - public ErrorCategory(ErrorCategoryEnum errorCategory, String subCategory) { + public ErrorCategory(ErrorCategoryEnum errorCategory, @Nullable String subCategory) { this.errorCategory = errorCategory; this.subCategory = subCategory; } @@ -52,8 +55,24 @@ public ErrorCategory(ErrorCategoryEnum errorCategory, String subCategory) { * Returns the category of the error. */ public String getErrorCategory() { - return errorCategory == null ? ErrorCategoryEnum.OTHERS.toString() : subCategory == null ? - errorCategory.toString() : String.format("%s-'%s'", errorCategory, subCategory); + return errorCategory == null ? ErrorCategoryEnum.OTHERS.toString() : subCategory == null + ? errorCategory.toString() : String.format("%s-'%s'", errorCategory, subCategory); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof ErrorCategory)) { + return false; + } + + ErrorCategory that = (ErrorCategory) o; + return Objects.equals(this.errorCategory, that.errorCategory) + && Objects.equals(this.subCategory, that.subCategory); + } + + @Override + public int hashCode() { + return Objects.hash(this.errorCategory, this.subCategory); } /* @@ -64,16 +83,26 @@ public String toString() { return getErrorCategory(); } + /** + * Returns the parent category of the error. + */ + public ErrorCategoryEnum getParentCategory() { + return errorCategory == null ? ErrorCategoryEnum.OTHERS : errorCategory; + } + /** * Enum representing the different categories of errors. */ public enum ErrorCategoryEnum { - PLUGIN("Plugin"), + ACCESS("Access"), + DEPROVISIONING("Deprovisioning"), NETWORKING("Networking"), + MACROS("Macros"), + OTHERS("Others"), + PLUGIN("Plugin"), PROVISIONING("Provisioning"), - ACCESS("Access"), SCHEDULES_AND_TRIGGERS("Schedules and Triggers"), - OTHERS("Others"); + STARTING("Starting"); private final String displayName; diff --git a/cdap-api-common/src/main/java/io/cdap/cdap/api/exception/ErrorUtils.java b/cdap-api-common/src/main/java/io/cdap/cdap/api/exception/ErrorUtils.java index cba5e11c397e..b868e89cc6d8 100644 --- a/cdap-api-common/src/main/java/io/cdap/cdap/api/exception/ErrorUtils.java +++ b/cdap-api-common/src/main/java/io/cdap/cdap/api/exception/ErrorUtils.java @@ -125,6 +125,56 @@ public static ActionErrorPair getActionErrorByStatusCode(Integer statusCode) { return new ActionErrorPair("Please ensure there are no network connectivity " + "issues between the proxy/gateway server and the upstream server or try again later.", ErrorType.SYSTEM); + case HttpURLConnection.HTTP_PROXY_AUTH: + return new ActionErrorPair("Proxy authentication required. Please check your " + + "proxy settings and provide valid credentials.", ErrorType.USER); + case HttpURLConnection.HTTP_NOT_ACCEPTABLE: + return new ActionErrorPair("Request cannot be processed in the requested format." + + " Please check the Accept headers.", ErrorType.USER); + case HttpURLConnection.HTTP_GONE: + return new ActionErrorPair("Requested resource is no longer available.", + ErrorType.USER); + case HttpURLConnection.HTTP_LENGTH_REQUIRED: + return new ActionErrorPair("Content-Length header is required. " + + "Please include it in your request.", ErrorType.USER); + case HttpURLConnection.HTTP_ENTITY_TOO_LARGE: + return new ActionErrorPair("Request entity too large. " + + "Please reduce payload size and try again.", ErrorType.USER); + case HttpURLConnection.HTTP_REQ_TOO_LONG: // 414 + return new ActionErrorPair("Request URL is too long. " + + "Consider shortening the URL.", ErrorType.USER); + case HttpURLConnection.HTTP_UNSUPPORTED_TYPE: + return new ActionErrorPair("Unsupported media type. " + + "Please use a supported format and try again.", ErrorType.USER); + case 416: // HTTP 416 Requested Range Not Satisfiable + return new ActionErrorPair("Requested range is not satisfiable. " + + "Please adjust range headers and try again.", ErrorType.USER); + case 417: // HTTP 417 Expectation Failed + return new ActionErrorPair("Expectation failed. " + + "Server cannot meet Expect header requirements.", ErrorType.USER); + case 421: // HTTP 421 Misdirected Request + return new ActionErrorPair("Request was misdirected. " + + "Please try sending it to the correct server.", ErrorType.USER); + case 422: // HTTP 422 Unprocessable Entity + return new ActionErrorPair("Request cannot be processed due to semantic errors. " + + "Please check the request syntax and try again.", ErrorType.USER); + case 423: // HTTP 423 Locked + return new ActionErrorPair("Resource is locked. Please try again later.", + ErrorType.USER); + case 424: // HTTP 424 Failed Dependency + return new ActionErrorPair("Request failed due to a failed dependency. " + + "Please ensure related actions are successful.", ErrorType.USER); + case 426: // HTTP 426 Upgrade Required + return new ActionErrorPair("Upgrade required. " + + "Please use a newer protocol version.", ErrorType.USER); + case 428: // HTTP 428 Precondition Required + return new ActionErrorPair("Request requires preconditions. " + + "Please ensure headers are set correctly.", ErrorType.USER); + case 431: // HTTP 431 Request Header Fields Too Large + return new ActionErrorPair("Request headers are too large. " + + "Please reduce the number or size of headers.", ErrorType.USER); + case 451: // HTTP 451 Unavailable For Legal Reasons + return new ActionErrorPair("Content is restricted due to legal reasons.", ErrorType.USER); default: return new ActionErrorPair(String.format("Request failed with error code: %s", statusCode), ErrorType.UNKNOWN); diff --git a/cdap-api-common/src/main/java/io/cdap/cdap/api/exception/FailureDetailsProvider.java b/cdap-api-common/src/main/java/io/cdap/cdap/api/exception/FailureDetailsProvider.java index c2aac29a017d..e94d012334a5 100644 --- a/cdap-api-common/src/main/java/io/cdap/cdap/api/exception/FailureDetailsProvider.java +++ b/cdap-api-common/src/main/java/io/cdap/cdap/api/exception/FailureDetailsProvider.java @@ -21,6 +21,8 @@ /** * Interface for providing failure details. + * While implementing the interface, please don't forget to add the class the in + * {io.cdap.cdap.logging.ErrorLogsClassifier#ALLOWLIST_CLASSES} list. */ public interface FailureDetailsProvider { @@ -69,4 +71,43 @@ default ErrorCategory getErrorCategory() { default ErrorType getErrorType() { return ErrorType.UNKNOWN; } + + /** + * Returns whether the error is coming from a dependent service. + * + * @return true if the error is a dependency service error, false otherwise. + */ + default boolean isDependency() { + return false; + } + + /** + * Returns the type of error code. + * + * @return the type of error code. + */ + @Nullable + default ErrorCodeType getErrorCodeType() { + return null; + } + + /** + * Returns the error code. + * + * @return the error code. + */ + @Nullable + default String getErrorCode() { + return null; + } + + /** + * Returns the URL to the documentation. + * + * @return the URL to the documentation. + */ + @Nullable + default String getSupportedDocumentationUrl() { + return null; + } } diff --git a/cdap-api-common/src/main/java/io/cdap/cdap/api/exception/ProgramFailureException.java b/cdap-api-common/src/main/java/io/cdap/cdap/api/exception/ProgramFailureException.java index ac22007992d5..6a1818ed7afb 100644 --- a/cdap-api-common/src/main/java/io/cdap/cdap/api/exception/ProgramFailureException.java +++ b/cdap-api-common/src/main/java/io/cdap/cdap/api/exception/ProgramFailureException.java @@ -70,46 +70,39 @@ public String getErrorReason() { @Override public ErrorCategory getErrorCategory() { + if (errorCategory == null) { + return FailureDetailsProvider.super.getErrorCategory(); + } return errorCategory; } @Override public ErrorType getErrorType() { - return errorType == null ? ErrorType.UNKNOWN : errorType; + if (errorType == null) { + return FailureDetailsProvider.super.getErrorType(); + } + return errorType; } - /** - * Returns whether the error is coming from a dependent service. - * - * @return true if the error is a dependency service error, false otherwise. - */ + @Override public boolean isDependency() { return dependency; } - /** - * Returns the type of error code. - * - * @return the type of error code. - */ + @Nullable + @Override public ErrorCodeType getErrorCodeType() { return errorCodeType; } - /** - * Returns the error code. - * - * @return the error code. - */ + @Nullable + @Override public String getErrorCode() { return errorCode; } - /** - * Returns the URL to the documentation. - * - * @return the URL to the documentation. - */ + @Nullable + @Override public String getSupportedDocumentationUrl() { return supportedDocumentationUrl; } diff --git a/cdap-api-common/src/main/java/io/cdap/cdap/api/macro/InvalidMacroException.java b/cdap-api-common/src/main/java/io/cdap/cdap/api/macro/InvalidMacroException.java index 013e147b3ac7..e4194ac278f4 100644 --- a/cdap-api-common/src/main/java/io/cdap/cdap/api/macro/InvalidMacroException.java +++ b/cdap-api-common/src/main/java/io/cdap/cdap/api/macro/InvalidMacroException.java @@ -16,21 +16,76 @@ package io.cdap.cdap.api.macro; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorCategory.ErrorCategoryEnum; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.FailureDetailsProvider; +import javax.annotation.Nullable; + /** * Indicates that there is an invalid macro. */ -public class InvalidMacroException extends RuntimeException { +public class InvalidMacroException extends RuntimeException implements FailureDetailsProvider { + + private final ErrorCategory errorCategory; + /** + * Constructor for {@link InvalidMacroException}. + */ public InvalidMacroException(String message) { - super(message); + this(message, null, null); } + /** + * Constructor for {@link InvalidMacroException}. + */ public InvalidMacroException(String message, Throwable cause) { - super(message, cause); + this(message, cause, null); } + /** + * Constructor for {@link InvalidMacroException}. + */ public InvalidMacroException(Throwable cause) { - super(cause); + this(cause.getMessage(), cause, null); + } + + /** + * Constructor for {@link InvalidMacroException}. + */ + public InvalidMacroException(Throwable cause, ErrorCategory errorCategory) { + this(cause.getMessage(), cause, errorCategory); + } + + /** + * Constructor for {@link InvalidMacroException}. + */ + public InvalidMacroException(String message, ErrorCategory errorCategory) { + this(message, null, errorCategory); + } + + /** + * Constructor for {@link InvalidMacroException}. + */ + public InvalidMacroException(String message, @Nullable Throwable cause, + @Nullable ErrorCategory errorCategory) { + super(message, cause); + this.errorCategory = errorCategory; + } + + + @Override + public ErrorCategory getErrorCategory() { + return errorCategory == null ? new ErrorCategory(ErrorCategoryEnum.MACROS) : errorCategory; + } + + @Override + public ErrorType getErrorType() { + return ErrorType.USER; } + @Override + public String getErrorReason() { + return getMessage(); + } } diff --git a/cdap-api-spark3_2.12/pom.xml b/cdap-api-spark3_2.12/pom.xml index 288b66e349df..38f36caf8b6e 100644 --- a/cdap-api-spark3_2.12/pom.xml +++ b/cdap-api-spark3_2.12/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-api-spark3_2.12 diff --git a/cdap-api/pom.xml b/cdap-api/pom.xml index a7bf5fe2b484..02bf1eebcb1c 100644 --- a/cdap-api/pom.xml +++ b/cdap-api/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-api diff --git a/cdap-api/src/main/java/io/cdap/cdap/api/artifact/ArtifactVersion.java b/cdap-api/src/main/java/io/cdap/cdap/api/artifact/ArtifactVersion.java index b3bd7ee6267c..70ffd72cd0b1 100644 --- a/cdap-api/src/main/java/io/cdap/cdap/api/artifact/ArtifactVersion.java +++ b/cdap-api/src/main/java/io/cdap/cdap/api/artifact/ArtifactVersion.java @@ -36,6 +36,9 @@ public final class ArtifactVersion implements Comparable { private static final String VERSION_REGEX = "(\\d+)(?:\\.(\\d+))?(?:\\.(\\d+))?(?:[.\\-](.*))?$"; private static final Pattern PATTERN = Pattern.compile(VERSION_REGEX); private static final Pattern SUFFIX_PATTERN = Pattern.compile("\\-" + VERSION_REGEX); + // Suffix can contain only alphanumeric characters and hyphen(-) + private static final String SUFFIX_VALIDATION_REGEX = "^[a-zA-Z0-9-.]+$"; + private static final Pattern SUFFIX_VALIDATION_PATTERN = Pattern.compile(SUFFIX_VALIDATION_REGEX); private final String version; private final Integer major; @@ -81,6 +84,14 @@ public ArtifactVersion(String str, boolean matchSuffix) { } } + if (suffix != null) { + Matcher suffixValidationMatcher = SUFFIX_VALIDATION_PATTERN.matcher(suffix); + if (!suffixValidationMatcher.matches()) { + throw new IllegalArgumentException("The suffix of the version is not valid. It should contain only " + + "alphanumeric characters, hyphen(-) and dot(.)"); + } + } + this.version = tmpVersion; this.major = major; this.minor = minor; diff --git a/cdap-api/src/test/java/io/cdap/cdap/api/artifact/ArtifactVersionTest.java b/cdap-api/src/test/java/io/cdap/cdap/api/artifact/ArtifactVersionTest.java index 70e1cf978d17..c26e9e9130ae 100644 --- a/cdap-api/src/test/java/io/cdap/cdap/api/artifact/ArtifactVersionTest.java +++ b/cdap-api/src/test/java/io/cdap/cdap/api/artifact/ArtifactVersionTest.java @@ -87,4 +87,19 @@ private void assertVersion(ArtifactVersion smaller, ArtifactVersion larger) { Assert.assertTrue(smaller.compareTo(larger) < 0); Assert.assertTrue(larger.compareTo(smaller) > 0); } + + @Test + public void testArtifactVersionValidationSuccess() { + // None of them should throw any exception. + new ArtifactVersion("1.5.0"); + new ArtifactVersion("1.5.0-SNAPSHOT"); + new ArtifactVersion("1.5.0-RC1"); + new ArtifactVersion("1.5.0-v2-beta3"); + new ArtifactVersion("8-1.0.16-jar-with-driver-and-dependencies"); + } + + @Test (expected = IllegalArgumentException.class) + public void testArtifactVersionValidationException() { + new ArtifactVersion("1.5.0./../../../tmp"); + } } diff --git a/cdap-app-fabric-tests/pom.xml b/cdap-app-fabric-tests/pom.xml index 6b174338858d..92febabab876 100644 --- a/cdap-app-fabric-tests/pom.xml +++ b/cdap-app-fabric-tests/pom.xml @@ -22,7 +22,7 @@ the License. cdap io.cdap.cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-app-fabric-tests @@ -134,11 +134,14 @@ the License. - org.apache.maven.plugins - maven-deploy-plugin - 2.8 + org.sonatype.central + central-publishing-maven-plugin + true - true + sonatype.release + false + true + true diff --git a/cdap-app-fabric-tests/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/DistributedWorkflowProgramRunnerTest.java b/cdap-app-fabric-tests/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/DistributedWorkflowProgramRunnerTest.java index 9d1e4724c985..997810f1546f 100644 --- a/cdap-app-fabric-tests/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/DistributedWorkflowProgramRunnerTest.java +++ b/cdap-app-fabric-tests/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/DistributedWorkflowProgramRunnerTest.java @@ -55,7 +55,7 @@ import io.cdap.cdap.internal.app.runtime.SimpleProgramOptions; import io.cdap.cdap.internal.app.runtime.SystemArguments; import io.cdap.cdap.logging.guice.LocalLogAppenderModule; -import io.cdap.cdap.messaging.guice.MessagingClientModule; +import io.cdap.cdap.messaging.guice.client.DefaultMessagingClientModule; import io.cdap.cdap.metrics.guice.MetricsClientRuntimeModule; import io.cdap.cdap.metrics.guice.MetricsStoreModule; import io.cdap.cdap.operations.guice.OperationalStatsModule; @@ -282,14 +282,15 @@ private static ProgramRunnerFactory createProgramRunnerFactory(CConfiguration cC new DataSetsModules().getDistributedModules(), new MetricsClientRuntimeModule().getDistributedModules(), new MetricsStoreModule(), - new MessagingClientModule(), + new DefaultMessagingClientModule(), new AuditModule(), CoreSecurityRuntimeModule.getDistributedModule(cConf), new AuthenticationContextModules().getNoOpModule(), new AuthorizationModule(), new AuthorizationEnforcementModule().getMasterModule(), new TwillModule(), - new AppFabricServiceRuntimeModule(cConf).getDistributedModules(), + new AppFabricServiceRuntimeModule(cConf, AppFabricServiceRuntimeModule.ALL_SERVICE_TYPES) + .getDistributedModules(), new ProgramRunnerRuntimeModule().getDistributedModules(), new SecureStoreServerModule(), new OperationalStatsModule(), diff --git a/cdap-app-fabric/pom.xml b/cdap-app-fabric/pom.xml index df6063e10bcc..375f82f279ea 100644 --- a/cdap-app-fabric/pom.xml +++ b/cdap-app-fabric/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-app-fabric @@ -149,6 +149,7 @@ junit junit + 4.13.2 test diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java index 8683d09d5556..3e169cbf07cc 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java @@ -19,6 +19,7 @@ import com.google.common.base.Supplier; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.inject.AbstractModule; import com.google.inject.Inject; import com.google.inject.Injector; @@ -46,6 +47,7 @@ import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.common.conf.Constants.AppFabric; +import io.cdap.cdap.common.conf.Constants.MessagingSystem; import io.cdap.cdap.common.conf.Constants.Service; import io.cdap.cdap.common.feature.DefaultFeatureFlagsProvider; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; @@ -76,6 +78,8 @@ import io.cdap.cdap.gateway.handlers.ProfileHttpHandler; import io.cdap.cdap.gateway.handlers.ProgramLifecycleHttpHandler; import io.cdap.cdap.gateway.handlers.ProgramLifecycleHttpHandlerInternal; +import io.cdap.cdap.gateway.handlers.ProgramRuntimeHttpHandler; +import io.cdap.cdap.gateway.handlers.ProgramScheduleHttpHandler; import io.cdap.cdap.gateway.handlers.ProvisionerHttpHandler; import io.cdap.cdap.gateway.handlers.SourceControlManagementHttpHandler; import io.cdap.cdap.gateway.handlers.TransactionHttpHandler; @@ -106,17 +110,20 @@ import io.cdap.cdap.internal.app.runtime.artifact.PluginFinder; import io.cdap.cdap.internal.app.runtime.schedule.DistributedTimeSchedulerService; import io.cdap.cdap.internal.app.runtime.schedule.ExecutorThreadPool; +import io.cdap.cdap.internal.app.runtime.schedule.LocalScheduleManager; import io.cdap.cdap.internal.app.runtime.schedule.LocalTimeSchedulerService; +import io.cdap.cdap.internal.app.runtime.schedule.RemoteScheduleManager; +import io.cdap.cdap.internal.app.runtime.schedule.ScheduleManager; import io.cdap.cdap.internal.app.runtime.schedule.TimeSchedulerService; import io.cdap.cdap.internal.app.runtime.schedule.store.DatasetBasedTimeScheduleStore; import io.cdap.cdap.internal.app.runtime.schedule.store.TriggerMisfireLogger; import io.cdap.cdap.internal.app.runtime.workflow.BasicWorkflowStateWriter; import io.cdap.cdap.internal.app.runtime.workflow.WorkflowStateWriter; +import io.cdap.cdap.internal.app.services.FlowControlService; import io.cdap.cdap.internal.app.services.LocalRunRecordCorrectorService; import io.cdap.cdap.internal.app.services.NoopRunRecordCorrectorService; import io.cdap.cdap.internal.app.services.ProgramLifecycleService; import io.cdap.cdap.internal.app.services.RunRecordCorrectorService; -import io.cdap.cdap.internal.app.services.RunRecordMonitorService; import io.cdap.cdap.internal.app.services.ScheduledRunRecordCorrectorService; import io.cdap.cdap.internal.app.store.DefaultStore; import io.cdap.cdap.internal.bootstrap.guice.BootstrapModules; @@ -147,10 +154,14 @@ import io.cdap.cdap.internal.tethering.TetheringClientHandler; import io.cdap.cdap.internal.tethering.TetheringHandler; import io.cdap.cdap.internal.tethering.TetheringServerHandler; +import io.cdap.cdap.messaging.server.FetchHandler; +import io.cdap.cdap.messaging.server.MetadataHandler; +import io.cdap.cdap.messaging.server.StoreHandler; import io.cdap.cdap.metadata.LocalPreferencesFetcherInternal; import io.cdap.cdap.metadata.PreferencesFetcher; import io.cdap.cdap.pipeline.PipelineFactory; import io.cdap.cdap.scheduler.CoreSchedulerService; +import io.cdap.cdap.scheduler.NoOpScheduler; import io.cdap.cdap.scheduler.Scheduler; import io.cdap.cdap.securestore.spi.SecretStore; import io.cdap.cdap.security.encryption.guice.DataStorageAeadEncryptionModule; @@ -166,7 +177,10 @@ import io.cdap.http.HttpHandler; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.quartz.SchedulerException; import org.quartz.core.JobRunShellFactory; import org.quartz.core.QuartzScheduler; @@ -184,18 +198,39 @@ */ public final class AppFabricServiceRuntimeModule extends RuntimeModule { + /** + * Service type for App Fabric. + */ + public enum ServiceType { + SERVER, + PROCESSOR, + } + public static final String NOAUTH_ARTIFACT_REPO = "noAuthArtifactRepo"; + public static final Set ALL_SERVICE_TYPES = ImmutableSet.copyOf(ServiceType.values()); + private final CConfiguration cConf; + private final Set serviceTypes; + + /** + * Create AppFabricServiceRuntimeModule for the provided service types. + * + * @param cConf CConfiguration + * @param serviceTypes Service types for which modules should be generated. + */ + public AppFabricServiceRuntimeModule(CConfiguration cConf, ServiceType... serviceTypes) { + this(cConf, new HashSet<>(Arrays.asList(serviceTypes))); + } - @Inject - public AppFabricServiceRuntimeModule(CConfiguration cConf) { + public AppFabricServiceRuntimeModule(CConfiguration cConf, Set serviceTypes) { this.cConf = cConf; + this.serviceTypes = serviceTypes; } @Override public Module getInMemoryModules() { - return Modules.combine(new AppFabricServiceModule(cConf), + return Modules.combine(new AppFabricServiceModule(cConf, serviceTypes), new CapabilityModule(), new NamespaceAdminModule().getInMemoryModules(), new ConfigStoreModule(), @@ -212,42 +247,41 @@ protected void configure() { .in(Scopes.SINGLETON); bind(TimeSchedulerService.class).to(LocalTimeSchedulerService.class) .in(Scopes.SINGLETON); + bind(ScheduleManager.class).to(LocalScheduleManager.class).in(Scopes.SINGLETON); bind(MRJobInfoFetcher.class).to(LocalMRJobInfoFetcher.class); bind(StorageProviderNamespaceAdmin.class).to(LocalStorageProviderNamespaceAdmin.class); bind(UGIProvider.class).toProvider(UgiProviderProvider.class); - Multibinder servicesNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.services.names")); - servicesNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); - - Multibinder processorNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.processor.services.names")); - processorNamesBinder.addBinding().toInstance(Service.APP_FABRIC_PROCESSOR); - - // TODO: Uncomment after CDAP-7688 is resolved - // servicesNamesBinder.addBinding().toInstance(Constants.Service.MESSAGING_SERVICE); - - Multibinder handlerHookNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.handler.hooks")); - handlerHookNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); - - // TODO: Uncomment after CDAP-7688 is resolved - // handlerHookNamesBinder.addBinding().toInstance(Constants.Service.MESSAGING_SERVICE); - - // TODO (CDAP-21112): Move HTTP handler from Appfabric processor to server after fixing - // ProgramRuntimeService and RunRecordMonitorService. - // Remove additional handlers added to server for in-memory module. - Multibinder handlerBinder = Multibinder.newSetBinder( - binder(), HttpHandler.class, Names.named(Constants.AppFabric.SERVER_HANDLERS_BINDING)); - handlerBinder.addBinding().to(BootstrapHttpHandler.class); - handlerBinder.addBinding().to(AppLifecycleHttpHandler.class); - handlerBinder.addBinding().to(AppLifecycleHttpHandlerInternal.class); - handlerBinder.addBinding().to(ProgramLifecycleHttpHandler.class); - handlerBinder.addBinding().to(ProgramLifecycleHttpHandlerInternal.class); - handlerBinder.addBinding().to(WorkflowHttpHandler.class); + if (serviceTypes.contains(ServiceType.SERVER)) { + Multibinder servicesNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.services.names")); + servicesNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); + + Multibinder handlerHookNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.handler.hooks")); + handlerHookNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); + + // For ProgramLifecycleHttpHandlerTest the ProgramRuntimeHttpHandler needs to be present + // in the appfabric service. + Multibinder handlerBinder = Multibinder.newSetBinder( + binder(), HttpHandler.class, Names.named(Constants.AppFabric.SERVER_HANDLERS_BINDING)); + handlerBinder.addBinding().to(ProgramRuntimeHttpHandler.class); + handlerBinder.addBinding().to(ProgramScheduleHttpHandler.class); + handlerBinder.addBinding().to(OperationsDashboardHttpHandler.class); + + // TODO: Uncomment after CDAP-7688 is resolved + // servicesNamesBinder.addBinding().toInstance(Constants.Service.MESSAGING_SERVICE); + // handlerHookNamesBinder.addBinding().toInstance(Constants.Service.MESSAGING_SERVICE); + } + + if (serviceTypes.contains(ServiceType.PROCESSOR)) { + Multibinder processorNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.processor.services.names")); + processorNamesBinder.addBinding().toInstance(Service.APP_FABRIC_PROCESSOR); + } } }); } @@ -255,7 +289,7 @@ protected void configure() { @Override public Module getStandaloneModules() { - return Modules.combine(new AppFabricServiceModule(cConf), + return Modules.combine(new AppFabricServiceModule(cConf, serviceTypes), new CapabilityModule(), new NamespaceAdminModule().getStandaloneModules(), new ConfigStoreModule(), @@ -265,7 +299,8 @@ public Module getStandaloneModules() { new MasterCredentialProviderModule(), new OperationModule(), new DataStorageAeadEncryptionModule(), - BootstrapModules.getFileBasedModule(), + serviceTypes.contains(ServiceType.PROCESSOR) ? BootstrapModules.getFileBasedModule() : + BootstrapModules.getNoOpModule(), new AbstractModule() { @Override protected void configure() { @@ -273,42 +308,41 @@ protected void configure() { .in(Scopes.SINGLETON); bind(TimeSchedulerService.class).to(LocalTimeSchedulerService.class) .in(Scopes.SINGLETON); + bind(ScheduleManager.class).to(LocalScheduleManager.class).in(Scopes.SINGLETON); bind(MRJobInfoFetcher.class).to(LocalMRJobInfoFetcher.class); bind(StorageProviderNamespaceAdmin.class).to(LocalStorageProviderNamespaceAdmin.class); bind(UGIProvider.class).toProvider(UgiProviderProvider.class); - Multibinder servicesNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.services.names")); - servicesNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); - - // for PingHandler - servicesNamesBinder.addBinding().toInstance(Constants.Service.METRICS_PROCESSOR); - servicesNamesBinder.addBinding().toInstance(Constants.Service.LOGSAVER); - servicesNamesBinder.addBinding().toInstance(Constants.Service.TRANSACTION_HTTP); - servicesNamesBinder.addBinding().toInstance(Constants.Service.RUNTIME); - - Multibinder processorNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.processor.services.names")); - processorNamesBinder.addBinding().toInstance(Service.APP_FABRIC_PROCESSOR); - - // TODO: Uncomment after CDAP-7688 is resolved - // servicesNamesBinder.addBinding().toInstance(Constants.Service.MESSAGING_SERVICE); - - Multibinder handlerHookNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.handler.hooks")); - handlerHookNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); - - // for PingHandler - handlerHookNamesBinder.addBinding().toInstance(Constants.Service.METRICS_PROCESSOR); - handlerHookNamesBinder.addBinding().toInstance(Constants.Service.LOGSAVER); - handlerHookNamesBinder.addBinding().toInstance(Constants.Service.TRANSACTION_HTTP); - handlerHookNamesBinder.addBinding().toInstance(Constants.Service.RUNTIME); - - // TODO: Uncomment after CDAP-7688 is resolved - // handlerHookNamesBinder.addBinding().toInstance(Constants.Service.MESSAGING_SERVICE); + if (serviceTypes.contains(ServiceType.SERVER)) { + Multibinder servicesNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.services.names")); + servicesNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); + + // for PingHandler + servicesNamesBinder.addBinding().toInstance(Constants.Service.TRANSACTION_HTTP); + servicesNamesBinder.addBinding().toInstance(Constants.Service.RUNTIME); + + Multibinder handlerHookNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.handler.hooks")); + handlerHookNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); + + // for PingHandler + handlerHookNamesBinder.addBinding().toInstance(Constants.Service.TRANSACTION_HTTP); + handlerHookNamesBinder.addBinding().toInstance(Constants.Service.RUNTIME); + + // TODO: Uncomment after CDAP-7688 is resolved + // servicesNamesBinder.addBinding().toInstance(Constants.Service.MESSAGING_SERVICE); + // handlerHookNamesBinder.addBinding().toInstance(Constants.Service.MESSAGING_SERVICE); + } + + if (serviceTypes.contains(ServiceType.PROCESSOR)) { + Multibinder processorNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.processor.services.names")); + processorNamesBinder.addBinding().toInstance(Service.APP_FABRIC_PROCESSOR); + } } }); } @@ -316,7 +350,7 @@ protected void configure() { @Override public Module getDistributedModules() { - return Modules.combine(new AppFabricServiceModule(cConf, ImpersonationHandler.class), + return Modules.combine(new AppFabricServiceModule(cConf, serviceTypes, ImpersonationHandler.class), new CapabilityModule(), new NamespaceAdminModule().getDistributedModules(), new ConfigStoreModule(), @@ -326,7 +360,8 @@ public Module getDistributedModules() { new MasterCredentialProviderModule(), new OperationModule(), new DataStorageAeadEncryptionModule(), - BootstrapModules.getFileBasedModule(), + serviceTypes.contains(ServiceType.PROCESSOR) ? BootstrapModules.getFileBasedModule() : + BootstrapModules.getNoOpModule(), new AbstractModule() { @Override protected void configure() { @@ -334,6 +369,7 @@ protected void configure() { .in(Scopes.SINGLETON); bind(TimeSchedulerService.class).to(DistributedTimeSchedulerService.class) .in(Scopes.SINGLETON); + bind(ScheduleManager.class).to(RemoteScheduleManager.class).in(Scopes.SINGLETON); bind(MRJobInfoFetcher.class).to(DistributedMRJobInfoFetcher.class); bind(StorageProviderNamespaceAdmin.class) .to(DistributedStorageProviderNamespaceAdmin.class); @@ -342,22 +378,26 @@ protected void configure() { bind(ProgramRunDispatcher.class).to(RemoteProgramRunDispatcher.class) .in(Scopes.SINGLETON); - Multibinder servicesNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.services.names")); - servicesNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); - servicesNamesBinder.addBinding().toInstance(Constants.Service.SECURE_STORE_SERVICE); - - Multibinder handlerHookNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.handler.hooks")); - handlerHookNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); - servicesNamesBinder.addBinding().toInstance(Constants.Service.SECURE_STORE_SERVICE); - - Multibinder processorNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.processor.services.names")); - processorNamesBinder.addBinding().toInstance(Service.APP_FABRIC_PROCESSOR); + if (serviceTypes.contains(ServiceType.SERVER)) { + Multibinder servicesNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.services.names")); + servicesNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); + servicesNamesBinder.addBinding().toInstance(Constants.Service.SECURE_STORE_SERVICE); + + Multibinder handlerHookNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.handler.hooks")); + handlerHookNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); + servicesNamesBinder.addBinding().toInstance(Constants.Service.SECURE_STORE_SERVICE); + } + + if (serviceTypes.contains(ServiceType.PROCESSOR)) { + Multibinder processorNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.processor.services.names")); + processorNamesBinder.addBinding().toInstance(Service.APP_FABRIC_PROCESSOR); + } } }); } @@ -369,11 +409,14 @@ private static final class AppFabricServiceModule extends AbstractModule { private final List> handlerClasses; private final CConfiguration cConf; + private final Set serviceTypes; private AppFabricServiceModule(CConfiguration cConf, + Set serviceTypes, Class... handlerClasses) { - this.handlerClasses = ImmutableList.copyOf(handlerClasses); this.cConf = cConf; + this.serviceTypes = serviceTypes; + this.handlerClasses = ImmutableList.copyOf(handlerClasses); } @Override @@ -420,12 +463,10 @@ protected void configure() { bind(ArtifactStore.class).in(Scopes.SINGLETON); bind(ProfileService.class).in(Scopes.SINGLETON); - bind(RunRecordMonitorService.class).in(Scopes.SINGLETON); + bind(FlowControlService.class).in(Scopes.SINGLETON); bind(ProgramLifecycleService.class).in(Scopes.SINGLETON); bind(SystemAppManagementService.class).in(Scopes.SINGLETON); bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class); - bind(CoreSchedulerService.class).in(Scopes.SINGLETON); - bind(Scheduler.class).to(CoreSchedulerService.class); install(new PrivateModule() { @Override protected void configure() { @@ -462,61 +503,80 @@ protected void configure() { bind(EventWriterProvider.class).to(EventWriterExtensionProvider.class); bind(MetricsProvider.class).to(SparkProgramStatusMetricsProvider.class); - Multibinder handlerBinder = Multibinder.newSetBinder( - binder(), HttpHandler.class, Names.named(Constants.AppFabric.SERVER_HANDLERS_BINDING)); - - CommonHandlers.add(handlerBinder); - handlerBinder.addBinding().to(ConfigHandler.class); - handlerBinder.addBinding().to(VersionHandler.class); - handlerBinder.addBinding().to(UsageHandler.class); - handlerBinder.addBinding().to(InstanceOperationHttpHandler.class); - handlerBinder.addBinding().to(NamespaceHttpHandler.class); - handlerBinder.addBinding().to(SourceControlManagementHttpHandler.class); - // TODO: [CDAP-13355] Move OperationsDashboardHttpHandler into report generation app - handlerBinder.addBinding().to(OperationsDashboardHttpHandler.class); - handlerBinder.addBinding().to(PreferencesHttpHandler.class); - handlerBinder.addBinding().to(PreferencesHttpHandlerInternal.class); - handlerBinder.addBinding().to(ConsoleSettingsHttpHandler.class); - handlerBinder.addBinding().to(TransactionHttpHandler.class); - handlerBinder.addBinding().to(ArtifactHttpHandler.class); - handlerBinder.addBinding().to(ArtifactHttpHandlerInternal.class); - handlerBinder.addBinding().to(WorkflowStatsSLAHttpHandler.class); - handlerBinder.addBinding().to(AuthorizationHandler.class); - handlerBinder.addBinding().to(SecureStoreHandler.class); - handlerBinder.addBinding().to(RemotePrivilegesHandler.class); - handlerBinder.addBinding().to(OperationalStatsHttpHandler.class); - handlerBinder.addBinding().to(ProfileHttpHandler.class); - handlerBinder.addBinding().to(ProvisionerHttpHandler.class); - handlerBinder.addBinding().to(FileFetcherHttpHandlerInternal.class); - handlerBinder.addBinding().to(TetheringHandler.class); - handlerBinder.addBinding().to(TetheringServerHandler.class); - handlerBinder.addBinding().to(TetheringClientHandler.class); - handlerBinder.addBinding().to(AppStateHandler.class); - handlerBinder.addBinding().to(CredentialProviderHttpHandler.class); - handlerBinder.addBinding().to(CredentialProviderHttpHandlerInternal.class); - handlerBinder.addBinding().to(OperationHttpHandler.class); - - FeatureFlagsProvider featureFlagsProvider = new DefaultFeatureFlagsProvider(cConf); - if (Feature.NAMESPACED_SERVICE_ACCOUNTS.isEnabled(featureFlagsProvider)) { - handlerBinder.addBinding().to(GcpWorkloadIdentityHttpHandler.class); - handlerBinder.addBinding().to(GcpWorkloadIdentityHttpHandlerInternal.class); - } + if (serviceTypes.contains(ServiceType.SERVER)) { + Multibinder handlerBinder = Multibinder.newSetBinder( + binder(), HttpHandler.class, Names.named(Constants.AppFabric.SERVER_HANDLERS_BINDING)); + + CommonHandlers.add(handlerBinder); + handlerBinder.addBinding().to(ConfigHandler.class); + handlerBinder.addBinding().to(VersionHandler.class); + handlerBinder.addBinding().to(UsageHandler.class); + handlerBinder.addBinding().to(InstanceOperationHttpHandler.class); + handlerBinder.addBinding().to(NamespaceHttpHandler.class); + handlerBinder.addBinding().to(SourceControlManagementHttpHandler.class); + handlerBinder.addBinding().to(PreferencesHttpHandler.class); + handlerBinder.addBinding().to(PreferencesHttpHandlerInternal.class); + handlerBinder.addBinding().to(ConsoleSettingsHttpHandler.class); + handlerBinder.addBinding().to(TransactionHttpHandler.class); + handlerBinder.addBinding().to(ArtifactHttpHandler.class); + handlerBinder.addBinding().to(ArtifactHttpHandlerInternal.class); + handlerBinder.addBinding().to(WorkflowStatsSLAHttpHandler.class); + handlerBinder.addBinding().to(AuthorizationHandler.class); + handlerBinder.addBinding().to(SecureStoreHandler.class); + handlerBinder.addBinding().to(RemotePrivilegesHandler.class); + handlerBinder.addBinding().to(OperationalStatsHttpHandler.class); + handlerBinder.addBinding().to(ProfileHttpHandler.class); + handlerBinder.addBinding().to(ProvisionerHttpHandler.class); + handlerBinder.addBinding().to(FileFetcherHttpHandlerInternal.class); + handlerBinder.addBinding().to(TetheringHandler.class); + handlerBinder.addBinding().to(TetheringServerHandler.class); + handlerBinder.addBinding().to(TetheringClientHandler.class); + handlerBinder.addBinding().to(AppStateHandler.class); + handlerBinder.addBinding().to(CredentialProviderHttpHandler.class); + handlerBinder.addBinding().to(CredentialProviderHttpHandlerInternal.class); + handlerBinder.addBinding().to(OperationHttpHandler.class); + handlerBinder.addBinding().to(AppLifecycleHttpHandler.class); + handlerBinder.addBinding().to(AppLifecycleHttpHandlerInternal.class); + handlerBinder.addBinding().to(ProgramLifecycleHttpHandler.class); + handlerBinder.addBinding().to(ProgramLifecycleHttpHandlerInternal.class); + handlerBinder.addBinding().to(WorkflowHttpHandler.class); + + if (!cConf.getBoolean(MessagingSystem.MESSAGING_SERVICE_ENABLED)) { + // Add these handlers only if messaging service endpoint doesn't exist and task workers need to + // communicate with messaging service via AppFabric. + handlerBinder.addBinding().to(MetadataHandler.class); + handlerBinder.addBinding().to(StoreHandler.class); + handlerBinder.addBinding().to(FetchHandler.class); + } - for (Class handlerClass : handlerClasses) { - handlerBinder.addBinding().to(handlerClass); + FeatureFlagsProvider featureFlagsProvider = new DefaultFeatureFlagsProvider(cConf); + if (Feature.NAMESPACED_SERVICE_ACCOUNTS.isEnabled(featureFlagsProvider)) { + handlerBinder.addBinding().to(GcpWorkloadIdentityHttpHandler.class); + handlerBinder.addBinding().to(GcpWorkloadIdentityHttpHandlerInternal.class); + } + + for (Class handlerClass : handlerClasses) { + handlerBinder.addBinding().to(handlerClass); + } } - Multibinder processorHandlerBinder = Multibinder.newSetBinder( - binder(), HttpHandler.class, Names.named(AppFabric.PROCESSOR_HANDLERS_BINDING)); - CommonHandlers.add(processorHandlerBinder); - // TODO (CDAP-21112): Move HTTP handler from Appfabric processor to server after fixing - // ProgramRuntimeService and RunRecordMonitorService. - processorHandlerBinder.addBinding().to(BootstrapHttpHandler.class); - processorHandlerBinder.addBinding().to(AppLifecycleHttpHandler.class); - processorHandlerBinder.addBinding().to(AppLifecycleHttpHandlerInternal.class); - processorHandlerBinder.addBinding().to(ProgramLifecycleHttpHandler.class); - processorHandlerBinder.addBinding().to(ProgramLifecycleHttpHandlerInternal.class); - processorHandlerBinder.addBinding().to(WorkflowHttpHandler.class); + if (serviceTypes.contains(ServiceType.PROCESSOR)) { + bind(CoreSchedulerService.class).in(Scopes.SINGLETON); + bind(Scheduler.class).to(CoreSchedulerService.class); + + Multibinder processorHandlerBinder = Multibinder.newSetBinder( + binder(), HttpHandler.class, Names.named(AppFabric.PROCESSOR_HANDLERS_BINDING)); + CommonHandlers.add(processorHandlerBinder); + processorHandlerBinder.addBinding().to(ProgramRuntimeHttpHandler.class); + processorHandlerBinder.addBinding().to(BootstrapHttpHandler.class); + processorHandlerBinder.addBinding().to(ProgramScheduleHttpHandler.class); + // TODO: [CDAP-13355] Move OperationsDashboardHttpHandler into report generation app + // TODO(CDAP-21134): Check feasibility of moving OperationsDashboardHttpHandler to Appfabric Server. + processorHandlerBinder.addBinding().to(OperationsDashboardHttpHandler.class); + } else { + bind(NoOpScheduler.class).in(Scopes.SINGLETON); + bind(Scheduler.class).to(NoOpScheduler.class); + } } @Provides diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/DistributedProgramContainerModule.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/DistributedProgramContainerModule.java index f563902c589e..3a2772a2126d 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/DistributedProgramContainerModule.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/DistributedProgramContainerModule.java @@ -64,8 +64,9 @@ import io.cdap.cdap.logging.guice.TMSLogAppenderModule; import io.cdap.cdap.master.environment.MasterEnvironments; import io.cdap.cdap.master.spi.environment.MasterEnvironment; -import io.cdap.cdap.messaging.client.ClientMessagingService; -import io.cdap.cdap.messaging.guice.MessagingClientModule; +import io.cdap.cdap.messaging.client.DefaultClientMessagingService; +import io.cdap.cdap.messaging.guice.client.DefaultMessagingClientModule; +import io.cdap.cdap.messaging.guice.MessagingServiceModule; import io.cdap.cdap.metadata.MetadataReaderWriterModules; import io.cdap.cdap.metadata.PreferencesFetcher; import io.cdap.cdap.metadata.RemotePreferencesFetcherInternal; @@ -166,7 +167,7 @@ private List getCoreModules() { modules.add(new IOModule()); modules.add(new DFSLocationModule()); modules.add(new MetricsClientRuntimeModule().getDistributedModules()); - modules.add(new MessagingClientModule()); + modules.add(getMessagingModules()); modules.add(new AuditModule()); modules.add(new AuthorizationEnforcementModule().getDistributedModules()); modules.add(new SecureStoreClientModule()); @@ -317,6 +318,18 @@ private Module getAuditLogModules() { return new NoOpAuditLogModule(); } + /** + * Return distributed module of Messaging service only if the program is of SERVICE in SYSTEM namespace. + */ + private Module getMessagingModules() { + if (programRunId.getNamespaceId().equals(NamespaceId.SYSTEM) + && programRunId.getType().equals(ProgramType.SERVICE)) { + return new MessagingServiceModule(cConf); + } + + return new DefaultMessagingClientModule(); + } + /** * A Guice module to provide {@link ProgramStateWriter} binding. In normal same cluster / remote * cluster execution, it just publishes program states to TMS based on the discovery service. For @@ -376,7 +389,7 @@ public ProgramStatePublisher get() { internalAuthenticator); return new MessagingProgramStatePublisher(cConf, - new ClientMessagingService(cConf, remoteClientFactory)); + new DefaultClientMessagingService(cConf, remoteClientFactory)); } } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/MonitorHandlerModule.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/MonitorHandlerModule.java index b487faa37e15..f042af1c0e7a 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/MonitorHandlerModule.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/MonitorHandlerModule.java @@ -29,7 +29,9 @@ import com.google.inject.name.Names; import io.cdap.cdap.api.dataset.module.DatasetModule; import io.cdap.cdap.app.store.ServiceStore; +import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.conf.Constants.MessagingSystem; import io.cdap.cdap.common.twill.MasterServiceManager; import io.cdap.cdap.data2.datafabric.dataset.DatasetExecutorServiceManager; import io.cdap.cdap.data2.datafabric.dataset.MetadataServiceManager; @@ -49,22 +51,23 @@ import io.cdap.cdap.internal.app.services.AppFabricServer; import io.cdap.cdap.logging.run.LogSaverServiceManager; import io.cdap.cdap.messaging.distributed.MessagingServiceManager; -import io.cdap.cdap.metrics.runtime.MetricsProcessorStatusServiceManager; import io.cdap.cdap.metrics.runtime.MetricsServiceManager; import io.cdap.http.HttpHandler; import java.util.Map; /** - * This Guice module providing binding of the {@link MonitorHandler} that get used inside {@link - * AppFabricServer}. + * This Guice module providing binding of the {@link MonitorHandler} that get used inside + * {@link AppFabricServer}. */ public class MonitorHandlerModule extends AbstractModule { private static final String SERVICE_STORE_DS_MODULES = "service.store.ds.modules"; private final boolean isHadoop; + private final CConfiguration cConf; - public MonitorHandlerModule(boolean isHadoop) { + public MonitorHandlerModule(boolean isHadoop, CConfiguration cConf) { this.isHadoop = isHadoop; + this.cConf = cConf; } @Override @@ -111,9 +114,6 @@ private void addNonHadoopBindings(Binder binder) { .toProvider(new NonHadoopMasterServiceManagerProvider(LogSaverServiceManager.class)); mapBinder.addBinding(Constants.Service.TRANSACTION) .toProvider(new NonHadoopMasterServiceManagerProvider(TransactionServiceManager.class)); - mapBinder.addBinding(Constants.Service.METRICS_PROCESSOR) - .toProvider( - new NonHadoopMasterServiceManagerProvider(MetricsProcessorStatusServiceManager.class)); mapBinder.addBinding(Constants.Service.METRICS) .toProvider(new NonHadoopMasterServiceManagerProvider(MetricsServiceManager.class)); mapBinder.addBinding(Constants.Service.APP_FABRIC_HTTP) @@ -126,11 +126,14 @@ private void addNonHadoopBindings(Binder binder) { .toProvider(new NonHadoopMasterServiceManagerProvider(DatasetExecutorServiceManager.class)); mapBinder.addBinding(Constants.Service.METADATA_SERVICE) .toProvider(new NonHadoopMasterServiceManagerProvider(MetadataServiceManager.class)); - mapBinder.addBinding(Constants.Service.MESSAGING_SERVICE) - .toProvider(new NonHadoopMasterServiceManagerProvider(MessagingServiceManager.class)); mapBinder.addBinding(Constants.Service.RUNTIME) .toProvider(new NonHadoopMasterServiceManagerProvider(RuntimeServiceManager.class)); + if (cConf.getBoolean(MessagingSystem.MESSAGING_SERVICE_ENABLED)) { + mapBinder.addBinding(Constants.Service.MESSAGING_SERVICE) + .toProvider(new NonHadoopMasterServiceManagerProvider(MessagingServiceManager.class)); + } + // The ServiceStore uses a special non-TX KV Table. bindDatasetModule(binder, new InMemoryKVTableDefinition.Module()); } @@ -145,11 +148,10 @@ private void addHadoopBindings(Binder binder) { MasterServiceManager.class); mapBinder.addBinding(Constants.Service.LOGSAVER).to(LogSaverServiceManager.class); mapBinder.addBinding(Constants.Service.TRANSACTION).to(TransactionServiceManager.class); - mapBinder.addBinding(Constants.Service.METRICS_PROCESSOR) - .to(MetricsProcessorStatusServiceManager.class); mapBinder.addBinding(Constants.Service.METRICS).to(MetricsServiceManager.class); mapBinder.addBinding(Constants.Service.APP_FABRIC_HTTP).to(AppFabricServiceManager.class); - mapBinder.addBinding(Constants.Service.APP_FABRIC_PROCESSOR).to(AppFabricProcessorManager.class); + mapBinder.addBinding(Constants.Service.APP_FABRIC_PROCESSOR) + .to(AppFabricProcessorManager.class); mapBinder.addBinding(Constants.Service.DATASET_EXECUTOR) .to(DatasetExecutorServiceManager.class); mapBinder.addBinding(Constants.Service.METADATA_SERVICE).to(MetadataServiceManager.class); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/DefaultPreviewRunnerManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/DefaultPreviewRunnerManager.java index 65af0b2fdc6c..a3558cfbe7f9 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/DefaultPreviewRunnerManager.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/DefaultPreviewRunnerManager.java @@ -166,6 +166,12 @@ public void stop(ApplicationId preview) throws Exception { newRunnerService.startAndWait(); } + @Override + public void stop(byte[] pollerInfo) throws Exception { + // This is a no-op in a non-distributed environment because there is no separate + // runner container to terminate. The preview runner is a thread in the same process. + } + /** * Create injector for the given application id. */ diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewDataPublisher.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewDataPublisher.java index 0477bdb9f008..d81c282a6b11 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewDataPublisher.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewDataPublisher.java @@ -30,4 +30,16 @@ public interface PreviewDataPublisher { * @param previewMessage preview message */ void publish(EntityId entityId, PreviewMessage previewMessage); + + /** + * Sets the identifying information for the publisher of the preview data. + * This is typically called once to associate the publisher with a specific preview runner. + * The provided information should be a serialized {@code PreviewRequestPollerInfo} object, + * which contains the unique identity of the runner pod. This information is used by the + * receiving service to authenticate the origin of the messages. + * + * @param publisherInfo a byte array containing the serialized identifying information + * for the publisher (the preview runner). + */ + void setPublisherInfo(byte[] publisherInfo); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewHttpServer.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewHttpServer.java index dd28b2fcf7c3..d93067bb3668 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewHttpServer.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewHttpServer.java @@ -31,8 +31,11 @@ import io.cdap.cdap.common.security.HttpsEnabler; import io.cdap.cdap.internal.app.services.AppFabricServer; import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.http.ChannelPipelineModifier; import io.cdap.http.HttpHandler; import io.cdap.http.NettyHttpService; +import io.netty.channel.ChannelPipeline; +import io.netty.handler.codec.http.HttpContentDecompressor; import java.util.Set; import org.apache.twill.common.Cancellable; import org.apache.twill.discovery.DiscoveryService; @@ -64,7 +67,13 @@ public class PreviewHttpServer extends AbstractIdleService { .setConnectionBacklog(cConf.getInt(Constants.Preview.BACKLOG_CONNECTIONS)) .setExecThreadPoolSize(cConf.getInt(Constants.Preview.EXEC_THREADS)) .setBossThreadPoolSize(cConf.getInt(Constants.Preview.BOSS_THREADS)) - .setWorkerThreadPoolSize(cConf.getInt(Constants.Preview.WORKER_THREADS)); + .setWorkerThreadPoolSize(cConf.getInt(Constants.Preview.WORKER_THREADS)) + .setChannelPipelineModifier(new ChannelPipelineModifier() { + @Override + public void modify(ChannelPipeline pipeline) { + pipeline.addAfter("compressor", "decompressor", new HttpContentDecompressor()); + } + }); if (cConf.getBoolean(Constants.Security.SSL.INTERNAL_ENABLED)) { new HttpsEnabler().configureKeyStore(cConf, sConf).enable(builder); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewManagerModule.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewManagerModule.java index 4978597a0334..c4bf1096aafb 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewManagerModule.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewManagerModule.java @@ -21,6 +21,8 @@ import com.google.inject.multibindings.Multibinder; import com.google.inject.name.Names; import io.cdap.cdap.app.store.preview.PreviewStore; +import io.cdap.cdap.common.conf.CConfiguration; +import io.cdap.cdap.common.conf.Constants.MessagingSystem; import io.cdap.cdap.data.runtime.DataSetsModules; import io.cdap.cdap.data2.datafabric.dataset.RemoteDatasetFramework; import io.cdap.cdap.data2.dataset2.DatasetDefinitionRegistryFactory; @@ -37,6 +39,9 @@ import io.cdap.cdap.internal.app.preview.PreviewDataCleanupService; import io.cdap.cdap.internal.app.preview.PreviewRunStopper; import io.cdap.cdap.internal.app.store.preview.DefaultPreviewStore; +import io.cdap.cdap.messaging.server.FetchHandler; +import io.cdap.cdap.messaging.server.MetadataHandler; +import io.cdap.cdap.messaging.server.StoreHandler; import io.cdap.http.HttpHandler; /** @@ -46,7 +51,10 @@ public class PreviewManagerModule extends PrivateModule { private final boolean distributedRunner; - public PreviewManagerModule(boolean distributedRunner) { + private final CConfiguration cConf; + + public PreviewManagerModule(CConfiguration cConf, boolean distributedRunner) { + this.cConf = cConf; this.distributedRunner = distributedRunner; } @@ -76,6 +84,15 @@ protected void configure() { handlerBinder.addBinding().to(PreviewHttpHandler.class); handlerBinder.addBinding().to(PreviewErrorClassificationHttpHandler.class); handlerBinder.addBinding().to(PreviewHttpHandlerInternal.class); + + if (!cConf.getBoolean(MessagingSystem.MESSAGING_SERVICE_ENABLED)) { + // Add these handlers only if messaging service endpoint doesn't exist and preview runners need to + // communicate with messaging service via preview manager. + handlerBinder.addBinding().to(MetadataHandler.class); + handlerBinder.addBinding().to(StoreHandler.class); + handlerBinder.addBinding().to(FetchHandler.class); + } + CommonHandlers.add(handlerBinder); bind(PreviewHttpServer.class); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewMessage.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewMessage.java index b6fa5f38b1f0..3564d19a398c 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewMessage.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewMessage.java @@ -23,14 +23,14 @@ import io.cdap.cdap.proto.id.ProgramRunId; /** - * A container for messages in the preview topic configured by {@link - * Constants.Preview#MESSAGING_TOPIC}. It carries the message type and the payload as {@link - * JsonElement}. + * A container for messages in the preview topic configured by + * {@link Constants.Preview#MESSAGING_TOPIC}. It carries the message type and the payload as + * {@link JsonElement}. */ public final class PreviewMessage { /** - * The message type + * The message type. */ public enum Type { DATA, @@ -41,13 +41,14 @@ public enum Type { private final Type type; private final EntityId entityId; private final JsonElement payload; + private byte[] publisherInfo; /** * Create an instance of message. * - * @param type type of the message + * @param type type of the message * @param entityId program run id associated with the message - * @param payload the payload + * @param payload the payload */ public PreviewMessage(Type type, EntityId entityId, JsonElement payload) { this.type = type; @@ -72,15 +73,36 @@ public EntityId getEntityId() { /** * Returns the payload by decoding the json to the given type. * - * @param gson the {@link Gson} for decoding the json element + * @param gson the {@link Gson} for decoding the json element * @param objType the resulting object type - * @param the resulting object type + * @param the resulting object type * @return the decode object */ public T getPayload(Gson gson, java.lang.reflect.Type objType) { return gson.fromJson(payload, objType); } + /** + * Returns the serialized information that identifies and authenticates the publisher. This is + * expected to be a serialized {@link PreviewRequestPollerInfo} object. + * + * @return the publisher information byte array, or {@code null} if not set. + */ + public byte[] getPublisherInfo() { + return publisherInfo; + } + + /** + * Sets the identifying information for the publisher of this message. This information is used by + * the receiving service to authenticate the message's origin. + * + * @param publisherInfo a byte array containing the serialized {@link PreviewRequestPollerInfo}, + * which includes the runner's identity and secret token for authentication. + */ + public void setPublisherInfo(byte[] publisherInfo) { + this.publisherInfo = publisherInfo; + } + @Override public String toString() { return "PreviewMessage{" diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewRequest.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewRequest.java index c4ee26eae7a0..97e5ffb9b012 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewRequest.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewRequest.java @@ -32,6 +32,7 @@ public class PreviewRequest { private final ProgramId program; private final AppRequest appRequest; private final Principal principal; + private byte[] runnerInfo; public PreviewRequest(ProgramId program, AppRequest appRequest, @Nullable Principal principal) { @@ -58,6 +59,14 @@ public Principal getPrincipal() { return principal; } + public byte[] getRunnerInfo() { + return runnerInfo; + } + + public void setRunnerInfo(byte[] runnerInfo) { + this.runnerInfo = runnerInfo; + } + private static ProgramId getProgramIdFromRequest(ApplicationId preview, AppRequest request) { PreviewConfig previewConfig = request.getPreview(); if (previewConfig == null) { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewRunnerModule.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewRunnerModule.java index f3d8e85c0524..70111a1a794c 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewRunnerModule.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewRunnerModule.java @@ -53,6 +53,8 @@ import io.cdap.cdap.internal.app.runtime.artifact.PluginFinder; import io.cdap.cdap.internal.app.runtime.artifact.RemoteArtifactRepositoryReaderWithLocalization; import io.cdap.cdap.internal.app.runtime.artifact.RemoteArtifactRepositoryWithLocalization; +import io.cdap.cdap.internal.app.runtime.schedule.RemoteScheduleManager; +import io.cdap.cdap.internal.app.runtime.schedule.ScheduleManager; import io.cdap.cdap.internal.app.runtime.workflow.BasicWorkflowStateWriter; import io.cdap.cdap.internal.app.runtime.workflow.WorkflowStateWriter; import io.cdap.cdap.internal.app.store.DefaultStore; @@ -189,6 +191,7 @@ protected void configure() { expose(PreviewRunner.class); bind(Scheduler.class).to(NoOpScheduler.class); + bind(ScheduleManager.class).to(RemoteScheduleManager.class); bind(DataTracerFactory.class).to(DefaultDataTracerFactory.class); expose(DataTracerFactory.class); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/runtime/AbstractProgramRuntimeService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/runtime/AbstractProgramRuntimeService.java index db1bb11f2dcc..d664d16171ff 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/runtime/AbstractProgramRuntimeService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/runtime/AbstractProgramRuntimeService.java @@ -18,6 +18,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.HashBasedTable; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Table; import com.google.common.io.Closeables; @@ -26,12 +27,14 @@ import com.google.inject.Inject; import io.cdap.cdap.app.deploy.ProgramRunDispatcherContext; import io.cdap.cdap.app.program.ProgramDescriptor; +import io.cdap.cdap.common.BadRequestException; import io.cdap.cdap.common.app.RunIds; import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.common.twill.TwillAppNames; import io.cdap.cdap.internal.app.deploy.ProgramRunDispatcherFactory; import io.cdap.cdap.internal.app.runtime.AbstractListener; +import io.cdap.cdap.internal.app.runtime.ProgramOptionConstants; import io.cdap.cdap.internal.app.runtime.service.SimpleRuntimeInfo; import io.cdap.cdap.proto.InMemoryProgramLiveInfo; import io.cdap.cdap.proto.NotRunningProgramLiveInfo; @@ -46,6 +49,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -59,6 +64,7 @@ import org.apache.twill.api.TwillController; import org.apache.twill.api.TwillRunner; import org.apache.twill.api.TwillRunnerService; +import org.apache.twill.api.logging.LogEntry; import org.apache.twill.common.Threads; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -121,27 +127,38 @@ void setRemoteTwillRunnerService( @Override public final RuntimeInfo run(ProgramDescriptor programDescriptor, ProgramOptions options, RunId runId) { - ProgramRunDispatcherContext dispatcherContext = new ProgramRunDispatcherContext( - programDescriptor, options, runId, - isDistributed()); ProgramId programId = programDescriptor.getProgramId(); ProgramRunId programRunId = programId.run(runId); - DelayedProgramController controller = new DelayedProgramController(programRunId); - RuntimeInfo runtimeInfo = createRuntimeInfo(controller, programId, - dispatcherContext::executeCleanupTasks); - updateRuntimeInfo(runtimeInfo); - executor.execute(() -> { - try { - controller.setProgramController( - programRunDispatcherFactory.getProgramRunDispatcher(programId.getType()) - .dispatchProgram(dispatcherContext)); - } catch (Exception e) { - controller.failed(e); - programStateWriter.error(programRunId, e); - LOG.error("Exception while trying to run program run {}", programRunId, e); + Lock lock = runtimeInfosLock.writeLock(); + lock.lock(); + try { + RuntimeInfo runtimeInfo = lookup(programRunId.getParent(), runId); + if (runtimeInfo != null) { + return runtimeInfo; } - }); - return runtimeInfo; + + ProgramRunDispatcherContext dispatcherContext = new ProgramRunDispatcherContext( + programDescriptor, options, runId, + isDistributed()); + DelayedProgramController controller = new DelayedProgramController(programRunId); + runtimeInfo = createRuntimeInfo(controller, programId, + dispatcherContext::executeCleanupTasks); + updateRuntimeInfo(runtimeInfo); + executor.execute(() -> { + try { + controller.setProgramController( + programRunDispatcherFactory.getProgramRunDispatcher(programId.getType()) + .dispatchProgram(dispatcherContext)); + } catch (Exception e) { + controller.failed(e); + programStateWriter.error(programRunId, e); + LOG.error("Exception while trying to run program run {}", programRunId, e); + } + }); + return runtimeInfo; + } finally { + lock.unlock(); + } } @Override @@ -239,6 +256,28 @@ public List listAll(ProgramType... types) { return runningPrograms; } + @Override + public void resetProgramLogLevels(ProgramId programId, Set loggerNames, + @Nullable String runId) throws Exception { + if (!EnumSet.of(ProgramType.SERVICE, ProgramType.WORKER).contains(programId.getType())) { + throw new BadRequestException( + String.format("Resetting log levels for program type %s is not supported", + programId.getType().getPrettyName())); + } + resetLogLevels(programId, loggerNames, runId); + } + + @Override + public void updateProgramLogLevels(ProgramId programId, Map logLevels, + @Nullable String runId) throws Exception { + if (!EnumSet.of(ProgramType.SERVICE, ProgramType.WORKER).contains(programId.getType())) { + throw new BadRequestException( + String.format("Updating log levels for program type %s is not supported", + programId.getType().getPrettyName())); + } + updateLogLevels(programId, logLevels, runId); + } + @Override protected void startUp() throws Exception { // Limits to at max poolSize number of concurrent program launch. @@ -436,4 +475,79 @@ private void cleanupRuntimeInfo(@Nullable RuntimeInfo info) { Closeables.closeQuietly((Closeable) info); } } + + /** + * Helper method to get the {@link LogLevelUpdater} for the program. + */ + private LogLevelUpdater getLogLevelUpdater(RuntimeInfo runtimeInfo) throws Exception { + ProgramController programController = runtimeInfo.getController(); + if (!(programController instanceof LogLevelUpdater)) { + throw new BadRequestException( + "Update log levels at runtime is only supported in distributed mode"); + } + return ((LogLevelUpdater) programController); + } + + /** + * Helper method to update log levels for Worker or Service. + */ + private void updateLogLevels(ProgramId programId, Map logLevels, + @Nullable String runId) throws Exception { + ProgramRuntimeService.RuntimeInfo runtimeInfo = findRuntimeInfo(programId, runId).values() + .stream() + .findFirst().orElse(null); + if (runtimeInfo != null) { + LogLevelUpdater logLevelUpdater = getLogLevelUpdater(runtimeInfo); + logLevelUpdater.updateLogLevels(logLevels, null); + } + } + + /** + * Helper method to reset log levels for Worker or Service. + */ + private void resetLogLevels(ProgramId programId, Set loggerNames, @Nullable String runId) + throws Exception { + ProgramRuntimeService.RuntimeInfo runtimeInfo = findRuntimeInfo(programId, runId).values() + .stream() + .findFirst().orElse(null); + if (runtimeInfo != null) { + LogLevelUpdater logLevelUpdater = getLogLevelUpdater(runtimeInfo); + logLevelUpdater.resetLogLevels(loggerNames, null); + } + } + + @Override + public void setInstances(ProgramId programId, int instances, int oldInstances) + throws ExecutionException, InterruptedException, BadRequestException { + ProgramRuntimeService.RuntimeInfo runtimeInfo = findRuntimeInfo(programId); + if (runtimeInfo != null) { + runtimeInfo.getController().command(ProgramOptionConstants.INSTANCES, + ImmutableMap.of("runnable", programId.getProgram(), + "newInstances", String.valueOf(instances), + "oldInstances", String.valueOf(oldInstances))).get(); + } + } + + private Map findRuntimeInfo( + ProgramId programId, @Nullable String runId) throws BadRequestException { + + if (runId != null) { + RunId run; + try { + run = RunIds.fromString(runId); + } catch (IllegalArgumentException e) { + throw new BadRequestException("Error parsing run-id.", e); + } + ProgramRuntimeService.RuntimeInfo runtimeInfo = lookup(programId, run); + return runtimeInfo == null ? Collections.emptyMap() + : Collections.singletonMap(run, runtimeInfo); + } + return new HashMap<>(list(programId)); + } + + @Nullable + protected ProgramRuntimeService.RuntimeInfo findRuntimeInfo(ProgramId programId) + throws BadRequestException { + return findRuntimeInfo(programId, null).values().stream().findFirst().orElse(null); + } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/runtime/ProgramRuntimeService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/runtime/ProgramRuntimeService.java index 67bcf04ae730..df9f321d854d 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/runtime/ProgramRuntimeService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/runtime/ProgramRuntimeService.java @@ -18,13 +18,19 @@ import com.google.common.util.concurrent.Service; import io.cdap.cdap.app.program.ProgramDescriptor; +import io.cdap.cdap.common.BadRequestException; import io.cdap.cdap.proto.ProgramLiveInfo; import io.cdap.cdap.proto.ProgramType; import io.cdap.cdap.proto.id.ProgramId; +import io.cdap.cdap.proto.security.StandardPermission; +import io.cdap.cdap.security.spi.authorization.UnauthorizedException; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; import javax.annotation.Nullable; import org.apache.twill.api.RunId; +import org.apache.twill.api.logging.LogEntry; /** * Service for interacting with the runtime system. @@ -47,7 +53,9 @@ interface RuntimeInfo { } /** - * Starts the given program and return a {@link RuntimeInfo} about the running program. + * Runs the given program and return a {@link RuntimeInfo} about the running program. The program + * is run if it is not already running, otherwise the {@link RuntimeInfo} of the already running + * program is returned. * * @param programDescriptor describing the program to run * @param options {@link ProgramOptions} that are needed by the program. @@ -97,4 +105,60 @@ interface RuntimeInfo { * @param types Types of program to check returns List of info about running programs. */ List listAll(ProgramType... types); + + /** + * Reset log levels for the given program. Only supported program types for this action are {@link + * ProgramType#SERVICE} and {@link ProgramType#WORKER}. + * + * @param programId the {@link ProgramId} of the program for which log levels are to be + * reset. + * @param loggerNames the {@link String} set of the logger names to be updated, empty means + * reset for all loggers. + * @param runId the run id of the program. + * @throws InterruptedException if there is an error while asynchronously resetting log + * levels. + * @throws ExecutionException if there is an error while asynchronously resetting log levels. + * @throws UnauthorizedException if the user does not have privileges to reset log levels for + * the specified program. To reset log levels for a program, a user needs {@link + * StandardPermission#UPDATE} on the program. + */ + void resetProgramLogLevels(ProgramId programId, Set loggerNames, @Nullable String runId) throws Exception; + + /** + * Update log levels for the given program. Only supported program types for this action are + * {@link ProgramType#SERVICE} and {@link ProgramType#WORKER}. + * + * @param programId the {@link ProgramId} of the program for which log levels are to be + * updated + * @param logLevels the {@link Map} of the log levels to be updated. + * @param runId the run id of the program. + * @throws InterruptedException if there is an error while asynchronously updating log + * levels. + * @throws ExecutionException if there is an error while asynchronously updating log levels. + * @throws BadRequestException if the log level is not valid or the program type is not + * supported. + * @throws UnauthorizedException if the user does not have privileges to update log levels for + * the specified program. To update log levels for a program, a user needs {@link + * StandardPermission#UPDATE} on the program. + */ + void updateProgramLogLevels(ProgramId programId, Map logLevels, @Nullable String runId) + throws Exception; + + /** + * Set instances for the given program. Only supported program types for this action are {@link + * ProgramType#SERVICE} and {@link ProgramType#WORKER}. + * + * @param programId the {@link ProgramId} of the program for which instances are to be + * updated + * @param instances the number of instances to be updated. + * @param instances the previous number of instances. + * + * @throws InterruptedException if there is an error while asynchronously updating instances + * @throws ExecutionException if there is an error while asynchronously updating instances + * @throws BadRequestException if the number of instances specified is less than 0 + * @throws UnauthorizedException if the user does not have privileges to set instances for the + * specified program. To set instances for a program, a user needs {@link + * StandardPermission#UPDATE} on the program. + */ + void setInstances(ProgramId programId, int instances, int oldInstances) throws Exception; } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/store/Store.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/store/Store.java index 6463e243fdc9..73c030f15e35 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/store/Store.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/store/Store.java @@ -464,6 +464,16 @@ void updateApplicationSourceControlMeta(Map up @Nullable ApplicationMeta getLatest(ApplicationReference appRef); + /** + * Gets the ApplicationId with the latest version for the given ApplicationReference. + * + * @param appRef ApplicationReference + * @return ApplicationId for the latest version + * + * @throws ApplicationNotFoundException if the app was not found for the given application reference. + */ + ApplicationId getLatestApp(ApplicationReference appRef) throws ApplicationNotFoundException; + /** * Scans for the latest applications across all namespaces. * @@ -728,6 +738,14 @@ Collection retrieveSpacedRecords(NamespaceId na */ void deleteAllStates(NamespaceId namespaceId, String appName) throws ApplicationNotFoundException; + /** + * Get the count of latest applications in the namespace. + * + * @param namespaceId NamespaceId of the application. + * @return the count of applications in the namespace + */ + long getApplicationCount(NamespaceId namespaceId); + /** * Ensures the given program exists in the given application spec. * diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/store/preview/PreviewStore.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/store/preview/PreviewStore.java index 052b6df957b2..3b936ce5c545 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/store/preview/PreviewStore.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/store/preview/PreviewStore.java @@ -125,6 +125,20 @@ void setPreviewRequestPollerInfo(ApplicationId applicationId, @Nullable byte[] p @Nullable byte[] getPreviewRequestPollerInfo(ApplicationId applicationId); + /** + * Retrieves the {@link ApplicationId} associated with a given preview runner's poller information. + *

+ * This method uses a secondary index for a fast, direct lookup to find the mapping from a + * {@code pollerInfo} to an {@code ApplicationId}. + * + * @param pollerInfo the {@link PreviewRequestPollerInfo} of the preview runner. + * @return the {@link ApplicationId} associated with the given {@code pollerInfo}, + * or {@code null} if no association is found. A {@code null} return value + * typically indicates that the runner is new and not yet assigned to a preview run. + */ + @Nullable + ApplicationId getApplicationId(byte[] pollerInfo); + /** * Deletes the preview data older than ttl. * diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandler.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandler.java index 89d27b4a2cdb..2f17e4c6b640 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandler.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandler.java @@ -244,7 +244,8 @@ public void getAllApps(HttpRequest request, HttpResponder responder, @QueryParam("nameFilter") String nameFilter, @QueryParam("nameFilterType") NameFilterType nameFilterType, @QueryParam("latestOnly") @DefaultValue("true") Boolean latestOnly, - @QueryParam("sortCreationTime") Boolean sortCreationTime) + @QueryParam("sortCreationTime") Boolean sortCreationTime, + @QueryParam("enableDefaultPagination") @DefaultValue("false") boolean enableDefaultPagination) throws Exception { validateNamespace(namespaceId); @@ -256,13 +257,12 @@ public void getAllApps(HttpRequest request, HttpResponder responder, } } - if (pageSize == null) { + if (pageSize == null && enableDefaultPagination) { pageSize = configuration.getInt(Constants.GET_APPS_DEFAULT_PAGE_SIZE); LOG.debug("Paginating GET apps call with page size as {}", pageSize); } - pageSize = Math.max(pageSize, 0); - if (Optional.ofNullable(pageSize).orElse(0) != 0) { + if (Optional.ofNullable(pageSize).orElse(0) > 0) { Integer finalPageSize = pageSize; JsonPaginatedListResponder.respond(GSON, responder, APP_LIST_PAGINATED_KEY, jsonListResponder -> { @@ -495,6 +495,20 @@ private Set getArtifactScopes(Set artifactScopes) return scopes; } + /** + * Gets count for all application the namespace. + * + *

This API returns the count for latest applications only. + */ + @GET + @Path("/apps/count") + public void getAppsCount(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId) throws Exception { + NamespaceId namespace = validateNamespace(namespaceId); + long count = applicationLifecycleService.getApplicationsCount(namespace); + responder.sendString(HttpResponseStatus.OK, String.valueOf(count)); + } + /** * Updates an existing application. * Deprecated : Unused - just another deploy action after introduction of edit versions diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandlerInternal.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandlerInternal.java index 0b02808f5573..e66680302b36 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandlerInternal.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandlerInternal.java @@ -56,6 +56,7 @@ import java.util.Collections; import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.PUT; @@ -109,19 +110,20 @@ public void getAllAppDetails(HttpRequest request, HttpResponder responder, @QueryParam("pageToken") String pageToken, @QueryParam("pageSize") Integer pageSize, @QueryParam("orderBy") SortOrder orderBy, - @QueryParam("nameFilter") String nameFilter) throws Exception { + @QueryParam("nameFilter") String nameFilter, + @QueryParam("enableDefaultPagination") @DefaultValue("false") boolean enableDefaultPagination + ) throws Exception { NamespaceId namespaceId = new NamespaceId(namespace); if (!namespaceQueryAdmin.exists(namespaceId)) { throw new NamespaceNotFoundException(namespaceId); } - if (pageSize == null) { + if (pageSize == null && enableDefaultPagination) { pageSize = configuration.getInt(Constants.GET_APPS_DEFAULT_PAGE_SIZE); } - pageSize = Math.max(pageSize, 0); - if (Optional.ofNullable(pageSize).orElse(0) != 0) { + if (Optional.ofNullable(pageSize).orElse(0) > 0) { Integer finalPageSize = pageSize; JsonPaginatedListResponder.respond(GSON, responder, APP_LIST_PAGINATED_KEY, jsonListResponder -> { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/ProgramLifecycleHttpHandler.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/ProgramLifecycleHttpHandler.java index 888bad896edf..46c7535a50ac 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/ProgramLifecycleHttpHandler.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/ProgramLifecycleHttpHandler.java @@ -16,115 +16,68 @@ package io.cdap.cdap.gateway.handlers; -import com.google.common.base.Charsets; -import com.google.common.base.Joiner; -import com.google.common.base.Objects; -import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonSyntaxException; import com.google.gson.reflect.TypeToken; import com.google.inject.Inject; import com.google.inject.Singleton; import io.cdap.cdap.api.ProgramSpecification; import io.cdap.cdap.api.app.ApplicationSpecification; -import io.cdap.cdap.api.schedule.Trigger; +import io.cdap.cdap.api.service.ServiceUnavailableException; import io.cdap.cdap.app.mapreduce.MRJobInfoFetcher; -import io.cdap.cdap.app.runtime.ProgramRuntimeService; import io.cdap.cdap.app.store.Store; -import io.cdap.cdap.common.ApplicationNotFoundException; import io.cdap.cdap.common.BadRequestException; import io.cdap.cdap.common.ConflictException; -import io.cdap.cdap.common.NamespaceNotFoundException; import io.cdap.cdap.common.NotFoundException; import io.cdap.cdap.common.NotImplementedException; -import io.cdap.cdap.api.service.ServiceUnavailableException; import io.cdap.cdap.common.app.RunIds; import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.common.discovery.EndpointStrategy; import io.cdap.cdap.common.discovery.RandomEndpointStrategy; import io.cdap.cdap.common.id.Id; -import io.cdap.cdap.common.io.CaseInsensitiveEnumTypeAdapterFactory; import io.cdap.cdap.common.namespace.NamespaceQueryAdmin; import io.cdap.cdap.common.security.AuditDetail; import io.cdap.cdap.common.security.AuditPolicy; import io.cdap.cdap.common.service.ServiceDiscoverable; import io.cdap.cdap.gateway.handlers.util.AbstractAppFabricHttpHandler; -import io.cdap.cdap.internal.app.ApplicationSpecificationAdapter; -import io.cdap.cdap.internal.app.runtime.schedule.ProgramSchedule; -import io.cdap.cdap.internal.app.runtime.schedule.ProgramScheduleRecord; -import io.cdap.cdap.internal.app.runtime.schedule.ProgramScheduleStatus; -import io.cdap.cdap.internal.app.runtime.schedule.SchedulerException; -import io.cdap.cdap.internal.app.runtime.schedule.constraint.ConstraintCodec; -import io.cdap.cdap.internal.app.runtime.schedule.store.Schedulers; -import io.cdap.cdap.internal.app.runtime.schedule.trigger.ProgramStatusTrigger; -import io.cdap.cdap.internal.app.runtime.schedule.trigger.SatisfiableTrigger; -import io.cdap.cdap.internal.app.runtime.schedule.trigger.TriggerCodec; +import io.cdap.cdap.gateway.handlers.util.NamespaceHelper; +import io.cdap.cdap.gateway.handlers.util.ProgramHandlerUtil; import io.cdap.cdap.internal.app.services.ProgramLifecycleService; -import io.cdap.cdap.internal.app.store.ApplicationMeta; import io.cdap.cdap.internal.app.store.RunRecordDetail; -import io.cdap.cdap.internal.schedule.constraint.Constraint; import io.cdap.cdap.proto.BatchProgram; import io.cdap.cdap.proto.BatchProgramCount; import io.cdap.cdap.proto.BatchProgramHistory; import io.cdap.cdap.proto.BatchProgramResult; -import io.cdap.cdap.proto.BatchProgramSchedule; import io.cdap.cdap.proto.BatchProgramStart; import io.cdap.cdap.proto.BatchProgramStatus; -import io.cdap.cdap.proto.BatchRunnable; -import io.cdap.cdap.proto.BatchRunnableInstances; -import io.cdap.cdap.proto.Containers; -import io.cdap.cdap.proto.Instances; import io.cdap.cdap.proto.MRJobInfo; -import io.cdap.cdap.proto.NotRunningProgramLiveInfo; import io.cdap.cdap.proto.ProgramHistory; -import io.cdap.cdap.proto.ProgramLiveInfo; import io.cdap.cdap.proto.ProgramRunStatus; import io.cdap.cdap.proto.ProgramStatus; import io.cdap.cdap.proto.ProgramType; -import io.cdap.cdap.proto.ProtoTrigger; import io.cdap.cdap.proto.RunCountResult; import io.cdap.cdap.proto.RunRecord; -import io.cdap.cdap.proto.ScheduleDetail; -import io.cdap.cdap.proto.ScheduledRuntime; -import io.cdap.cdap.proto.ServiceInstances; import io.cdap.cdap.proto.id.ApplicationId; import io.cdap.cdap.proto.id.ApplicationReference; -import io.cdap.cdap.proto.id.NamespaceId; import io.cdap.cdap.proto.id.ProgramId; import io.cdap.cdap.proto.id.ProgramReference; import io.cdap.cdap.proto.id.ProgramRunId; -import io.cdap.cdap.proto.id.ScheduleId; -import io.cdap.cdap.scheduler.ProgramScheduleService; import io.cdap.cdap.security.spi.authorization.UnauthorizedException; import io.cdap.http.HttpResponder; -import io.netty.buffer.ByteBufInputStream; import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponseStatus; import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; import java.lang.reflect.Type; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.function.Predicate; import java.util.stream.Collectors; -import javax.annotation.Nullable; -import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.POST; @@ -137,7 +90,7 @@ import org.slf4j.LoggerFactory; /** - * {@link io.cdap.http.HttpHandler} to manage program lifecycle for v3 REST APIs + * {@link io.cdap.http.HttpHandler} to manage program lifecycle for v3 REST APIs. */ @Singleton @Path(Constants.Gateway.API_VERSION_3 + "/namespaces/{namespace-id}") @@ -146,66 +99,26 @@ public class ProgramLifecycleHttpHandler extends AbstractAppFabricHttpHandler { private static final Logger LOG = LoggerFactory.getLogger(ProgramLifecycleHttpHandler.class); private static final Type BATCH_PROGRAMS_TYPE = new TypeToken>() { }.getType(); - private static final Type BATCH_RUNNABLES_TYPE = new TypeToken>() { - }.getType(); private static final Type BATCH_STARTS_TYPE = new TypeToken>() { }.getType(); - private static final List NO_CONSTRAINTS = Collections.emptyList(); - - private static final String SCHEDULES = "schedules"; - - /** - * Json serializer/deserializer. - */ - private static final Gson GSON = ApplicationSpecificationAdapter - .addTypeAdapters(new GsonBuilder()) - .registerTypeAdapter(Trigger.class, new TriggerCodec()) - .registerTypeAdapter(SatisfiableTrigger.class, new TriggerCodec()) - .registerTypeAdapter(Constraint.class, new ConstraintCodec()) - .create(); - - /** - * Json serde for decoding request. It uses a case insensitive enum adapter. - */ - private static final Gson DECODE_GSON = ApplicationSpecificationAdapter - .addTypeAdapters(new GsonBuilder()) - .registerTypeAdapterFactory(new CaseInsensitiveEnumTypeAdapterFactory()) - .registerTypeAdapter(Trigger.class, new TriggerCodec()) - .registerTypeAdapter(SatisfiableTrigger.class, new TriggerCodec()) - .registerTypeAdapter(Constraint.class, new ConstraintCodec()) - .create(); - - private final ProgramScheduleService programScheduleService; private final ProgramLifecycleService lifecycleService; private final DiscoveryServiceClient discoveryServiceClient; private final MRJobInfoFetcher mrJobInfoFetcher; private final NamespaceQueryAdmin namespaceQueryAdmin; - - /** - * Store manages non-runtime lifecycle. - */ private final Store store; - /** - * Runtime program service for running and managing programs. - */ - private final ProgramRuntimeService runtimeService; - @Inject - ProgramLifecycleHttpHandler(Store store, ProgramRuntimeService runtimeService, + ProgramLifecycleHttpHandler(Store store, DiscoveryServiceClient discoveryServiceClient, ProgramLifecycleService lifecycleService, MRJobInfoFetcher mrJobInfoFetcher, - NamespaceQueryAdmin namespaceQueryAdmin, - ProgramScheduleService programScheduleService) { + NamespaceQueryAdmin namespaceQueryAdmin) { this.store = store; - this.runtimeService = runtimeService; this.discoveryServiceClient = discoveryServiceClient; this.lifecycleService = lifecycleService; this.mrJobInfoFetcher = mrJobInfoFetcher; this.namespaceQueryAdmin = namespaceQueryAdmin; - this.programScheduleService = programScheduleService; } /** @@ -256,7 +169,7 @@ public void getMapReduceInfo(HttpRequest request, HttpResponder responder, } /** - * Returns status of a type specified by the type{flows,workflows,mapreduce,spark,services,schedules}. + * Returns status of a type specified by the type{flows,workflows,mapreduce,spark,services}. */ @GET @Path("/apps/{app-id}/{program-type}/{program-id}/status") @@ -270,8 +183,7 @@ public void getStatus(HttpRequest request, HttpResponder responder, } /** - * Returns status of a type specified by the type{flows,workflows,mapreduce,spark,services,schedules}. - * + * Returns status of a type specified by the type{flows,workflows,mapreduce,spark,services}. * Deprecated: Only allowing program info retrieval for the latest app version (active app). */ @Deprecated @@ -284,23 +196,7 @@ public void getStatusVersioned(HttpRequest request, HttpResponder responder, @PathParam("program-type") String type, @PathParam("program-id") String programId) throws Exception { ApplicationReference appReference = new ApplicationReference(namespaceId, appId); - if (SCHEDULES.equals(type)) { - JsonObject json = new JsonObject(); - // ScheduleId is versionless (always "-SNAPSHOT") - ScheduleId scheduleId = appReference.app(ApplicationId.DEFAULT_VERSION).schedule(programId); - ApplicationSpecification appSpec = ApplicationId.DEFAULT_VERSION.equals(versionId) - ? Optional.ofNullable(store.getLatest(appReference)).map(ApplicationMeta::getSpec) - .orElse(null) - : store.getApplication(appReference.app(versionId)); - if (appSpec == null) { - throw new NotFoundException(appReference); - } - json.addProperty("status", programScheduleService.getStatus(scheduleId).toString()); - responder.sendJson(HttpResponseStatus.OK, json.toString()); - return; - } - - ProgramType programType = getProgramType(type); + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); ProgramStatus programStatus; if (ApplicationId.DEFAULT_VERSION.equals(versionId)) { programStatus = lifecycleService.getProgramStatus( @@ -311,7 +207,7 @@ public void getStatusVersioned(HttpRequest request, HttpResponder responder, } Map status = ImmutableMap.of("status", programStatus.name()); - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(status)); + responder.sendJson(HttpResponseStatus.OK, ProgramHandlerUtil.toJson(status)); } /** @@ -325,7 +221,7 @@ public void performRunLevelStop(HttpRequest request, HttpResponder responder, @PathParam("program-type") String type, @PathParam("program-id") String programId, @PathParam("run-id") String runId) throws Exception { - ProgramType programType = getProgramType(type); + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); ProgramReference program = new ApplicationReference(namespaceId, appId).program(programType, programId); @@ -378,12 +274,10 @@ public void performActionVersioned(FullHttpRequest request, HttpResponder respon * @param appVersion app version of the program * @param type type of the program * @param programId programId of the program - * @param action action to be performed. The value can be one of enable/disable/suspend/resume - * for schedule and one of start/stop/debug for other program types. "debug" can only be - * applied on type {@link ProgramType#SERVICE} and {@link ProgramType#WORKER} + * @param action action to be performed. The value can be one of start/stop/debug for other + * program types. "debug" can only be applied on type {@link ProgramType#SERVICE} and + * {@link ProgramType#WORKER} * @throws NotFoundException if action is an unknown action - * @throws BadRequestException if type is schedule and action is not any of - * enable/disable/suspend/resume * @throws BadRequestException if action is "start" or "debug" and appVersion is not the * latest version * @throws NotImplementedException if action is debug and type is not {@link @@ -393,22 +287,7 @@ private void doPerformAction(FullHttpRequest request, HttpResponder responder, S String appId, String appVersion, String type, String programId, String action) throws Exception { ApplicationId applicationId = new ApplicationId(namespaceId, appId, appVersion); - if (SCHEDULES.equals(type)) { - ScheduleId scheduleId = applicationId.schedule(programId); - if (action.equals("disable") || action.equals("suspend")) { - programScheduleService.suspend(scheduleId); - } else if (action.equals("enable") || action.equals("resume")) { - programScheduleService.resume(scheduleId); - } else { - throw new BadRequestException( - "Action for schedules may only be 'enable', 'disable', 'suspend', or 'resume' but is'" - + action + "'"); - } - responder.sendJson(HttpResponseStatus.OK, "OK"); - return; - } - - ProgramType programType = getProgramType(type); + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); ProgramReference programReference = new ProgramReference(namespaceId, appId, programType, programId); ProgramId program = applicationId.program(programType, programId); @@ -491,7 +370,7 @@ public void programHistory(HttpRequest request, HttpResponder responder, @QueryParam("end") String endTs, @QueryParam("limit") @DefaultValue("100") final int resultLimit) throws Exception { - ProgramType programType = getProgramType(type); + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); long start = parseAndValidate(startTs, 0L, "start"); long end = parseAndValidate(endTs, Long.MAX_VALUE, "end"); @@ -508,7 +387,7 @@ public void programHistory(HttpRequest request, HttpResponder responder, List records = lifecycleService.getAllRunRecords(programReference, runStatus, start, end, resultLimit, record -> !isTetheredRunRecord(record)); - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(records)); + responder.sendJson(HttpResponseStatus.OK, ProgramHandlerUtil.toJson(records)); } /** @@ -528,7 +407,7 @@ public void programHistoryVersioned(HttpRequest request, HttpResponder responder @QueryParam("end") String endTs, @QueryParam("limit") @DefaultValue("100") final int resultLimit) throws Exception { - ProgramType programType = getProgramType(type); + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); long start = parseAndValidate(startTs, 0L, "start"); long end = parseAndValidate(endTs, Long.MAX_VALUE, "end"); @@ -553,7 +432,7 @@ public void programHistoryVersioned(HttpRequest request, HttpResponder responder } records = records.stream().filter(record -> !isTetheredRunRecord(record)) .collect(Collectors.toList()); - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(records)); + responder.sendJson(HttpResponseStatus.OK, ProgramHandlerUtil.toJson(records)); } private long parseAndValidate(String strVal, long defaultVal, String paramName) @@ -578,12 +457,19 @@ public void programRunRecord(HttpRequest request, HttpResponder responder, @PathParam("app-name") String appName, @PathParam("program-type") String type, @PathParam("program-name") String programName, - @PathParam("run-id") String runid) throws NotFoundException, BadRequestException { - RunRecordDetail runRecordMeta = getRunRecordDetailFromId(namespaceId, appName, type, - programName, runid); + @PathParam("run-id") String runId) throws NotFoundException, BadRequestException { + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); + ProgramReference programRef = new ApplicationReference(namespaceId, appName).program(programType, + programName); + RunRecordDetail runRecordMeta = store.getRun(programRef, runId); + if (runRecordMeta == null) { + throw new NotFoundException( + String.format("No run record found for program %s and runID: %s", programRef, runId)); + } + if (!isTetheredRunRecord(runRecordMeta)) { RunRecord runRecord = RunRecord.builder(runRecordMeta).build(); - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(runRecord)); + responder.sendJson(HttpResponseStatus.OK, ProgramHandlerUtil.toJson(runRecord)); return; } throw new NotFoundException(runRecordMeta.getProgramRunId()); @@ -605,13 +491,13 @@ public void programRunRecordVersioned(HttpRequest request, @PathParam("program-name") String programName, @PathParam("run-id") String runid) throws NotFoundException, BadRequestException { - ProgramType programType = getProgramType(type); + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); ProgramId progId = new ApplicationId(namespaceId, appName, appVersion).program(programType, programName); RunRecordDetail runRecordMeta = store.getRun(progId.run(runid)); if (runRecordMeta != null && !isTetheredRunRecord(runRecordMeta)) { RunRecord runRecord = RunRecord.builder(runRecordMeta).build(); - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(runRecord)); + responder.sendJson(HttpResponseStatus.OK, ProgramHandlerUtil.toJson(runRecord)); return; } throw new NotFoundException(progId.run(runid)); @@ -627,10 +513,9 @@ public void getProgramRuntimeArgs(HttpRequest request, HttpResponder responder, @PathParam("app-name") String appName, @PathParam("program-type") String type, @PathParam("program-name") String programName) throws Exception { - ProgramType programType = getProgramType(type); - String appVersion = getLatestAppVersion(new NamespaceId(namespaceId), appName); - ProgramId programId = new ApplicationId(namespaceId, appName, appVersion).program(programType, - programName); + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); + ProgramId programId = store.getLatestApp(new ApplicationReference(namespaceId, appName)) + .program(programType, programName); getProgramIdRuntimeArgs(programId, responder); } @@ -645,9 +530,8 @@ public void saveProgramRuntimeArgs(FullHttpRequest request, HttpResponder respon @PathParam("app-name") String appName, @PathParam("program-type") String type, @PathParam("program-name") String programName) throws Exception { - ProgramType programType = getProgramType(type); - String appVersion = getLatestAppVersion(new NamespaceId(namespaceId), appName); - ProgramId programId = new ApplicationId(namespaceId, appName, appVersion).program(programType, + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); + ProgramId programId = store.getLatestApp(new ApplicationReference(namespaceId, appName)).program(programType, programName); saveProgramIdRuntimeArgs(programId, request, responder); } @@ -666,7 +550,7 @@ public void getProgramRuntimeArgsVersioned(HttpRequest request, HttpResponder re @PathParam("app-version") String appVersion, @PathParam("program-type") String type, @PathParam("program-name") String programName) throws Exception { - ProgramType programType = getProgramType(type); + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); ProgramId programId = new ApplicationId(namespaceId, appName, appVersion).program(programType, programName); getProgramIdRuntimeArgs(programId, responder); @@ -687,12 +571,12 @@ public void saveProgramRuntimeArgsVersioned(FullHttpRequest request, HttpRespond @PathParam("app-version") String appVersion, @PathParam("program-type") String type, @PathParam("program-name") String programName) throws Exception { - String latestVersion = getLatestAppVersion(new NamespaceId(namespaceId), appName); + String latestVersion = store.getLatestApp(new ApplicationReference(namespaceId, appName)).getVersion(); if (!appVersion.equals(latestVersion) && !ApplicationId.DEFAULT_VERSION.equals(appVersion)) { throw new BadRequestException( "Runtime arguments can only be changed on the latest program version"); } - ProgramType programType = getProgramType(type); + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); ProgramId programId = new ApplicationId(namespaceId, appName, appVersion).program(programType, programName); saveProgramIdRuntimeArgs(programId, request, responder); @@ -701,7 +585,7 @@ public void saveProgramRuntimeArgsVersioned(FullHttpRequest request, HttpRespond private void getProgramIdRuntimeArgs(ProgramId programId, HttpResponder responder) throws Exception { responder.sendJson(HttpResponseStatus.OK, - GSON.toJson(lifecycleService.getRuntimeArgs(programId))); + ProgramHandlerUtil.toJson(lifecycleService.getRuntimeArgs(programId))); } private void saveProgramIdRuntimeArgs(ProgramId programId, FullHttpRequest request, @@ -733,7 +617,7 @@ public void programSpecificationVersioned(HttpRequest request, HttpResponder res @PathParam("app-version") String appVersion, @PathParam("program-type") String type, @PathParam("program-name") String programName) throws Exception { - ProgramType programType = getProgramType(type); + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); ApplicationId application = new ApplicationId(namespaceId, appName, appVersion); ProgramId programId = application.program(programType, programName); ProgramSpecification specification; @@ -747,712 +631,7 @@ public void programSpecificationVersioned(HttpRequest request, HttpResponder res if (specification == null) { throw new NotFoundException(programId); } - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(specification)); - } - - /** - * Update schedules which were suspended between startTimeMillis and endTimeMillis - * - * @param startTimeMillis lower bound in millis of the update time for schedules (inclusive) - * @param endTimeMillis upper bound in millis of the update time for schedules (exclusive) - */ - @PUT - @Path("schedules/re-enable") - public void reEnableSuspendedSchedules(HttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @QueryParam("start-time-millis") long startTimeMillis, - @QueryParam("end-time-millis") long endTimeMillis) throws Exception { - programScheduleService.reEnableSchedules(new NamespaceId(namespaceId), startTimeMillis, - endTimeMillis); - responder.sendStatus(HttpResponseStatus.OK); - } - - /** - * Get schedules containing {@link ProgramStatusTrigger} filtered by triggering program, and - * optionally by triggering program statuses or schedule status - * - * @param triggerNamespaceId namespace of the triggering program in {@link - * ProgramStatusTrigger} - * @param triggerAppName application name of the triggering program in {@link - * ProgramStatusTrigger} - * @param triggerAppVersion application version of the triggering program in {@link - * ProgramStatusTrigger} - * @param triggerProgramType program type of the triggering program in {@link - * ProgramStatusTrigger} - * @param triggerProgramName program name of the triggering program in {@link - * ProgramStatusTrigger} - * @param triggerProgramStatuses comma separated {@link ProgramStatus} in {@link - * ProgramStatusTrigger}. Schedules with {@link ProgramStatusTrigger} triggered by none of the - * {@link ProgramStatus} in triggerProgramStatuses will be filtered out. If not specified, - * schedules will be returned regardless of triggering program status. - * @param scheduleStatus status of the schedule. Can only be one of "SCHEDULED" or - * "SUSPENDED". If specified, only schedules with matching status will be returned. - */ - @GET - @Path("schedules/trigger-type/program-status") - public void getProgramStatusSchedules(HttpRequest request, HttpResponder responder, - @QueryParam("trigger-namespace-id") String triggerNamespaceId, - @QueryParam("trigger-app-name") String triggerAppName, - @QueryParam("trigger-app-version") @DefaultValue(ApplicationId.DEFAULT_VERSION) - String triggerAppVersion, - @QueryParam("trigger-program-type") String triggerProgramType, - @QueryParam("trigger-program-name") String triggerProgramName, - @QueryParam("trigger-program-statuses") String triggerProgramStatuses, - @QueryParam("schedule-status") String scheduleStatus) throws Exception { - if (triggerNamespaceId == null) { - throw new BadRequestException("Must specify trigger-namespace-id as a query param"); - } - if (triggerAppName == null) { - throw new BadRequestException("Must specify trigger-app-name as a query param"); - } - if (triggerProgramType == null) { - throw new BadRequestException("Must specify trigger-program-type as a query param"); - } - if (triggerProgramName == null) { - throw new BadRequestException("Must specify trigger-program-name as a query param"); - } - - ProgramType programType = getProgramType(triggerProgramType); - ProgramScheduleStatus programScheduleStatus; - try { - programScheduleStatus = - scheduleStatus == null ? null : ProgramScheduleStatus.valueOf(scheduleStatus); - } catch (IllegalArgumentException e) { - throw new BadRequestException( - String.format("Invalid schedule status '%s'. Must be one of %s.", - scheduleStatus, Joiner.on(',').join(ProgramScheduleStatus.values())), - e); - } - - ProgramId triggerProgramId = new NamespaceId(triggerNamespaceId) - .app(triggerAppName, triggerAppVersion) - .program(programType, triggerProgramName); - - Set queryProgramStatuses = new HashSet<>(); - if (triggerProgramStatuses != null) { - try { - for (String status : triggerProgramStatuses.split(",")) { - queryProgramStatuses.add(io.cdap.cdap.api.ProgramStatus.valueOf(status)); - } - } catch (Exception e) { - throw new BadRequestException( - String.format("Unable to parse program statuses '%s'. Must be comma separated " - + "valid ProgramStatus names such as COMPLETED, FAILED, KILLED.", - triggerProgramStatuses), e); - } - } else { - // Query for schedules with all the statuses if no query status is specified - Collections.addAll(queryProgramStatuses, io.cdap.cdap.api.ProgramStatus.values()); - } - - List details = programScheduleService.findTriggeredBy(triggerProgramId, - queryProgramStatuses) - .stream() - .filter(record -> programScheduleStatus == null || record.getMeta().getStatus() - .equals(programScheduleStatus)) - .map(ProgramScheduleRecord::toScheduleDetail) - .collect(Collectors.toList()); - responder.sendJson(HttpResponseStatus.OK, - GSON.toJson(details, Schedulers.SCHEDULE_DETAILS_TYPE)); - } - - @GET - @Path("apps/{app-name}/schedules/{schedule-name}") - public void getSchedule(HttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @PathParam("app-name") String appName, - @PathParam("schedule-name") String scheduleName) throws Exception { - doGetSchedule(responder, namespaceId, appName, scheduleName); - } - - /* - * Deprecated: Schedules are versionless. - * */ - @Deprecated - @GET - @Path("apps/{app-name}/versions/{app-version}/schedules/{schedule-name}") - public void getScheduleVersioned(HttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @PathParam("app-name") String appName, - @PathParam("app-version") String appVersion, - @PathParam("schedule-name") String scheduleName) throws Exception { - doGetSchedule(responder, namespaceId, appName, scheduleName); - } - - private void doGetSchedule(HttpResponder responder, String namespace, String app, - String scheduleName) - throws Exception { - ScheduleId scheduleId = new ApplicationId(namespace, app).schedule(scheduleName); - ProgramScheduleRecord record = programScheduleService.getRecord(scheduleId); - ScheduleDetail detail = record.toScheduleDetail(); - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(detail, ScheduleDetail.class)); - } - - /** - * See {@link #getAllSchedulesVersioned(HttpRequest, HttpResponder, String, String, String, - * String, String)} - */ - @GET - @Path("apps/{app-name}/schedules") - public void getAllSchedules(HttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @PathParam("app-name") String appName, - @QueryParam("trigger-type") String triggerType, - @QueryParam("schedule-status") String scheduleStatus) throws Exception { - doGetSchedules(responder, new NamespaceId(namespaceId).app(appName), null, triggerType, - scheduleStatus); - } - - /** - * Get schedules in a given application, optionally filtered by the given {@link - * io.cdap.cdap.proto.ProtoTrigger.Type}. - * - * @param namespaceId namespace of the application to get schedules from - * @param appName name of the application to get schedules from - * @param appVersion version of the application to get schedules from - * @param triggerType trigger type of returned schedules. If not specified, all schedules are - * returned regardless of trigger type - * @param scheduleStatus the status of the schedule, must be values in {@link - * ProgramScheduleStatus}. - * - * Deprecated : Schedules are versionless. - */ - @Deprecated - @GET - @Path("apps/{app-name}/versions/{app-version}/schedules") - public void getAllSchedulesVersioned(HttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @PathParam("app-name") String appName, - @PathParam("app-version") String appVersion, - @QueryParam("trigger-type") String triggerType, - @QueryParam("schedule-status") String scheduleStatus) throws Exception { - doGetSchedules(responder, new NamespaceId(namespaceId).app(appName), null, triggerType, - scheduleStatus); - } - - /** - * Get program schedules - */ - @GET - @Path("/apps/{app-name}/{program-type}/{program-name}/schedules") - public void getProgramSchedules(HttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespace, - @PathParam("app-name") String application, - @PathParam("program-type") String type, - @PathParam("program-name") String program, - @QueryParam("trigger-type") String triggerType, - @QueryParam("schedule-status") String scheduleStatus) throws Exception { - getProgramSchedulesVersioned(request, responder, namespace, application, - ApplicationId.DEFAULT_VERSION, - type, program, triggerType, scheduleStatus); - } - - /** - * Get Workflow schedules - * - * Deprecated : Schedules are versionless. - */ - @Deprecated - @GET - @Path("/apps/{app-name}/versions/{app-version}/{program-type}/{program-name}/schedules") - public void getProgramSchedulesVersioned(HttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespace, - @PathParam("app-name") String application, - @PathParam("app-version") String appVersion, - @PathParam("program-type") String type, - @PathParam("program-name") String program, - @QueryParam("trigger-type") String triggerType, - @QueryParam("schedule-status") String scheduleStatus) throws Exception { - ProgramType programType = getProgramType(type); - if (programType.getSchedulableType() == null) { - throw new BadRequestException("Program type " + programType + " cannot have schedule"); - } - - ProgramId programId = new ApplicationId(namespace, application).program(programType, program); - doGetSchedules(responder, new NamespaceId(namespace).app(application), programId, - triggerType, scheduleStatus); - } - - private void doGetSchedules(HttpResponder responder, ApplicationId applicationId, - @Nullable ProgramId programId, @Nullable String triggerTypeStr, - @Nullable String statusStr) throws Exception { - ApplicationSpecification appSpec = Optional.ofNullable( - store.getLatest(applicationId.getAppReference())) - .map(ApplicationMeta::getSpec) - .orElse(null); - if (appSpec == null) { - throw new NotFoundException(applicationId); - } - ProgramScheduleStatus status; - try { - status = statusStr == null ? null : ProgramScheduleStatus.valueOf(statusStr); - } catch (IllegalArgumentException e) { - throw new BadRequestException( - String.format("Invalid schedule status '%s'. Must be one of %s.", - statusStr, Joiner.on(',').join(ProgramScheduleStatus.values())), e); - } - - ProtoTrigger.Type triggerType; - try { - triggerType = - triggerTypeStr == null ? null : ProtoTrigger.Type.valueOfCategoryName(triggerTypeStr); - } catch (IllegalArgumentException e) { - throw new BadRequestException(e.getMessage(), e); - } - - Predicate predicate = record -> true; - if (status != null) { - predicate = predicate.and(record -> record.getMeta().getStatus().equals(status)); - } - if (triggerType != null) { - predicate = predicate.and( - record -> record.getSchedule().getTrigger().getType().equals(triggerType)); - } - - Collection schedules; - if (programId != null) { - if (!appSpec.getProgramsByType(programId.getType().getApiProgramType()) - .contains(programId.getProgram())) { - throw new NotFoundException(programId); - } - schedules = programScheduleService.list(programId, predicate); - } else { - schedules = programScheduleService.list(applicationId, predicate); - } - - List details = schedules.stream() - .map(ProgramScheduleRecord::toScheduleDetail) - .collect(Collectors.toList()); - responder.sendJson(HttpResponseStatus.OK, - GSON.toJson(details, Schedulers.SCHEDULE_DETAILS_TYPE)); - } - - /** - * Returns the previous runtime when the scheduled program ran. - */ - @GET - @Path("/apps/{app-name}/{program-type}/{program-name}/previousruntime") - public void getPreviousScheduledRunTime(HttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @PathParam("app-name") String appId, - @PathParam("program-type") String type, - @PathParam("program-name") String program) throws Exception { - ProgramType programType = getProgramType(type); - String appVersion = getLatestAppVersion(new NamespaceId(namespaceId), appId); - handleScheduleRunTime(responder, - new NamespaceId(namespaceId).app(appId, appVersion).program(programType, program), - true); - } - - /** - * Returns next scheduled runtime of a workflow. - */ - @GET - @Path("/apps/{app-name}/{program-type}/{program-name}/nextruntime") - public void getNextScheduledRunTime(HttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @PathParam("app-name") String appId, - @PathParam("program-type") String type, - @PathParam("program-name") String program) throws Exception { - ProgramType programType = getProgramType(type); - String appVersion = getLatestAppVersion(new NamespaceId(namespaceId), appId); - handleScheduleRunTime(responder, - new NamespaceId(namespaceId).app(appId, appVersion).program(programType, program), - false); - } - - private void handleScheduleRunTime(HttpResponder responder, ProgramId programId, - boolean previousRuntimeRequested) throws Exception { - try { - lifecycleService.ensureProgramExists(programId); - responder.sendJson(HttpResponseStatus.OK, - GSON.toJson(getScheduledRunTimes(programId, previousRuntimeRequested))); - } catch (SecurityException e) { - responder.sendStatus(HttpResponseStatus.UNAUTHORIZED); - } - } - - /** - * Returns the previous scheduled run time for all programs that are passed into the data. The - * data is an array of JSON objects where each object must contain the following three elements: - * appId, programType, and programId (flow name, service name, etc.). - *

- * Example input: - * 


-   * [{"appId": "App1", "programType": "Workflow", "programId": "WF1"},
-   * {"appId": "App1", "programType": "Workflow", "programId": "WF2"}]
-   *
- *

- * The response will be an array of JsonObjects each of which will contain the three input - * parameters as well as a "schedules" field, which is a list of {@link ScheduledRuntime} object. - *

- * If an error occurs in the input (for the example above, App1 does not exist), then all - * JsonObjects for which the parameters have a valid status will have the status field but all - * JsonObjects for which the parameters do not have a valid status will have an error message and - * statusCode. - */ - @POST - @Path("/previousruntime") - public void batchPreviousRunTimes(FullHttpRequest request, - HttpResponder responder, - @PathParam("namespace-id") String namespaceId) throws Exception { - List batchPrograms = validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE); - responder.sendJson(HttpResponseStatus.OK, - GSON.toJson(batchRunTimes(namespaceId, batchPrograms, true))); - } - - /** - * Returns the next scheduled run time for all programs that are passed into the data. The data is - * an array of JSON objects where each object must contain the following three elements: appId, - * programType, and programId (flow name, service name, etc.). - *

- * Example input: - * 


-   * [{"appId": "App1", "programType": "Workflow", "programId": "WF1"},
-   * {"appId": "App1", "programType": "Workflow", "programId": "WF2"}]
-   *
- *

- * The response will be an array of JsonObjects each of which will contain the three input - * parameters as well as a "schedules" field, which is a list of {@link ScheduledRuntime} object. - *

- * If an error occurs in the input (for the example above, App1 does not exist), then all - * JsonObjects for which the parameters have a valid status will have the status field but all - * JsonObjects for which the parameters do not have a valid status will have an error message and - * statusCode. - */ - @POST - @Path("/nextruntime") - public void batchNextRunTimes(FullHttpRequest request, - HttpResponder responder, - @PathParam("namespace-id") String namespaceId) throws Exception { - List batchPrograms = validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE); - responder.sendJson(HttpResponseStatus.OK, - GSON.toJson(batchRunTimes(namespaceId, batchPrograms, false))); - } - - /** - * Fetches scheduled run times for a set of programs. - * - * @param namespace namespace of the programs - * @param programs the list of programs to fetch scheduled run times - * @param previous {@code true} to get the previous scheduled times; {@code false} to get the - * next scheduled times - * @return a list of {@link BatchProgramSchedule} containing the result - * @throws SchedulerException if failed to fetch schedules - */ - private List batchRunTimes(String namespace, - Collection programs, - boolean previous) throws Exception { - List programReferences = programs.stream() - .map(p -> new ProgramReference(namespace, p.getAppId(), p.getProgramType(), - p.getProgramId())) - .collect(Collectors.toList()); - Map programMap = store.getPrograms(programReferences); - - List result = new ArrayList<>(); - for (ProgramReference programReference : programReferences) { - if (programMap.containsKey(programReference)) { - ProgramId programId = programMap.get(programReference); - result.add(new BatchProgramSchedule(programId, HttpResponseStatus.OK.code(), null, - getScheduledRunTimes(programId, previous))); - } else { - result.add(new BatchProgramSchedule(programReference, HttpResponseStatus.NOT_FOUND.code(), - new NotFoundException(programReference).getMessage(), null)); - } - } - return result; - } - - /** - * Returns a list of {@link ScheduledRuntime} for the given program. - * - * @param programId the program to fetch schedules for - * @param previous {@code true} to get the previous scheduled times; {@code false} to get the - * next scheduled times - * @return a list of {@link ScheduledRuntime} - * @throws SchedulerException if failed to fetch the schedule - */ - private List getScheduledRunTimes(ProgramId programId, - boolean previous) throws Exception { - if (programId.getType().getSchedulableType() == null) { - throw new BadRequestException("Program " + programId + " cannot have schedule"); - } - - if (previous) { - return programScheduleService.getPreviousScheduledRuntimes(programId); - } else { - return programScheduleService.getNextScheduledRuntimes(programId); - } - } - - @PUT - @Path("apps/{app-name}/schedules/{schedule-name}") - @AuditPolicy(AuditDetail.REQUEST_BODY) - public void addSchedule(FullHttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @PathParam("app-name") String appName, - @PathParam("schedule-name") String scheduleName) - throws Exception { - doAddSchedule(request, responder, namespaceId, appName, scheduleName); - } - - /* - * Deprecated : Schedules are versionless. - * */ - @Deprecated - @PUT - @Path("apps/{app-name}/versions/{app-version}/schedules/{schedule-name}") - @AuditPolicy(AuditDetail.REQUEST_BODY) - public void addScheduleVersioned(FullHttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @PathParam("app-name") String appName, - @PathParam("app-version") String appVersion, - @PathParam("schedule-name") String scheduleName) - throws Exception { - // Schedules are versionless (all are "-SNAPSHOT" version) - doAddSchedule(request, responder, namespaceId, appName, scheduleName); - } - - private void doAddSchedule(FullHttpRequest request, HttpResponder responder, String namespace, - String appName, - String scheduleName) throws Exception { - - final ApplicationId applicationId = new ApplicationId(namespace, appName); - ScheduleDetail scheduleFromRequest = readScheduleDetailBody(request, scheduleName); - - if (scheduleFromRequest.getProgram() == null) { - throw new BadRequestException("No program was specified for the schedule"); - } - if (scheduleFromRequest.getProgram().getProgramType() == null) { - throw new BadRequestException("No program type was specified for the schedule"); - } - if (scheduleFromRequest.getProgram().getProgramName() == null) { - throw new BadRequestException("No program name was specified for the schedule"); - } - if (scheduleFromRequest.getTrigger() == null) { - throw new BadRequestException("No trigger was specified for the schedule"); - } - ProgramType programType = ProgramType.valueOfSchedulableType( - scheduleFromRequest.getProgram().getProgramType()); - String programName = scheduleFromRequest.getProgram().getProgramName(); - ProgramId programId = applicationId.program(programType, programName); - - // Schedules are versionless - lifecycleService.ensureLatestProgramExists(programId.getProgramReference()); - - String description = Objects.firstNonNull(scheduleFromRequest.getDescription(), ""); - Map properties = Objects.firstNonNull(scheduleFromRequest.getProperties(), - Collections.emptyMap()); - List constraints = Objects.firstNonNull( - scheduleFromRequest.getConstraints(), NO_CONSTRAINTS); - long timeoutMillis = - Objects.firstNonNull(scheduleFromRequest.getTimeoutMillis(), - Schedulers.JOB_QUEUE_TIMEOUT_MILLIS); - ProgramSchedule schedule = new ProgramSchedule(scheduleName, description, programId, properties, - scheduleFromRequest.getTrigger(), constraints, timeoutMillis); - programScheduleService.add(schedule); - responder.sendStatus(HttpResponseStatus.OK); - } - - @POST - @Path("apps/{app-name}/schedules/{schedule-name}/update") - @AuditPolicy(AuditDetail.REQUEST_BODY) - public void updateSchedule(FullHttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @PathParam("app-name") String appName, - @PathParam("schedule-name") String scheduleName) throws Exception { - doUpdateSchedule(request, responder, namespaceId, appName, scheduleName); - } - - /* - * Deprecated : Schedules are versionless. - * */ - @Deprecated - @POST - @Path("apps/{app-name}/versions/{app-version}/schedules/{schedule-name}/update") - @AuditPolicy(AuditDetail.REQUEST_BODY) - public void updateScheduleVersioned(FullHttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @PathParam("app-name") String appName, - @PathParam("app-version") String appVersion, - @PathParam("schedule-name") String scheduleName) throws Exception { - doUpdateSchedule(request, responder, namespaceId, appName, scheduleName); - } - - private void doUpdateSchedule(FullHttpRequest request, HttpResponder responder, - String namespaceId, String appId, - String scheduleName) throws Exception { - - ScheduleId scheduleId = new ApplicationId(namespaceId, appId).schedule(scheduleName); - ScheduleDetail scheduleDetail = readScheduleDetailBody(request, scheduleName); - - programScheduleService.update(scheduleId, scheduleDetail); - responder.sendStatus(HttpResponseStatus.OK); - } - - private ScheduleDetail readScheduleDetailBody(FullHttpRequest request, String scheduleName) - throws BadRequestException, IOException { - - JsonElement json; - try (Reader reader = new InputStreamReader(new ByteBufInputStream(request.content()), - Charsets.UTF_8)) { - // The schedule spec in the request body does not contain the program information - json = DECODE_GSON.fromJson(reader, JsonElement.class); - } catch (IOException e) { - throw new IOException("Error reading request body", e); - } catch (JsonSyntaxException e) { - throw new BadRequestException("Request body is invalid json: " + e.getMessage()); - } - if (!json.isJsonObject()) { - throw new BadRequestException( - "Expected a json object in the request body but received " + GSON.toJson(json)); - } - ScheduleDetail scheduleDetail; - try { - scheduleDetail = DECODE_GSON.fromJson(json, ScheduleDetail.class); - } catch (JsonSyntaxException e) { - throw new BadRequestException( - "Error parsing request body as a schedule specification: " + e.getMessage()); - } - - // If the schedule name is present in the request body, it should match the name in path params - if (scheduleDetail.getName() != null && !scheduleName.equals(scheduleDetail.getName())) { - throw new BadRequestException(String.format( - "Schedule name in the body of the request (%s) does not match the schedule name in the path parameter (%s)", - scheduleDetail.getName(), scheduleName)); - } - return scheduleDetail; - } - - @DELETE - @Path("apps/{app-name}/schedules/{schedule-name}") - public void deleteSchedule(HttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @PathParam("app-name") String appName, - @PathParam("schedule-name") String scheduleName) throws Exception { - doDeleteSchedule(responder, namespaceId, appName, scheduleName); - } - - /* - * Deprecated : Schedules are versionless. - * */ - @Deprecated - @DELETE - @Path("apps/{app-name}/versions/{app-version}/schedules/{schedule-name}") - public void deleteScheduleVersioned(HttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @PathParam("app-name") String appName, - @PathParam("app-version") String appVersion, - @PathParam("schedule-name") String scheduleName) throws Exception { - doDeleteSchedule(responder, namespaceId, appName, scheduleName); - } - - private void doDeleteSchedule(HttpResponder responder, String namespaceId, String appName, - String scheduleName) throws Exception { - ScheduleId scheduleId = new ApplicationId(namespaceId, appName).schedule(scheduleName); - programScheduleService.delete(scheduleId); - responder.sendStatus(HttpResponseStatus.OK); - } - - /** - * Update the log level for a running program according to the request body. Currently supported - * program types are {@link ProgramType#SERVICE} and {@link ProgramType#WORKER}. The request body - * is expected to contain a map of log levels, where key is loggername, value is one of the valid - * {@link org.apache.twill.api.logging.LogEntry.Level} or null. - * - */ - @PUT - @Path("/apps/{app-name}/{program-type}/{program-name}/runs/{run-id}/loglevels") - @AuditPolicy(AuditDetail.REQUEST_BODY) - public void updateProgramLogLevels(FullHttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespace, - @PathParam("app-name") String appName, - @PathParam("program-type") String type, - @PathParam("program-name") String programName, - @PathParam("run-id") String runId) throws Exception { - RunRecordDetail run = getRunRecordDetailFromId(namespace, appName, type, programName, runId); - updateLogLevels(request, responder, namespace, appName, run.getVersion(), type, programName, - runId); - } - - /** - * Update the log level for a running program according to the request body. - * Deprecated : Run-id is sufficient to identify a program run. - */ - @Deprecated - @PUT - @Path("/apps/{app-name}/versions/{app-version}/{program-type}/{program-name}/runs/{run-id}/loglevels") - @AuditPolicy(AuditDetail.REQUEST_BODY) - public void updateProgramLogLevelsVersioned(FullHttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespace, - @PathParam("app-name") String appName, - @PathParam("app-version") String appVersion, - @PathParam("program-type") String type, - @PathParam("program-name") String programName, - @PathParam("run-id") String runId) throws Exception { - updateLogLevels(request, responder, namespace, appName, appVersion, type, programName, runId); - } - - /** - * Reset the log level for a running program back to where it starts. Currently supported program - * types are {@link ProgramType#SERVICE} and {@link ProgramType#WORKER}. The request body can - * either be empty, which will reset all loggers for the program, or contain a list of logger - * names, which will reset for these particular logger names for the program. - */ - @POST - @Path("/apps/{app-name}/{program-type}/{program-name}/runs/{run-id}/resetloglevels") - @AuditPolicy(AuditDetail.REQUEST_BODY) - public void resetProgramLogLevels(FullHttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespace, - @PathParam("app-name") String appName, - @PathParam("program-type") String type, - @PathParam("program-name") String programName, - @PathParam("run-id") String runId) throws Exception { - RunRecordDetail run = getRunRecordDetailFromId(namespace, appName, type, programName, runId); - resetLogLevels(request, responder, namespace, appName, run.getVersion(), type, programName, - runId); - } - - /** - * Reset the log level for a running program back to where it starts. - * - * Deprecated : Run-id is sufficient to identify a program run. - */ - @POST - @Path("/apps/{app-name}/versions/{app-version}/{program-type}/{program-name}/runs/{run-id}/resetloglevels") - @AuditPolicy(AuditDetail.REQUEST_BODY) - public void resetProgramLogLevelsVersioned(FullHttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespace, - @PathParam("app-name") String appName, - @PathParam("app-version") String appVersion, - @PathParam("program-type") String type, - @PathParam("program-name") String programName, - @PathParam("run-id") String runId) throws Exception { - resetLogLevels(request, responder, namespace, appName, appVersion, type, programName, runId); - } - - /** - * Fetches the run record for particular run of a program without version. - * - * @param namespace namespace id - * @param appName application name - * @param type program type - * @param programName program name - * @param runId the run id - * @return run record for the specified program and runRef, null if not found - */ - private RunRecordDetail getRunRecordDetailFromId(String namespace, String appName, - String type, String programName, String runId) - throws BadRequestException, NotFoundException { - ProgramType programType = getProgramType(type); - ProgramReference programRef = new ApplicationReference(namespace, appName).program(programType, - programName); - RunRecordDetail runRecordMeta = store.getRun(programRef, runId); - if (runRecordMeta == null) { - throw new NotFoundException( - String.format("No run record found for program %s and runID: %s", programRef, runId)); - } - return runRecordMeta; + responder.sendJson(HttpResponseStatus.OK, ProgramHandlerUtil.toJson(specification)); } /** @@ -1488,7 +667,7 @@ private RunRecordDetail getRunRecordDetailFromId(String namespace, String appNam @AuditPolicy(AuditDetail.REQUEST_BODY) public void getStatuses(FullHttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception { - List batchPrograms = validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE); + List batchPrograms = ProgramHandlerUtil.validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE); List programs = batchPrograms.stream() .map(p -> new ProgramReference(namespaceId, p.getAppId(), p.getProgramType(), p.getProgramId())) @@ -1512,7 +691,7 @@ public void getStatuses(FullHttpRequest request, HttpResponder responder, null, statuses.get(programId).name())); } } - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(result)); + responder.sendJson(HttpResponseStatus.OK, ProgramHandlerUtil.toJson(result)); } /** @@ -1548,13 +727,11 @@ public void getStatuses(FullHttpRequest request, HttpResponder responder, @AuditPolicy({AuditDetail.REQUEST_BODY, AuditDetail.RESPONSE_BODY}) public void stopPrograms(FullHttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception { - List programs = validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE); + List programs = ProgramHandlerUtil.validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE); List issuedStops = new ArrayList<>(programs.size()); for (final BatchProgram program : programs) { - ApplicationId applicationId = new ApplicationId(namespaceId, program.getAppId(), - getLatestAppVersion(new NamespaceId(namespaceId), - program.getAppId())); + ApplicationId applicationId = store.getLatestApp(new ApplicationReference(namespaceId, program.getAppId())); ProgramId programId = new ProgramId(applicationId, program.getProgramType(), program.getProgramId()); try { @@ -1572,7 +749,7 @@ public void stopPrograms(FullHttpRequest request, HttpResponder responder, } } - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(issuedStops)); + responder.sendJson(HttpResponseStatus.OK, ProgramHandlerUtil.toJson(issuedStops)); } /** @@ -1611,13 +788,11 @@ public void stopPrograms(FullHttpRequest request, HttpResponder responder, public void startPrograms(FullHttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception { - List programs = validateAndGetBatchInput(request, BATCH_STARTS_TYPE); + List programs = ProgramHandlerUtil.validateAndGetBatchInput(request, BATCH_STARTS_TYPE); List output = new ArrayList<>(programs.size()); for (BatchProgramStart program : programs) { - ApplicationId applicationId = new ApplicationId(namespaceId, program.getAppId(), - getLatestAppVersion(new NamespaceId(namespaceId), - program.getAppId())); + ApplicationId applicationId = store.getLatestApp(new ApplicationReference(namespaceId, program.getAppId())); ProgramId programId = new ProgramId(applicationId, program.getProgramType(), program.getProgramId()); try { @@ -1634,85 +809,7 @@ public void startPrograms(FullHttpRequest request, HttpResponder responder, new BatchProgramResult(program, HttpResponseStatus.CONFLICT.code(), e.getMessage())); } } - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(output)); - } - - /** - * Returns the number of instances for all program runnables that are passed into the data. The - * data is an array of Json objects where each object must contain the following three elements: - * appId, programType, and programId (flow name, service name). Retrieving instances only applies - * to flows, and user services. For flows, another parameter, "runnableId", must be provided. This - * corresponds to the flowlet/runnable for which to retrieve the instances. - *

- * Example input: - * 


-   * [{"appId": "App1", "programType": "Service", "programId": "Service1", "runnableId": "Runnable1"},
-   *  {"appId": "App1", "programType": "Mapreduce", "programId": "Mapreduce2"}]
-   *
- *

- * The response will be an array of JsonObjects each of which will contain the three input - * parameters as well as 3 fields: - *

    - *
  • "provisioned" which maps to the number of instances actually provided for the input runnable;
    • - *
  • "requested" which maps to the number of instances the user has requested for the input runnable; and
    • - *
  • "statusCode" which maps to the http status code for the data in that JsonObjects (200, 400, 404).
    • - *
- *

- * If an error occurs in the input (for the example above, Flowlet1 does not exist), then all JsonObjects for - * which the parameters have a valid instances will have the provisioned and requested fields status code fields - * but all JsonObjects for which the parameters are not valid will have an error message and statusCode. - *

- * For example, if there is no Flowlet1 in the above data, then the response could be 200 OK with the following data: - *

- * 

-   * [{"appId": "App1", "programType": "Service", "programId": "Service1", "runnableId": "Runnable1",
-   *   "statusCode": 200, "provisioned": 2, "requested": 2},
-   *  {"appId": "App1", "programType": "Mapreduce", "programId": "Mapreduce2", "statusCode": 400,
-   *   "error": "Program type 'Mapreduce' is not a valid program type to get instances"}]
-   *
- */ - @POST - @Path("/instances") - @AuditPolicy(AuditDetail.REQUEST_BODY) - public void getInstances(FullHttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId) throws IOException, BadRequestException { - - List runnables = validateAndGetBatchInput(request, BATCH_RUNNABLES_TYPE); - - // cache app specs to perform fewer store lookups - Map appSpecs = new HashMap<>(); - - List output = new ArrayList<>(runnables.size()); - for (BatchRunnable runnable : runnables) { - // cant get instances for things that are not flows, services, or workers - if (!canHaveInstances(runnable.getProgramType())) { - output.add( - new BatchRunnableInstances(runnable, HttpResponseStatus.BAD_REQUEST.code(), - String.format("Program type '%s' is not a valid program type to get instances", - runnable.getProgramType().getPrettyName()))); - continue; - } - - ApplicationId appId = new ApplicationId(namespaceId, runnable.getAppId()); - try { - appId = new ApplicationId(namespaceId, runnable.getAppId(), - getLatestAppVersion(new NamespaceId(namespaceId), runnable.getAppId())); - } catch (ApplicationNotFoundException e) { - output.add(new BatchRunnableInstances(runnable, HttpResponseStatus.NOT_FOUND.code(), - String.format("App: %s not found", appId))); - continue; - } - - // populate spec cache if this is the first time we've seen the appid. - if (!appSpecs.containsKey(appId)) { - appSpecs.put(appId, store.getApplication(appId)); - } - - ApplicationSpecification spec = appSpecs.get(appId); - ProgramId programId = appId.program(runnable.getProgramType(), runnable.getProgramId()); - output.add(getProgramInstances(runnable, spec, programId)); - } - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(output)); + responder.sendJson(HttpResponseStatus.OK, ProgramHandlerUtil.toJson(output)); } /** @@ -1753,7 +850,7 @@ public void getInstances(FullHttpRequest request, HttpResponder responder, @Path("/runcount") public void getRunCounts(FullHttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception { - List programs = validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE); + List programs = ProgramHandlerUtil.validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE); if (programs.size() > 100) { throw new BadRequestException( String.format("%d programs found in the request, the maximum number " @@ -1784,7 +881,7 @@ public void getRunCounts(FullHttpRequest request, HttpResponder responder, exception.getMessage(), null)); } } - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(counts)); + responder.sendJson(HttpResponseStatus.OK, ProgramHandlerUtil.toJson(counts)); } /** @@ -1825,7 +922,7 @@ public void getRunCounts(FullHttpRequest request, HttpResponder responder, @Path("/runs") public void getLatestRuns(FullHttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception { - List programs = validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE); + List programs = ProgramHandlerUtil.validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE); List programRefs = programs.stream().map( p -> new ProgramReference(namespaceId, p.getAppId(), p.getProgramType(), @@ -1856,11 +953,11 @@ public void getLatestRuns(FullHttpRequest request, HttpResponder responder, exception.getMessage(), Collections.emptyList())); } } - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(response)); + responder.sendJson(HttpResponseStatus.OK, ProgramHandlerUtil.toJson(response)); } /** - * Returns the count of the given program runs + * Returns the count of the given program runs. */ @GET @Path("/apps/{app-name}/{program-type}/{program-name}/runcount") @@ -1869,15 +966,15 @@ public void getProgramRunCount(HttpRequest request, HttpResponder responder, @PathParam("app-name") String appName, @PathParam("program-type") String type, @PathParam("program-name") String programName) throws Exception { - ProgramType programType = getProgramType(type); + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); ProgramReference programReference = new ProgramReference(namespaceId, appName, programType, programName); long runCount = lifecycleService.getProgramTotalRunCount(programReference); - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(runCount)); + responder.sendJson(HttpResponseStatus.OK, ProgramHandlerUtil.toJson(runCount)); } /** - * Returns the count of the given program runs + * Returns the count of the given program runs. */ @GET @Path("/apps/{app-name}/versions/{app-version}/{program-type}/{program-name}/runcount") @@ -1887,7 +984,7 @@ public void getProgramRunCountVersioned(HttpRequest request, HttpResponder respo @PathParam("app-version") String appVersion, @PathParam("program-type") String type, @PathParam("program-name") String programName) throws Exception { - ProgramType programType = getProgramType(type); + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); long runCount; if (ApplicationId.DEFAULT_VERSION.equals(appVersion)) { ProgramReference programReference = new ProgramReference(namespaceId, appName, programType, @@ -1898,7 +995,7 @@ public void getProgramRunCountVersioned(HttpRequest request, HttpResponder respo programName); runCount = lifecycleService.getProgramRunCount(programId); } - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(runCount)); + responder.sendJson(HttpResponseStatus.OK, ProgramHandlerUtil.toJson(runCount)); } /* @@ -1913,8 +1010,9 @@ public void getProgramRunCountVersioned(HttpRequest request, HttpResponder respo public void getAllMapReduce(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception { responder.sendJson(HttpResponseStatus.OK, - GSON.toJson( - lifecycleService.list(validateAndGetNamespace(namespaceId), ProgramType.MAPREDUCE))); + ProgramHandlerUtil.toJson( + lifecycleService.list(NamespaceHelper.validateNamespace(namespaceQueryAdmin,namespaceId), + ProgramType.MAPREDUCE))); } /** @@ -1925,8 +1023,9 @@ public void getAllMapReduce(HttpRequest request, HttpResponder responder, public void getAllSpark(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception { responder.sendJson(HttpResponseStatus.OK, - GSON.toJson( - lifecycleService.list(validateAndGetNamespace(namespaceId), ProgramType.SPARK))); + ProgramHandlerUtil.toJson( + lifecycleService.list(NamespaceHelper.validateNamespace(namespaceQueryAdmin,namespaceId), + ProgramType.SPARK))); } /** @@ -1937,8 +1036,9 @@ public void getAllSpark(HttpRequest request, HttpResponder responder, public void getAllWorkflows(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception { responder.sendJson(HttpResponseStatus.OK, - GSON.toJson( - lifecycleService.list(validateAndGetNamespace(namespaceId), ProgramType.WORKFLOW))); + ProgramHandlerUtil.toJson( + lifecycleService.list(NamespaceHelper.validateNamespace(namespaceQueryAdmin,namespaceId), + ProgramType.WORKFLOW))); } /** @@ -1949,8 +1049,9 @@ public void getAllWorkflows(HttpRequest request, HttpResponder responder, public void getAllServices(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception { responder.sendJson(HttpResponseStatus.OK, - GSON.toJson( - lifecycleService.list(validateAndGetNamespace(namespaceId), ProgramType.SERVICE))); + ProgramHandlerUtil.toJson( + lifecycleService.list(NamespaceHelper.validateNamespace(namespaceQueryAdmin,namespaceId), + ProgramType.SERVICE))); } @GET @@ -1958,107 +1059,9 @@ public void getAllServices(HttpRequest request, HttpResponder responder, public void getAllWorkers(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception { responder.sendJson(HttpResponseStatus.OK, - GSON.toJson( - lifecycleService.list(validateAndGetNamespace(namespaceId), ProgramType.WORKER))); - } - - /** - * Returns number of instances of a worker. - */ - @GET - @Path("/apps/{app-id}/workers/{worker-id}/instances") - public void getWorkerInstances(HttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @PathParam("app-id") String appId, - @PathParam("worker-id") String workerId) throws Exception { - try { - ProgramId programId = validateAndGetNamespace(namespaceId) - .app(appId, getLatestAppVersion(new NamespaceId(namespaceId), appId)) - .worker(workerId); - lifecycleService.ensureProgramExists(programId); - int count = store.getWorkerInstances(programId); - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(new Instances(count))); - } catch (SecurityException e) { - responder.sendStatus(HttpResponseStatus.UNAUTHORIZED); - } catch (Throwable e) { - if (respondIfElementNotFound(e, responder)) { - return; - } - throw e; - } - } - - /** - * Sets the number of instances of a worker. - */ - @PUT - @Path("/apps/{app-id}/workers/{worker-id}/instances") - @AuditPolicy(AuditDetail.REQUEST_BODY) - public void setWorkerInstances(FullHttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @PathParam("app-id") String appId, - @PathParam("worker-id") String workerId) throws Exception { - int instances = getInstances(request); - try { - lifecycleService.setInstances(new ApplicationId(namespaceId, appId, - getLatestAppVersion(new NamespaceId(namespaceId), appId)) - .program(ProgramType.WORKER, workerId), instances); - responder.sendStatus(HttpResponseStatus.OK); - } catch (SecurityException e) { - responder.sendStatus(HttpResponseStatus.UNAUTHORIZED); - } catch (Throwable e) { - if (respondIfElementNotFound(e, responder)) { - return; - } - throw e; - } - } - - @GET - @Path("/apps/{app-id}/{program-category}/{program-id}/live-info") - @SuppressWarnings("unused") - public void liveInfo(HttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @PathParam("app-id") String appId, @PathParam("program-category") String programCategory, - @PathParam("program-id") String programId) - throws BadRequestException, ApplicationNotFoundException { - ProgramType type = getProgramType(programCategory); - ProgramId program = new ApplicationId(namespaceId, appId, - getLatestAppVersion(new NamespaceId(namespaceId), appId)) - .program(type, programId); - getLiveInfo(responder, program, runtimeService); - } - - - private void getLiveInfo(HttpResponder responder, ProgramId programId, - ProgramRuntimeService runtimeService) { - try { - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(runtimeService.getLiveInfo(programId))); - } catch (SecurityException e) { - responder.sendStatus(HttpResponseStatus.UNAUTHORIZED); - } - } - - /** - * Return the number of instances of a service. - */ - @GET - @Path("/apps/{app-id}/services/{service-id}/instances") - public void getServiceInstances(HttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @PathParam("app-id") String appId, - @PathParam("service-id") String serviceId) throws Exception { - try { - ProgramId programId = validateAndGetNamespace(namespaceId) - .app(appId, getLatestAppVersion(new NamespaceId(namespaceId), appId)) - .service(serviceId); - lifecycleService.ensureProgramExists(programId); - int instances = store.getServiceInstances(programId); - responder.sendJson(HttpResponseStatus.OK, - GSON.toJson(new ServiceInstances(instances, getInstanceCount(programId, serviceId)))); - } catch (SecurityException e) { - responder.sendStatus(HttpResponseStatus.UNAUTHORIZED); - } + ProgramHandlerUtil.toJson( + lifecycleService.list(NamespaceHelper.validateNamespace(namespaceQueryAdmin,namespaceId), + ProgramType.WORKER))); } /** @@ -2091,7 +1094,7 @@ public void getServiceAvailabilityVersioned(HttpRequest request, HttpResponder r @PathParam("service-type") String serviceType, @PathParam("program-name") String programName) throws Exception { // Currently, we only support services and sparks as the service-type - ProgramType programType = getProgramType(serviceType); + ProgramType programType = ProgramType.valueOfCategoryName(serviceType, BadRequestException::new); if (!ServiceDiscoverable.getUserServiceTypes().contains(programType)) { throw new BadRequestException( "Only service or spark is support for service availability check"); @@ -2128,230 +1131,14 @@ public void getServiceAvailabilityVersioned(HttpRequest request, HttpResponder r responder.sendString(HttpResponseStatus.OK, "Service is available to accept requests."); } - /** - * Set instances of a service. - */ - @PUT - @Path("/apps/{app-id}/services/{service-id}/instances") - @AuditPolicy(AuditDetail.REQUEST_BODY) - public void setServiceInstances(FullHttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespaceId, - @PathParam("app-id") String appId, - @PathParam("service-id") String serviceId) throws Exception { - try { - ProgramId programId = new ApplicationId(namespaceId, appId, - getLatestAppVersion(new NamespaceId(namespaceId), appId)) - .program(ProgramType.SERVICE, serviceId); - Store.ensureProgramExists(programId, store.getApplication(programId.getParent())); - - int instances = getInstances(request); - lifecycleService.setInstances(programId, instances); - responder.sendStatus(HttpResponseStatus.OK); - } catch (SecurityException e) { - responder.sendStatus(HttpResponseStatus.UNAUTHORIZED); - } catch (Throwable throwable) { - if (respondIfElementNotFound(throwable, responder)) { - return; - } - throw throwable; - } - } - - /** - * Get requested and provisioned instances for a program type. The program type passed here should - * be one that can have instances (flows, services, ...) Requires caller to do this validation. - */ - private BatchRunnableInstances getProgramInstances(BatchRunnable runnable, - ApplicationSpecification spec, - ProgramId programId) { - int requested; - String programName = programId.getProgram(); - ProgramType programType = programId.getType(); - if (programType == ProgramType.WORKER) { - if (!spec.getWorkers().containsKey(programName)) { - return new BatchRunnableInstances(runnable, HttpResponseStatus.NOT_FOUND.code(), - "Worker: " + programName + " not found"); - } - requested = spec.getWorkers().get(programName).getInstances(); - - } else if (programType == ProgramType.SERVICE) { - if (!spec.getServices().containsKey(programName)) { - return new BatchRunnableInstances(runnable, HttpResponseStatus.NOT_FOUND.code(), - "Service: " + programName + " not found"); - } - requested = spec.getServices().get(programName).getInstances(); - - } else { - return new BatchRunnableInstances(runnable, HttpResponseStatus.BAD_REQUEST.code(), - "Instances not supported for program type + " + programType); - } - int provisioned = getInstanceCount(programId, programName); - // use the pretty name of program types to be consistent - return new BatchRunnableInstances(runnable, HttpResponseStatus.OK.code(), provisioned, - requested); - } - - /** - * Returns the number of instances currently running for different runnables for different - * programs - */ - private int getInstanceCount(ProgramId programId, String runnableId) { - ProgramLiveInfo info = runtimeService.getLiveInfo(programId); - int count = 0; - if (info instanceof NotRunningProgramLiveInfo) { - return count; - } - if (info instanceof Containers) { - Containers containers = (Containers) info; - for (Containers.ContainerInfo container : containers.getContainers()) { - if (container.getName().equals(runnableId)) { - count++; - } - } - return count; - } - // TODO: CDAP-1091: For standalone mode, returning the requested instances instead of provisioned only for services. - // Doing this only for services to keep it consistent with the existing contract for flowlets right now. - // The get instances contract for both flowlets and services should be re-thought and fixed as part of CDAP-1091 - if (programId.getType() == ProgramType.SERVICE) { - return getRequestedServiceInstances(programId); - } - - // Not running on YARN default 1 - return 1; - } - - private int getRequestedServiceInstances(ProgramId serviceId) { - // Not running on YARN, get it from store - return store.getServiceInstances(serviceId); - } - private boolean isDebugAllowed(ProgramType programType) { return EnumSet.of(ProgramType.SERVICE, ProgramType.WORKER).contains(programType); } - private boolean canHaveInstances(ProgramType programType) { - return EnumSet.of(ProgramType.SERVICE, ProgramType.WORKER).contains(programType); - } - - private List validateAndGetBatchInput(FullHttpRequest request, - Type type) - throws BadRequestException, IOException { - - List programs; - try (Reader reader = new InputStreamReader(new ByteBufInputStream(request.content()), - StandardCharsets.UTF_8)) { - try { - programs = DECODE_GSON.fromJson(reader, type); - if (programs == null) { - throw new BadRequestException( - "Request body is invalid json, please check that it is a json array."); - } - } catch (JsonSyntaxException e) { - throw new BadRequestException("Request body is invalid json: " + e.getMessage()); - } - } - - // validate input - for (BatchProgram program : programs) { - try { - program.validate(); - } catch (IllegalArgumentException e) { - throw new BadRequestException( - "Must provide valid appId, programType, and programId for each object: " - + e.getMessage()); - } - } - return programs; - } - - private void updateLogLevels(FullHttpRequest request, HttpResponder responder, String namespace, - String appName, - String appVersion, String type, String programName, - String runId) throws Exception { - ProgramType programType = getProgramType(type); - try { - // we are decoding the body to Map instead of Map here since Gson will - // serialize invalid enum values to null, which is allowed for log level, instead of throw an Exception. - lifecycleService.updateProgramLogLevels( - new ApplicationId(namespace, appName, appVersion).program(programType, programName), - transformLogLevelsMap(decodeArguments(request)), runId); - responder.sendStatus(HttpResponseStatus.OK); - } catch (JsonSyntaxException e) { - throw new BadRequestException("Invalid JSON in body"); - } catch (IllegalArgumentException e) { - throw new BadRequestException(e.getMessage()); - } catch (SecurityException e) { - throw new UnauthorizedException("Unauthorized to update the log levels"); - } - } - - private void resetLogLevels(FullHttpRequest request, HttpResponder responder, String namespace, - String appName, - String appVersion, String type, String programName, - String runId) throws Exception { - ProgramType programType = getProgramType(type); - try { - Set loggerNames = parseBody(request, SET_STRING_TYPE); - lifecycleService.resetProgramLogLevels( - new ApplicationId(namespace, appName, appVersion).program(programType, programName), - loggerNames == null ? Collections.emptySet() : loggerNames, runId); - responder.sendStatus(HttpResponseStatus.OK); - } catch (JsonSyntaxException e) { - throw new BadRequestException("Invalid JSON in body"); - } catch (SecurityException e) { - throw new UnauthorizedException("Unauthorized to reset the log levels"); - } - } - - private NamespaceId validateAndGetNamespace(String namespace) throws NamespaceNotFoundException { - NamespaceId namespaceId = new NamespaceId(namespace); - try { - namespaceQueryAdmin.get(namespaceId); - } catch (NamespaceNotFoundException e) { - throw e; - } catch (Exception e) { - // This can only happen when NamespaceAdmin uses HTTP to interact with namespaces. - // Within AppFabric, NamespaceAdmin is bound to DefaultNamespaceAdmin which directly interacts with MDS. - // Hence, this should never happen. - throw Throwables.propagate(e); - } - return namespaceId; - } - - /** - * Parses the give program type into {@link ProgramType} object. - * - * @param programType the program type to parse. - * @throws BadRequestException if the given program type is not a valid {@link ProgramType}. - */ - private ProgramType getProgramType(String programType) throws BadRequestException { - try { - return ProgramType.valueOfCategoryName(programType); - } catch (Exception e) { - throw new BadRequestException(String.format("Invalid program type '%s'", programType), e); - } - } - /** * Used to filter out RunRecords initiated by a tethered instance */ private boolean isTetheredRunRecord(RunRecord runRecord) { return runRecord.getPeerName() != null; } - - /** - * @param namespaceId namespace Id - * @param appId app Id - * @return latest app version - */ - private String getLatestAppVersion(NamespaceId namespaceId, String appId) - throws ApplicationNotFoundException { - ApplicationMeta latestApplicationMeta = store.getLatest(namespaceId.appReference(appId)); - if (latestApplicationMeta == null) { - throw new ApplicationNotFoundException( - new ApplicationReference(namespaceId.getNamespace(), appId)); - } - return latestApplicationMeta.getSpec().getAppVersion(); - } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/ProgramRuntimeHttpHandler.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/ProgramRuntimeHttpHandler.java new file mode 100644 index 000000000000..a6cc26a7d0ce --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/ProgramRuntimeHttpHandler.java @@ -0,0 +1,566 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.gateway.handlers; + +import com.google.gson.JsonSyntaxException; +import com.google.gson.reflect.TypeToken; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import io.cdap.cdap.api.app.ApplicationSpecification; +import io.cdap.cdap.app.runtime.ProgramRuntimeService; +import io.cdap.cdap.app.store.Store; +import io.cdap.cdap.common.ApplicationNotFoundException; +import io.cdap.cdap.common.BadRequestException; +import io.cdap.cdap.common.NotFoundException; +import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.namespace.NamespaceQueryAdmin; +import io.cdap.cdap.common.security.AuditDetail; +import io.cdap.cdap.common.security.AuditPolicy; +import io.cdap.cdap.gateway.handlers.util.AbstractAppFabricHttpHandler; +import io.cdap.cdap.gateway.handlers.util.NamespaceHelper; +import io.cdap.cdap.gateway.handlers.util.ProgramHandlerUtil; +import io.cdap.cdap.internal.app.services.ProgramLifecycleService; +import io.cdap.cdap.internal.app.store.RunRecordDetail; +import io.cdap.cdap.proto.BatchRunnable; +import io.cdap.cdap.proto.BatchRunnableInstances; +import io.cdap.cdap.proto.Containers; +import io.cdap.cdap.proto.Instances; +import io.cdap.cdap.proto.NotRunningProgramLiveInfo; +import io.cdap.cdap.proto.ProgramLiveInfo; +import io.cdap.cdap.proto.ProgramType; +import io.cdap.cdap.proto.ServiceInstances; +import io.cdap.cdap.proto.id.ApplicationId; +import io.cdap.cdap.proto.id.ApplicationReference; +import io.cdap.cdap.proto.id.ProgramId; +import io.cdap.cdap.proto.id.ProgramReference; +import io.cdap.cdap.proto.security.StandardPermission; +import io.cdap.cdap.security.spi.authentication.AuthenticationContext; +import io.cdap.cdap.security.spi.authorization.AccessEnforcer; +import io.cdap.cdap.security.spi.authorization.UnauthorizedException; +import io.cdap.http.HttpResponder; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponseStatus; +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; + +/** + * {@link io.cdap.http.HttpHandler} to manage runtime of Programs for v3 REST APIs + * + * Only supported program types for this handler are {@link ProgramType#SERVICE} and {@link ProgramType#WORKER}. + */ +@Singleton +@Path(Constants.Gateway.API_VERSION_3 + "/namespaces/{namespace-id}") +public class ProgramRuntimeHttpHandler extends AbstractAppFabricHttpHandler { + + private final ProgramLifecycleService lifecycleService; + private final ProgramRuntimeService runtimeService; + private final Store store; + private final NamespaceQueryAdmin namespaceQueryAdmin; + private final AccessEnforcer accessEnforcer; + private final AuthenticationContext authenticationContext; + + @Inject + public ProgramRuntimeHttpHandler(ProgramLifecycleService lifecycleService, Store store, + ProgramRuntimeService runtimeService, NamespaceQueryAdmin namespaceQueryAdmin, AccessEnforcer accessEnforcer, + AuthenticationContext authenticationContext) { + this.lifecycleService = lifecycleService; + this.runtimeService = runtimeService; + this.store = store; + this.namespaceQueryAdmin = namespaceQueryAdmin; + this.accessEnforcer = accessEnforcer; + this.authenticationContext = authenticationContext; + } + + private static final Type BATCH_RUNNABLES_TYPE = new TypeToken>() { + }.getType(); + + /** + * Returns the number of instances for all program runnables that are passed into the data. The + * data is an array of Json objects where each object must contain the following three elements: + * appId, programType, and programId (flow name, service name). Retrieving instances only applies + * to flows, and user services. For flows, another parameter, "runnableId", must be provided. This + * corresponds to the flowlet/runnable for which to retrieve the instances. + *

+ * Example input: + * 


+   * [{"appId": "App1", "programType": "Service", "programId": "Service1", "runnableId": "Runnable1"},
+   *  {"appId": "App1", "programType": "Mapreduce", "programId": "Mapreduce2"}]
+   *
+ *

+ * The response will be an array of JsonObjects each of which will contain the three input + * parameters as well as 3 fields: + *

    + *
  • "provisioned" which maps to the number of instances actually provided for the input runnable;
    • + *
  • "requested" which maps to the number of instances the user has requested for the input runnable; and
    • + *
  • "statusCode" which maps to the http status code for the data in that JsonObjects (200, 400, 404).
    • + *
+ *

+ * If an error occurs in the input (for the example above, Flowlet1 does not exist), then all JsonObjects for + * which the parameters have a valid instances will have the provisioned and requested fields status code fields + * but all JsonObjects for which the parameters are not valid will have an error message and statusCode. + *

+ * For example, if there is no Flowlet1 in the above data, then the response could be 200 OK with the following data: + *

+ * 

+   * [{"appId": "App1", "programType": "Service", "programId": "Service1", "runnableId": "Runnable1",
+   *   "statusCode": 200, "provisioned": 2, "requested": 2},
+   *  {"appId": "App1", "programType": "Mapreduce", "programId": "Mapreduce2", "statusCode": 400,
+   *   "error": "Program type 'Mapreduce' is not a valid program type to get instances"}]
+   *
+ */ + @POST + @Path("/instances") + @AuditPolicy(AuditDetail.REQUEST_BODY) + public void getInstances(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId) throws IOException, BadRequestException { + + List runnables = ProgramHandlerUtil.validateAndGetBatchInput(request, BATCH_RUNNABLES_TYPE); + + // cache app specs to perform fewer store lookups + Map appSpecs = new HashMap<>(); + + List output = new ArrayList<>(runnables.size()); + for (BatchRunnable runnable : runnables) { + // cant get instances for things that are not services, or workers + if (!canHaveInstances(runnable.getProgramType())) { + output.add( + new BatchRunnableInstances(runnable, HttpResponseStatus.BAD_REQUEST.code(), + String.format("Program type '%s' is not a valid program type to get instances", + runnable.getProgramType().getPrettyName()))); + continue; + } + + ApplicationId appId = new ApplicationId(namespaceId, runnable.getAppId()); + try { + appId = store.getLatestApp(new ApplicationReference(namespaceId, runnable.getAppId())); + } catch (ApplicationNotFoundException e) { + output.add(new BatchRunnableInstances(runnable, HttpResponseStatus.NOT_FOUND.code(), + String.format("App: %s not found", appId))); + continue; + } + + // populate spec cache if this is the first time we've seen the appid. + if (!appSpecs.containsKey(appId)) { + appSpecs.put(appId, store.getApplication(appId)); + } + + ApplicationSpecification spec = appSpecs.get(appId); + ProgramId programId = appId.program(runnable.getProgramType(), runnable.getProgramId()); + output.add(getProgramInstances(runnable, spec, programId)); + } + responder.sendJson(HttpResponseStatus.OK, ProgramHandlerUtil.toJson(output)); + } + + /** + * Return the number of instances of a service. + */ + @GET + @Path("/apps/{app-id}/services/{service-id}/instances") + public void getServiceInstances(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-id") String appId, + @PathParam("service-id") String serviceId) throws Exception { + try { + NamespaceHelper.validateNamespace(namespaceQueryAdmin, namespaceId); + ProgramId programId = store.getLatestApp(new ApplicationReference(namespaceId, appId)).service(serviceId); + lifecycleService.ensureProgramExists(programId); + int instances = store.getServiceInstances(programId); + responder.sendJson(HttpResponseStatus.OK, + ProgramHandlerUtil.toJson(new ServiceInstances(instances, getInstanceCount(programId, serviceId)))); + } catch (SecurityException e) { + responder.sendStatus(HttpResponseStatus.UNAUTHORIZED); + } + } + + /** + * Set instances of a service. + */ + @PUT + @Path("/apps/{app-id}/services/{service-id}/instances") + @AuditPolicy(AuditDetail.REQUEST_BODY) + public void setServiceInstances(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-id") String appId, + @PathParam("service-id") String serviceId) throws Exception { + try { + ProgramId programId = store.getLatestApp(new ApplicationReference(namespaceId, appId)) + .program(ProgramType.SERVICE, serviceId); + Store.ensureProgramExists(programId, store.getApplication(programId.getParent())); + int instances = getInstances(request); + accessEnforcer.enforce(programId, authenticationContext.getPrincipal(), StandardPermission.UPDATE); + setInstances(programId, instances); + responder.sendStatus(HttpResponseStatus.OK); + } catch (SecurityException e) { + responder.sendStatus(HttpResponseStatus.UNAUTHORIZED); + } catch (Throwable throwable) { + if (respondIfElementNotFound(throwable, responder)) { + return; + } + throw throwable; + } + } + + /** + * Returns number of instances of a worker. + */ + @GET + @Path("/apps/{app-id}/workers/{worker-id}/instances") + public void getWorkerInstances(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-id") String appId, + @PathParam("worker-id") String workerId) throws Exception { + try { + NamespaceHelper.validateNamespace(namespaceQueryAdmin, namespaceId); + ProgramId programId = store.getLatestApp(new ApplicationReference(namespaceId, appId)).worker(workerId); + lifecycleService.ensureProgramExists(programId); + int count = store.getWorkerInstances(programId); + responder.sendJson(HttpResponseStatus.OK, ProgramHandlerUtil.toJson(new Instances(count))); + } catch (SecurityException e) { + responder.sendStatus(HttpResponseStatus.UNAUTHORIZED); + } catch (Throwable e) { + if (respondIfElementNotFound(e, responder)) { + return; + } + throw e; + } + } + + /** + * Sets the number of instances of a worker. + */ + @PUT + @Path("/apps/{app-id}/workers/{worker-id}/instances") + @AuditPolicy(AuditDetail.REQUEST_BODY) + public void setWorkerInstances(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-id") String appId, + @PathParam("worker-id") String workerId) throws Exception { + int instances = getInstances(request); + try { + ProgramId programId = store.getLatestApp(new ApplicationReference(namespaceId, appId)) + .program(ProgramType.WORKER, workerId); + Store.ensureProgramExists(programId, store.getApplication(programId.getParent())); + accessEnforcer.enforce(programId, authenticationContext.getPrincipal(), StandardPermission.UPDATE); + setInstances(programId, instances); + responder.sendStatus(HttpResponseStatus.OK); + } catch (SecurityException e) { + responder.sendStatus(HttpResponseStatus.UNAUTHORIZED); + } catch (Throwable e) { + if (respondIfElementNotFound(e, responder)) { + return; + } + throw e; + } + } + + /** + * Gets runtime information about a running program. + * + * @param request the HTTP request + * @param responder the HTTP responder + * @param namespaceId namespaceId for the program + * @param appId appId for the program + * @param programCategory program type + * @param programId the program Id + * + * @throws BadRequestException + * @throws ApplicationNotFoundException + */ + @GET + @Path("/apps/{app-id}/{program-category}/{program-id}/live-info") + @SuppressWarnings("unused") + public void liveInfo(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-id") String appId, @PathParam("program-category") String programCategory, + @PathParam("program-id") String programId) + throws BadRequestException, ApplicationNotFoundException { + ProgramType type = ProgramType.valueOfCategoryName(programCategory, BadRequestException::new); + ProgramId program = store.getLatestApp(new ApplicationReference(namespaceId, appId)) + .program(type, programId); + getLiveInfo(responder, program, runtimeService); + } + + /** + * Update the log level for a running program according to the request body. Currently supported + * program types are {@link ProgramType#SERVICE} and {@link ProgramType#WORKER}. The request body + * is expected to contain a map of log levels, where key is loggername, value is one of the valid + * {@link org.apache.twill.api.logging.LogEntry.Level} or null. + * + */ + @PUT + @Path("/apps/{app-name}/{program-type}/{program-name}/runs/{run-id}/loglevels") + @AuditPolicy(AuditDetail.REQUEST_BODY) + public void updateProgramLogLevels(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespace, + @PathParam("app-name") String appName, + @PathParam("program-type") String type, + @PathParam("program-name") String programName, + @PathParam("run-id") String runId) throws Exception { + RunRecordDetail run = getRunRecordDetailFromId(namespace, appName, type, programName, runId); + updateLogLevels(request, responder, namespace, appName, run.getVersion(), type, programName, + runId); + } + + /** + * Update the log level for a running program according to the request body. + * Deprecated : Run-id is sufficient to identify a program run. + */ + @Deprecated + @PUT + @Path("/apps/{app-name}/versions/{app-version}/{program-type}/{program-name}/runs/{run-id}/loglevels") + @AuditPolicy(AuditDetail.REQUEST_BODY) + public void updateProgramLogLevelsVersioned(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespace, + @PathParam("app-name") String appName, + @PathParam("app-version") String appVersion, + @PathParam("program-type") String type, + @PathParam("program-name") String programName, + @PathParam("run-id") String runId) throws Exception { + updateLogLevels(request, responder, namespace, appName, appVersion, type, programName, runId); + } + + /** + * Reset the log level for a running program back to where it starts. Currently supported program + * types are {@link ProgramType#SERVICE} and {@link ProgramType#WORKER}. The request body can + * either be empty, which will reset all loggers for the program, or contain a list of logger + * names, which will reset for these particular logger names for the program. + */ + @POST + @Path("/apps/{app-name}/{program-type}/{program-name}/runs/{run-id}/resetloglevels") + @AuditPolicy(AuditDetail.REQUEST_BODY) + public void resetProgramLogLevels(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespace, + @PathParam("app-name") String appName, + @PathParam("program-type") String type, + @PathParam("program-name") String programName, + @PathParam("run-id") String runId) throws Exception { + RunRecordDetail run = getRunRecordDetailFromId(namespace, appName, type, programName, runId); + resetLogLevels(request, responder, namespace, appName, run.getVersion(), type, programName, + runId); + } + + /** + * Reset the log level for a running program back to where it starts. + * + * Deprecated : Run-id is sufficient to identify a program run. + */ + @POST + @Path("/apps/{app-name}/versions/{app-version}/{program-type}/{program-name}/runs/{run-id}/resetloglevels") + @AuditPolicy(AuditDetail.REQUEST_BODY) + public void resetProgramLogLevelsVersioned(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespace, + @PathParam("app-name") String appName, + @PathParam("app-version") String appVersion, + @PathParam("program-type") String type, + @PathParam("program-name") String programName, + @PathParam("run-id") String runId) throws Exception { + resetLogLevels(request, responder, namespace, appName, appVersion, type, programName, runId); + } + + private void getLiveInfo(HttpResponder responder, ProgramId programId, + ProgramRuntimeService runtimeService) { + try { + responder.sendJson(HttpResponseStatus.OK, ProgramHandlerUtil.toJson(runtimeService.getLiveInfo(programId))); + } catch (SecurityException e) { + responder.sendStatus(HttpResponseStatus.UNAUTHORIZED); + } + } + + /** + * Returns the number of instances currently running for different runnables for different + * programs + */ + private int getInstanceCount(ProgramId programId, String runnableId) { + ProgramLiveInfo info = runtimeService.getLiveInfo(programId); + int count = 0; + if (info instanceof NotRunningProgramLiveInfo) { + return count; + } + if (info instanceof Containers) { + Containers containers = (Containers) info; + for (Containers.ContainerInfo container : containers.getContainers()) { + if (container.getName().equals(runnableId)) { + count++; + } + } + return count; + } + // TODO: CDAP-1091: For standalone mode, returning the requested instances instead of provisioned only for services. + // Doing this only for services to keep it consistent with the existing contract for flowlets right now. + // The get instances contract for both flowlets and services should be re-thought and fixed as part of CDAP-1091 + if (programId.getType() == ProgramType.SERVICE) { + return getRequestedServiceInstances(programId); + } + + // Not running on YARN default 1 + return 1; + } + + /** + * Get requested and provisioned instances for a program type. The program type passed here should + * be one that can have instances (flows, services, ...) Requires caller to do this validation. + */ + private BatchRunnableInstances getProgramInstances(BatchRunnable runnable, + ApplicationSpecification spec, + ProgramId programId) { + int requested; + String programName = programId.getProgram(); + ProgramType programType = programId.getType(); + if (programType == ProgramType.WORKER) { + if (!spec.getWorkers().containsKey(programName)) { + return new BatchRunnableInstances(runnable, HttpResponseStatus.NOT_FOUND.code(), + "Worker: " + programName + " not found"); + } + requested = spec.getWorkers().get(programName).getInstances(); + + } else if (programType == ProgramType.SERVICE) { + if (!spec.getServices().containsKey(programName)) { + return new BatchRunnableInstances(runnable, HttpResponseStatus.NOT_FOUND.code(), + "Service: " + programName + " not found"); + } + requested = spec.getServices().get(programName).getInstances(); + + } else { + return new BatchRunnableInstances(runnable, HttpResponseStatus.BAD_REQUEST.code(), + "Instances not supported for program type + " + programType); + } + int provisioned = getInstanceCount(programId, programName); + // use the pretty name of program types to be consistent + return new BatchRunnableInstances(runnable, HttpResponseStatus.OK.code(), provisioned, + requested); + } + + private int getRequestedServiceInstances(ProgramId serviceId) { + // Not running on YARN, get it from store + return store.getServiceInstances(serviceId); + } + + private boolean canHaveInstances(ProgramType programType) { + return EnumSet.of(ProgramType.SERVICE, ProgramType.WORKER).contains(programType); + } + + private void resetLogLevels(FullHttpRequest request, HttpResponder responder, String namespace, + String appName, + String appVersion, String type, String programName, + String runId) throws Exception { + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); + try { + Set loggerNames = parseBody(request, SET_STRING_TYPE); + ProgramId programId = new ApplicationId(namespace, appName, appVersion).program(programType, programName); + accessEnforcer.enforce(programId, authenticationContext.getPrincipal(), StandardPermission.UPDATE); + runtimeService.resetProgramLogLevels( + new ApplicationId(namespace, appName, appVersion).program(programType, programName), + loggerNames == null ? Collections.emptySet() : loggerNames, runId); + responder.sendStatus(HttpResponseStatus.OK); + } catch (JsonSyntaxException e) { + throw new BadRequestException("Invalid JSON in body"); + } catch (SecurityException e) { + throw new UnauthorizedException("Unauthorized to reset the log levels"); + } + } + + private void updateLogLevels(FullHttpRequest request, HttpResponder responder, String namespace, + String appName, + String appVersion, String type, String programName, + String runId) throws Exception { + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); + try { + // we are decoding the body to Map instead of Map here since Gson will + // serialize invalid enum values to null, which is allowed for log level, instead of throw an Exception. + ProgramId programId = new ApplicationId(namespace, appName, appVersion).program(programType, programName); + accessEnforcer.enforce(programId, authenticationContext.getPrincipal(), StandardPermission.UPDATE); + runtimeService.updateProgramLogLevels( + new ApplicationId(namespace, appName, appVersion).program(programType, programName), + transformLogLevelsMap(decodeArguments(request)), runId); + responder.sendStatus(HttpResponseStatus.OK); + } catch (JsonSyntaxException e) { + throw new BadRequestException("Invalid JSON in body"); + } catch (IllegalArgumentException e) { + throw new BadRequestException(e.getMessage()); + } catch (SecurityException e) { + throw new UnauthorizedException("Unauthorized to update the log levels"); + } + } + + /** + * Set instances for the given program. Only supported program types for this action are {@link + * ProgramType#SERVICE} and {@link ProgramType#WORKER}. + * + * @param programId the {@link ProgramId} of the program for which instances are to be + * updated + * @param instances the number of instances to be updated. + * @throws InterruptedException if there is an error while asynchronously updating instances + * @throws ExecutionException if there is an error while asynchronously updating instances + * @throws BadRequestException if the number of instances specified is less than 0 + * @throws UnauthorizedException if the user does not have privileges to set instances for the + * specified program. To set instances for a program, a user needs {@link + * StandardPermission#UPDATE} on the program. + */ + private void setInstances(ProgramId programId, int instances) throws Exception { + if (instances < 1) { + throw new BadRequestException( + String.format("Instance count should be greater than 0. Got %s.", instances)); + } + switch (programId.getType()) { + case SERVICE: + int oldInstances = store.getServiceInstances(programId); + if (oldInstances != instances) { + store.setServiceInstances(programId, instances); + runtimeService.setInstances(programId, instances, instances); + } + break; + case WORKER: + oldInstances = store.getWorkerInstances(programId); + if (oldInstances != instances) { + store.setWorkerInstances(programId, instances); + runtimeService.setInstances(programId, instances, instances); + } + break; + default: + throw new BadRequestException( + String.format("Setting instances for program type %s is not supported", + programId.getType().getPrettyName())); + } + } + + private RunRecordDetail getRunRecordDetailFromId(String namespaceId, String appName, String type, + String programName, String runId) throws NotFoundException, BadRequestException { + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); + ProgramReference programRef = new ApplicationReference(namespaceId, appName).program(programType, + programName); + RunRecordDetail runRecordMeta = store.getRun(programRef, runId); + if (runRecordMeta == null) { + throw new NotFoundException( + String.format("No run record found for program %s and runID: %s", programRef, runId)); + } + return runRecordMeta; + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/ProgramScheduleHttpHandler.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/ProgramScheduleHttpHandler.java new file mode 100644 index 000000000000..cf3028ba0c8e --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/ProgramScheduleHttpHandler.java @@ -0,0 +1,824 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.gateway.handlers; + +import com.google.common.base.Charsets; +import com.google.common.base.Joiner; +import com.google.common.base.Objects; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonSyntaxException; +import com.google.gson.reflect.TypeToken; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import io.cdap.cdap.api.app.ApplicationSpecification; +import io.cdap.cdap.app.store.Store; +import io.cdap.cdap.common.BadRequestException; +import io.cdap.cdap.common.NotFoundException; +import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.security.AuditDetail; +import io.cdap.cdap.common.security.AuditPolicy; +import io.cdap.cdap.gateway.handlers.util.AbstractAppFabricHttpHandler; +import io.cdap.cdap.gateway.handlers.util.ProgramHandlerUtil; +import io.cdap.cdap.internal.app.runtime.schedule.ProgramSchedule; +import io.cdap.cdap.internal.app.runtime.schedule.ProgramScheduleRecord; +import io.cdap.cdap.internal.app.runtime.schedule.ProgramScheduleStatus; +import io.cdap.cdap.internal.app.runtime.schedule.SchedulerException; +import io.cdap.cdap.internal.app.runtime.schedule.store.Schedulers; +import io.cdap.cdap.internal.app.runtime.schedule.trigger.ProgramStatusTrigger; +import io.cdap.cdap.internal.app.services.ProgramLifecycleService; +import io.cdap.cdap.internal.app.store.ApplicationMeta; +import io.cdap.cdap.internal.schedule.constraint.Constraint; +import io.cdap.cdap.proto.BatchProgram; +import io.cdap.cdap.proto.BatchProgramSchedule; +import io.cdap.cdap.proto.ProgramStatus; +import io.cdap.cdap.proto.ProgramType; +import io.cdap.cdap.proto.ProtoTrigger; +import io.cdap.cdap.proto.ScheduleDetail; +import io.cdap.cdap.proto.ScheduledRuntime; +import io.cdap.cdap.proto.id.ApplicationId; +import io.cdap.cdap.proto.id.ApplicationReference; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.proto.id.ProgramId; +import io.cdap.cdap.proto.id.ProgramReference; +import io.cdap.cdap.proto.id.ScheduleId; +import io.cdap.cdap.scheduler.ProgramScheduleService; +import io.cdap.http.HttpResponder; +import io.netty.buffer.ByteBufInputStream; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponseStatus; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.QueryParam; + +/** + * {@link io.cdap.http.HttpHandler} to manage program schedule for v3 REST APIs. + */ +@Singleton +@Path(Constants.Gateway.API_VERSION_3 + "/namespaces/{namespace-id}") +public class ProgramScheduleHttpHandler extends AbstractAppFabricHttpHandler { + + private static final Type BATCH_PROGRAMS_TYPE = new TypeToken>() { + }.getType(); + private static final List NO_CONSTRAINTS = Collections.emptyList(); + + private final ProgramScheduleService programScheduleService; + private final ProgramLifecycleService lifecycleService; + private final Store store; + + /** + * Constructor for ProgramScheduleHttpHandler. + * @param programScheduleService ProgramScheduleService + * @param lifecycleService ProgramLifecycleService + * @param store Store + */ + @Inject + public ProgramScheduleHttpHandler(ProgramScheduleService programScheduleService, + ProgramLifecycleService lifecycleService, + Store store) { + this.programScheduleService = programScheduleService; + this.lifecycleService = lifecycleService; + this.store = store; + } + + /** + * Returns status of a schedule. + */ + @GET + @Path("/apps/{app-id}/schedules/{schedule-name}/status") + public void getStatus(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-id") String appId, + @PathParam("schedule-name") String scheduleName) throws Exception { + getStatusVersioned(request, responder, namespaceId, appId, ApplicationId.DEFAULT_VERSION, scheduleName); + } + + /** + * Returns status of a schedule. + * Deprecated: Only schedule info retrieval for the latest app version (active app). + */ + @Deprecated + @GET + @Path("/apps/{app-id}/versions/{version-id}/schedules/{schedule-name}/status") + public void getStatusVersioned(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-id") String appId, + @PathParam("version-id") String versionId, + @PathParam("schedule-name") String scheduleName) throws Exception { + ApplicationReference appReference = new ApplicationReference(namespaceId, appId); + JsonObject json = new JsonObject(); + // ScheduleId is versionless (always "-SNAPSHOT") + ScheduleId scheduleId = appReference.app(ApplicationId.DEFAULT_VERSION).schedule(scheduleName); + ApplicationSpecification appSpec = ApplicationId.DEFAULT_VERSION.equals(versionId) + ? Optional.ofNullable(store.getLatest(appReference)).map(ApplicationMeta::getSpec) + .orElse(null) + : store.getApplication(appReference.app(versionId)); + if (appSpec == null) { + throw new NotFoundException(appReference); + } + json.addProperty("status", programScheduleService.getStatus(scheduleId).toString()); + responder.sendJson(HttpResponseStatus.OK, json.toString()); + } + + /** + * Perform an action on the latest version of a schedule. + */ + @POST + @Path("/apps/{app-id}/schedules/{schedule-name}/{action}") + @AuditPolicy(AuditDetail.REQUEST_BODY) + public void performAction(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-id") String appId, + @PathParam("schedule-name") String scheduleName, + @PathParam("action") String action) throws Exception { + doPerformAction(responder, namespaceId, appId, ApplicationId.DEFAULT_VERSION, scheduleName, action); + } + + /** + * Perform an action on the schedule of a specific {@code appVersion}. + */ + @POST + @Path("/apps/{app-id}/versions/{app-version}/schedules/{schedule-name}/{action}") + @AuditPolicy(AuditDetail.REQUEST_BODY) + public void performActionVersioned(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-id") String appId, + @PathParam("app-version") String appVersion, + @PathParam("schedule-name") String scheduleName, + @PathParam("action") String action) throws Exception { + doPerformAction(responder, namespaceId, appId, appVersion, scheduleName, action); + } + + /** + * Perform an action on the schedule of a specific {@code appVersion}. + * + * @param namespaceId namespace of the program + * @param appId appId of the program + * @param appVersion app version of the program + * @param programId programId of the program + * @param action action to be performed. The value can be one of enable/disable/suspend/resume + * for schedule. + * + * @throws NotFoundException if action is an unknown action + * @throws BadRequestException if type is schedule and action is not any of + * enable/disable/suspend/resume + * @throws BadRequestException if action is not supported. + */ + private void doPerformAction(HttpResponder responder, String namespaceId, + String appId, String appVersion, String programId, String action) throws Exception { + ApplicationId applicationId = new ApplicationId(namespaceId, appId, appVersion); + ScheduleId scheduleId = applicationId.schedule(programId); + if (action.equals("disable") || action.equals("suspend")) { + programScheduleService.suspend(scheduleId); + } else if (action.equals("enable") || action.equals("resume")) { + programScheduleService.resume(scheduleId); + } else { + throw new BadRequestException( + "Action for schedules may only be 'enable', 'disable', 'suspend', or 'resume' but is'" + + action + "'"); + } + responder.sendJson(HttpResponseStatus.OK, "OK"); + } + + /** + * Update schedules which were suspended between startTimeMillis and endTimeMillis. + * + * @param startTimeMillis lower bound in millis of the update time for schedules (inclusive) + * @param endTimeMillis upper bound in millis of the update time for schedules (exclusive) + */ + @PUT + @Path("schedules/re-enable") + public void reEnableSuspendedSchedules(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @QueryParam("start-time-millis") long startTimeMillis, + @QueryParam("end-time-millis") long endTimeMillis) throws Exception { + programScheduleService.reEnableSchedules(new NamespaceId(namespaceId), startTimeMillis, + endTimeMillis); + responder.sendStatus(HttpResponseStatus.OK); + } + + /** + * Get schedules containing {@link ProgramStatusTrigger} filtered by triggering program, and + * optionally by triggering program statuses or schedule status. + * + * @param triggerNamespaceId namespace of the triggering program in {@link + * ProgramStatusTrigger} + * @param triggerAppName application name of the triggering program in {@link + * ProgramStatusTrigger} + * @param triggerAppVersion application version of the triggering program in {@link + * ProgramStatusTrigger} + * @param triggerProgramType program type of the triggering program in {@link + * ProgramStatusTrigger} + * @param triggerProgramName program name of the triggering program in {@link + * ProgramStatusTrigger} + * @param triggerProgramStatuses comma separated {@link ProgramStatus} in {@link + * ProgramStatusTrigger}. Schedules with {@link ProgramStatusTrigger} triggered by none of the + * {@link ProgramStatus} in triggerProgramStatuses will be filtered out. If not specified, + * schedules will be returned regardless of triggering program status. + * @param scheduleStatus status of the schedule. Can only be one of "SCHEDULED" or + * "SUSPENDED". If specified, only schedules with matching status will be returned. + */ + @GET + @Path("schedules/trigger-type/program-status") + public void getProgramStatusSchedules(HttpRequest request, HttpResponder responder, + @QueryParam("trigger-namespace-id") String triggerNamespaceId, + @QueryParam("trigger-app-name") String triggerAppName, + @QueryParam("trigger-app-version") @DefaultValue(ApplicationId.DEFAULT_VERSION) + String triggerAppVersion, + @QueryParam("trigger-program-type") String triggerProgramType, + @QueryParam("trigger-program-name") String triggerProgramName, + @QueryParam("trigger-program-statuses") String triggerProgramStatuses, + @QueryParam("schedule-status") String scheduleStatus) throws Exception { + if (triggerNamespaceId == null) { + throw new BadRequestException("Must specify trigger-namespace-id as a query param"); + } + if (triggerAppName == null) { + throw new BadRequestException("Must specify trigger-app-name as a query param"); + } + if (triggerProgramType == null) { + throw new BadRequestException("Must specify trigger-program-type as a query param"); + } + if (triggerProgramName == null) { + throw new BadRequestException("Must specify trigger-program-name as a query param"); + } + + ProgramType programType = ProgramType.valueOfCategoryName(triggerProgramType, BadRequestException::new); + ProgramScheduleStatus programScheduleStatus; + try { + programScheduleStatus = + scheduleStatus == null ? null : ProgramScheduleStatus.valueOf(scheduleStatus); + } catch (IllegalArgumentException e) { + throw new BadRequestException( + String.format("Invalid schedule status '%s'. Must be one of %s.", + scheduleStatus, Joiner.on(',').join(ProgramScheduleStatus.values())), + e); + } + + ProgramId triggerProgramId = new NamespaceId(triggerNamespaceId) + .app(triggerAppName, triggerAppVersion) + .program(programType, triggerProgramName); + + Set queryProgramStatuses = new HashSet<>(); + if (triggerProgramStatuses != null) { + try { + for (String status : triggerProgramStatuses.split(",")) { + queryProgramStatuses.add(io.cdap.cdap.api.ProgramStatus.valueOf(status)); + } + } catch (Exception e) { + throw new BadRequestException( + String.format("Unable to parse program statuses '%s'. Must be comma separated " + + "valid ProgramStatus names such as COMPLETED, FAILED, KILLED.", + triggerProgramStatuses), e); + } + } else { + // Query for schedules with all the statuses if no query status is specified + Collections.addAll(queryProgramStatuses, io.cdap.cdap.api.ProgramStatus.values()); + } + + List details = programScheduleService.findTriggeredBy(triggerProgramId, + queryProgramStatuses) + .stream() + .filter(record -> programScheduleStatus == null || record.getMeta().getStatus() + .equals(programScheduleStatus)) + .map(ProgramScheduleRecord::toScheduleDetail) + .collect(Collectors.toList()); + responder.sendJson(HttpResponseStatus.OK, + ProgramHandlerUtil.toJson(details, Schedulers.SCHEDULE_DETAILS_TYPE)); + } + + /** + * Gets the schedule for the given {@code appName}. + */ + @GET + @Path("apps/{app-name}/schedules/{schedule-name}") + public void getSchedule(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-name") String appName, + @PathParam("schedule-name") String scheduleName) throws Exception { + doGetSchedule(responder, namespaceId, appName, scheduleName); + } + + /* + * Deprecated: Schedules are versionless. + * */ + @Deprecated + @GET + @Path("apps/{app-name}/versions/{app-version}/schedules/{schedule-name}") + public void getScheduleVersioned(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-name") String appName, + @PathParam("app-version") String appVersion, + @PathParam("schedule-name") String scheduleName) throws Exception { + doGetSchedule(responder, namespaceId, appName, scheduleName); + } + + private void doGetSchedule(HttpResponder responder, String namespace, String app, + String scheduleName) + throws Exception { + ScheduleId scheduleId = new ApplicationId(namespace, app).schedule(scheduleName); + ProgramScheduleRecord record = programScheduleService.getRecord(scheduleId); + ScheduleDetail detail = record.toScheduleDetail(); + responder.sendJson(HttpResponseStatus.OK, ProgramHandlerUtil.toJson(detail, ScheduleDetail.class)); + } + + /** + * See {@link #getAllSchedulesVersioned(HttpRequest, HttpResponder, String, String, String, + * String, String)}. + */ + @GET + @Path("apps/{app-name}/schedules") + public void getAllSchedules(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-name") String appName, + @QueryParam("trigger-type") String triggerType, + @QueryParam("schedule-status") String scheduleStatus) throws Exception { + doGetSchedules(responder, new NamespaceId(namespaceId).app(appName), null, triggerType, + scheduleStatus); + } + + /** + * Get schedules in a given application, optionally filtered by the given {@link + * io.cdap.cdap.proto.ProtoTrigger.Type}. + * Deprecated : Schedules are versionless. + * @param namespaceId namespace of the application to get schedules from + * @param appName name of the application to get schedules from + * @param appVersion version of the application to get schedules from + * @param triggerType trigger type of returned schedules. If not specified, all schedules are + * returned regardless of trigger type + * @param scheduleStatus the status of the schedule, must be values in {@link + * ProgramScheduleStatus}. + * + */ + @Deprecated + @GET + @Path("apps/{app-name}/versions/{app-version}/schedules") + public void getAllSchedulesVersioned(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-name") String appName, + @PathParam("app-version") String appVersion, + @QueryParam("trigger-type") String triggerType, + @QueryParam("schedule-status") String scheduleStatus) throws Exception { + doGetSchedules(responder, new NamespaceId(namespaceId).app(appName), null, triggerType, + scheduleStatus); + } + + /** + * Get program schedules. + */ + @GET + @Path("/apps/{app-name}/{program-type}/{program-name}/schedules") + public void getProgramSchedules(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespace, + @PathParam("app-name") String application, + @PathParam("program-type") String type, + @PathParam("program-name") String program, + @QueryParam("trigger-type") String triggerType, + @QueryParam("schedule-status") String scheduleStatus) throws Exception { + getProgramSchedulesVersioned(request, responder, namespace, application, + ApplicationId.DEFAULT_VERSION, + type, program, triggerType, scheduleStatus); + } + + /** + * Get Workflow schedules. + * Deprecated : Schedules are versionless. + */ + @Deprecated + @GET + @Path("/apps/{app-name}/versions/{app-version}/{program-type}/{program-name}/schedules") + public void getProgramSchedulesVersioned(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespace, + @PathParam("app-name") String application, + @PathParam("app-version") String appVersion, + @PathParam("program-type") String type, + @PathParam("program-name") String program, + @QueryParam("trigger-type") String triggerType, + @QueryParam("schedule-status") String scheduleStatus) throws Exception { + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); + if (programType.getSchedulableType() == null) { + throw new BadRequestException("Program type " + programType + " cannot have schedule"); + } + + ProgramId programId = new ApplicationId(namespace, application).program(programType, program); + doGetSchedules(responder, new NamespaceId(namespace).app(application), programId, + triggerType, scheduleStatus); + } + + private void doGetSchedules(HttpResponder responder, ApplicationId applicationId, + @Nullable ProgramId programId, @Nullable String triggerTypeStr, + @Nullable String statusStr) throws Exception { + ApplicationSpecification appSpec = Optional.ofNullable( + store.getLatest(applicationId.getAppReference())) + .map(ApplicationMeta::getSpec) + .orElse(null); + if (appSpec == null) { + throw new NotFoundException(applicationId); + } + ProgramScheduleStatus status; + try { + status = statusStr == null ? null : ProgramScheduleStatus.valueOf(statusStr); + } catch (IllegalArgumentException e) { + throw new BadRequestException( + String.format("Invalid schedule status '%s'. Must be one of %s.", + statusStr, Joiner.on(',').join(ProgramScheduleStatus.values())), e); + } + + ProtoTrigger.Type triggerType; + try { + triggerType = + triggerTypeStr == null ? null : ProtoTrigger.Type.valueOfCategoryName(triggerTypeStr); + } catch (IllegalArgumentException e) { + throw new BadRequestException(e.getMessage(), e); + } + + Predicate predicate = record -> true; + if (status != null) { + predicate = predicate.and(record -> record.getMeta().getStatus().equals(status)); + } + if (triggerType != null) { + predicate = predicate.and( + record -> record.getSchedule().getTrigger().getType().equals(triggerType)); + } + + Collection schedules; + if (programId != null) { + if (!appSpec.getProgramsByType(programId.getType().getApiProgramType()) + .contains(programId.getProgram())) { + throw new NotFoundException(programId); + } + schedules = programScheduleService.list(programId, predicate); + } else { + schedules = programScheduleService.list(applicationId, predicate); + } + + List details = schedules.stream() + .map(ProgramScheduleRecord::toScheduleDetail) + .collect(Collectors.toList()); + responder.sendJson(HttpResponseStatus.OK, + ProgramHandlerUtil.toJson(details, Schedulers.SCHEDULE_DETAILS_TYPE)); + } + + /** + * Returns the previous runtime when the scheduled program ran. + */ + @GET + @Path("/apps/{app-name}/{program-type}/{program-name}/previousruntime") + public void getPreviousScheduledRunTime(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-name") String appName, + @PathParam("program-type") String type, + @PathParam("program-name") String program) throws Exception { + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); + ApplicationId appId = store.getLatestApp(new ApplicationReference(namespaceId, appName)); + handleScheduleRunTime(responder, appId.program(programType, program), true); + } + + /** + * Returns next scheduled runtime of a workflow. + */ + @GET + @Path("/apps/{app-name}/{program-type}/{program-name}/nextruntime") + public void getNextScheduledRunTime(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-name") String appName, + @PathParam("program-type") String type, + @PathParam("program-name") String program) throws Exception { + ProgramType programType = ProgramType.valueOfCategoryName(type, BadRequestException::new); + ApplicationId appId = store.getLatestApp(new ApplicationReference(namespaceId, appName)); + handleScheduleRunTime(responder, appId.program(programType, program), false); + } + + private void handleScheduleRunTime(HttpResponder responder, ProgramId programId, + boolean previousRuntimeRequested) throws Exception { + try { + lifecycleService.ensureProgramExists(programId); + responder.sendJson(HttpResponseStatus.OK, + ProgramHandlerUtil.toJson(getScheduledRunTimes(programId, previousRuntimeRequested))); + } catch (SecurityException e) { + responder.sendStatus(HttpResponseStatus.UNAUTHORIZED); + } + } + + /** + * Adds a schedule for a given {@code appName}. + */ + @PUT + @Path("apps/{app-name}/schedules/{schedule-name}") + @AuditPolicy(AuditDetail.REQUEST_BODY) + public void addSchedule(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-name") String appName, + @PathParam("schedule-name") String scheduleName) + throws Exception { + doAddSchedule(request, responder, namespaceId, appName, scheduleName); + } + + /* + * Deprecated : Schedules are versionless. + */ + @Deprecated + @PUT + @Path("apps/{app-name}/versions/{app-version}/schedules/{schedule-name}") + @AuditPolicy(AuditDetail.REQUEST_BODY) + public void addScheduleVersioned(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-name") String appName, + @PathParam("app-version") String appVersion, + @PathParam("schedule-name") String scheduleName) + throws Exception { + // Schedules are versionless (all are "-SNAPSHOT" version) + doAddSchedule(request, responder, namespaceId, appName, scheduleName); + } + + private void doAddSchedule(FullHttpRequest request, HttpResponder responder, String namespace, + String appName, + String scheduleName) throws Exception { + + final ApplicationId applicationId = new ApplicationId(namespace, appName); + ScheduleDetail scheduleFromRequest = readScheduleDetailBody(request, scheduleName); + + if (scheduleFromRequest.getProgram() == null) { + throw new BadRequestException("No program was specified for the schedule"); + } + if (scheduleFromRequest.getProgram().getProgramType() == null) { + throw new BadRequestException("No program type was specified for the schedule"); + } + if (scheduleFromRequest.getProgram().getProgramName() == null) { + throw new BadRequestException("No program name was specified for the schedule"); + } + if (scheduleFromRequest.getTrigger() == null) { + throw new BadRequestException("No trigger was specified for the schedule"); + } + ProgramType programType = ProgramType.valueOfSchedulableType( + scheduleFromRequest.getProgram().getProgramType()); + String programName = scheduleFromRequest.getProgram().getProgramName(); + ProgramId programId = applicationId.program(programType, programName); + + // Schedules are versionless + lifecycleService.ensureLatestProgramExists(programId.getProgramReference()); + + String description = Objects.firstNonNull(scheduleFromRequest.getDescription(), ""); + Map properties = Objects.firstNonNull(scheduleFromRequest.getProperties(), + Collections.emptyMap()); + List constraints = Objects.firstNonNull( + scheduleFromRequest.getConstraints(), NO_CONSTRAINTS); + long timeoutMillis = + Objects.firstNonNull(scheduleFromRequest.getTimeoutMillis(), + Schedulers.JOB_QUEUE_TIMEOUT_MILLIS); + ProgramSchedule schedule = new ProgramSchedule(scheduleName, description, programId, properties, + scheduleFromRequest.getTrigger(), constraints, timeoutMillis); + programScheduleService.add(schedule); + responder.sendStatus(HttpResponseStatus.OK); + } + + /** + * Updates the schedule for a given {@code appName}. + */ + @POST + @Path("apps/{app-name}/schedules/{schedule-name}/update") + @AuditPolicy(AuditDetail.REQUEST_BODY) + public void updateSchedule(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-name") String appName, + @PathParam("schedule-name") String scheduleName) throws Exception { + doUpdateSchedule(request, responder, namespaceId, appName, scheduleName); + } + + /* + * Deprecated : Schedules are versionless. + * */ + @Deprecated + @POST + @Path("apps/{app-name}/versions/{app-version}/schedules/{schedule-name}/update") + @AuditPolicy(AuditDetail.REQUEST_BODY) + public void updateScheduleVersioned(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-name") String appName, + @PathParam("app-version") String appVersion, + @PathParam("schedule-name") String scheduleName) throws Exception { + doUpdateSchedule(request, responder, namespaceId, appName, scheduleName); + } + + private void doUpdateSchedule(FullHttpRequest request, HttpResponder responder, + String namespaceId, String appId, + String scheduleName) throws Exception { + + ScheduleId scheduleId = new ApplicationId(namespaceId, appId).schedule(scheduleName); + ScheduleDetail scheduleDetail = readScheduleDetailBody(request, scheduleName); + + programScheduleService.update(scheduleId, scheduleDetail); + responder.sendStatus(HttpResponseStatus.OK); + } + + private ScheduleDetail readScheduleDetailBody(FullHttpRequest request, String scheduleName) + throws BadRequestException, IOException { + + JsonElement json; + try (Reader reader = new InputStreamReader(new ByteBufInputStream(request.content()), + Charsets.UTF_8)) { + // The schedule spec in the request body does not contain the program information + json = ProgramHandlerUtil.fromJson(reader, JsonElement.class); + } catch (IOException e) { + throw new IOException("Error reading request body", e); + } catch (JsonSyntaxException e) { + throw new BadRequestException("Request body is invalid json: " + e.getMessage()); + } + if (!json.isJsonObject()) { + throw new BadRequestException( + "Expected a json object in the request body but received " + ProgramHandlerUtil.toJson(json)); + } + ScheduleDetail scheduleDetail; + try { + scheduleDetail = ProgramHandlerUtil.fromJson(json, ScheduleDetail.class); + } catch (JsonSyntaxException e) { + throw new BadRequestException( + "Error parsing request body as a schedule specification: " + e.getMessage()); + } + + // If the schedule name is present in the request body, it should match the name in path params + if (scheduleDetail.getName() != null && !scheduleName.equals(scheduleDetail.getName())) { + throw new BadRequestException(String.format( + "Schedule name in the body of the request (%s) does not match the schedule name in the path parameter (%s)", + scheduleDetail.getName(), scheduleName)); + } + return scheduleDetail; + } + + @DELETE + @Path("apps/{app-name}/schedules/{schedule-name}") + public void deleteSchedule(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-name") String appName, + @PathParam("schedule-name") String scheduleName) throws Exception { + doDeleteSchedule(responder, namespaceId, appName, scheduleName); + } + + /* + * Deprecated : Schedules are versionless. + * */ + @Deprecated + @DELETE + @Path("apps/{app-name}/versions/{app-version}/schedules/{schedule-name}") + public void deleteScheduleVersioned(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId, + @PathParam("app-name") String appName, + @PathParam("app-version") String appVersion, + @PathParam("schedule-name") String scheduleName) throws Exception { + doDeleteSchedule(responder, namespaceId, appName, scheduleName); + } + + private void doDeleteSchedule(HttpResponder responder, String namespaceId, String appName, + String scheduleName) throws Exception { + ScheduleId scheduleId = new ApplicationId(namespaceId, appName).schedule(scheduleName); + programScheduleService.delete(scheduleId); + responder.sendStatus(HttpResponseStatus.OK); + } + + /** + * Returns the previous scheduled run time for all programs that are passed into the data. The + * data is an array of JSON objects where each object must contain the following three elements: + * appId, programType, and programId (flow name, service name, etc.). + *

+ * Example input: + * 


+   * [{"appId": "App1", "programType": "Workflow", "programId": "WF1"},
+   * {"appId": "App1", "programType": "Workflow", "programId": "WF2"}]
+   *
+ *

+ *

The response will be an array of JsonObjects each of which will contain the three input + * parameters as well as a "schedules" field, which is a list of {@link ScheduledRuntime} object. + *

+ *

If an error occurs in the input (for the example above, App1 does not exist), then all + * JsonObjects for which the parameters have a valid status will have the status field but all + * JsonObjects for which the parameters do not have a valid status will have an error message and + * statusCode. + */ + @POST + @Path("/previousruntime") + public void batchPreviousRunTimes(FullHttpRequest request, + HttpResponder responder, + @PathParam("namespace-id") String namespaceId) throws Exception { + List batchPrograms = ProgramHandlerUtil.validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE); + responder.sendJson(HttpResponseStatus.OK, + ProgramHandlerUtil.toJson(batchRunTimes(namespaceId, batchPrograms, true))); + } + + /** + * Returns the next scheduled run time for all programs that are passed into the data. The data is + * an array of JSON objects where each object must contain the following three elements: appId, + * programType, and programId (flow name, service name, etc.). + *

+ * Example input: + * 


+   * [{"appId": "App1", "programType": "Workflow", "programId": "WF1"},
+   * {"appId": "App1", "programType": "Workflow", "programId": "WF2"}]
+   *
+ *

+ *

The response will be an array of JsonObjects each of which will contain the three input + * parameters as well as a "schedules" field, which is a list of {@link ScheduledRuntime} object. + *

+ *

If an error occurs in the input (for the example above, App1 does not exist), then all + * JsonObjects for which the parameters have a valid status will have the status field but all + * JsonObjects for which the parameters do not have a valid status will have an error message and + * statusCode. + */ + @POST + @Path("/nextruntime") + public void batchNextRunTimes(FullHttpRequest request, + HttpResponder responder, + @PathParam("namespace-id") String namespaceId) throws Exception { + List batchPrograms = ProgramHandlerUtil.validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE); + responder.sendJson(HttpResponseStatus.OK, + ProgramHandlerUtil.toJson(batchRunTimes(namespaceId, batchPrograms, false))); + } + + /** + * Fetches scheduled run times for a set of programs. + * + * @param namespace namespace of the programs + * @param programs the list of programs to fetch scheduled run times + * @param previous {@code true} to get the previous scheduled times; {@code false} to get the + * next scheduled times + * @return a list of {@link BatchProgramSchedule} containing the result + * @throws SchedulerException if failed to fetch schedules + */ + private List batchRunTimes(String namespace, + Collection programs, + boolean previous) throws Exception { + List programReferences = programs.stream() + .map(p -> new ProgramReference(namespace, p.getAppId(), p.getProgramType(), + p.getProgramId())) + .collect(Collectors.toList()); + Map programMap = store.getPrograms(programReferences); + + List result = new ArrayList<>(); + for (ProgramReference programReference : programReferences) { + if (programMap.containsKey(programReference)) { + ProgramId programId = programMap.get(programReference); + result.add(new BatchProgramSchedule(programId, HttpResponseStatus.OK.code(), null, + getScheduledRunTimes(programId, previous))); + } else { + result.add(new BatchProgramSchedule(programReference, HttpResponseStatus.NOT_FOUND.code(), + new NotFoundException(programReference).getMessage(), null)); + } + } + return result; + } + + /** + * Returns a list of {@link ScheduledRuntime} for the given program. + * + * @param programId the program to fetch schedules for + * @param previous {@code true} to get the previous scheduled times; {@code false} to get the + * next scheduled times + * @return a list of {@link ScheduledRuntime} + * @throws SchedulerException if failed to fetch the schedule + */ + private List getScheduledRunTimes(ProgramId programId, + boolean previous) throws Exception { + if (programId.getType().getSchedulableType() == null) { + throw new BadRequestException("Program " + programId + " cannot have schedule"); + } + + if (previous) { + return programScheduleService.getPreviousScheduledRuntimes(programId); + } else { + return programScheduleService.getNextScheduledRuntimes(programId); + } + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/meta/RemotePrivilegesHandler.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/meta/RemotePrivilegesHandler.java index 396cd45f12b5..cf4c5c737d75 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/meta/RemotePrivilegesHandler.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/meta/RemotePrivilegesHandler.java @@ -34,6 +34,7 @@ import io.cdap.cdap.security.spi.authorization.AuditLogContext; import io.cdap.cdap.security.spi.authorization.AuthorizationResponse; import io.cdap.cdap.security.spi.authorization.PermissionManager; +import io.cdap.cdap.security.spi.authorization.UnauthorizedException; import io.cdap.http.HttpResponder; import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.codec.http.HttpResponseStatus; @@ -81,25 +82,32 @@ public void enforce(FullHttpRequest request, HttpResponder responder) throws Exc AuthorizationPrivilege authorizationPrivilege = GSON.fromJson( request.content().toString(StandardCharsets.UTF_8), AuthorizationPrivilege.class); - LOG.debug("Enforcing for {}", authorizationPrivilege); + LOG.trace("Enforcing for {}", authorizationPrivilege); Set permissions = authorizationPrivilege.getPermissions(); - if (authorizationPrivilege.getChildEntityType() != null) { - //It's expected that we'll always have one, but let's handle generic case - for (Permission permission : permissions) { - accessEnforcer.enforceOnParent(authorizationPrivilege.getChildEntityType(), - authorizationPrivilege.getEntity(), - authorizationPrivilege.getPrincipal(), permission); + HttpResponseStatus responseStatus = HttpResponseStatus.OK; + try { + if (authorizationPrivilege.getChildEntityType() != null) { + //It's expected that we'll always have one, but let's handle generic case + for (Permission permission : permissions) { + accessEnforcer.enforceOnParent(authorizationPrivilege.getChildEntityType(), + authorizationPrivilege.getEntity(), + authorizationPrivilege.getPrincipal(), permission); + } + } else { + accessEnforcer.enforce(authorizationPrivilege.getEntity(), + authorizationPrivilege.getPrincipal(), + permissions); } - } else { - accessEnforcer.enforce(authorizationPrivilege.getEntity(), - authorizationPrivilege.getPrincipal(), - permissions); + } catch (UnauthorizedException e) { + // In case of UnauthorizedException, we have the audit logs. So, propagating them with proper response instead of + // throwing it here as we will loose the audit logs. The caller should handle the logs and throw as needed. + responseStatus = HttpResponseStatus.valueOf(e.getStatusCode()); } Queue auditLogContextQueue = SecurityRequestContext.getAuditLogQueue(); //Clearing this so it doesn't get double write to messaging queue //This should be written by the Client who is calling this service. SecurityRequestContext.clearAuditLogQueue(); - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(auditLogContextQueue)); + responder.sendJson(responseStatus, GSON.toJson(auditLogContextQueue)); } @POST diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/preview/PreviewErrorClassificationHttpHandler.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/preview/PreviewErrorClassificationHttpHandler.java index 6708e2b2a146..71f4bcb54ba6 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/preview/PreviewErrorClassificationHttpHandler.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/preview/PreviewErrorClassificationHttpHandler.java @@ -20,37 +20,24 @@ import com.google.inject.Singleton; import io.cdap.cdap.api.dataset.lib.CloseableIterator; import io.cdap.cdap.app.preview.PreviewManager; -import io.cdap.cdap.app.preview.PreviewStatus; import io.cdap.cdap.app.preview.PreviewStatus.Status; import io.cdap.cdap.common.NotFoundException; import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.common.logging.LoggingContext; -import io.cdap.cdap.internal.app.store.RunRecordDetail; import io.cdap.cdap.logging.ErrorLogsClassifier; import io.cdap.cdap.logging.context.LoggingContextHelper; import io.cdap.cdap.logging.filter.Filter; import io.cdap.cdap.logging.filter.FilterParser; -import io.cdap.cdap.logging.gateway.handlers.AbstractLogHttpHandler; -import io.cdap.cdap.logging.gateway.handlers.ProgramRunRecordFetcher; import io.cdap.cdap.logging.read.LogEvent; import io.cdap.cdap.logging.read.LogOffset; import io.cdap.cdap.logging.read.LogReader; import io.cdap.cdap.logging.read.ReadRange; -import io.cdap.cdap.proto.ProgramRunStatus; -import io.cdap.cdap.proto.ProgramType; import io.cdap.cdap.proto.id.ApplicationId; -import io.cdap.cdap.proto.id.ProgramId; -import io.cdap.cdap.proto.id.ProgramReference; import io.cdap.cdap.proto.id.ProgramRunId; -import io.cdap.cdap.proto.security.StandardPermission; -import io.cdap.cdap.security.spi.authentication.AuthenticationContext; -import io.cdap.cdap.security.spi.authorization.AccessEnforcer; -import io.cdap.cdap.security.spi.authorization.UnauthorizedException; import io.cdap.http.HttpResponder; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponseStatus; -import java.io.IOException; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -88,6 +75,10 @@ private ProgramRunId getProgramRunId(ApplicationId applicationId) throws Excepti return programRunId; } + /** + * Returns the list of {@link io.cdap.cdap.proto.ErrorClassificationResponse} for + * failed preview run. + */ @POST @Path("/namespaces/{namespace-id}/previews/{preview-id}/classify") public void classifyRunIdLogs(HttpRequest request, HttpResponder responder, @@ -110,7 +101,7 @@ public void classifyRunIdLogs(HttpRequest request, HttpResponder responder, try (CloseableIterator logIter = logReader.getLog(loggingContext, readRange.getFromMillis(), readRange.getToMillis(), filter)) { // the iterator is closed by the BodyProducer passed to the HttpResponder - errorLogsClassifier.classify(logIter, responder); + errorLogsClassifier.classify(logIter, responder, namespaceId, null, "preview", previewId); } catch (Exception ex) { LOG.debug("Exception while classifying logs for logging context {}", loggingContext, ex); responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/preview/PreviewHttpHandlerInternal.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/preview/PreviewHttpHandlerInternal.java index e35f453f2e7b..5fce15212830 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/preview/PreviewHttpHandlerInternal.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/preview/PreviewHttpHandlerInternal.java @@ -23,6 +23,7 @@ import io.cdap.cdap.app.preview.PreviewManager; import io.cdap.cdap.app.preview.PreviewRequest; import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.internal.app.runtime.k8s.PreviewRequestPollerInfo; import io.cdap.http.AbstractHttpHandler; import io.cdap.http.HttpHandler; import io.cdap.http.HttpResponder; @@ -57,7 +58,7 @@ public void poll(FullHttpRequest request, HttpResponder responder) { if (previewRequest != null) { LOG.debug("Send preview request {} to poller {}", previewRequest.getProgram(), - Bytes.toString(pollerInfo)); + GSON.fromJson(Bytes.toString(pollerInfo), PreviewRequestPollerInfo.class)); responder.sendString(HttpResponseStatus.OK, GSON.toJson(previewRequest)); } else { responder.sendStatus(HttpResponseStatus.OK); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/util/NamespaceHelper.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/util/NamespaceHelper.java new file mode 100644 index 000000000000..63224673bef9 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/util/NamespaceHelper.java @@ -0,0 +1,56 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.gateway.handlers.util; + +import com.google.common.base.Throwables; +import io.cdap.cdap.common.NamespaceNotFoundException; +import io.cdap.cdap.common.namespace.NamespaceQueryAdmin; +import io.cdap.cdap.proto.id.NamespaceId; + +/** + * Helper class for Namespace operations. + */ +public class NamespaceHelper { + + private NamespaceHelper() { + } + + /** + * Validates that the namespace exists and gets the NamespaceId + * + * @param namespaceQueryAdmin query admin for namespace operations + * @param namespace the namespace to validate + * @return NamespaceId + * + * @throws NamespaceNotFoundException if namespace is not found + */ + public static NamespaceId validateNamespace(NamespaceQueryAdmin namespaceQueryAdmin, String namespace) + throws NamespaceNotFoundException { + NamespaceId namespaceId = new NamespaceId(namespace); + try { + namespaceQueryAdmin.get(namespaceId); + } catch (NamespaceNotFoundException e) { + throw e; + } catch (Exception e) { + // This can only happen when NamespaceAdmin uses HTTP to interact with namespaces. + // Within AppFabric, NamespaceAdmin is bound to DefaultNamespaceAdmin which directly interacts with MDS. + // Hence, this should never happen. + throw Throwables.propagate(e); + } + return namespaceId; + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/util/ProgramHandlerUtil.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/util/ProgramHandlerUtil.java new file mode 100644 index 000000000000..24055e70eeea --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/util/ProgramHandlerUtil.java @@ -0,0 +1,119 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.gateway.handlers.util; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonSyntaxException; +import com.google.inject.Inject; +import io.cdap.cdap.api.schedule.Trigger; +import io.cdap.cdap.app.store.Store; +import io.cdap.cdap.common.BadRequestException; +import io.cdap.cdap.common.io.CaseInsensitiveEnumTypeAdapterFactory; +import io.cdap.cdap.internal.app.ApplicationSpecificationAdapter; +import io.cdap.cdap.internal.app.runtime.schedule.constraint.ConstraintCodec; +import io.cdap.cdap.internal.app.runtime.schedule.trigger.SatisfiableTrigger; +import io.cdap.cdap.internal.app.runtime.schedule.trigger.TriggerCodec; +import io.cdap.cdap.internal.schedule.constraint.Constraint; +import io.cdap.cdap.proto.BatchProgram; +import io.cdap.cdap.security.spi.authentication.AuthenticationContext; +import io.cdap.cdap.security.spi.authorization.AccessEnforcer; +import io.netty.buffer.ByteBufInputStream; +import io.netty.handler.codec.http.FullHttpRequest; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.lang.reflect.Type; +import java.nio.charset.StandardCharsets; +import java.util.List; +import javax.annotation.Nullable; +import javax.validation.constraints.NotNull; + +public class ProgramHandlerUtil { + + private ProgramHandlerUtil() { + } + + /** + * Json serializer/deserializer. + */ + private static final Gson GSON = ApplicationSpecificationAdapter + .addTypeAdapters(new GsonBuilder()) + .registerTypeAdapter(Trigger.class, new TriggerCodec()) + .registerTypeAdapter(SatisfiableTrigger.class, new TriggerCodec()) + .registerTypeAdapter(Constraint.class, new ConstraintCodec()) + .create(); + + /** + * Json serde for decoding request. It uses a case insensitive enum adapter. + */ + private static final Gson DECODE_GSON = ApplicationSpecificationAdapter + .addTypeAdapters(new GsonBuilder()) + .registerTypeAdapterFactory(new CaseInsensitiveEnumTypeAdapterFactory()) + .registerTypeAdapter(Trigger.class, new TriggerCodec()) + .registerTypeAdapter(SatisfiableTrigger.class, new TriggerCodec()) + .registerTypeAdapter(Constraint.class, new ConstraintCodec()) + .create(); + + public static String toJson(Object object) { + return GSON.toJson(object); + } + + public static String toJson(Object object, @NotNull Type type) { + return GSON.toJson(object, type); + } + + public static T fromJson(@NotNull Reader reader, Class type) { + return DECODE_GSON.fromJson(reader, type); + } + + public static T fromJson(@Nullable JsonElement json, Class type) { + return DECODE_GSON.fromJson(json, type); + } + + public static List validateAndGetBatchInput(FullHttpRequest request, + Type type) + throws BadRequestException, IOException { + + List programs; + try (Reader reader = new InputStreamReader(new ByteBufInputStream(request.content()), + StandardCharsets.UTF_8)) { + try { + programs = DECODE_GSON.fromJson(reader, type); + if (programs == null) { + throw new BadRequestException( + "Request body is invalid json, please check that it is a json array."); + } + } catch (JsonSyntaxException e) { + throw new BadRequestException("Request body is invalid json: " + e.getMessage()); + } + } + + // validate input + for (BatchProgram program : programs) { + try { + program.validate(); + } catch (IllegalArgumentException e) { + throw new BadRequestException( + "Must provide valid appId, programType, and programId for each object: " + + e.getMessage()); + } + } + return programs; + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/ApplicationSpecificationCodec.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/ApplicationSpecificationCodec.java index 298fc8f3752d..7592a25fccd6 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/ApplicationSpecificationCodec.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/ApplicationSpecificationCodec.java @@ -40,7 +40,7 @@ /** * TODO: Move to cdap-proto */ -final class ApplicationSpecificationCodec extends +public final class ApplicationSpecificationCodec extends AbstractSpecificationCodec { @Override diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/InMemoryProgramRunDispatcher.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/InMemoryProgramRunDispatcher.java index 78199ddd9234..3bbe1bf42171 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/InMemoryProgramRunDispatcher.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/InMemoryProgramRunDispatcher.java @@ -287,7 +287,7 @@ public ProgramController dispatchProgram(ProgramRunDispatcherContext dispatcherC appSpec = generatedAppSpec != null ? generatedAppSpec : appSpec; newProgramDescriptor = new ProgramDescriptor(programDescriptor.getProgramId(), appSpec); } catch (Exception e) { - LOG.warn("Failed to regenerate the app spec for program {}, using the existing app spec", + LOG.error("Failed to regenerate the app spec for program {}, using the existing app spec", programId, e); } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/LocalApplicationManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/LocalApplicationManager.java index ffc4ba0efaad..f270eb68ec17 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/LocalApplicationManager.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/LocalApplicationManager.java @@ -40,10 +40,10 @@ import io.cdap.cdap.internal.app.deploy.pipeline.MetadataWriterStage; import io.cdap.cdap.internal.app.deploy.pipeline.ProgramGenerationStage; import io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository; +import io.cdap.cdap.internal.app.runtime.schedule.ScheduleManager; import io.cdap.cdap.internal.capability.CapabilityReader; import io.cdap.cdap.pipeline.Pipeline; import io.cdap.cdap.pipeline.PipelineFactory; -import io.cdap.cdap.scheduler.Scheduler; import io.cdap.cdap.security.impersonation.Impersonator; import io.cdap.cdap.security.impersonation.OwnerAdmin; import io.cdap.cdap.security.spi.authentication.AuthenticationContext; @@ -71,7 +71,7 @@ public class LocalApplicationManager implements Manager { private final MetadataServiceClient metadataServiceClient; private final Impersonator impersonator; private final AuthenticationContext authenticationContext; - private final io.cdap.cdap.scheduler.Scheduler programScheduler; + private final ScheduleManager scheduleManager; private final AccessEnforcer accessEnforcer; private final StructuredTableAdmin structuredTableAdmin; private final CapabilityReader capabilityReader; @@ -87,7 +87,7 @@ public class LocalApplicationManager implements Manager { UsageRegistry usageRegistry, ArtifactRepository artifactRepository, MetadataServiceClient metadataServiceClient, Impersonator impersonator, AuthenticationContext authenticationContext, - Scheduler programScheduler, + ScheduleManager scheduleManager, AccessEnforcer accessEnforcer, StructuredTableAdmin structuredTableAdmin, CapabilityReader capabilityReader, @@ -105,7 +105,7 @@ public class LocalApplicationManager implements Manager { this.metadataServiceClient = metadataServiceClient; this.impersonator = impersonator; this.authenticationContext = authenticationContext; - this.programScheduler = programScheduler; + this.scheduleManager = scheduleManager; this.accessEnforcer = accessEnforcer; this.structuredTableAdmin = structuredTableAdmin; this.capabilityReader = capabilityReader; @@ -128,11 +128,11 @@ public ListenableFuture deploy(I input) throws Exception { pipeline.addLast(new CreateDatasetInstancesStage(cConf, datasetFramework, ownerAdmin, authenticationContext)); pipeline.addLast(new DeletedProgramHandlerStage(store, programTerminator, - metricsSystemClient, metadataServiceClient, programScheduler)); + metricsSystemClient, metadataServiceClient, scheduleManager)); pipeline.addLast(new ProgramGenerationStage()); pipeline.addLast(new ApplicationRegistrationStage(store, usageRegistry, ownerAdmin, metricsCollectionService)); - pipeline.addLast(new DeleteAndCreateSchedulesStage(programScheduler)); + pipeline.addLast(new DeleteAndCreateSchedulesStage(scheduleManager)); pipeline.addLast(new MetadataWriterStage(metadataServiceClient)); pipeline.setFinally(new DeploymentCleanupStage()); return pipeline.execute(input); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/DeleteAndCreateSchedulesStage.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/DeleteAndCreateSchedulesStage.java index 00b8d83ff7ab..bbdab7b86217 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/DeleteAndCreateSchedulesStage.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/DeleteAndCreateSchedulesStage.java @@ -20,11 +20,11 @@ import io.cdap.cdap.api.app.ApplicationSpecification; import io.cdap.cdap.api.schedule.Trigger; import io.cdap.cdap.internal.app.runtime.schedule.ProgramSchedule; +import io.cdap.cdap.internal.app.runtime.schedule.ScheduleManager; import io.cdap.cdap.internal.schedule.ScheduleCreationSpec; import io.cdap.cdap.pipeline.AbstractStage; import io.cdap.cdap.proto.id.ApplicationId; import io.cdap.cdap.proto.id.ProgramId; -import io.cdap.cdap.scheduler.Scheduler; import java.util.HashSet; import java.util.Set; @@ -33,11 +33,11 @@ */ public class DeleteAndCreateSchedulesStage extends AbstractStage { - private final Scheduler programScheduler; + private final ScheduleManager scheduleManager; - public DeleteAndCreateSchedulesStage(Scheduler programScheduler) { + public DeleteAndCreateSchedulesStage(ScheduleManager scheduleManager) { super(TypeToken.of(ApplicationWithPrograms.class)); - this.programScheduler = programScheduler; + this.scheduleManager = scheduleManager; } @Override @@ -52,17 +52,17 @@ public void process(final ApplicationWithPrograms input) throws Exception { ApplicationId appId = input.getApplicationId(); // Get a set of new schedules from the app spec Set newSchedules = getProgramScheduleSet(appId, input.getSpecification()); - for (ProgramSchedule schedule : programScheduler.listSchedules(appId)) { + for (ProgramSchedule schedule : scheduleManager.listSchedules(appId)) { if (newSchedules.contains(schedule)) { newSchedules.remove(schedule); // Remove the existing schedule from the newSchedules continue; } // Delete the existing schedule if it is not present in newSchedules - programScheduler.deleteSchedule(schedule.getScheduleId()); + scheduleManager.deleteSchedule(schedule.getScheduleId()); } // Add new schedules - programScheduler.addSchedules(newSchedules); + scheduleManager.addSchedules(newSchedules); // Emit the input to next stage. emit(input); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/DeletedProgramHandlerStage.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/DeletedProgramHandlerStage.java index 0259d9965dec..fd9fc92e6bd4 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/DeletedProgramHandlerStage.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/DeletedProgramHandlerStage.java @@ -24,11 +24,11 @@ import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.data2.metadata.writer.MetadataServiceClient; import io.cdap.cdap.internal.app.deploy.ProgramTerminator; +import io.cdap.cdap.internal.app.runtime.schedule.ScheduleManager; import io.cdap.cdap.pipeline.AbstractStage; import io.cdap.cdap.proto.ProgramType; import io.cdap.cdap.proto.ProgramTypes; import io.cdap.cdap.proto.id.ProgramId; -import io.cdap.cdap.scheduler.Scheduler; import io.cdap.cdap.spi.metadata.MetadataMutation; import java.io.IOException; import java.util.ArrayList; @@ -53,18 +53,18 @@ public class DeletedProgramHandlerStage extends AbstractStage startPreview(PreviewRequest previewRequest) throws Exception { + public Future startPreview(PreviewRequest previewRequest) + throws Exception { + previewDataPublisher.setPublisherInfo(previewRequest.getRunnerInfo()); ProgramId programId = previewRequest.getProgram(); long submitTimeMillis = RunIds.getTime(programId.getApplication(), TimeUnit.MILLISECONDS); previewStarted(programId); @@ -208,7 +211,10 @@ public Future startPreview(PreviewRequest previewRequest) throws } LOG.debug("Starting preview for {}", programId); - ProgramController controller = programLifecycleService.start(programId, userProps, false, true); + ProgramStartRequest startRequest = programLifecycleService.prepareStart(programId, userProps, false, true); + ProgramController controller = programRuntimeService.run( + startRequest.getProgramDescriptor(), startRequest.getProgramOptions(), startRequest.getRunId()) + .getController(); long startTimeMillis = System.currentTimeMillis(); AtomicBoolean timeout = new AtomicBoolean(); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DistributedPreviewManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DistributedPreviewManager.java index 08be41066108..b52c311e8a0f 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DistributedPreviewManager.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DistributedPreviewManager.java @@ -23,7 +23,9 @@ import io.cdap.cdap.api.metrics.MetricsCollectionService; import io.cdap.cdap.app.preview.PreviewConfigModule; import io.cdap.cdap.app.preview.PreviewManager; +import io.cdap.cdap.app.preview.PreviewRequest; import io.cdap.cdap.app.preview.PreviewRequestQueue; +import io.cdap.cdap.app.preview.PreviewStatus; import io.cdap.cdap.app.store.preview.PreviewStore; import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; @@ -46,6 +48,8 @@ import io.cdap.cdap.master.spi.twill.StatefulDisk; import io.cdap.cdap.master.spi.twill.StatefulTwillPreparer; import io.cdap.cdap.messaging.spi.MessagingService; +import io.cdap.cdap.proto.BasicThrowable; +import io.cdap.cdap.proto.id.ApplicationId; import io.cdap.cdap.proto.id.NamespaceId; import io.cdap.cdap.security.authorization.AccessControllerInstantiator; import io.cdap.cdap.security.spi.authentication.AuthenticationContext; @@ -65,6 +69,7 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; import org.apache.hadoop.conf.Configuration; import org.apache.tephra.TransactionSystemClient; import org.apache.twill.api.ResourceSpecification; @@ -88,6 +93,8 @@ public class DistributedPreviewManager extends DefaultPreviewManager implements private final Configuration hConf; private final FeatureFlagsProvider featureFlagsProvider; private final TwillRunner twillRunner; + private final PreviewRunStopper previewRunStopper; + private final PreviewStore previewStore; private ScheduledExecutorService scheduler; private TwillController controller; @@ -119,6 +126,8 @@ public class DistributedPreviewManager extends DefaultPreviewManager implements this.hConf = hConf; this.twillRunner = twillRunner; this.featureFlagsProvider = new DefaultFeatureFlagsProvider(cConf); + this.previewRunStopper = previewRunStopper; + this.previewStore = previewStore; } @Override @@ -308,6 +317,66 @@ public void run() { controller = activeController; } + /** + * Overrides the default poll behavior to enforce a "one-request-per-runner" policy. + *

+ * This implementation first checks if the polling runner is already associated with an active + * preview run. If it is, this indicates a protocol violation or a state inconsistency. In this + * case, the existing preview run is immediately marked as {@link PreviewStatus.Status#KILLED} + * with a reason indicating an invalid state. A best-effort attempt is then made to terminate the + * runner's container, and the poll request is denied by returning an empty {@link Optional}. + *

+ * If the runner is not associated with an existing run, this method delegates to the parent + * {@code poll} method to retrieve the next available preview request from the queue. + * + * @param pollerInfo Information that uniquely identifies the polling preview runner. + * @return {@link Optional} containing a {@link PreviewRequest} if a job is available for a valid + * runner, or {@link Optional#empty()} if the request is denied due to a protocol violation or if + * no job is currently available in the queue. + */ + @Override + public Optional poll(@Nullable byte[] pollerInfo) { + // In a distributed environment, pollerInfo is always expected. + // Return empty to deny the request without throwing an exception. + if (pollerInfo == null) { + LOG.warn("poll() was called with a null pollerInfo in a distributed environment. " + + "This is unexpected. Denying the request."); + return Optional.empty(); + } + + // Check if the runner is already registered to an application. + // If no app is associated with this runner, it's a valid new runner. + // Proceed to the default polling behavior. + ApplicationId existingAppId = previewStore.getApplicationId(pollerInfo); + if (existingAppId == null) { + return super.poll(pollerInfo); + } + + // The runner is already registered, which is a state violation. + // Handle the violation and deny the request. + handlePollingViolation(existingAppId, pollerInfo); + return Optional.empty(); + } + + private void handlePollingViolation(ApplicationId appId, byte[] pollerInfo) { + try { + PreviewStatus status = previewStore.getPreviewStatus(appId); + if (status != null && !status.getStatus().isEndState()) { + LOG.warn("Runner for application '{}' polled for a new request, indicating a state violation. " + + "Setting status to KILLED.", appId); + previewStore.setPreviewStatus(appId, + new PreviewStatus(PreviewStatus.Status.KILLED, status.getSubmitTime(), + new BasicThrowable(new IllegalStateException( + "Preview run stopped due to an invalid state. " + + "A runner can only be assigned one preview run at a time.")), null, null)); + } + previewRunStopper.stop(pollerInfo); + } catch (Exception e) { + LOG.warn("Attempted to stop a runner due to a state violation but failed. " + + "The runner was still denied a new request.", e); + } + } + /** * Deletes the given directory {@link Path} recursively. */ diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DistributedPreviewRunStopper.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DistributedPreviewRunStopper.java index 52ba1e472472..8b377adcbf22 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DistributedPreviewRunStopper.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DistributedPreviewRunStopper.java @@ -56,7 +56,12 @@ public void stop(ApplicationId previewApp) throws Exception { throw new IllegalStateException( "Preview cannot be stopped. Please try stopping again or run the new preview."); } + stop(info); + LOG.info("Force stopped preview run {}", previewApp); + } + @Override + public void stop(byte[] info) throws Exception { PreviewRequestPollerInfo pollerInfo = GSON.fromJson(new String(info, StandardCharsets.UTF_8), PreviewRequestPollerInfo.class); Iterator controllers = twillRunner.lookup(PreviewRunnerTwillApplication.NAME) @@ -65,20 +70,18 @@ public void stop(ApplicationId previewApp) throws Exception { throw new IllegalStateException("Preview runners cannot be stopped. Please try again."); } - LOG.debug("Stopping preview run {} with poller info {}", previewApp, pollerInfo); + LOG.debug("Stopping preview run with poller info {}", pollerInfo); TwillController controller = controllers.next(); Future future; if (controller instanceof ExtendedTwillController) { future = ((ExtendedTwillController) controller).restartInstance( - PreviewRunnerTwillRunnable.class.getSimpleName(), - pollerInfo.getInstanceId(), + PreviewRunnerTwillRunnable.class.getSimpleName(), pollerInfo.getInstanceId(), pollerInfo.getInstanceUid()); } else { future = controller.restartInstances(PreviewRunnerTwillRunnable.class.getSimpleName(), pollerInfo.getInstanceId()); } future.get(); - LOG.info("Force stopped preview run {}", previewApp); } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/MessagingPreviewDataPublisher.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/MessagingPreviewDataPublisher.java index a4fa0401df50..f030d91bcbf5 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/MessagingPreviewDataPublisher.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/MessagingPreviewDataPublisher.java @@ -28,9 +28,9 @@ import io.cdap.cdap.common.service.Retries; import io.cdap.cdap.common.service.RetryStrategies; import io.cdap.cdap.common.service.RetryStrategy; +import io.cdap.cdap.messaging.client.StoreRequestBuilder; import io.cdap.cdap.messaging.spi.MessagingService; import io.cdap.cdap.messaging.spi.StoreRequest; -import io.cdap.cdap.messaging.client.StoreRequestBuilder; import io.cdap.cdap.proto.id.EntityId; import io.cdap.cdap.proto.id.NamespaceId; import io.cdap.cdap.proto.id.TopicId; @@ -46,6 +46,7 @@ public class MessagingPreviewDataPublisher implements PreviewDataPublisher { private final TopicId topic; private final MessagingService messagingService; private final RetryStrategy retryStrategy; + private byte[] publisherInfo; @Inject MessagingPreviewDataPublisher(CConfiguration cConf, @@ -57,6 +58,11 @@ public class MessagingPreviewDataPublisher implements PreviewDataPublisher { @Override public void publish(EntityId entityId, PreviewMessage previewMessage) { + if (publisherInfo != null) { + // The publisherInfo is null in case of non-distributed environments, because there is no separate + // runner container. The preview runner is a thread in the same process. + previewMessage.setPublisherInfo(publisherInfo); + } StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(previewMessage)) .build(); try { @@ -68,4 +74,8 @@ public void publish(EntityId entityId, PreviewMessage previewMessage) { e); } } + + public void setPublisherInfo(byte[] publisherInfo) { + this.publisherInfo = publisherInfo; + } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/PreviewDataSubscriberService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/PreviewDataSubscriberService.java index 1cc348d26130..ad4e17df15ab 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/PreviewDataSubscriberService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/PreviewDataSubscriberService.java @@ -35,9 +35,10 @@ import io.cdap.cdap.common.utils.ImmutablePair; import io.cdap.cdap.internal.app.runtime.ProgramRunners; import io.cdap.cdap.internal.app.store.AppMetadataStore; -import io.cdap.cdap.messaging.spi.MessagingService; import io.cdap.cdap.messaging.context.MultiThreadMessagingContext; +import io.cdap.cdap.messaging.spi.MessagingService; import io.cdap.cdap.messaging.subscriber.AbstractMessagingSubscriberService; +import io.cdap.cdap.proto.BasicThrowable; import io.cdap.cdap.proto.codec.EntityIdTypeAdapter; import io.cdap.cdap.proto.id.ApplicationId; import io.cdap.cdap.proto.id.EntityId; @@ -45,6 +46,7 @@ import io.cdap.cdap.proto.id.ProgramRunId; import io.cdap.cdap.spi.data.StructuredTableContext; import io.cdap.cdap.spi.data.transaction.TransactionRunner; +import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -68,6 +70,7 @@ public class PreviewDataSubscriberService extends private final MultiThreadMessagingContext messagingContext; private final TransactionRunner transactionRunner; private final int maxRetriesOnError; + private final PreviewRunStopper previewRunStopper; private int errorCount; private String erroredMessageId; private MetricsCollectionService metricsCollectionService; @@ -81,7 +84,8 @@ public class PreviewDataSubscriberService extends @Named(PreviewConfigModule.GLOBAL_METRICS) MetricsCollectionService metricsCollectionService, PreviewStore previewStore, - TransactionRunner transactionRunner) { + TransactionRunner transactionRunner, + PreviewRunStopper previewRunStopper) { super( NamespaceId.SYSTEM.topic(cConf.get(Constants.Preview.MESSAGING_TOPIC)), cConf.getInt(Constants.Metadata.MESSAGING_FETCH_SIZE), @@ -101,6 +105,7 @@ public class PreviewDataSubscriberService extends this.transactionRunner = transactionRunner; this.maxRetriesOnError = cConf.getInt(Constants.Metadata.MESSAGING_RETRIES_ON_CONFLICT); this.metricsCollectionService = metricsCollectionService; + this.previewRunStopper = previewRunStopper; } @Override @@ -131,6 +136,9 @@ protected void processMessages(StructuredTableContext structuredTableContext, ImmutablePair next = messages.next(); String messageId = next.getFirst(); PreviewMessage message = next.getSecond(); + if (!validateMessage(message)) { + continue; + } PreviewMessageProcessor processor = processors.computeIfAbsent(message.getType(), type -> { switch (type) { @@ -201,13 +209,6 @@ private final class PreviewDataProcessor implements PreviewMessageProcessor { @Override public void processMessage(PreviewMessage message) { - if (!(message.getEntityId() instanceof ApplicationId)) { - LOG.warn( - "Missing application id from the preview data information. Ignoring the message {}", - message); - return; - } - ApplicationId applicationId = (ApplicationId) message.getEntityId(); PreviewDataPayload payload; try { @@ -230,13 +231,6 @@ private final class PreviewStatusWriter implements PreviewMessageProcessor { @Override public void processMessage(PreviewMessage message) { - if (!(message.getEntityId() instanceof ApplicationId)) { - LOG.warn( - "Missing application id from the preview status information. Ignoring the message {}", - message); - return; - } - ApplicationId applicationId = (ApplicationId) message.getEntityId(); PreviewStatus payload; try { @@ -268,12 +262,6 @@ private final class PreviewProgramRunIdWriter implements PreviewMessageProcessor @Override public void processMessage(PreviewMessage message) { - if (!(message.getEntityId() instanceof ApplicationId)) { - LOG.warn("Missing application id from the preview run information. Ignoring the message {}", - message); - return; - } - ProgramRunId payload; try { payload = message.getPayload(GSON, ProgramRunId.class); @@ -286,4 +274,76 @@ public void processMessage(PreviewMessage message) { previewStore.setProgramId(payload); } } + + /** + * Validates an incoming {@link PreviewMessage} to enforce security policies. This method performs + * the following checks: + *

    + *
  • Ensures the message is associated with a valid {@link ApplicationId}.
    • + *
  • Allows messages that do not contain publisher information to pass (for non-authenticated flows).
    • + *
  • If publisher information is present, it validates that the message's {@code ApplicationId} + * matches the one registered for that publisher in the {@link PreviewStore}.
    • + *
+ * If a mismatch is found, it is treated as a violation, and an attempt is made to + * terminate the offending runner. + * + * @param message the message to be validated. + * @return {@code true} if the message is valid, {@code false} otherwise. + */ + private boolean validateMessage(PreviewMessage message) { + if (!(message.getEntityId() instanceof ApplicationId)) { + LOG.warn("Missing application id from the preview run information. Ignoring message: {}", + message); + return false; + } + + byte[] messageRunnerInfo = message.getPublisherInfo(); + // If there's no publisher info, the message is considered valid but unauthenticated. + // This allows for backward compatibility or simpler, non-secure, non-distributed environments. + if (messageRunnerInfo == null) { + return true; + } + + ApplicationId appIdFromMessage = (ApplicationId) message.getEntityId(); + byte[] registeredRunnerInfo = previewStore.getPreviewRequestPollerInfo(appIdFromMessage); + + // Happy Path: If no runner is registered for this app yet, or if the messageRunnerInfo from the message + // matches the registered messageRunnerInfo, the message is valid. + if (Arrays.equals(registeredRunnerInfo, messageRunnerInfo)) { + return true; + } + + // Failure Path: The publisher information does not match. + LOG.warn( + "Authentication failure: A message for application '{}' was received with a non-registered " + + "runner. Terminating the runner.", appIdFromMessage); + terminateRunner(messageRunnerInfo); + + return false; + } + + /** + * This method finds the application the suspicious runner was registered to, marks its status as + * KILLED, and then stops the runner's container. + */ + private void terminateRunner(byte[] runnerInfo) { + try { + ApplicationId appId = previewStore.getApplicationId(runnerInfo); + if (appId != null) { + PreviewStatus status = previewStore.getPreviewStatus(appId); + if (status != null && !status.getStatus().isEndState()) { + previewStore.setPreviewStatus(appId, + new PreviewStatus(PreviewStatus.Status.KILLED, + status.getSubmitTime(), new BasicThrowable(new IllegalStateException( + "Preview run stopped due to an authentication failure." + + "Please try running preview again.")), null, null)); + } + } + previewRunStopper.stop(runnerInfo); + } catch (Exception e) { + LOG.warn( + "Failed to stop runner after an authentication failure. The invalid message was still rejected.", + e); + } + } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/PreviewRunStopper.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/PreviewRunStopper.java index 01b2faea26fd..de090b92c208 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/PreviewRunStopper.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/PreviewRunStopper.java @@ -30,4 +30,21 @@ public interface PreviewRunStopper { * @throws Exception if any error while stopping */ void stop(ApplicationId preview) throws Exception; + + /** + * Stops the preview runner associated with the given poller information. + *

+ * In a distributed environment (e.g., Kubernetes), this method is responsible for finding the + * runner's container and terminating it. This is a critical action for enforcing the + * "one-request-per-runner" security policy when a suspicious runner is detected. + *

+ * In a non-distributed (standalone) environment, this is a no-op as there is no separate container + * to terminate. + *

+ * + * @param pollerInfo The poller information identifying the preview runner to be stopped. + * @throws Exception if there is an error while attempting to stop the runner in a distributed + * environment. + */ + void stop(byte[] pollerInfo) throws Exception; } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/PreviewRunnerTwillRunnable.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/PreviewRunnerTwillRunnable.java index a8731b20c784..4fff6e20336e 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/PreviewRunnerTwillRunnable.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/PreviewRunnerTwillRunnable.java @@ -67,7 +67,7 @@ import io.cdap.cdap.master.environment.MasterEnvironments; import io.cdap.cdap.master.spi.environment.MasterEnvironment; import io.cdap.cdap.master.spi.twill.ExtendedTwillContext; -import io.cdap.cdap.messaging.guice.MessagingServiceModule; +import io.cdap.cdap.messaging.guice.client.PreviewRunnerMessagingClientModule; import io.cdap.cdap.proto.id.NamespaceId; import io.cdap.cdap.security.auth.context.AuthenticationContextModules; import io.cdap.cdap.security.authorization.AuthorizationEnforcementModule; @@ -76,6 +76,7 @@ import io.cdap.cdap.security.impersonation.UGIProvider; import io.cdap.cdap.spi.data.StorageProvider; import java.io.File; +import java.security.SecureRandom; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -168,12 +169,13 @@ private void doInitialize(TwillContext context) throws Exception { hConf.clear(); hConf.addResource(new File(getArgument("hConf")).toURI().toURL()); + byte[] secureToken = generateSecureToken(); PreviewRequestPollerInfo pollerInfo; if (context instanceof ExtendedTwillContext) { pollerInfo = new PreviewRequestPollerInfo(context.getInstanceId(), - ((ExtendedTwillContext) context).getUID()); + ((ExtendedTwillContext) context).getUID(), secureToken); } else { - pollerInfo = new PreviewRequestPollerInfo(context.getInstanceId(), null); + pollerInfo = new PreviewRequestPollerInfo(context.getInstanceId(), null, secureToken); } CConfiguration cConf = CConfiguration.create(new File(getArgument("cConf")).toURI().toURL()); @@ -237,7 +239,7 @@ protected void configure() { } modules.add(new PreviewRunnerManagerModule().getDistributedModules()); - modules.add(new MessagingServiceModule(cConf)); + modules.add(new PreviewRunnerMessagingClientModule(cConf)); modules.add(new SecureStoreClientModule()); // Needed for InMemoryProgramRunnerModule. We use local metadata reader/publisher to avoid conflicting with // metadata stored in AppFabric. @@ -274,4 +276,14 @@ protected void configure() { return Guice.createInjector(modules); } + + /** + * Generates a cryptographically strong random byte array to be used as a secret token. + */ + private byte[] generateSecureToken() { + SecureRandom random = new SecureRandom(); + byte[] token = new byte[32]; // 256 bits + random.nextBytes(token); + return token; + } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/RemotePreviewRequestFetcher.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/RemotePreviewRequestFetcher.java index d563de6bd78b..82023f1fc64a 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/RemotePreviewRequestFetcher.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/RemotePreviewRequestFetcher.java @@ -55,19 +55,24 @@ public class RemotePreviewRequestFetcher implements PreviewRequestFetcher { @Override public Optional fetch() throws IOException, UnauthorizedException { + byte[] runnerInfo = pollerInfoProvider.get(); HttpRequest request = remoteClientInternal.requestBuilder(HttpMethod.POST, "requests/pull") - .withBody(ByteBuffer.wrap(pollerInfoProvider.get())) + .withBody(ByteBuffer.wrap(runnerInfo)) .build(); HttpResponse httpResponse = remoteClientInternal.execute(request); if (httpResponse.getResponseCode() == 200) { PreviewRequest previewRequest = GSON.fromJson(httpResponse.getResponseBodyAsString(), PreviewRequest.class); - if (previewRequest != null && previewRequest.getPrincipal() != null) { - SecurityRequestContext.setUserId(previewRequest.getPrincipal().getName()); - SecurityRequestContext.setUserCredential(previewRequest.getPrincipal().getFullCredential()); - } - return Optional.ofNullable(previewRequest); + return Optional.ofNullable(previewRequest) + .map(req -> { + if (req.getPrincipal() != null) { + SecurityRequestContext.setUserId(req.getPrincipal().getName()); + SecurityRequestContext.setUserCredential(req.getPrincipal().getFullCredential()); + } + req.setRunnerInfo(runnerInfo); + return req; + }); } throw new IOException( String.format("Received status code:%s and body: %s", httpResponse.getResponseCode(), diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/program/MessagingProgramStatePublisher.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/program/MessagingProgramStatePublisher.java index 20067a04f654..c1cba7557f60 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/program/MessagingProgramStatePublisher.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/program/MessagingProgramStatePublisher.java @@ -42,6 +42,7 @@ import io.cdap.cdap.proto.id.ProgramRunId; import io.cdap.cdap.proto.id.TopicId; import java.io.IOException; +import java.net.SocketTimeoutException; import java.util.Collections; import java.util.List; import java.util.Map; @@ -142,9 +143,7 @@ public void publish(Notification.Type notificationType, Map prop .build()); LOG.trace("Published program status notification: {}", programStatusNotification); done = true; - } catch (IOException | AccessException e) { - throw Throwables.propagate(e); - } catch (TopicNotFoundException | ServiceUnavailableException e) { + } catch (TopicNotFoundException | ServiceUnavailableException | SocketTimeoutException e) { // These exceptions are retry-able due to TMS not completely started if (startTime < 0) { startTime = System.currentTimeMillis(); @@ -164,6 +163,8 @@ public void publish(Notification.Type notificationType, Map prop Thread.currentThread().interrupt(); done = true; } + } catch (AccessException | IOException e) { + throw Throwables.propagate(e); } } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/ProgramControllerServiceAdapter.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/ProgramControllerServiceAdapter.java index 24196f81e235..5a1c4b8159a0 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/ProgramControllerServiceAdapter.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/ProgramControllerServiceAdapter.java @@ -16,8 +16,8 @@ package io.cdap.cdap.internal.app.runtime; -import com.google.common.base.Throwables; import com.google.common.util.concurrent.Service; +import io.cdap.cdap.api.exception.WrappedStageException; import io.cdap.cdap.app.runtime.ProgramController; import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.common.logging.Loggers; @@ -94,7 +94,7 @@ public void running() { @Override public void failed(Service.State from, Throwable failure) { - Throwable rootCause = Throwables.getRootCause(failure); + Throwable rootCause = getRootCause(failure); LOG.error("{} Program '{}' failed.", getProgramRunId().getType(), getProgramRunId().getProgram(), failure); USER_LOG.error( @@ -104,6 +104,18 @@ public void failed(Service.State from, Throwable failure) { error(failure); } + private Throwable getRootCause(Throwable failure) { + Throwable cause; + while ((cause = failure.getCause()) != null) { + failure = cause; + if (WrappedStageException.class.isAssignableFrom(failure.getClass())) { + // to prevent stage information from getting lost from the log. + break; + } + } + return failure; + } + @Override public void terminated(Service.State from) { if (from != Service.State.STOPPING) { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/ProgramStartRequest.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/ProgramStartRequest.java new file mode 100644 index 000000000000..84814d5a8f10 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/ProgramStartRequest.java @@ -0,0 +1,53 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.runtime; + +import io.cdap.cdap.app.program.ProgramDescriptor; +import io.cdap.cdap.app.runtime.ProgramOptions; +import io.cdap.cdap.common.app.RunIds; +import io.cdap.cdap.proto.id.ProgramRunId; +import org.apache.twill.api.RunId; + +/** + * Request object for starting a new Program run. + */ +public class ProgramStartRequest { + + private final ProgramOptions programOptions; + private final ProgramDescriptor programDescriptor; + private final RunId runId; + + public ProgramStartRequest(ProgramOptions programOptions, + ProgramDescriptor programDescriptor, + ProgramRunId programRunId) { + this.programOptions = programOptions; + this.programDescriptor = programDescriptor; + this.runId = RunIds.fromString(programRunId.getRun()); + } + + public ProgramOptions getProgramOptions() { + return programOptions; + } + + public ProgramDescriptor getProgramDescriptor() { + return programDescriptor; + } + + public RunId getRunId() { + return runId; + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/Artifacts.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/Artifacts.java index e23a7e788d81..7ddd4722a3f7 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/Artifacts.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/Artifacts.java @@ -43,9 +43,12 @@ public static String getFileName(ArtifactId artifactId) { * @return the resolved config type * @throws IllegalArgumentException if the config type is not a valid type */ - public static Type getConfigType(Class appClass) { + public static Type getConfigType(Class appClass) { + // Class has to be a child of Application + Preconditions.checkArgument(Application.class.isAssignableFrom(appClass), "The given class : " + appClass + + " is not supported. Type must be a child class of Application"); TypeToken configType = TypeToken.of(appClass) - .resolveType(Application.class.getTypeParameters()[0]); + .resolveType(Application.class.getTypeParameters()[0]); if (Reflections.isResolved(configType.getType())) { // Default the type to Config.class if the resolved type is not subclass of Config. // It normally won't happen, unless someone generate the bytecode directly. diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/DefaultArtifactInspector.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/DefaultArtifactInspector.java index 354ea62030d7..662695ff324d 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/DefaultArtifactInspector.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/DefaultArtifactInspector.java @@ -241,13 +241,19 @@ private ArtifactClasses.Builder inspectApplications(Id.Artifact artifactId, return builder; } - Application app = (Application) mainClass.newInstance(); + //If it's application assignable class, then ensure it has a public no-arg constructor + try { + mainClass.getConstructor(); + } catch (NoSuchMethodException e) { + throw new InvalidArtifactException(String.format( + "Application class %s in artifact %s must have a public no-arg constructor.", mainClassName, artifactId)); + } java.lang.reflect.Type configType; // if the user parameterized their application, like 'xyz extends Application', // we can deserialize the config into that object. Otherwise it'll just be a Config try { - configType = Artifacts.getConfigType(app.getClass()); + configType = Artifacts.getConfigType(mainClass); } catch (Exception e) { throw new InvalidArtifactException(String.format( "Could not resolve config type for Application class %s in artifact %s. " @@ -258,7 +264,7 @@ private ArtifactClasses.Builder inspectApplications(Id.Artifact artifactId, Schema configSchema = configType == Config.class ? null : schemaGenerator.generate(configType); builder.addApp(new ApplicationClass(mainClassName, "", configSchema, - getArtifactRequirements(app.getClass()))); + getArtifactRequirements(mainClass))); } catch (ClassNotFoundException e) { throw new InvalidArtifactException(String.format( "Could not find Application main class %s in artifact %s.", mainClassName, artifactId)); @@ -267,10 +273,6 @@ private ArtifactClasses.Builder inspectApplications(Id.Artifact artifactId, "Config for Application %s in artifact %s has an unsupported schema. " + "The type must extend Config and cannot be parameterized.", mainClassName, artifactId)); - } catch (InstantiationException | IllegalAccessException e) { - throw new InvalidArtifactException(String.format( - "Could not instantiate Application class %s in artifact %s.", mainClassName, artifactId), - e); } return builder; @@ -459,8 +461,8 @@ private String getPluginDescription(Class cls) { } /** - * Returns the metadata mutation for this plugin, return {@code null} if no metadata annotation is - * there + * Returns the metadata mutation for this plugin. + * return {@code null} if no metadata annotation is there. */ @Nullable private MetadataMutation getMetadataMutation(PluginId pluginId, Class cls) @@ -622,7 +624,7 @@ private boolean isPlugin(String className, ClassLoader classLoader) { // Use ASM to inspect the class bytecode to see if it is annotated with @Plugin final boolean[] isPlugin = new boolean[1]; ClassReader cr = new ClassReader(is); - cr.accept(new ClassVisitor(Opcodes.ASM5) { + cr.accept(new ClassVisitor(Opcodes.ASM7) { @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) { if (Plugin.class.getName().equals(Type.getType(desc).getClassName()) && visible) { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/runtimejob/DefaultRuntimeJob.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/runtimejob/DefaultRuntimeJob.java index 9298e30b5f65..94d1d33eba49 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/runtimejob/DefaultRuntimeJob.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/runtimejob/DefaultRuntimeJob.java @@ -243,7 +243,7 @@ public void run(RuntimeJobEnvironment runtimeJobEnv) throws Exception { programDescriptor = new ProgramDescriptor( programDescriptor.getProgramId(), appSpec); } catch (Exception e) { - LOG.warn("Failed to regenerate the app spec for program {}, using the existing app spec", + LOG.error("Failed to regenerate the app spec for program {}, using the existing app spec", programId, e); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/k8s/PreviewRequestPollerInfo.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/k8s/PreviewRequestPollerInfo.java index 08db55df55ed..a0b38469b0f9 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/k8s/PreviewRequestPollerInfo.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/k8s/PreviewRequestPollerInfo.java @@ -16,19 +16,32 @@ package io.cdap.cdap.internal.app.runtime.k8s; +import com.google.common.base.Objects; +import java.util.Arrays; import javax.annotation.Nullable; /** - * Poller information holder. + * A holder for information that uniquely identifies and authenticates a preview runner. This object + * is serialized and stored in the PreviewStore. */ public class PreviewRequestPollerInfo { private final int instanceId; private final String instanceUid; + private final byte[] secureToken; - public PreviewRequestPollerInfo(int instanceId, @Nullable String instanceUid) { + /** + * Constructs a new PreviewRequestPollerInfo. + * + * @param instanceId the instance id of the runner + * @param instanceUid the unique UID of the runner pod + * @param secureToken a secret token used to authenticate requests from the runner + */ + public PreviewRequestPollerInfo(int instanceId, @Nullable String instanceUid, + @Nullable byte[] secureToken) { this.instanceId = instanceId; this.instanceUid = instanceUid; + this.secureToken = secureToken; } public int getInstanceId() { @@ -42,9 +55,25 @@ public String getInstanceUid() { @Override public String toString() { - return "PreviewRequestPollerInfo{" - + "instanceId=" + instanceId - + ", instanceUid='" + instanceUid + '\'' - + '}'; + return "PreviewRequestPollerInfo{" + "instanceId=" + instanceId + ", instanceUid='" + + instanceUid + '\'' + '}'; + } + + @Override + public int hashCode() { + return Objects.hashCode(instanceId, instanceUid, Arrays.hashCode(secureToken)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PreviewRequestPollerInfo that = (PreviewRequestPollerInfo) o; + return instanceId == that.instanceId && Objects.equal(instanceUid, that.instanceUid) + && Arrays.equals(secureToken, that.secureToken); } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/LocalScheduleManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/LocalScheduleManager.java new file mode 100644 index 000000000000..7ba652f8dff5 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/LocalScheduleManager.java @@ -0,0 +1,163 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.runtime.schedule; + +import com.google.common.base.Throwables; +import com.google.inject.Inject; +import io.cdap.cdap.common.AlreadyExistsException; +import io.cdap.cdap.common.BadRequestException; +import io.cdap.cdap.common.ConflictException; +import io.cdap.cdap.common.NotFoundException; +import io.cdap.cdap.common.ProfileConflictException; +import io.cdap.cdap.common.conf.CConfiguration; +import io.cdap.cdap.internal.app.runtime.SystemArguments; +import io.cdap.cdap.internal.app.runtime.schedule.queue.JobQueueTable; +import io.cdap.cdap.internal.app.runtime.schedule.store.ProgramScheduleStoreDataset; +import io.cdap.cdap.internal.app.runtime.schedule.store.Schedulers; +import io.cdap.cdap.internal.app.store.profile.ProfileStore; +import io.cdap.cdap.messaging.spi.MessagingService; +import io.cdap.cdap.proto.ProgramType; +import io.cdap.cdap.proto.id.ProfileId; +import io.cdap.cdap.proto.id.ScheduleId; +import io.cdap.cdap.runtime.spi.profile.ProfileStatus; +import io.cdap.cdap.spi.data.transaction.TransactionRunner; +import io.cdap.cdap.spi.data.transaction.TransactionRunners; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * {@code ScheduleManager} to manage program schedules. This class is meant to be used by in-memory + * modules and tests. + */ +public class LocalScheduleManager extends ScheduleManager { + + private static final Logger LOG = LoggerFactory.getLogger(LocalScheduleManager.class); + private final TimeSchedulerService timeSchedulerService; + + /** + * Parameterized constructor for LocalScheduleManager. + * + * @param transactionRunner TransactionRunner + * @param messagingService MessagingService + * @param cConf CConfiguration + * @param timeSchedulerService TimeSchedulerService + */ + @Inject + public LocalScheduleManager(TransactionRunner transactionRunner, MessagingService messagingService, + CConfiguration cConf, TimeSchedulerService timeSchedulerService) { + super(transactionRunner, messagingService, cConf); + this.timeSchedulerService = timeSchedulerService; + } + + @Override + public void addSchedules(Iterable schedules) + throws BadRequestException, NotFoundException, IOException, ConflictException { + for (ProgramSchedule schedule : schedules) { + if (!schedule.getProgramId().getType().equals(ProgramType.WORKFLOW)) { + throw new BadRequestException(String.format( + "Cannot schedule program %s of type %s: Only workflows can be scheduled", + schedule.getProgramId().getProgram(), schedule.getProgramId().getType())); + } + } + + try { + TransactionRunners.run(transactionRunner, context -> { + ProgramScheduleStoreDataset store = Schedulers.getScheduleStore(context); + ProfileStore profileStore = ProfileStore.get(context); + long updatedTime = store.addSchedules(schedules); + for (ProgramSchedule schedule : schedules) { + if (schedule.getProperties() != null) { + Optional profile = SystemArguments.getProfileIdFromArgs( + schedule.getProgramId().getNamespaceId(), schedule.getProperties()); + if (profile.isPresent()) { + ProfileId profileId = profile.get(); + if (profileStore.getProfile(profileId).getStatus() == ProfileStatus.DISABLED) { + throw new ProfileConflictException( + String.format("Profile %s in namespace %s is disabled. It cannot " + + "be assigned to schedule %s", + profileId.getProfile(), profileId.getNamespace(), + schedule.getName()), profileId); + } + } + } + try { + timeSchedulerService.addProgramSchedule(schedule); + } catch (SchedulerException e) { + LOG.error("Exception occurs when adding schedule {}", schedule, e); + throw new RuntimeException(e); + } + } + for (ProgramSchedule schedule : schedules) { + ScheduleId scheduleId = schedule.getScheduleId(); + + // If the added properties contains profile assignment, add the assignment. + Optional profileId = SystemArguments.getProfileIdFromArgs( + scheduleId.getNamespaceId(), + schedule.getProperties()); + if (profileId.isPresent()) { + profileStore.addProfileAssignment(profileId.get(), scheduleId); + } + } + // Publish the messages at the end of transaction. + for (ProgramSchedule schedule : schedules) { + adminEventPublisher.publishScheduleCreation(schedule.getScheduleId(), updatedTime); + } + return null; + }, Exception.class); + } catch (NotFoundException | ProfileConflictException | AlreadyExistsException e) { + throw e; + } catch (Exception e) { + throw Throwables.propagate(e); + } + } + + @Override + public void deleteSchedule(ScheduleId scheduleId) + throws NotFoundException, BadRequestException, IOException, ConflictException { + TransactionRunners.run(transactionRunner, context -> { + ProgramScheduleStoreDataset store = Schedulers.getScheduleStore(context); + ProfileStore profileStore = ProfileStore.get(context); + JobQueueTable queue = JobQueueTable.getJobQueue(context, cConf); + long deleteTime = System.currentTimeMillis(); + List toNotify = new ArrayList<>(); + ProgramSchedule schedule = store.getSchedule(scheduleId); + timeSchedulerService.deleteProgramSchedule(schedule); + queue.markJobsForDeletion(scheduleId, deleteTime); + toNotify.add(schedule); + // If the deleted schedule has properties with profile assignment, remove the assignment. + Optional profileId = SystemArguments.getProfileIdFromArgs( + scheduleId.getNamespaceId(), + schedule.getProperties()); + if (profileId.isPresent()) { + try { + profileStore.removeProfileAssignment(profileId.get(), scheduleId); + } catch (NotFoundException e) { + // This should not happen since the profile cannot be deleted if there is a schedule who is using it. + LOG.warn("Unable to find the profile {} when deleting schedule {}, " + + "skipping assignment deletion.", profileId.get(), scheduleId); + } + } + store.deleteSchedule(scheduleId); + toNotify.forEach(adminEventPublisher::publishScheduleDeletion); + return null; + }, NotFoundException.class); + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/RemoteScheduleManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/RemoteScheduleManager.java new file mode 100644 index 000000000000..1b421c6a6306 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/RemoteScheduleManager.java @@ -0,0 +1,110 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.runtime.schedule; + +import com.google.inject.Inject; +import io.cdap.cdap.common.BadRequestException; +import io.cdap.cdap.common.ConflictException; +import io.cdap.cdap.common.NotFoundException; +import io.cdap.cdap.common.conf.CConfiguration; +import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.conf.Constants.Gateway; +import io.cdap.cdap.common.http.DefaultHttpRequestConfig; +import io.cdap.cdap.common.internal.remote.RemoteClient; +import io.cdap.cdap.common.internal.remote.RemoteClientFactory; +import io.cdap.cdap.gateway.handlers.util.ProgramHandlerUtil; +import io.cdap.cdap.messaging.spi.MessagingService; +import io.cdap.cdap.proto.ScheduleDetail; +import io.cdap.cdap.proto.id.ScheduleId; +import io.cdap.cdap.security.spi.authorization.UnauthorizedException; +import io.cdap.cdap.spi.data.transaction.TransactionRunner; +import io.cdap.common.http.HttpMethod; +import io.cdap.common.http.HttpRequest; +import io.cdap.common.http.HttpResponse; +import java.io.IOException; +import java.net.HttpURLConnection; + +/** + * {@code ScheduleManager} to manage program schedules. This class uses remote client to communicate with + * {@code ProgramScheduleHttpHandler}. + */ +public class RemoteScheduleManager extends ScheduleManager { + + private RemoteClient remoteClient; + + /** + * Parameterized constructor for RemoteScheduleManager. + * + * @param transactionRunner TransactionRunner + * @param messagingService MessagingService + * @param cConf CConfiguration + * @param clientFactory RemoteClientFactory + */ + @Inject + public RemoteScheduleManager(TransactionRunner transactionRunner, MessagingService messagingService, + CConfiguration cConf, RemoteClientFactory clientFactory) { + super(transactionRunner, messagingService, cConf); + this.remoteClient = clientFactory.createRemoteClient(Constants.Service.APP_FABRIC_PROCESSOR, + new DefaultHttpRequestConfig(false), + Gateway.API_VERSION_3); + } + + @Override + public void addSchedules(Iterable schedules) + throws BadRequestException, NotFoundException, IOException, ConflictException { + for (ProgramSchedule schedule : schedules) { + ScheduleId scheduleId = schedule.getScheduleId(); + String url = String.format("namespaces/%s/apps/%s/schedules/%s", + scheduleId.getNamespace(), + scheduleId.getApplication(), + scheduleId.getSchedule()); + HttpRequest.Builder requestBuilder = remoteClient.requestBuilder(HttpMethod.PUT, url); + ScheduleDetail scheduleDetail = schedule.toScheduleDetail(); + requestBuilder.withBody(ProgramHandlerUtil.toJson(scheduleDetail, ScheduleDetail.class)); + execute(requestBuilder.build()); + } + } + + @Override + public void deleteSchedule(ScheduleId scheduleId) + throws NotFoundException, BadRequestException, IOException, ConflictException { + String url = String.format("namespaces/%s/apps/%s/schedules/%s", + scheduleId.getNamespace(), + scheduleId.getApplication(), + scheduleId.getSchedule()); + HttpRequest.Builder requestBuilder = remoteClient.requestBuilder(HttpMethod.DELETE, url); + execute(requestBuilder.build()); + } + + private HttpResponse execute(HttpRequest request) + throws IOException, NotFoundException, UnauthorizedException, BadRequestException, ConflictException { + HttpResponse httpResponse = remoteClient.execute(request); + switch (httpResponse.getResponseCode()) { + case HttpURLConnection.HTTP_OK: + return httpResponse; + case HttpURLConnection.HTTP_BAD_REQUEST: + throw new BadRequestException(httpResponse.getResponseBodyAsString()); + case HttpURLConnection.HTTP_NOT_FOUND: + throw new NotFoundException(httpResponse.getResponseBodyAsString()); + case HttpURLConnection.HTTP_CONFLICT: + throw new ConflictException(httpResponse.getResponseBodyAsString()); + default: + throw new IOException( + String.format("Request failed %s", httpResponse.getResponseBodyAsString())); + } + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/ScheduleManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/ScheduleManager.java new file mode 100644 index 000000000000..182f8f16dfd2 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/ScheduleManager.java @@ -0,0 +1,144 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.runtime.schedule; + +import io.cdap.cdap.common.BadRequestException; +import io.cdap.cdap.common.ConflictException; +import io.cdap.cdap.common.NotFoundException; +import io.cdap.cdap.common.conf.CConfiguration; +import io.cdap.cdap.internal.app.runtime.schedule.store.ProgramScheduleStoreDataset; +import io.cdap.cdap.internal.app.runtime.schedule.store.Schedulers; +import io.cdap.cdap.internal.app.runtime.schedule.trigger.ProgramStatusTrigger; +import io.cdap.cdap.internal.profile.AdminEventPublisher; +import io.cdap.cdap.messaging.context.MultiThreadMessagingContext; +import io.cdap.cdap.messaging.spi.MessagingService; +import io.cdap.cdap.proto.id.ApplicationId; +import io.cdap.cdap.proto.id.ProgramId; +import io.cdap.cdap.proto.id.ScheduleId; +import io.cdap.cdap.spi.data.transaction.TransactionRunner; +import io.cdap.cdap.spi.data.transaction.TransactionRunners; +import java.io.IOException; +import java.util.List; + +/** + * Abstract class to manager program schedules. + */ +public abstract class ScheduleManager { + + protected final CConfiguration cConf; + protected final TransactionRunner transactionRunner; + protected final AdminEventPublisher adminEventPublisher; + + /** + * Parameterized constructor for ScheduleManager. + * + * @param transactionRunner TransactionRunner + * @param messagingService MessagingService + * @param cConf CConfiguration + */ + public ScheduleManager(TransactionRunner transactionRunner, + MessagingService messagingService, CConfiguration cConf) { + this.cConf = cConf; + this.transactionRunner = transactionRunner; + MultiThreadMessagingContext messagingContext = new MultiThreadMessagingContext(messagingService); + this.adminEventPublisher = new AdminEventPublisher(cConf, messagingContext); + } + + /** + * Add program schedules for the given collection of Program Schedules. + * + * @param schedules Collection of Program Schedules. + * + * @throws BadRequestException if the program type or not workflow. + * @throws NotFoundException if the program was not found. + * @throws IOException if there was an internal error in adding schedules. + * @throws ConflictException if the profile is disabled. + */ + public abstract void addSchedules(Iterable schedules) + throws Exception; + + /** + * Deleted a program schedule for the given {@code scheduleId}. + * + * @param scheduleId for the schedule to be deleted. + * + * @throws NotFoundException if the schedule is not found. + * @throws IOException if an internal error occurred when deleting the schedule. + */ + public abstract void deleteSchedule(ScheduleId scheduleId) + throws Exception; + + /** + * Deletes all the schedules for the given {@code ApplicationId}. + * + * @param appId the {@code ApplicationId} whose schedules need to be deleted. + */ + public void deleteSchedules(ApplicationId appId) { + TransactionRunners.run(transactionRunner, context -> { + ProgramScheduleStoreDataset store = Schedulers.getScheduleStore(context); + List programSchedules = store.listSchedules(appId); + for (ProgramSchedule programSchedule : programSchedules) { + deleteSchedule(programSchedule.getScheduleId()); + } + }, RuntimeException.class); + } + + /** + * Deletes all the schedules for the given {@code ProgramId}. + * + * @param programId the {@code ProgramId} whose schedules need to be deleted. + */ + public void deleteSchedules(ProgramId programId) { + TransactionRunners.run(transactionRunner, context -> { + ProgramScheduleStoreDataset store = Schedulers.getScheduleStore(context); + List programSchedules = store.listSchedules(programId); + for (ProgramSchedule programSchedule : programSchedules) { + deleteSchedule(programSchedule.getScheduleId()); + } + }, RuntimeException.class); + } + + /** + * Update all schedules that can be triggered by the given deleted program. A schedule will be + * removed if the only {@link ProgramStatusTrigger} in it is triggered by the deleted program. + * Schedules with composite triggers will be updated if the composite trigger can still be + * satisfied after the program is deleted, otherwise the schedules will be deleted. + * + * @param programId the program id for which to delete the schedules. + */ + public void modifySchedulesTriggeredByDeletedProgram(ProgramId programId) { + TransactionRunners.run(transactionRunner, context -> { + ProgramScheduleStoreDataset store = Schedulers.getScheduleStore(context); + List deletedSchedules = store.modifySchedulesTriggeredByDeletedProgram(programId); + deletedSchedules.forEach(adminEventPublisher::publishScheduleDeletion); + }, RuntimeException.class); + } + + /** + * Lists all the schedules for the given {@code ApplicationId}. + * + * @param appId the {@code ApplicationId}. + * + * @return List of {@code ProgramSchedule}. + */ + public List listSchedules(ApplicationId appId) { + return TransactionRunners.run(transactionRunner, context -> { + ProgramScheduleStoreDataset store = Schedulers.getScheduleStore(context); + return store.listSchedules(appId); + }, RuntimeException.class); + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/InMemoryProgramRuntimeService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/InMemoryProgramRuntimeService.java index 38a1019ce455..eef7abc6eed7 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/InMemoryProgramRuntimeService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/InMemoryProgramRuntimeService.java @@ -47,9 +47,8 @@ public final class InMemoryProgramRuntimeService extends AbstractProgramRuntimeS @Inject InMemoryProgramRuntimeService(CConfiguration cConf, ProgramRunnerFactory programRunnerFactory, - ProgramStateWriter programStateWriter, - ProgramRunDispatcherFactory programRunDispatcherFactory) { - super(cConf, programRunnerFactory, programStateWriter, programRunDispatcherFactory); + ProgramStateWriter programStateWriter, ProgramRunDispatcherFactory programRunDispatcherFactory) { + super(cConf, programRunnerFactory, programStateWriter, programRunDispatcherFactory ); } @Override diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricProcessorService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricProcessorService.java index d2a2d891db66..97fafa95f819 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricProcessorService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricProcessorService.java @@ -26,6 +26,7 @@ import io.cdap.cdap.app.runtime.ProgramRuntimeService; import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.conf.Constants.AppFabric; import io.cdap.cdap.common.conf.Constants.Service; import io.cdap.cdap.common.conf.SConfiguration; import io.cdap.cdap.common.discovery.ResolvingDiscoverable; @@ -42,7 +43,6 @@ import io.cdap.cdap.internal.sysapp.SystemAppManagementService; import io.cdap.cdap.proto.id.NamespaceId; import io.cdap.cdap.scheduler.CoreSchedulerService; -import io.cdap.cdap.scheduler.ScheduleNotificationSubscriberService; import io.cdap.cdap.security.auth.AuditLogSubscriberService; import io.cdap.http.HttpHandler; import io.cdap.http.NettyHttpService; @@ -51,7 +51,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; -import javax.annotation.Nullable; import org.apache.twill.common.Cancellable; import org.apache.twill.discovery.DiscoveryService; import org.slf4j.Logger; @@ -75,13 +74,12 @@ public class AppFabricProcessorService extends AbstractIdleService { private final RunRecordCorrectorService runRecordCorrectorService; private final RunDataTimeToLiveService runDataTimeToLiveService; private final ProgramRunStatusMonitorService programRunStatusMonitorService; - private final RunRecordMonitorService runRecordCounterService; + private final FlowControlService runRecordCounterService; private final CoreSchedulerService coreSchedulerService; private final ProvisioningService provisioningService; private final BootstrapService bootstrapService; private final SystemAppManagementService systemAppManagementService; private final OperationNotificationSubscriberService operationNotificationSubscriberService; - private final ScheduleNotificationSubscriberService scheduleNotificationSubscriberService; private final CConfiguration cConf; private final SConfiguration sConf; private final boolean sslEnabled; @@ -111,10 +109,9 @@ public AppFabricProcessorService(CConfiguration cConf, ProvisioningService provisioningService, BootstrapService bootstrapService, SystemAppManagementService systemAppManagementService, - RunRecordMonitorService runRecordCounterService, + FlowControlService runRecordCounterService, RunDataTimeToLiveService runDataTimeToLiveService, - OperationNotificationSubscriberService operationNotificationSubscriberService, - ScheduleNotificationSubscriberService scheduleNotificationSubscriberService) { + OperationNotificationSubscriberService operationNotificationSubscriberService) { this.hostname = hostname; this.discoveryService = discoveryService; this.handlers = handlers; @@ -137,7 +134,6 @@ public AppFabricProcessorService(CConfiguration cConf, this.runRecordCounterService = runRecordCounterService; this.runDataTimeToLiveService = runDataTimeToLiveService; this.operationNotificationSubscriberService = operationNotificationSubscriberService; - this.scheduleNotificationSubscriberService = scheduleNotificationSubscriberService; } /** @@ -167,7 +163,6 @@ protected void startUp() throws Exception { programStopSubscriberService.start(), runRecordCorrectorService.start(), programRunStatusMonitorService.start(), - scheduleNotificationSubscriberService.start(), coreSchedulerService.start(), runRecordCounterService.start(), runDataTimeToLiveService.start(), @@ -188,7 +183,7 @@ protected void startUp() throws Exception { Constants.AppFabric.DEFAULT_BOSS_THREADS)) .setWorkerThreadPoolSize(cConf.getInt(Constants.AppFabric.WORKER_THREADS, Constants.AppFabric.DEFAULT_WORKER_THREADS)) - .setPort(cConf.getInt(Constants.AppFabric.SERVER_PORT)); + .setPort(cConf.getInt(Constants.AppFabric.PROCESSOR_PORT)); if (sslEnabled) { new HttpsEnabler().configureKeyStore(cConf, sConf).enable(httpServiceBuilder); } @@ -201,7 +196,6 @@ protected void startUp() throws Exception { protected void shutDown() throws Exception { LOG.info("Stopping AppFabric processor service."); cancelHttpService.cancel(); - scheduleNotificationSubscriberService.stopAndWait(); coreSchedulerService.stopAndWait(); bootstrapService.stopAndWait(); systemAppManagementService.stopAndWait(); @@ -224,7 +218,7 @@ private Cancellable startHttpService(NettyHttpService httpService) throws Except String announceAddress = cConf.get(Constants.Service.MASTER_SERVICES_ANNOUNCE_ADDRESS, httpService.getBindAddress().getHostName()); - int announcePort = cConf.getInt(Constants.AppFabric.SERVER_ANNOUNCE_PORT, + int announcePort = cConf.getInt(AppFabric.PROCESSOR_ANNOUNCE_PORT, httpService.getBindAddress().getPort()); final InetSocketAddress socketAddress = new InetSocketAddress(announceAddress, announcePort); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java index 3f6e6418a484..0d584668b8e4 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java @@ -43,7 +43,6 @@ import io.cdap.cdap.internal.namespace.credential.NamespaceCredentialProviderService; import io.cdap.cdap.internal.provision.ProvisioningService; import io.cdap.cdap.proto.id.NamespaceId; -import io.cdap.cdap.scheduler.CoreSchedulerService; import io.cdap.cdap.sourcecontrol.RepositoryCleanupService; import io.cdap.cdap.sourcecontrol.operationrunner.SourceControlOperationRunner; import io.cdap.cdap.spi.data.transaction.TransactionRunner; @@ -78,7 +77,6 @@ public class AppFabricServer extends AbstractIdleService { private final ApplicationLifecycleService applicationLifecycleService; private final Set servicesNames; private final Set handlerHookNames; - private final CoreSchedulerService coreSchedulerService; private final CredentialProviderService credentialProviderService; private final NamespaceCredentialProviderService namespaceCredentialProviderService; private final ProvisioningService provisioningService; @@ -107,7 +105,6 @@ public AppFabricServer(CConfiguration cConf, SConfiguration sConf, ApplicationLifecycleService applicationLifecycleService, @Named("appfabric.services.names") Set servicesNames, @Named("appfabric.handler.hooks") Set handlerHookNames, - CoreSchedulerService coreSchedulerService, CredentialProviderService credentialProviderService, NamespaceCredentialProviderService namespaceCredentialProviderService, ProvisioningService provisioningService, @@ -126,7 +123,6 @@ public AppFabricServer(CConfiguration cConf, SConfiguration sConf, this.handlerHookNames = handlerHookNames; this.applicationLifecycleService = applicationLifecycleService; this.sslEnabled = cConf.getBoolean(Constants.Security.SSL.INTERNAL_ENABLED); - this.coreSchedulerService = coreSchedulerService; this.credentialProviderService = credentialProviderService; this.namespaceCredentialProviderService = namespaceCredentialProviderService; this.provisioningService = provisioningService; @@ -155,7 +151,6 @@ protected void startUp() throws Exception { provisioningService.start(), applicationLifecycleService.start(), bootstrapService.start(), - coreSchedulerService.start(), credentialProviderService.start(), sourceControlOperationRunner.start(), repositoryCleanupService.start() @@ -209,7 +204,6 @@ protected void startUp() throws Exception { @Override protected void shutDown() throws Exception { - coreSchedulerService.stopAndWait(); cancelHttpService.cancel(); applicationLifecycleService.stopAndWait(); bootstrapService.stopAndWait(); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleService.java index f67673bf097b..22eef1f32806 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleService.java @@ -79,6 +79,7 @@ import io.cdap.cdap.internal.app.runtime.artifact.ArtifactDetail; import io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository; import io.cdap.cdap.internal.app.runtime.artifact.Artifacts; +import io.cdap.cdap.internal.app.runtime.schedule.ScheduleManager; import io.cdap.cdap.internal.app.store.ApplicationMeta; import io.cdap.cdap.internal.app.store.RunRecordDetail; import io.cdap.cdap.internal.app.store.state.AppStateKey; @@ -112,7 +113,6 @@ import io.cdap.cdap.proto.security.Principal; import io.cdap.cdap.proto.security.StandardPermission; import io.cdap.cdap.proto.sourcecontrol.SourceControlMeta; -import io.cdap.cdap.scheduler.Scheduler; import io.cdap.cdap.security.impersonation.EntityImpersonator; import io.cdap.cdap.security.impersonation.Impersonator; import io.cdap.cdap.security.impersonation.OwnerAdmin; @@ -165,7 +165,7 @@ public class ApplicationLifecycleService extends AbstractIdleService { */ private final CConfiguration cConf; private final Store store; - private final Scheduler scheduler; + private final ScheduleManager scheduleManager; private final UsageRegistry usageRegistry; private final PreferencesService preferencesService; private final MetricsSystemClient metricsSystemClient; @@ -189,7 +189,7 @@ public class ApplicationLifecycleService extends AbstractIdleService { */ @Inject public ApplicationLifecycleService(CConfiguration cConf, - Store store, Scheduler scheduler, UsageRegistry usageRegistry, + Store store, ScheduleManager scheduleManager, UsageRegistry usageRegistry, PreferencesService preferencesService, MetricsSystemClient metricsSystemClient, OwnerAdmin ownerAdmin, ArtifactRepository artifactRepository, ManagerFactory managerFactory, @@ -203,7 +203,7 @@ public ApplicationLifecycleService(CConfiguration cConf, Constants.AppFabric.DEFAULT_APP_UPDATE_SCHEDULES); this.batchSize = cConf.getInt(AppFabric.STREAMING_BATCH_SIZE); this.store = store; - this.scheduler = scheduler; + this.scheduleManager = scheduleManager; this.usageRegistry = usageRegistry; this.preferencesService = preferencesService; this.metricsSystemClient = metricsSystemClient; @@ -299,6 +299,22 @@ public boolean scanApplications(ScanApplicationsRequest request, } } + /** + * Get application count of the latest versions in the namespace. + * + * @param namespace namespace for which count is to be returned. + * @return Count of the applications in the namespace. + */ + public long getApplicationsCount(NamespaceId namespace) { + if (namespace == null) { + throw new IllegalStateException("Application scan request without namespace"); + } + // Get count should have the same permissions as list apps since its used in tandem. + accessEnforcer.enforceOnParent(EntityType.APPLICATION, namespace, + authenticationContext.getPrincipal(), StandardPermission.LIST); + return store.getApplicationCount(namespace); + } + private void processApplications(List> list, Consumer consumer) { @@ -777,7 +793,6 @@ private ApplicationId updateApplicationInternal(ApplicationId appId, ApplicationUpdateResult updateResult = app.updateConfig(updateContext); updatedAppConfig = GSON.toJson(updateResult.getNewConfig(), configType); } - Principal requestingUser = authenticationContext.getPrincipal(); String versionId = appId.getVersion(); // If LCM flow is enabled - we generate specific versions of the app. @@ -796,8 +811,8 @@ private ApplicationId updateApplicationInternal(ApplicationId appId, .setConfigString(updatedAppConfig) .setOwnerPrincipal(ownerPrincipal) .setUpdateSchedules(false) - .setChangeDetail(new ChangeDetail(null, appId.getVersion(), requestingUser == null ? null : - requestingUser.getName(), System.currentTimeMillis())) + .setChangeDetail(new ChangeDetail(null, appId.getVersion(), + decodeUserId(authenticationContext), System.currentTimeMillis())) .setDeployedApplicationSpec(appSpec) .setIsUpgrade(true) .build(); @@ -1102,7 +1117,7 @@ private ApplicationWithPrograms deployApp(NamespaceId namespaceId, @Nullable Str ChangeDetail change = new ChangeDetail( changeSummary == null ? null : changeSummary.getDescription(), changeSummary == null ? null : changeSummary.getParentVersion(), - requestingUser == null ? null : requestingUser.getName(), + decodeUserId(authenticationContext), System.currentTimeMillis()); // deploy application with newly added artifact AppDeploymentInfo deploymentInfo = AppDeploymentInfo.builder() @@ -1339,9 +1354,9 @@ private void deletePreferences(ApplicationId appId, ApplicationSpecification app */ private void deleteApp(ApplicationId appId, ApplicationSpecification spec) throws IOException { //Delete the schedules - scheduler.deleteSchedules(appId); + scheduleManager.deleteSchedules(appId); for (WorkflowSpecification workflowSpec : spec.getWorkflows().values()) { - scheduler.modifySchedulesTriggeredByDeletedProgram(appId.workflow(workflowSpec.getName())); + scheduleManager.modifySchedulesTriggeredByDeletedProgram(appId.workflow(workflowSpec.getName())); } deleteMetrics(appId, spec); @@ -1379,9 +1394,9 @@ private void deleteApp(ApplicationId appId, ApplicationSpecification spec) throw */ private void deleteAppVersion(ApplicationId appId, ApplicationSpecification spec) { //Delete the schedules - scheduler.deleteSchedules(appId); + scheduleManager.deleteSchedules(appId); for (WorkflowSpecification workflowSpec : spec.getWorkflows().values()) { - scheduler.modifySchedulesTriggeredByDeletedProgram(appId.workflow(workflowSpec.getName())); + scheduleManager.modifySchedulesTriggeredByDeletedProgram(appId.workflow(workflowSpec.getName())); } store.removeApplication(appId); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/FlowControlService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/FlowControlService.java new file mode 100644 index 000000000000..921cbe588e40 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/FlowControlService.java @@ -0,0 +1,167 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.services; + +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.AbstractIdleService; +import com.google.inject.Inject; +import io.cdap.cdap.api.metrics.MetricsCollectionService; +import io.cdap.cdap.app.program.ProgramDescriptor; +import io.cdap.cdap.app.runtime.ProgramOptions; +import io.cdap.cdap.common.app.RunIds; +import io.cdap.cdap.common.conf.CConfiguration; +import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.internal.app.store.AppMetadataStore; +import io.cdap.cdap.proto.ProgramRunStatus; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.proto.id.ProgramRunId; +import io.cdap.cdap.spi.data.transaction.TransactionRunner; +import io.cdap.cdap.spi.data.transaction.TransactionRunners; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Maintain and provides total number of launching and running run-records. This class is used by + * flow-control mechanism for launch requests. + */ +public class FlowControlService extends AbstractIdleService { + + private static final Logger LOG = LoggerFactory.getLogger(FlowControlService.class); + + private final MetricsCollectionService metricsCollectionService; + private final TransactionRunner transactionRunner; + private final int readStalenessSeconds; + + private static final Map tags = ImmutableMap.of( + Constants.Metrics.Tag.NAMESPACE, NamespaceId.SYSTEM.getNamespace() + ); + + /** + * Monitors the program flow control. + * + * @param metricsCollectionService collect metrics + */ + @Inject + public FlowControlService( + MetricsCollectionService metricsCollectionService, + TransactionRunner transactionRunner, CConfiguration cConf) { + this.metricsCollectionService = metricsCollectionService; + this.transactionRunner = transactionRunner; + this.readStalenessSeconds = cConf.getInt(Constants.Metrics.FlowControl.READ_STALENESS_SECONDS, 0); + } + + @Override + protected void startUp() throws Exception { + LOG.info("FlowControlService started."); + } + + @Override + protected void shutDown() throws Exception { + LOG.info("FlowControlService successfully shut down."); + } + + /** + * Add a new in-flight launch request and return total number of launching and running programs. + * + * @param programRunId run id associated with the launch request + * @return total number of launching and running program runs. + */ + public Counter addRequestAndGetCounter(ProgramRunId programRunId, ProgramOptions programOptions, + ProgramDescriptor programDescriptor) throws Exception { + if (RunIds.getTime(programRunId.getRun(), TimeUnit.MILLISECONDS) == -1) { + throw new Exception("None time-based UUIDs are not supported"); + } + + Counter counter = TransactionRunners.run(transactionRunner, context -> { + AppMetadataStore store = AppMetadataStore.create(context); + store.recordProgramPending(programRunId, + programOptions.getArguments().asMap(), + programOptions.getUserArguments().asMap(), + programDescriptor.getArtifactId().toApiArtifactId()); + int launchingCount = store.getFlowControlLaunchingCount(readStalenessSeconds); + int runningCount = store.getFlowControlRunningCount(readStalenessSeconds); + return new Counter(launchingCount, runningCount); + }); + LOG.info("Added request with runId {}.", programRunId); + emitMetrics(Constants.Metrics.FlowControl.LAUNCHING_COUNT, counter.getLaunchingCount()); + + LOG.info( + "Counter has {} concurrent launching and {} running programs.", + counter.getLaunchingCount(), + counter.getRunningCount()); + return counter; + } + + /** + * Get total number of launching and running programs. + * + * @return Counter with total number of launching and running program runs. + */ + public Counter getCounter() { + return TransactionRunners.run(transactionRunner, context -> { + AppMetadataStore store = AppMetadataStore.create(context); + return new Counter(store.getFlowControlLaunchingCount(readStalenessSeconds), + store.getFlowControlRunningCount(readStalenessSeconds)); + }); + } + + public void emitFlowControlMetrics() { + Counter counter = getCounter(); + emitMetrics(Constants.Metrics.FlowControl.LAUNCHING_COUNT, counter.getLaunchingCount()); + emitMetrics(Constants.Metrics.FlowControl.RUNNING_COUNT, counter.getRunningCount()); + } + + private void emitMetrics(String metricName, long value) { + LOG.trace("Setting metric {} to value {}", metricName, value); + metricsCollectionService.getContext(tags).gauge(metricName, value); + } + + /** + * Counts the concurrent program runs. + */ + public class Counter { + + /** + * Total number of launch requests that have been accepted but still missing in metadata store + + * * total number of run records with {@link ProgramRunStatus#PENDING} status + total number of + * run records with {@link ProgramRunStatus#STARTING} status. + */ + private final int launchingCount; + + /** + * Total number of run records with {@link ProgramRunStatus#RUNNING} status + Total number of run + * records with {@link ProgramRunStatus#SUSPENDED} status + Total number of run records with + * {@link ProgramRunStatus#RESUMING} status. + */ + private final int runningCount; + + Counter(int launchingCount, int runningCount) { + this.launchingCount = launchingCount; + this.runningCount = runningCount; + } + + public int getLaunchingCount() { + return launchingCount; + } + + public int getRunningCount() { + return runningCount; + } + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ProgramLifecycleService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ProgramLifecycleService.java index 8bd33edccd14..8ee093574b07 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ProgramLifecycleService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ProgramLifecycleService.java @@ -17,7 +17,6 @@ package io.cdap.cdap.internal.app.services; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ListenableFuture; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -29,11 +28,8 @@ import io.cdap.cdap.api.plugin.Plugin; import io.cdap.cdap.app.guice.ClusterMode; import io.cdap.cdap.app.program.ProgramDescriptor; -import io.cdap.cdap.app.runtime.LogLevelUpdater; -import io.cdap.cdap.app.runtime.ProgramController; import io.cdap.cdap.app.runtime.ProgramOptions; import io.cdap.cdap.app.runtime.ProgramRuntimeService; -import io.cdap.cdap.app.runtime.ProgramRuntimeService.RuntimeInfo; import io.cdap.cdap.app.runtime.ProgramStateWriter; import io.cdap.cdap.app.store.ScanApplicationsRequest; import io.cdap.cdap.app.store.Store; @@ -54,6 +50,7 @@ import io.cdap.cdap.internal.app.ApplicationSpecificationAdapter; import io.cdap.cdap.internal.app.runtime.BasicArguments; import io.cdap.cdap.internal.app.runtime.ProgramOptionConstants; +import io.cdap.cdap.internal.app.runtime.ProgramStartRequest; import io.cdap.cdap.internal.app.runtime.SimpleProgramOptions; import io.cdap.cdap.internal.app.runtime.SystemArguments; import io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository; @@ -67,7 +64,6 @@ import io.cdap.cdap.internal.provision.ProvisioningService; import io.cdap.cdap.proto.ProgramHistory; import io.cdap.cdap.proto.ProgramRecord; -import io.cdap.cdap.proto.ProgramRunClusterStatus; import io.cdap.cdap.proto.ProgramRunStatus; import io.cdap.cdap.proto.ProgramStatus; import io.cdap.cdap.proto.ProgramType; @@ -112,7 +108,6 @@ import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.twill.api.RunId; -import org.apache.twill.api.logging.LogEntry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -134,71 +129,52 @@ public class ProgramLifecycleService { ProgramRunStatus.SUSPENDED); private final Store store; - private final ProfileService profileService; - private final ProgramRuntimeService runtimeService; - private final PropertiesResolver propertiesResolver; - private final PreferencesService preferencesService; + private final ProgramStateWriter programStateWriter; private final AccessEnforcer accessEnforcer; private final AuthenticationContext authenticationContext; + private final PropertiesResolver propertiesResolver; + private final CapabilityReader capabilityReader; + private final ArtifactRepository artifactRepository; + private final ProfileService profileService; + private final PreferencesService preferencesService; private final ProvisionerNotifier provisionerNotifier; private final ProvisioningService provisioningService; - private final ProgramStateWriter programStateWriter; - private final CapabilityReader capabilityReader; + private final FlowControlService flowControlService; private final int maxConcurrentRuns; private final int maxConcurrentLaunching; private final int defaultStopTimeoutSecs; private final int batchSize; - private final ArtifactRepository artifactRepository; - private final RunRecordMonitorService runRecordMonitorService; + private final boolean userProgramLaunchDisabled; @Inject ProgramLifecycleService(CConfiguration cConf, - Store store, ProfileService profileService, ProgramRuntimeService runtimeService, + Store store, ProfileService profileService, PropertiesResolver propertiesResolver, PreferencesService preferencesService, AccessEnforcer accessEnforcer, AuthenticationContext authenticationContext, ProvisionerNotifier provisionerNotifier, ProvisioningService provisioningService, ProgramStateWriter programStateWriter, CapabilityReader capabilityReader, ArtifactRepository artifactRepository, - RunRecordMonitorService runRecordMonitorService) { + FlowControlService flowControlService) { + this.store = store; + this.programStateWriter = programStateWriter; + this.accessEnforcer = accessEnforcer; + this.authenticationContext = authenticationContext; + this.propertiesResolver = propertiesResolver; + this.capabilityReader = capabilityReader; + this.artifactRepository = artifactRepository; this.maxConcurrentRuns = cConf.getInt(Constants.AppFabric.MAX_CONCURRENT_RUNS); this.maxConcurrentLaunching = cConf.getInt(Constants.AppFabric.MAX_CONCURRENT_LAUNCHING); this.defaultStopTimeoutSecs = cConf.getInt(Constants.AppFabric.PROGRAM_MAX_STOP_SECONDS); this.userProgramLaunchDisabled = cConf.getBoolean( Constants.AppFabric.USER_PROGRAM_LAUNCH_DISABLED, false); this.batchSize = cConf.getInt(Constants.AppFabric.STREAMING_BATCH_SIZE); - this.store = store; this.profileService = profileService; - this.runtimeService = runtimeService; - this.propertiesResolver = propertiesResolver; this.preferencesService = preferencesService; - this.accessEnforcer = accessEnforcer; - this.authenticationContext = authenticationContext; this.provisionerNotifier = provisionerNotifier; this.provisioningService = provisioningService; - this.programStateWriter = programStateWriter; - this.capabilityReader = capabilityReader; - this.artifactRepository = artifactRepository; - this.runRecordMonitorService = runRecordMonitorService; - } - - /** - * Returns the program status. - * - * @param programId the id of the program for which the status call is made - * @return the status of the program - * @throws NotFoundException if the application to which this program belongs was not found - */ - public ProgramStatus getProgramStatus(ProgramId programId) throws Exception { - // check that app exists - ApplicationId appId = programId.getParent(); - ApplicationSpecification appSpec = store.getApplication(appId); - if (appSpec == null) { - throw new NotFoundException(appId); - } - - return getExistingAppProgramStatus(appSpec, programId); + this.flowControlService = flowControlService; } /** @@ -214,26 +190,21 @@ public ProgramStatus getProgramStatus(ProgramReference programReference) throws } /** - * Returns the program status based on the active run records of a program. A program is RUNNING - * if there are any RUNNING, STOPPING, or SUSPENDED run records. A program is starting if there - * are any PENDING or STARTING run records and no RUNNING run records. Otherwise, it is STOPPED. + * Returns the program status. * - * @param runRecords run records for the program - * @return the program status + * @param programId the id of the program for which the status call is made + * @return the status of the program + * @throws NotFoundException if the application to which this program belongs was not found */ - @VisibleForTesting - static ProgramStatus getProgramStatus(Collection runRecords) { - boolean hasStarting = false; - for (RunRecordDetail runRecord : runRecords) { - ProgramRunStatus runStatus = runRecord.getStatus(); - if (runStatus == ProgramRunStatus.RUNNING || runStatus == ProgramRunStatus.SUSPENDED - || runStatus == ProgramRunStatus.STOPPING) { - return ProgramStatus.RUNNING; - } - hasStarting = hasStarting || runStatus == ProgramRunStatus.STARTING - || runStatus == ProgramRunStatus.PENDING; + public ProgramStatus getProgramStatus(ProgramId programId) throws Exception { + // check that app exists + ApplicationId appId = programId.getParent(); + ApplicationSpecification appSpec = store.getApplication(appId); + if (appSpec == null) { + throw new NotFoundException(appId); } - return hasStarting ? ProgramStatus.STARTING : ProgramStatus.STOPPED; + + return getExistingAppProgramStatus(appSpec, programId); } /** @@ -243,8 +214,7 @@ static ProgramStatus getProgramStatus(Collection runRecords) { * @return a {@link Map} from the {@link ProgramId} to the corresponding status; there will be no * entry for programs that do not exist. */ - public Map getProgramStatuses(Collection programRefs) - throws Exception { + public Map getProgramStatuses(Collection programRefs) { // filter the result Set visibleEntities = accessEnforcer.isVisible( new LinkedHashSet<>(programRefs), @@ -506,27 +476,6 @@ private void addProgramHistory(List histories, List overrides, boolean deb } authorizePipelineRuntimeImpersonation(userArgs); - return runInternal(programId, userArgs, sysArgs, debug); } @@ -728,10 +645,9 @@ public RunId runInternal(ProgramId programId, Map userArgs, userId = userId == null ? "" : userId; checkCapability(programDescriptor); - ProgramRunId programRunId = programId.run(runId); - RunRecordMonitorService.Counter counter = runRecordMonitorService.addRequestAndGetCount( - programRunId); + FlowControlService.Counter counter = flowControlService.addRequestAndGetCounter( + programRunId, programOptions, programDescriptor); boolean done = false; try { @@ -765,7 +681,7 @@ public RunId runInternal(ProgramId programId, Map userArgs, done = true; } finally { if (!done) { - runRecordMonitorService.removeRequest(programRunId, false); + flowControlService.emitFlowControlMetrics(); } } @@ -822,7 +738,11 @@ ProgramOptions createProgramOptions(ProgramId programId, Map use /** * Starts a Program with the specified argument overrides, skipping cluster lifecycle steps in the - * run. NOTE: This method should only be called from preview runner. + * run. + * + * NOTE: {@Link ProgramRuntimeService#run} needs be called to start the program run. + * + * NOTE: This method should only be called from preview runner. * * @param programId the {@link ProgramId} to start/stop * @param overrides the arguments to override in the program's configured user arguments @@ -831,7 +751,7 @@ ProgramOptions createProgramOptions(ProgramId programId, Map use * otherwise * @param isPreview true if the program is for preview run, for preview run, the app is * already deployed with resolved properties, so no need to regenerate app spec again - * @return {@link ProgramController} + * * @throws ConflictException if the specified program is already running, and if concurrent * runs are not allowed * @throws NotFoundException if the specified program or the app it belongs to is not found in @@ -842,7 +762,7 @@ ProgramOptions createProgramOptions(ProgramId programId, Map use * @throws Exception if there were other exceptions checking if the current user is authorized * to start the program */ - public ProgramController start(ProgramId programId, Map overrides, boolean debug, + public ProgramStartRequest prepareStart(ProgramId programId, Map overrides, boolean debug, boolean isPreview) throws Exception { accessEnforcer.enforce(programId, authenticationContext.getPrincipal(), ApplicationPermission.EXECUTE); @@ -859,62 +779,16 @@ public ProgramController start(ProgramId programId, Map override } authorizePipelineRuntimeImpersonation(userArgs); - BasicArguments systemArguments = new BasicArguments(sysArgs); BasicArguments userArguments = new BasicArguments(userArgs); - ProgramOptions options = new SimpleProgramOptions(programId, systemArguments, userArguments, - debug); - ProgramDescriptor programDescriptor = store.loadProgram(programId); - ProgramRunId programRunId = programId.run(RunIds.generate()); + ProgramOptions programOptions = new SimpleProgramOptions(programId, systemArguments, userArguments, debug); + ProgramDescriptor programDescriptor = store.loadProgram(programId); checkCapability(programDescriptor); + ProgramRunId programRunId = programId.run(RunIds.generate()); - programStateWriter.start(programRunId, options, null, programDescriptor); - return startInternal(programDescriptor, options, programRunId); - } - - private void checkCapability(ProgramDescriptor programDescriptor) throws Exception { - //check for capability at application class level - Set applicationClasses = artifactRepository - .getArtifact(Id.Artifact.fromEntityId(programDescriptor.getArtifactId())).getMeta() - .getClasses() - .getApps(); - for (ApplicationClass applicationClass : applicationClasses) { - Set capabilities = applicationClass.getRequirements().getCapabilities(); - capabilityReader.checkAllEnabled(capabilities); - } - for (Map.Entry pluginEntry : programDescriptor.getApplicationSpecification() - .getPlugins() - .entrySet()) { - Set capabilities = pluginEntry.getValue().getPluginClass().getRequirements() - .getCapabilities(); - capabilityReader.checkAllEnabled(capabilities); - } - } - - /** - * Starts a Program run with the given arguments. This method skips cluster lifecycle steps and - * does not perform authorization checks. If the program is already started, returns the - * controller for the program. NOTE: This method should only be used from this service and the - * {@link ProgramNotificationSubscriberService} upon receiving a {@link - * ProgramRunClusterStatus#PROVISIONED} state. - * - * @param programDescriptor descriptor of the program to run - * @param programOptions options for the program run - * @param programRunId program run id - * @return controller for the program - */ - ProgramController startInternal(ProgramDescriptor programDescriptor, - ProgramOptions programOptions, ProgramRunId programRunId) { - RunId runId = RunIds.fromString(programRunId.getRun()); - - synchronized (this) { - RuntimeInfo runtimeInfo = runtimeService.lookup(programRunId.getParent(), runId); - if (runtimeInfo != null) { - return runtimeInfo.getController(); - } - return runtimeService.run(programDescriptor, programOptions, runId).getController(); - } + programStateWriter.start(programRunId, programOptions, null, programDescriptor); + return new ProgramStartRequest(programOptions, programDescriptor, programRunId); } /** @@ -1137,63 +1011,6 @@ public Map getRuntimeArgs(@Name("programId") ProgramId programId return preferencesService.getProperties(programId); } - /** - * Update log levels for the given program. Only supported program types for this action are - * {@link ProgramType#SERVICE} and {@link ProgramType#WORKER}. - * - * @param programId the {@link ProgramId} of the program for which log levels are to be - * updated - * @param logLevels the {@link Map} of the log levels to be updated. - * @param runId the run id of the program. - * @throws InterruptedException if there is an error while asynchronously updating log - * levels. - * @throws ExecutionException if there is an error while asynchronously updating log levels. - * @throws BadRequestException if the log level is not valid or the program type is not - * supported. - * @throws UnauthorizedException if the user does not have privileges to update log levels for - * the specified program. To update log levels for a program, a user needs {@link - * StandardPermission#UPDATE} on the program. - */ - public void updateProgramLogLevels(ProgramId programId, Map logLevels, - @Nullable String runId) throws Exception { - accessEnforcer.enforce(programId, authenticationContext.getPrincipal(), - StandardPermission.UPDATE); - if (!EnumSet.of(ProgramType.SERVICE, ProgramType.WORKER).contains(programId.getType())) { - throw new BadRequestException( - String.format("Updating log levels for program type %s is not supported", - programId.getType().getPrettyName())); - } - updateLogLevels(programId, logLevels, runId); - } - - /** - * Reset log levels for the given program. Only supported program types for this action are {@link - * ProgramType#SERVICE} and {@link ProgramType#WORKER}. - * - * @param programId the {@link ProgramId} of the program for which log levels are to be - * reset. - * @param loggerNames the {@link String} set of the logger names to be updated, empty means - * reset for all loggers. - * @param runId the run id of the program. - * @throws InterruptedException if there is an error while asynchronously resetting log - * levels. - * @throws ExecutionException if there is an error while asynchronously resetting log levels. - * @throws UnauthorizedException if the user does not have privileges to reset log levels for - * the specified program. To reset log levels for a program, a user needs {@link - * StandardPermission#UPDATE} on the program. - */ - public void resetProgramLogLevels(ProgramId programId, Set loggerNames, - @Nullable String runId) throws Exception { - accessEnforcer.enforce(programId, authenticationContext.getPrincipal(), - StandardPermission.UPDATE); - if (!EnumSet.of(ProgramType.SERVICE, ProgramType.WORKER).contains(programId.getType())) { - throw new BadRequestException( - String.format("Resetting log levels for program type %s is not supported", - programId.getType().getPrettyName())); - } - resetLogLevels(programId, loggerNames, runId); - } - /** * Ensures the caller is authorized to check if the given program exists. */ @@ -1225,10 +1042,6 @@ public void ensureLatestProgramExists(ProgramReference programRef) throws Except Store.ensureProgramExists(programRef.id(appSpec.getAppVersion()), appSpec); } - private boolean isStopped(ProgramId programId) throws Exception { - return ProgramStatus.STOPPED == getProgramStatus(programId); - } - /** * Checks if the given program is running and is allowed for concurrent execution. * @@ -1237,9 +1050,9 @@ private boolean isStopped(ProgramId programId) throws Exception { * @throws NotFoundException if the program is not found * @throws Exception if failed to determine the state */ - private synchronized void checkConcurrentExecution(ProgramId programId) throws Exception { + public void checkConcurrentExecution(ProgramId programId) throws Exception { + Map runs = store.getActiveRuns(programId); if (isConcurrentRunsInSameAppForbidden(programId.getType())) { - Map runs = runtimeService.list(programId); if (!runs.isEmpty() || !isStoppedInSameProgram(programId)) { throw new ConflictException( String.format( @@ -1248,13 +1061,8 @@ private synchronized void checkConcurrentExecution(ProgramId programId) throws E } } if (!isConcurrentRunsAllowed(programId.getType())) { - List runIds = new ArrayList<>(); - for (Map.Entry entry : runtimeService.list(programId.getType()) - .entrySet()) { - if (programId.equals(entry.getValue().getProgramId())) { - runIds.add(entry.getKey()); - } - } + List runIds = runs.keySet().stream().map(r -> programId.run(r.getRun())) + .collect(Collectors.toList()); if (!runIds.isEmpty() || !isStopped(programId)) { throw new ConflictException( String.format("Program %s is already running with run ids %s", programId, runIds)); @@ -1262,101 +1070,6 @@ private synchronized void checkConcurrentExecution(ProgramId programId) throws E } } - /** - * Returns whether the given program is stopped in all versions of the app. - * - * @param programId the id of the program for which the stopped status in all versions of the - * app is found - * @return whether the given program is stopped in all versions of the app - * @throws NotFoundException if the application to which this program belongs was not found - */ - private boolean isStoppedInSameProgram(ProgramId programId) throws Exception { - // check that app exists - Collection appIds = store.getAllAppVersionsAppIds( - programId.getParent().getAppReference()); - if (appIds == null || appIds.isEmpty()) { - throw new NotFoundException( - Id.Application.from(programId.getNamespace(), programId.getApplication())); - } - ApplicationSpecification appSpec = store.getApplication(programId.getParent()); - for (ApplicationId appId : appIds) { - ProgramId pId = appId.program(programId.getType(), programId.getProgram()); - if (!getExistingAppProgramStatus(appSpec, pId).equals(ProgramStatus.STOPPED)) { - return false; - } - } - return true; - } - - private boolean isConcurrentRunsInSameAppForbidden(ProgramType type) { - // Concurrent runs in different (or same) versions of an application are forbidden for worker - return EnumSet.of(ProgramType.WORKER).contains(type); - } - - private boolean isConcurrentRunsAllowed(ProgramType type) { - // Concurrent runs are only allowed for the Workflow, MapReduce and Spark - return EnumSet.of(ProgramType.WORKFLOW, ProgramType.MAPREDUCE, ProgramType.SPARK) - .contains(type); - } - - private Map findRuntimeInfo( - ProgramId programId, @Nullable String runId) throws BadRequestException { - - if (runId != null) { - RunId run; - try { - run = RunIds.fromString(runId); - } catch (IllegalArgumentException e) { - throw new BadRequestException("Error parsing run-id.", e); - } - ProgramRuntimeService.RuntimeInfo runtimeInfo = runtimeService.lookup(programId, run); - return runtimeInfo == null ? Collections.emptyMap() - : Collections.singletonMap(run, runtimeInfo); - } - return new HashMap<>(runtimeService.list(programId)); - } - - @Nullable - private ProgramRuntimeService.RuntimeInfo findRuntimeInfo(ProgramId programId) - throws BadRequestException { - return findRuntimeInfo(programId, null).values().stream().findFirst().orElse(null); - } - - /** - * Set instances for the given program. Only supported program types for this action are {@link - * ProgramType#SERVICE} and {@link ProgramType#WORKER}. - * - * @param programId the {@link ProgramId} of the program for which instances are to be - * updated - * @param instances the number of instances to be updated. - * @throws InterruptedException if there is an error while asynchronously updating instances - * @throws ExecutionException if there is an error while asynchronously updating instances - * @throws BadRequestException if the number of instances specified is less than 0 - * @throws UnauthorizedException if the user does not have privileges to set instances for the - * specified program. To set instances for a program, a user needs {@link - * StandardPermission#UPDATE} on the program. - */ - public void setInstances(ProgramId programId, int instances) throws Exception { - accessEnforcer.enforce(programId, authenticationContext.getPrincipal(), - StandardPermission.UPDATE); - if (instances < 1) { - throw new BadRequestException( - String.format("Instance count should be greater than 0. Got %s.", instances)); - } - switch (programId.getType()) { - case SERVICE: - setServiceInstances(programId, instances); - break; - case WORKER: - setWorkerInstances(programId, instances); - break; - default: - throw new BadRequestException( - String.format("Setting instances for program type %s is not supported", - programId.getType().getPrettyName())); - } - } - /** * Lists all programs with the specified program type in a namespace. If perimeter security and * authorization are enabled, only returns the programs that the current user has access to. @@ -1416,76 +1129,6 @@ private boolean hasAccess(ProgramId programId) { return !accessEnforcer.isVisible(Collections.singleton(programId), principal).isEmpty(); } - private void setWorkerInstances(ProgramId programId, int instances) - throws ExecutionException, InterruptedException, BadRequestException { - int oldInstances = store.getWorkerInstances(programId); - if (oldInstances != instances) { - store.setWorkerInstances(programId, instances); - ProgramRuntimeService.RuntimeInfo runtimeInfo = findRuntimeInfo(programId); - if (runtimeInfo != null) { - runtimeInfo.getController().command(ProgramOptionConstants.INSTANCES, - ImmutableMap.of("runnable", programId.getProgram(), - "newInstances", String.valueOf(instances), - "oldInstances", String.valueOf(oldInstances))).get(); - } - } - } - - private void setServiceInstances(ProgramId programId, int instances) - throws ExecutionException, InterruptedException, BadRequestException { - int oldInstances = store.getServiceInstances(programId); - if (oldInstances != instances) { - store.setServiceInstances(programId, instances); - ProgramRuntimeService.RuntimeInfo runtimeInfo = findRuntimeInfo(programId); - if (runtimeInfo != null) { - runtimeInfo.getController().command(ProgramOptionConstants.INSTANCES, - ImmutableMap.of("runnable", programId.getProgram(), - "newInstances", String.valueOf(instances), - "oldInstances", String.valueOf(oldInstances))).get(); - } - } - } - - /** - * Helper method to update log levels for Worker or Service. - */ - private void updateLogLevels(ProgramId programId, Map logLevels, - @Nullable String runId) throws Exception { - ProgramRuntimeService.RuntimeInfo runtimeInfo = findRuntimeInfo(programId, runId).values() - .stream() - .findFirst().orElse(null); - if (runtimeInfo != null) { - LogLevelUpdater logLevelUpdater = getLogLevelUpdater(runtimeInfo); - logLevelUpdater.updateLogLevels(logLevels, null); - } - } - - /** - * Helper method to reset log levels for Worker or Service. - */ - private void resetLogLevels(ProgramId programId, Set loggerNames, @Nullable String runId) - throws Exception { - ProgramRuntimeService.RuntimeInfo runtimeInfo = findRuntimeInfo(programId, runId).values() - .stream() - .findFirst().orElse(null); - if (runtimeInfo != null) { - LogLevelUpdater logLevelUpdater = getLogLevelUpdater(runtimeInfo); - logLevelUpdater.resetLogLevels(loggerNames, null); - } - } - - /** - * Helper method to get the {@link LogLevelUpdater} for the program. - */ - private LogLevelUpdater getLogLevelUpdater(RuntimeInfo runtimeInfo) throws Exception { - ProgramController programController = runtimeInfo.getController(); - if (!(programController instanceof LogLevelUpdater)) { - throw new BadRequestException( - "Update log levels at runtime is only supported in distributed mode"); - } - return ((LogLevelUpdater) programController); - } - /** * Returns the active run records (PENDING / STARTING / RUNNING / SUSPENDED) based on the given * program id and an optional run id. @@ -1536,22 +1179,6 @@ private ProgramSpecification getLatestProgramSpecificationWithoutAuthz( return getExistingAppProgramSpecification(appMeta.getSpec(), programReference); } - /** - * Adds {@link Constants#APP_CDAP_VERSION} system argument to the argument map if known. - * - * @param programId program that corresponds to application with version information - * @param systemArgs map to add version information to - */ - public void addAppCdapVersion(ProgramId programId, Map systemArgs) { - ApplicationSpecification appSpec = store.getApplication(programId.getParent()); - if (appSpec != null) { - String appCDAPVersion = appSpec.getAppCDAPVersion(); - if (appCDAPVersion != null) { - systemArgs.put(Constants.APP_CDAP_VERSION, appCDAPVersion); - } - } - } - private Set getPluginRequirements(ProgramSpecification programSpecification) { return programSpecification.getPlugins().values() .stream().map(plugin -> new PluginRequirement(plugin.getPluginClass().getName(), @@ -1560,19 +1187,6 @@ private Set getPluginRequirements(ProgramSpecification progra .collect(Collectors.toSet()); } - private void authorizePipelineRuntimeImpersonation(Map userArgs) - throws Exception { - if ((userArgs.containsKey(SystemArguments.RUNTIME_PRINCIPAL_NAME)) - && (userArgs.containsKey(SystemArguments.RUNTIME_KEYTAB_PATH))) { - String principal = userArgs.get(SystemArguments.RUNTIME_PRINCIPAL_NAME); - LOG.debug("Checking authorisation for user: {}, using runtime config principal: {}", - authenticationContext.getPrincipal(), principal); - KerberosPrincipalId kid = new KerberosPrincipalId(principal); - accessEnforcer.enforce(kid, authenticationContext.getPrincipal(), - AccessPermission.IMPERSONATE); - } - } - private String decodeUserId(@Nullable String encodedUserId) { if (encodedUserId == null) { return ""; @@ -1612,4 +1226,168 @@ private ProgramId getLatestProgramId(ProgramReference programReference) ApplicationId applicationId = getLatestApplicationId(programReference.getParent()); return applicationId.program(programReference.getType(), programReference.getProgram()); } + + public void checkCapability(ProgramDescriptor programDescriptor) throws Exception { + // Check for capability at application class level. + Set applicationClasses = artifactRepository + .getArtifact(Id.Artifact.fromEntityId(programDescriptor.getArtifactId())).getMeta() + .getClasses() + .getApps(); + for (ApplicationClass applicationClass : applicationClasses) { + Set capabilities = applicationClass.getRequirements().getCapabilities(); + capabilityReader.checkAllEnabled(capabilities); + } + for (Map.Entry pluginEntry : programDescriptor.getApplicationSpecification() + .getPlugins() + .entrySet()) { + Set capabilities = pluginEntry.getValue().getPluginClass().getRequirements() + .getCapabilities(); + capabilityReader.checkAllEnabled(capabilities); + } + } + + /** + * Adds {@link Constants#APP_CDAP_VERSION} system argument to the argument map if known. + * + * @param programId program that corresponds to application with version information + * @param systemArgs map to add version information to + */ + private void addAppCdapVersion(ProgramId programId, Map systemArgs) { + ApplicationSpecification appSpec = store.getApplication(programId.getParent()); + if (appSpec != null) { + String appCDAPVersion = appSpec.getAppCDAPVersion(); + if (appCDAPVersion != null) { + systemArgs.put(Constants.APP_CDAP_VERSION, appCDAPVersion); + } + } + } + + private void authorizePipelineRuntimeImpersonation(Map userArgs) { + if ((userArgs.containsKey(SystemArguments.RUNTIME_PRINCIPAL_NAME)) + && (userArgs.containsKey(SystemArguments.RUNTIME_KEYTAB_PATH))) { + String principal = userArgs.get(SystemArguments.RUNTIME_PRINCIPAL_NAME); + LOG.debug("Checking authorisation for user: {}, using runtime config principal: {}", + authenticationContext.getPrincipal(), principal); + KerberosPrincipalId kid = new KerberosPrincipalId(principal); + accessEnforcer.enforce(kid, authenticationContext.getPrincipal(), + AccessPermission.IMPERSONATE); + } + } + + private static boolean isConcurrentRunsInSameAppForbidden(ProgramType type) { + // Concurrent runs in different (or same) versions of an application are forbidden for worker + return EnumSet.of(ProgramType.WORKER).contains(type); + } + + private static boolean isConcurrentRunsAllowed(ProgramType type) { + // Concurrent runs are only allowed for the Workflow, MapReduce and Spark + return EnumSet.of(ProgramType.WORKFLOW, ProgramType.MAPREDUCE, ProgramType.SPARK) + .contains(type); + } + + private boolean isStopped(ProgramId programId) throws Exception { + return ProgramStatus.STOPPED == getProgramStatus(programId); + } + + /** + * Returns whether the given program is stopped in all versions of the app. + * + * @param programId the id of the program for which the stopped status in all versions of the + * app is found + * @return whether the given program is stopped in all versions of the app + * @throws NotFoundException if the application to which this program belongs was not found + */ + private boolean isStoppedInSameProgram(ProgramId programId) throws Exception { + // check that app exists + Collection appIds = store.getAllAppVersionsAppIds( + programId.getParent().getAppReference()); + if (appIds == null || appIds.isEmpty()) { + throw new NotFoundException( + Id.Application.from(programId.getNamespace(), programId.getApplication())); + } + ApplicationSpecification appSpec = store.getApplication(programId.getParent()); + for (ApplicationId appId : appIds) { + ProgramId pId = appId.program(programId.getType(), programId.getProgram()); + if (!getExistingAppProgramStatus(appSpec, pId).equals(ProgramStatus.STOPPED)) { + return false; + } + } + return true; + } + + /** + * Returns the program status with no need of application existence check. + * + * @param appSpec the ApplicationSpecification of the existing application + * @param programId the id of the program for which the status call is made + * @return the status of the program + * @throws NotFoundException if the application to which this program belongs was not found + */ + private ProgramStatus getExistingAppProgramStatus(ApplicationSpecification appSpec, + ProgramId programId) throws Exception { + // TODO(CDAP-21126): Review access enforcement in this auxiliary function. + accessEnforcer.enforce(programId, authenticationContext.getPrincipal(), StandardPermission.GET); + ProgramSpecification spec = getExistingAppProgramSpecification(appSpec, + programId.getProgramReference()); + if (spec == null) { + // program doesn't exist + throw new NotFoundException(programId); + } + + return getProgramStatus(store.getActiveRuns(programId).values()); + } + + /** + * Returns the program status based on the active run records of a program. A program is RUNNING + * if there are any RUNNING, STOPPING, or SUSPENDED run records. A program is starting if there + * are any PENDING or STARTING run records and no RUNNING run records. Otherwise, it is STOPPED. + * + * @param runRecords run records for the program + * @return the program status + */ + @VisibleForTesting + static ProgramStatus getProgramStatus(Collection runRecords) { + boolean hasStarting = false; + for (RunRecordDetail runRecord : runRecords) { + ProgramRunStatus runStatus = runRecord.getStatus(); + if (runStatus == ProgramRunStatus.RUNNING || runStatus == ProgramRunStatus.SUSPENDED + || runStatus == ProgramRunStatus.STOPPING) { + return ProgramStatus.RUNNING; + } + hasStarting = hasStarting || runStatus == ProgramRunStatus.STARTING + || runStatus == ProgramRunStatus.PENDING; + } + return hasStarting ? ProgramStatus.STARTING : ProgramStatus.STOPPED; + } + + /** + * Returns the {@link ProgramSpecification} for the specified {@link ProgramId program}. + * + * @param appSpec the {@link ApplicationSpecification} of the existing application + * @param programReference the {@link ProgramReference program} for which the {@link + * ProgramSpecification} is requested + * @return the {@link ProgramSpecification} for the specified {@link ProgramId program}, or {@code + * null} if it does not exist + */ + @Nullable + private ProgramSpecification getExistingAppProgramSpecification(ApplicationSpecification appSpec, + ProgramReference programReference) { + String programName = programReference.getProgram(); + ProgramType type = programReference.getType(); + ProgramSpecification programSpec; + if (type == ProgramType.MAPREDUCE && appSpec.getMapReduce().containsKey(programName)) { + programSpec = appSpec.getMapReduce().get(programName); + } else if (type == ProgramType.SPARK && appSpec.getSpark().containsKey(programName)) { + programSpec = appSpec.getSpark().get(programName); + } else if (type == ProgramType.WORKFLOW && appSpec.getWorkflows().containsKey(programName)) { + programSpec = appSpec.getWorkflows().get(programName); + } else if (type == ProgramType.SERVICE && appSpec.getServices().containsKey(programName)) { + programSpec = appSpec.getServices().get(programName); + } else if (type == ProgramType.WORKER && appSpec.getWorkers().containsKey(programName)) { + programSpec = appSpec.getWorkers().get(programName); + } else { + programSpec = null; + } + return programSpec; + } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ProgramNotificationSubscriberService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ProgramNotificationSubscriberService.java index 88fe9dc5ff0e..4b5b377fa234 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ProgramNotificationSubscriberService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ProgramNotificationSubscriberService.java @@ -33,6 +33,7 @@ import io.cdap.cdap.api.workflow.WorkflowSpecification; import io.cdap.cdap.app.program.ProgramDescriptor; import io.cdap.cdap.app.runtime.ProgramOptions; +import io.cdap.cdap.app.runtime.ProgramRuntimeService; import io.cdap.cdap.app.runtime.ProgramStateWriter; import io.cdap.cdap.app.store.Store; import io.cdap.cdap.common.app.RunIds; @@ -91,6 +92,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; import javax.annotation.Nullable; +import org.apache.twill.api.RunId; import org.apache.twill.internal.CompositeService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,11 +110,12 @@ public class ProgramNotificationSubscriberService extends AbstractIdleService { private final MetricsCollectionService metricsCollectionService; private final ProvisionerNotifier provisionerNotifier; private final ProgramLifecycleService programLifecycleService; + private final ProgramRuntimeService runtimeService; private final ProvisioningService provisioningService; private final ProgramStateWriter programStateWriter; private final TransactionRunner transactionRunner; private final Store store; - private final RunRecordMonitorService runRecordMonitorService; + private final FlowControlService flowControlService; private Service delegate; private Set programCompletionNotifiers; @@ -123,22 +126,24 @@ public class ProgramNotificationSubscriberService extends AbstractIdleService { MetricsCollectionService metricsCollectionService, ProvisionerNotifier provisionerNotifier, ProgramLifecycleService programLifecycleService, + ProgramRuntimeService runtimeService, ProvisioningService provisioningService, ProgramStateWriter programStateWriter, TransactionRunner transactionRunner, Store store, - RunRecordMonitorService runRecordMonitorService) { + FlowControlService flowControlService) { this.messagingService = messagingService; this.cConf = cConf; this.metricsCollectionService = metricsCollectionService; this.provisionerNotifier = provisionerNotifier; this.programLifecycleService = programLifecycleService; + this.runtimeService = runtimeService; this.provisioningService = provisioningService; this.programStateWriter = programStateWriter; this.transactionRunner = transactionRunner; this.store = store; - this.runRecordMonitorService = runRecordMonitorService; + this.flowControlService = flowControlService; this.programCompletionNotifiers = Collections.emptySet(); } @@ -163,8 +168,7 @@ protected void startUp() throws Exception { } private void emitFlowControlMetrics() { - runRecordMonitorService.emitLaunchingMetrics(); - runRecordMonitorService.emitRunningMetrics(); + flowControlService.emitFlowControlMetrics(); } private void restoreActiveRuns() { @@ -184,23 +188,22 @@ private void restoreActiveRuns() { } try { LOG.info("Found active run: {}", runRecordDetail.getProgramRunId()); - if (runRecordDetail.getStatus() == ProgramRunStatus.PENDING) { - runRecordMonitorService.addRequest(runRecordDetail.getProgramRunId()); - } else if (runRecordDetail.getStatus() == ProgramRunStatus.STARTING) { - runRecordMonitorService.addRequest(runRecordDetail.getProgramRunId()); - // It is unknown what is the state of program runs in STARTING state. - // A STARTING message is published again to retry STARTING logic. + if (runRecordDetail.getStatus() == ProgramRunStatus.STARTING) { ProgramOptions programOptions = new SimpleProgramOptions( runRecordDetail.getProgramRunId().getParent(), new BasicArguments(runRecordDetail.getSystemArgs()), new BasicArguments(runRecordDetail.getUserArgs())); + ProgramDescriptor programDescriptor = this.store.loadProgram( + runRecordDetail.getProgramRunId().getParent()); + // It is unknown what is the state of program runs in STARTING state. + // A STARTING message is published again to retry STARTING logic. LOG.debug("Retrying to start run {}.", runRecordDetail.getProgramRunId()); programStateWriter.start( runRecordDetail.getProgramRunId(), programOptions, null, - this.store.loadProgram(runRecordDetail.getProgramRunId().getParent())); + programDescriptor); } } catch (Exception e) { ProgramRunId programRunId = runRecordDetail.getProgramRunId(); @@ -231,10 +234,11 @@ private ProgramNotificationSingleTopicSubscriberService createChildService( metricsCollectionService, provisionerNotifier, programLifecycleService, + runtimeService, provisioningService, programStateWriter, transactionRunner, - runRecordMonitorService, + flowControlService, name, topicName, programCompletionNotifiers); @@ -270,12 +274,13 @@ class ProgramNotificationSingleTopicSubscriberService private final String recordedProgramStatusPublishTopic; private final ProvisionerNotifier provisionerNotifier; private final ProgramLifecycleService programLifecycleService; + private final ProgramRuntimeService runtimeService; private final ProvisioningService provisioningService; private final ProgramStateWriter programStateWriter; private final Queue tasks; private final MetricsCollectionService metricsCollectionService; private Set programCompletionNotifiers; - private final RunRecordMonitorService runRecordMonitorService; + private final FlowControlService flowControlService; private final boolean checkTxSeparation; ProgramNotificationSingleTopicSubscriberService( @@ -284,10 +289,11 @@ class ProgramNotificationSingleTopicSubscriberService MetricsCollectionService metricsCollectionService, ProvisionerNotifier provisionerNotifier, ProgramLifecycleService programLifecycleService, + ProgramRuntimeService runtimeService, ProvisioningService provisioningService, ProgramStateWriter programStateWriter, TransactionRunner transactionRunner, - RunRecordMonitorService runRecordMonitorService, + FlowControlService flowControlService, String name, String topicName, Set programCompletionNotifiers) { @@ -305,12 +311,13 @@ class ProgramNotificationSingleTopicSubscriberService cConf.get(Constants.AppFabric.PROGRAM_STATUS_RECORD_EVENT_TOPIC); this.provisionerNotifier = provisionerNotifier; this.programLifecycleService = programLifecycleService; + this.runtimeService = runtimeService; this.provisioningService = provisioningService; this.programStateWriter = programStateWriter; this.tasks = new LinkedList<>(); this.metricsCollectionService = metricsCollectionService; this.programCompletionNotifiers = programCompletionNotifiers; - this.runRecordMonitorService = runRecordMonitorService; + this.flowControlService = flowControlService; // If number of partitions equals 1, DB deadlock cannot happen as a result of concurrent // modifications to @@ -545,7 +552,8 @@ private void handleProgramEvent( SecurityRequestContext.setUserId( prgOptions.getArguments().getOption(ProgramOptionConstants.USER_ID)); try { - programLifecycleService.startInternal(prgDescriptor, prgOptions, programRunId); + RunId runId = RunIds.fromString(programRunId.getRun()); + runtimeService.run(prgDescriptor, prgOptions, runId); } catch (Exception e) { LOG.error("Failed to start program {}", programRunId, e); programStateWriter.error(programRunId, e); @@ -582,7 +590,7 @@ private void handleProgramEvent( appMetadataStore.recordProgramRunning( programRunId, logicalStartTimeSecs, twillRunId, messageIdBytes); writeToHeartBeatTable(recordedRunRecord, logicalStartTimeSecs, programHeartbeatTable); - runRecordMonitorService.removeRequest(programRunId, true); + flowControlService.emitFlowControlMetrics(); long startDelayTime = logicalStartTimeSecs - RunIds.getTime(programRunId.getRun(), TimeUnit.SECONDS); emitStartingTimeMetric(programRunId, startDelayTime, recordedRunRecord); @@ -660,7 +668,7 @@ private void handleProgramEvent( Constants.Metrics.Program.PROGRAM_REJECTED_RUNS, null) .ifPresent(runnables::add); - runRecordMonitorService.removeRequest(programRunId, true); + flowControlService.emitFlowControlMetrics(); break; default: // This should not happen @@ -774,7 +782,7 @@ private RunRecordDetail handleProgramCompletion( programCompletionNotifiers.forEach( notifier -> notifier.onProgramCompleted(programRunId, recordedRunRecord.getStatus())); - runRecordMonitorService.removeRequest(programRunId, true); + flowControlService.emitFlowControlMetrics(); }); } return recordedRunRecord; diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/RunRecordMonitorService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/RunRecordMonitorService.java deleted file mode 100644 index 89c2fb0c8a9b..000000000000 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/RunRecordMonitorService.java +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Copyright © 2022 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.cdap.internal.app.services; - -import com.google.common.util.concurrent.AbstractScheduledService; -import com.google.inject.Inject; -import io.cdap.cdap.api.metrics.MetricsCollectionService; -import io.cdap.cdap.app.runtime.ProgramRuntimeService; -import io.cdap.cdap.common.app.RunIds; -import io.cdap.cdap.common.conf.CConfiguration; -import io.cdap.cdap.common.conf.Constants; -import io.cdap.cdap.common.conf.Constants.Metrics.FlowControl; -import io.cdap.cdap.proto.ProgramRunStatus; -import io.cdap.cdap.proto.ProgramType; -import io.cdap.cdap.proto.id.ProgramRunId; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executors; -import java.util.concurrent.PriorityBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import org.apache.twill.common.Threads; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Maintain and return total number of launching and running run-records. This class is used by - * flow-control mechanism for launch requests. It also has a cleanup mechanism to automatically - * remove old (i.e., configurable) entries from the counter as a safe-guard mechanism. - */ -public class RunRecordMonitorService extends AbstractScheduledService { - - private static final Logger LOG = LoggerFactory.getLogger(RunRecordMonitorService.class); - - /** - * Contains ProgramRunIds of runs that have been accepted, but have not been added to metadata - * store plus all run records with {@link ProgramRunStatus#PENDING} or {@link - * ProgramRunStatus#STARTING} status. - */ - private final BlockingQueue launchingQueue; - - private final ProgramRuntimeService runtimeService; - private final long ageThresholdSec; - private final CConfiguration cConf; - private final MetricsCollectionService metricsCollectionService; - private final int maxConcurrentRuns; - private ScheduledExecutorService executor; - - /** - * Tracks the program runs. - * - * @param cConf configuration - * @param runtimeService service to get info on programs - * @param metricsCollectionService collect metrics - */ - @Inject - public RunRecordMonitorService( - CConfiguration cConf, - ProgramRuntimeService runtimeService, - MetricsCollectionService metricsCollectionService) { - this.cConf = cConf; - this.runtimeService = runtimeService; - this.metricsCollectionService = metricsCollectionService; - - this.launchingQueue = - new PriorityBlockingQueue<>( - 128, Comparator.comparingLong(o -> RunIds.getTime(o.getRun(), TimeUnit.MILLISECONDS))); - this.ageThresholdSec = cConf.getLong(Constants.AppFabric.MONITOR_RECORD_AGE_THRESHOLD_SECONDS); - this.maxConcurrentRuns = cConf.getInt(Constants.AppFabric.MAX_CONCURRENT_RUNS); - } - - @Override - protected void startUp() throws Exception { - LOG.info("RunRecordMonitorService started."); - } - - @Override - protected void shutDown() throws Exception { - if (executor != null) { - executor.shutdownNow(); - } - LOG.info("RunRecordMonitorService successfully shut down."); - } - - @Override - protected void runOneIteration() throws Exception { - cleanupQueue(); - } - - @Override - protected Scheduler scheduler() { - return Scheduler.newFixedRateSchedule( - 0, cConf.getInt(Constants.AppFabric.MONITOR_CLEANUP_INTERVAL_SECONDS), TimeUnit.SECONDS); - } - - @Override - protected final ScheduledExecutorService executor() { - executor = - Executors.newSingleThreadScheduledExecutor( - Threads.createDaemonThreadFactory("run-record-monitor-service-cleanup-scheduler")); - return executor; - } - - /** - * Add a new in-flight launch request and return total number of launching and running programs. - * - * @param programRunId run id associated with the launch request - * @return total number of launching and running program runs. - */ - public Counter addRequestAndGetCount(ProgramRunId programRunId) throws Exception { - if (RunIds.getTime(programRunId.getRun(), TimeUnit.MILLISECONDS) == -1) { - throw new Exception("None time-based UUIDs are not supported"); - } - - int launchingCount = addRequest(programRunId); - int runningCount = getProgramsRunningCount(); - - LOG.info( - "Counter has {} concurrent launching and {} running programs.", - launchingCount, - runningCount); - return new Counter(launchingCount, runningCount); - } - - /** - * Get imprecise (due to data races) total number of launching and running programs. - * - * @return total number of launching and running program runs. - */ - public Counter getCount() { - int launchingCount = launchingQueue.size(); - int runningCount = getProgramsRunningCount(); - - return new Counter(launchingCount, runningCount); - } - - /** - * Add a new in-flight launch request. - * - * @param programRunId run id associated with the launch request - */ - public int addRequest(ProgramRunId programRunId) { - int result; - synchronized (launchingQueue) { - launchingQueue.add(programRunId); - result = launchingQueue.size(); - } - emitMetrics(Constants.Metrics.FlowControl.LAUNCHING_COUNT, result); - LOG.info("Added request with runId {}.", programRunId); - return result; - } - - /** - * Remove the request with the provided programRunId when the request is no longer launching. - * I.e., not in-flight, not in {@link ProgramRunStatus#PENDING} and not in {@link - * ProgramRunStatus#STARTING} - * - * @param programRunId of the request to be removed from launching queue. - * @param emitRunningChange if true, also updates {@link - * Constants.Metrics.FlowControl#RUNNING_COUNT} - */ - public void removeRequest(ProgramRunId programRunId, boolean emitRunningChange) { - if (launchingQueue.remove(programRunId)) { - LOG.info( - "Removed request with runId {}. Counter has {} concurrent launching requests.", - programRunId, - launchingQueue.size()); - emitMetrics(Constants.Metrics.FlowControl.LAUNCHING_COUNT, launchingQueue.size()); - } - - if (emitRunningChange) { - emitRunningMetrics(); - } - } - - public void emitLaunchingMetrics(long value) { - emitMetrics(Constants.Metrics.FlowControl.LAUNCHING_COUNT, value); - } - - /** - * Emit the {@link Constants.Metrics.FlowControl#LAUNCHING_COUNT} metric for runs. - */ - public void emitLaunchingMetrics() { - emitMetrics(Constants.Metrics.FlowControl.LAUNCHING_COUNT, launchingQueue.size()); - } - - - /** - * Emit the {@link Constants.Metrics.FlowControl#RUNNING_COUNT} metric for runs. - */ - public void emitRunningMetrics() { - emitMetrics(FlowControl.RUNNING_COUNT, getProgramsRunningCount()); - } - - private void emitMetrics(String metricName, long value) { - LOG.trace("Setting metric {} to value {}", metricName, value); - metricsCollectionService.getContext(Collections.emptyMap()).gauge(metricName, value); - } - - private void cleanupQueue() { - while (true) { - ProgramRunId programRunId = launchingQueue.peek(); - if (programRunId == null - || RunIds.getTime(programRunId.getRun(), TimeUnit.MILLISECONDS) + (ageThresholdSec * 1000) - >= System.currentTimeMillis()) { - // Queue is empty or queue head has not expired yet. - break; - } - // Queue head might have already been removed. So instead of calling poll, we call remove. - if (launchingQueue.remove(programRunId)) { - LOG.info("Removing request with runId {} due to expired retention time.", programRunId); - } - } - // Always emit both metrics after cleanup. - emitLaunchingMetrics(); - emitRunningMetrics(); - } - - /** - * Returns the total number of programs in running state. The count includes batch (i.e., {@link - * ProgramType#WORKFLOW}), streaming (i.e., {@link ProgramType#SPARK}) with no parent and - * replication (i.e., {@link ProgramType#WORKER}) jobs. - */ - private int getProgramsRunningCount() { - List list = - runtimeService.listAll( - ProgramType.WORKFLOW, ProgramType.WORKER, ProgramType.SPARK, ProgramType.MAPREDUCE); - - int launchingCount = launchingQueue.size(); - - // We use program controllers (instead of querying metadata store) to count the total number of - // programs in running state. - // A program controller is created when a launch request is in the middle of starting state. - // Therefore, the returning running count is NOT precise. - int impreciseRunningCount = - (int) list.stream() - .filter(r -> isRunning(r.getController().getState().getRunStatus())) - .count(); - - if (maxConcurrentRuns < 0 || (launchingCount + impreciseRunningCount < maxConcurrentRuns)) { - // It is safe to return the imprecise value since either flow control for runs is disabled - // (i.e., -1) or flow control will not reject an incoming request yet. - return impreciseRunningCount; - } - - // Flow control is at the threshold. We return the precise count. - return (int) list.stream() - .filter( - r -> - isRunning(r.getController().getState().getRunStatus()) - && !launchingQueue.contains(r.getController().getProgramRunId())) - .count(); - } - - private boolean isRunning(ProgramRunStatus status) { - if (status == ProgramRunStatus.RUNNING - || status == ProgramRunStatus.SUSPENDED - || status == ProgramRunStatus.RESUMING) { - return true; - } - - return false; - } - - /** - * Counts the concurrent program runs. - */ - public class Counter { - - /** - * Total number of launch requests that have been accepted but still missing in metadata store + - * * total number of run records with {@link ProgramRunStatus#PENDING} status + total number of - * run records with {@link ProgramRunStatus#STARTING} status. - */ - private final int launchingCount; - - /** - * Total number of run records with {@link ProgramRunStatus#RUNNING} status + Total number of run - * records with {@link ProgramRunStatus#SUSPENDED} status + Total number of run records with - * {@link ProgramRunStatus#RESUMING} status. - */ - private final int runningCount; - - Counter(int launchingCount, int runningCount) { - this.launchingCount = launchingCount; - this.runningCount = runningCount; - } - - public int getLaunchingCount() { - return launchingCount; - } - - public int getRunningCount() { - return runningCount; - } - } -} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/SystemProgramManagementService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/SystemProgramManagementService.java index 4757c066b420..da79722933e6 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/SystemProgramManagementService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/SystemProgramManagementService.java @@ -25,6 +25,7 @@ import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.common.service.AbstractRetryableScheduledService; import io.cdap.cdap.common.service.RetryStrategies; +import io.cdap.cdap.internal.app.runtime.ProgramStartRequest; import io.cdap.cdap.proto.ProgramType; import io.cdap.cdap.proto.id.NamespaceId; import io.cdap.cdap.proto.id.ProgramId; @@ -121,7 +122,10 @@ private void startPrograms(Map enabledProgramsMap) { Map overrides = enabledProgramsMap.get(programId).asMap(); LOG.debug("Starting program {} with args {}", programId, overrides); try { - programLifecycleService.start(programId, overrides, false, false); + ProgramStartRequest startRequest = programLifecycleService.prepareStart(programId, overrides, false, false); + programRuntimeService.run( + startRequest.getProgramDescriptor(), startRequest.getProgramOptions(), startRequest.getRunId()) + .getController(); } catch (ConflictException ex) { // Ignore if the program is already running. LOG.debug("Program {} is already running.", programId); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/AppMetadataStore.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/AppMetadataStore.java index c8d5e1def038..d895e5ccc1a4 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/AppMetadataStore.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/AppMetadataStore.java @@ -20,6 +20,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import com.google.common.reflect.TypeToken; import com.google.gson.Gson; @@ -39,10 +40,12 @@ import io.cdap.cdap.common.ConflictException; import io.cdap.cdap.common.app.RunIds; import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.conf.Constants.AppMetaStore; import io.cdap.cdap.internal.app.ApplicationSpecificationAdapter; import io.cdap.cdap.internal.app.runtime.ProgramOptionConstants; import io.cdap.cdap.internal.app.runtime.SystemArguments; import io.cdap.cdap.internal.app.runtime.workflow.BasicWorkflowToken; +import io.cdap.cdap.internal.app.store.adapters.ApplicationMetaAdapter; import io.cdap.cdap.proto.BasicThrowable; import io.cdap.cdap.proto.ProgramRunCluster; import io.cdap.cdap.proto.ProgramRunClusterStatus; @@ -59,6 +62,7 @@ import io.cdap.cdap.proto.id.ProgramReference; import io.cdap.cdap.proto.id.ProgramRunId; import io.cdap.cdap.proto.sourcecontrol.SourceControlMeta; +import io.cdap.cdap.spi.data.FieldSizeLimitExceededException; import io.cdap.cdap.spi.data.SortOrder; import io.cdap.cdap.spi.data.StructuredRow; import io.cdap.cdap.spi.data.StructuredTable; @@ -67,6 +71,7 @@ import io.cdap.cdap.spi.data.table.field.Field; import io.cdap.cdap.spi.data.table.field.Fields; import io.cdap.cdap.spi.data.table.field.Range; +import io.cdap.cdap.spi.data.table.options.StaleReadOption; import io.cdap.cdap.store.StoreDefinition; import java.io.IOException; import java.io.StringReader; @@ -150,7 +155,18 @@ public class AppMetadataStore { .put(ProgramRunStatus.REJECTED, TYPE_RUN_RECORD_COMPLETED) .build(); + private static final String TYPE_FLOW_CONTROL_LAUNCHING = "launching"; + private static final String TYPE_FLOW_CONTROL_RUNNING = "running"; + private static final String TYPE_FLOW_CONTROL_NONE = ""; + + // Program types controlled by flow-control mechanism. + private static final Set CONTROL_FLOW_PROGRAM_TYPES = ImmutableSet.of(ProgramType.MAPREDUCE, + ProgramType.WORKFLOW, + ProgramType.SPARK, + ProgramType.WORKER); + private final StructuredTableContext context; + private final boolean appSpecReductionEnabled; private StructuredTable applicationSpecificationTable; private StructuredTable applicationEditTable; private StructuredTable workflowNodeStateTable; @@ -158,6 +174,8 @@ public class AppMetadataStore { private StructuredTable workflowsTable; private StructuredTable programCountsTable; private StructuredTable subscriberStateTable; + private StructuredTable pluginDataTable; + private StructuredTable universalPluginDataTable; /** * Static method for creating an instance of {@link AppMetadataStore}. @@ -168,6 +186,8 @@ public static AppMetadataStore create(StructuredTableContext context) { private AppMetadataStore(StructuredTableContext context) { this.context = context; + this.appSpecReductionEnabled = AppMetaStore.APPSPEC_REDUCTION_SUPPORTED_STORAGE_PROVIDERS.contains( + context.getStorageProvider()); } private StructuredTable getApplicationSpecificationTable() { @@ -249,6 +269,29 @@ private StructuredTable getSubscriberStateTable() { return subscriberStateTable; } + @VisibleForTesting + StructuredTable getPluginDataTable() { + try { + if (pluginDataTable == null) { + pluginDataTable = context.getTable(StoreDefinition.ArtifactStore.PLUGIN_DATA_TABLE); + } + } catch (TableNotFoundException e) { + throw new RuntimeException(e); + } + return pluginDataTable; + } + + private StructuredTable getUniversalPluginDataTable() { + try { + if (universalPluginDataTable == null) { + universalPluginDataTable = context.getTable(StoreDefinition.ArtifactStore.UNIV_PLUGIN_DATA_TABLE); + } + } catch (TableNotFoundException e) { + throw new RuntimeException(e); + } + return universalPluginDataTable; + } + /** * Gets the {@link ApplicationMeta} of the given application. * @@ -348,7 +391,8 @@ public void scanApplications(ScanApplicationsRequest request, boolean keepScanning = true; while (iterator.hasNext() && keepScanning && limit > 0) { StructuredRow row = iterator.next(); - AppScanEntry scanEntry = new AppScanEntry(row); + AppScanEntry scanEntry = new AppScanEntry(row, getPluginDataTable(), + getUniversalPluginDataTable(), appSpecReductionEnabled); if (scanEntryPredicate.test(scanEntry)) { keepScanning = func.apply(scanEntry); limit--; @@ -391,13 +435,29 @@ public long getApplicationCount() throws IOException { Range.Bound.EXCLUSIVE), Range.create(fields, Range.Bound.EXCLUSIVE, null, Range.Bound.INCLUSIVE)); - // Count the latest version of app, - // we treat latest=["true",null] as latest for backward compatibility. + return getApplicationSpecificationTable().count(ranges, createLatestFilterIndex()); + } + + /** + * Returns the number of applications in the given namespace. + * + * @param namespaceId namespace ID for which to count. + * @return the count of application in the namespace. + * @throws IOException if the count fails. + */ + public long getNamespaceApplicationCount(NamespaceId namespaceId) throws IOException { + Collection ranges = ImmutableList.of(Range.singleton( + ImmutableList.of(Fields.stringField(StoreDefinition.AppMetadataStore.NAMESPACE_FIELD, + namespaceId.getNamespace())))); + return getApplicationSpecificationTable().count(ranges, createLatestFilterIndex()); + } + + private Collection> createLatestFilterIndex() { + // We treat latest=["true",null] as latest for backward compatibility. // Prior to 6.8, all versions of an application were returned in the list apps api, not just the latest version. - Collection> filterIndexes = - ImmutableList.of(Fields.booleanField(StoreDefinition.AppMetadataStore.LATEST_FIELD, null), - Fields.booleanField(StoreDefinition.AppMetadataStore.LATEST_FIELD, true)); - return getApplicationSpecificationTable().count(ranges, filterIndexes); + return ImmutableList.of( + Fields.booleanField(StoreDefinition.AppMetadataStore.LATEST_FIELD, null), + Fields.booleanField(StoreDefinition.AppMetadataStore.LATEST_FIELD, true)); } @Nullable @@ -697,9 +757,16 @@ public int createApplicationVersion(ApplicationId id, ApplicationMeta appMeta, b } // Add a new version of the app - writeApplication(id.getNamespace(), id.getApplication(), id.getVersion(), appMeta.getSpec(), - appMeta.getChange(), - appMeta.getSourceControlMeta(), markAsLatest); + try { + writeApplication(id.getNamespace(), id.getApplication(), id.getVersion(), appMeta.getSpec(), + appMeta.getChange(), + appMeta.getSourceControlMeta(), markAsLatest); + } catch (FieldSizeLimitExceededException e) { + LOG.error("Application deployment failed: ", e); + throw new IllegalArgumentException( + "Application deployment failed as size exceeds the supported limit of " + e.getLimit() + + " characters"); + } return getApplicationEditNumber( new ApplicationReference(id.getNamespaceId(), id.getApplication())); } @@ -715,10 +782,10 @@ void writeApplication(String namespaceId, String appId, String versionId, void writeApplication(String namespaceId, String appId, String versionId, ApplicationSpecification spec, @Nullable ChangeDetail change, @Nullable SourceControlMeta sourceControlMeta, boolean markAsLatest) throws IOException { + ApplicationMeta meta = new ApplicationMeta(appId, spec, null, null); + String json = ApplicationMetaAdapter.toJson(meta, appSpecReductionEnabled); writeApplicationSerialized(namespaceId, appId, versionId, - GSON.toJson( - new ApplicationMeta(appId, spec, null, null)), - change, sourceControlMeta, markAsLatest); + json, change, sourceControlMeta, markAsLatest); updateApplicationEdit(namespaceId, appId); } @@ -795,8 +862,9 @@ public void updateAppSpec(ApplicationId appId, ApplicationSpecification spec) th } // creation time cannot be null - will be written to app-spec but won't be added to table ApplicationMeta updated = new ApplicationMeta(existing.getId(), spec, null); + String json = ApplicationMetaAdapter.toJson(updated, appSpecReductionEnabled); updateApplicationSerialized(appId.getNamespace(), appId.getApplication(), appId.getVersion(), - GSON.toJson(updated)); + json); } /** @@ -898,11 +966,58 @@ private void addWorkflowNodeState(ProgramRunId programRunId, Map // Update the parent Workflow run record by adding node id and program run id in the properties Map properties = new HashMap<>(record.getProperties()); properties.put(workflowNodeId, programRunId.getRun()); - writeToStructuredTableWithPrimaryKeys( - runRecordFields, - RunRecordDetail.builder(record).setProperties(properties).setSourceId(sourceId).build(), - getRunRecordsTable(), StoreDefinition.AppMetadataStore.RUN_RECORD_DATA); + writeRunRecordWithPrimaryKeys(runRecordFields, + RunRecordDetail.builder(record).setProperties(properties).setSourceId(sourceId).build()); + } + } + + /** + * Record that the program run is pending run. + * + * @param programRunId program run + * + * @return {@link RunRecordDetail} with status Pending. + */ + @Nullable + public RunRecordDetail recordProgramPending(ProgramRunId programRunId, Map runtimeArgs, + Map systemArgs, @Nullable ArtifactId artifactId) + throws IOException { + long startTs = RunIds.getTime(programRunId.getRun(), TimeUnit.SECONDS); + if (startTs == -1L) { + throw new IllegalArgumentException(String.format( + "Provisioning state for program run that does not have a timestamp in the run id: '%s'", programRunId)); + } + + Optional profileId = SystemArguments.getProfileIdFromArgs( + programRunId.getNamespaceId(), systemArgs); + RunRecordDetail existing = getRun(programRunId); + + // If for some reason, there is an existing run record then return null. + if (existing != null) { + LOG.warn( + "Ignoring unexpected request to record pending state for program run {} that has an " + + "existing run record in run state {} and cluster state {}.", + programRunId, existing.getStatus()); + return null; } + + ProgramRunCluster cluster = new ProgramRunCluster(ProgramRunClusterStatus.PROVISIONING, null, null); + RunRecordDetail meta = RunRecordDetail.builder() + .setProgramRunId(programRunId) + .setStartTime(startTs) + .setProperties(getRecordProperties(systemArgs, runtimeArgs)) + .setSystemArgs(systemArgs) + .setCluster(cluster) + .setStatus(ProgramRunStatus.PENDING) + .setPeerName(systemArgs.get(ProgramOptionConstants.PEER_NAME)) + .setArtifactId(artifactId) + .setPrincipal(systemArgs.get(ProgramOptionConstants.PRINCIPAL)) + .setProfileId(profileId.orElse(null)) + .setFlowControlStatus(getFlowControlStatus(programRunId, ProgramRunStatus.PENDING, systemArgs)) + .build(); + writeNewRunRecord(meta, TYPE_RUN_RECORD_ACTIVE); + LOG.trace("Recorded {} for program {}", ProgramRunStatus.PENDING, programRunId); + return meta; } /** @@ -927,25 +1042,6 @@ public RunRecordDetail recordProgramProvisioning(ProgramRunId programRunId, Map systemArgs, byte[] sourceId, @Nullable ArtifactId artifactId) throws IOException { - long startTs = RunIds.getTime(programRunId.getRun(), TimeUnit.SECONDS); - if (startTs == -1L) { - LOG.error( - "Ignoring unexpected request to record provisioning state for program run {} that does not have " - - + "a timestamp in the run id.", programRunId); - return null; - } - - RunRecordDetail existing = getRun(programRunId); - // for some reason, there is an existing run record. - if (existing != null) { - LOG.error( - "Ignoring unexpected request to record provisioning state for program run {} that has an existing " - + "run record in run state {} and cluster state {}.", - programRunId, existing.getStatus(), existing.getCluster().getStatus()); - return null; - } - Optional profileId = SystemArguments.getProfileIdFromArgs( programRunId.getNamespaceId(), systemArgs); if (!profileId.isPresent()) { @@ -957,20 +1053,60 @@ public RunRecordDetail recordProgramProvisioning(ProgramRunId programRunId, ProgramRunCluster cluster = new ProgramRunCluster(ProgramRunClusterStatus.PROVISIONING, null, null); - RunRecordDetail meta = RunRecordDetail.builder() - .setProgramRunId(programRunId) - .setStartTime(startTs) - .setStatus(ProgramRunStatus.PENDING) - .setProperties(getRecordProperties(systemArgs, runtimeArgs)) - .setSystemArgs(systemArgs) - .setCluster(cluster) - .setProfileId(profileId.get()) - .setPeerName(systemArgs.get(ProgramOptionConstants.PEER_NAME)) - .setSourceId(sourceId) - .setArtifactId(artifactId) - .setPrincipal(systemArgs.get(ProgramOptionConstants.PRINCIPAL)) - .build(); - writeNewRunRecord(meta, TYPE_RUN_RECORD_ACTIVE); + + RunRecordDetail existing = getRun(programRunId); + RunRecordDetail meta; + if (existing == null) { + // Create a new run record if it doesn't exist. + long startTs = RunIds.getTime(programRunId.getRun(), TimeUnit.SECONDS); + if (startTs == -1L) { + LOG.error( + "Ignoring unexpected request to record provisioning state for program run {} that does not have " + + + "a timestamp in the run id.", programRunId); + return null; + } + meta = RunRecordDetail.builder() + .setProgramRunId(programRunId) + .setStartTime(startTs) + .setStatus(ProgramRunStatus.PENDING) + .setProperties(getRecordProperties(systemArgs, runtimeArgs)) + .setSystemArgs(systemArgs) + .setCluster(cluster) + .setProfileId(profileId.get()) + .setPeerName(systemArgs.get(ProgramOptionConstants.PEER_NAME)) + .setSourceId(sourceId) + .setArtifactId(artifactId) + .setPrincipal(systemArgs.get(ProgramOptionConstants.PRINCIPAL)) + .setFlowControlStatus(getFlowControlStatus(programRunId, ProgramRunStatus.PENDING, systemArgs)) + .build(); + writeNewRunRecord(meta, TYPE_RUN_RECORD_ACTIVE); + } else { + if (existing.getStatus() != ProgramRunStatus.PENDING && existing.getStatus() != ProgramRunStatus.SUSPENDED) { + LOG.error( + "Ignoring unexpected request to record provisioning state for program run {} that has " + + "a status in end state {}.", programRunId, existing.getStatus()); + return null; + } + delete(existing); + meta = RunRecordDetail.builder(existing) + .setStatus(ProgramRunStatus.PENDING) + .setProperties(getRecordProperties(systemArgs, runtimeArgs)) + .setSystemArgs(systemArgs) + .setCluster(cluster) + .setProfileId(profileId.get()) + .setPeerName(systemArgs.get(ProgramOptionConstants.PEER_NAME)) + .setSourceId(sourceId) + .setArtifactId(artifactId) + .setPrincipal(systemArgs.get(ProgramOptionConstants.PRINCIPAL)) + .setFlowControlStatus(getFlowControlStatus(programRunId, ProgramRunStatus.PENDING, systemArgs)) + .build(); + List> key = getProgramRunInvertedTimeKey(TYPE_RUN_RECORD_ACTIVE, + existing.getProgramRunId(), + existing.getStartTs()); + writeRunRecordWithPrimaryKeys(key, meta); + } + LOG.trace("Recorded {} for program {}", ProgramRunClusterStatus.PROVISIONING, programRunId); return meta; } @@ -1031,8 +1167,7 @@ public RunRecordDetail recordProgramProvisioned(ProgramRunId programRunId, int n .setCluster(cluster) .setSourceId(sourceId) .build(); - writeToStructuredTableWithPrimaryKeys( - key, meta, getRunRecordsTable(), StoreDefinition.AppMetadataStore.RUN_RECORD_DATA); + writeRunRecordWithPrimaryKeys(key, meta); LOG.trace("Recorded {} for program {}", ProgramRunClusterStatus.PROVISIONED, existing.getProgramRunId()); return meta; @@ -1076,8 +1211,7 @@ public RunRecordDetail recordProgramDeprovisioning(ProgramRunId programRunId, by .setCluster(cluster) .setSourceId(sourceId) .build(); - writeToStructuredTableWithPrimaryKeys( - key, meta, getRunRecordsTable(), StoreDefinition.AppMetadataStore.RUN_RECORD_DATA); + writeRunRecordWithPrimaryKeys(key, meta); LOG.trace("Recorded {} for program {}", ProgramRunClusterStatus.DEPROVISIONING, existing.getProgramRunId()); return meta; @@ -1122,8 +1256,7 @@ public RunRecordDetail recordProgramDeprovisioned(ProgramRunId programRunId, @Nu .setCluster(cluster) .setSourceId(sourceId) .build(); - writeToStructuredTableWithPrimaryKeys( - key, meta, getRunRecordsTable(), StoreDefinition.AppMetadataStore.RUN_RECORD_DATA); + writeRunRecordWithPrimaryKeys(key, meta); LOG.trace("Recorded {} for program {}", ProgramRunClusterStatus.DEPROVISIONED, existing.getProgramRunId()); return meta; @@ -1167,10 +1300,9 @@ public RunRecordDetail recordProgramOrphaned(ProgramRunId programRunId, long end .setCluster(cluster) .setSourceId(sourceId) .build(); - writeToStructuredTableWithPrimaryKeys( - key, meta, getRunRecordsTable(), StoreDefinition.AppMetadataStore.RUN_RECORD_DATA); + writeRunRecordWithPrimaryKeys(key, meta); LOG.trace("Recorded {} for program {}", ProgramRunClusterStatus.ORPHANED, - existing.getProgramRunId()); + meta.getProgramRunId()); return meta; } @@ -1179,32 +1311,30 @@ public RunRecordDetail recordProgramRejected(ProgramRunId programRunId, Map runtimeArgs, Map systemArgs, byte[] sourceId, @Nullable ArtifactId artifactId) throws IOException { - long startTs = RunIds.getTime(programRunId.getRun(), TimeUnit.SECONDS); - if (startTs == -1L) { - LOG.error( - "Ignoring unexpected request to record provisioning state for program run {} that does not have " - - + "a timestamp in the run id.", programRunId); - return null; - } - RunRecordDetail existing = getRun(programRunId); - // for some reason, there is an existing run record? - if (existing != null) { - LOG.error( - "Ignoring unexpected request to record rejected state for program run {} that has an existing " - + "run record in run state {} and cluster state {}.", - programRunId, existing.getStatus(), existing.getCluster().getStatus()); - return null; + + RunRecordDetail.Builder builder; + if (existing == null) { + LOG.warn( + "Unexpected request to record rejected state for program run {} that has no existing run record.", + programRunId); + long startTs = RunIds.getTime(programRunId.getRun(), TimeUnit.SECONDS); + if (startTs == -1L) { + LOG.error( + "Ignoring unexpected request to record provisioning state for program run {} that does not have " + + + "a timestamp in the run id.", programRunId); + return null; + } + builder = RunRecordDetail.builder().setProgramRunId(programRunId).setStartTime(startTs).setStopTime(startTs); + } else { + delete(existing); + builder = RunRecordDetail.builder(existing).setStopTime(existing.getStartTs()); } Optional profileId = SystemArguments.getProfileIdFromArgs( programRunId.getNamespaceId(), systemArgs); - RunRecordDetail meta = RunRecordDetail.builder() - .setProgramRunId(programRunId) - .setStartTime(startTs) - .setStopTime(startTs) // rejected: stop time == start time - .setStatus(ProgramRunStatus.REJECTED) + RunRecordDetail meta = builder.setStatus(ProgramRunStatus.REJECTED) .setProperties(getRecordProperties(systemArgs, runtimeArgs)) .setSystemArgs(systemArgs) .setProfileId(profileId.orElse(null)) @@ -1212,9 +1342,13 @@ public RunRecordDetail recordProgramRejected(ProgramRunId programRunId, .setArtifactId(artifactId) .setSourceId(sourceId) .setPrincipal(systemArgs.get(ProgramOptionConstants.PRINCIPAL)) + .setFlowControlStatus(getFlowControlStatus(programRunId, ProgramRunStatus.REJECTED, systemArgs)) .build(); - writeNewRunRecord(meta, TYPE_RUN_RECORD_COMPLETED); + List> key = getProgramRunInvertedTimeKey(TYPE_RUN_RECORD_COMPLETED, + meta.getProgramRunId(), + meta.getStartTs()); + writeRunRecordWithPrimaryKeys(key, meta); LOG.trace("Recorded {} for program {}", ProgramRunStatus.REJECTED, programRunId); return meta; } @@ -1226,8 +1360,7 @@ private void writeNewRunRecord(RunRecordDetail meta, String typeRunRecordComplet throws IOException { List> fields = getProgramRunInvertedTimeKey(typeRunRecordCompleted, meta.getProgramRunId(), meta.getStartTs()); - writeToStructuredTableWithPrimaryKeys(fields, meta, getRunRecordsTable(), - StoreDefinition.AppMetadataStore.RUN_RECORD_DATA); + writeRunRecordWithPrimaryKeys(fields, meta); List> countKey = getProgramCountPrimaryKeys(TYPE_COUNT, meta.getProgramRunId().getParent()); getProgramCountsTable().increment(countKey, StoreDefinition.AppMetadataStore.COUNTS, 1L); @@ -1280,9 +1413,9 @@ public RunRecordDetail recordProgramStart(ProgramRunId programRunId, @Nullable S .setSystemArgs(newSystemArgs) .setTwillRunId(twillRunId) .setSourceId(sourceId) + .setFlowControlStatus(getFlowControlStatus(programRunId, ProgramRunStatus.STARTING, newSystemArgs)) .build(); - writeToStructuredTableWithPrimaryKeys( - key, meta, getRunRecordsTable(), StoreDefinition.AppMetadataStore.RUN_RECORD_DATA); + writeRunRecordWithPrimaryKeys(key, meta); LOG.trace("Recorded {} for program {}", ProgramRunStatus.STARTING, existing.getProgramRunId()); return meta; } @@ -1333,9 +1466,9 @@ public RunRecordDetail recordProgramRunning(ProgramRunId programRunId, long stat .setStatus(ProgramRunStatus.RUNNING) .setTwillRunId(twillRunId) .setSourceId(sourceId) + .setFlowControlStatus(getFlowControlStatus(programRunId, ProgramRunStatus.RUNNING, systemArgs)) .build(); - writeToStructuredTableWithPrimaryKeys( - key, meta, getRunRecordsTable(), StoreDefinition.AppMetadataStore.RUN_RECORD_DATA); + writeRunRecordWithPrimaryKeys(key, meta); LOG.trace("Recorded {} for program {}", ProgramRunStatus.RUNNING, existing.getProgramRunId()); return meta; } @@ -1410,7 +1543,9 @@ private RunRecordDetail recordProgramSuspendResume(byte[] sourceId, RunRecordDet List> key = getProgramRunInvertedTimeKey(TYPE_RUN_RECORD_ACTIVE, existing.getProgramRunId(), existing.getStartTs()); - RunRecordDetail.Builder builder = RunRecordDetail.builder(existing).setStatus(toStatus) + RunRecordDetail.Builder builder = RunRecordDetail.builder(existing) + .setStatus(toStatus) + .setFlowControlStatus(getFlowControlStatus(existing.getProgramRunId(), toStatus, existing.getSystemArgs())) .setSourceId(sourceId); if (timestamp != -1) { if (action.equals("resume")) { @@ -1420,8 +1555,7 @@ private RunRecordDetail recordProgramSuspendResume(byte[] sourceId, RunRecordDet } } RunRecordDetail meta = builder.build(); - writeToStructuredTableWithPrimaryKeys( - key, meta, getRunRecordsTable(), StoreDefinition.AppMetadataStore.RUN_RECORD_DATA); + writeRunRecordWithPrimaryKeys(key, meta); LOG.trace("Recorded {} for program {}", toStatus, existing.getProgramRunId()); return meta; } @@ -1475,9 +1609,9 @@ public RunRecordDetail recordProgramStopping(ProgramRunId programRunId, byte[] s .setStoppingTime(stoppingTsSecs) .setTerminateTs(terminateTsSecs) .setSourceId(sourceId) + .setFlowControlStatus(getFlowControlStatus(programRunId, ProgramRunStatus.STOPPING, systemArgs)) .build(); - writeToStructuredTableWithPrimaryKeys( - key, meta, getRunRecordsTable(), StoreDefinition.AppMetadataStore.RUN_RECORD_DATA); + writeRunRecordWithPrimaryKeys(key, meta); LOG.trace("Recorded {} for program {}", ProgramRunStatus.STOPPING, existing.getProgramRunId()); return meta; } @@ -1529,9 +1663,9 @@ public RunRecordDetailWithExistingStatus recordProgramStop(ProgramRunId programR .setStopTime(stopTs) .setStatus(runStatus) .setSourceId(sourceId) + .setFlowControlStatus(getFlowControlStatus(programRunId, runStatus, systemArgs)) .build(); - writeToStructuredTableWithPrimaryKeys( - key, meta, getRunRecordsTable(), StoreDefinition.AppMetadataStore.RUN_RECORD_DATA); + writeRunRecordWithPrimaryKeys(key, meta); LOG.trace("Recorded {} for program {}", runStatus, existing.getProgramRunId()); return meta; } @@ -1643,6 +1777,38 @@ public int countActiveRuns(@Nullable Integer limit) throws IOException { return count.get(); } + /** + * Count all records in launching state. + * + * @return Count of records in launching state. + */ + public int getFlowControlLaunchingCount(int readStalenessSeconds) throws IOException { + ImmutableList> keyPrefix = ImmutableList.of( + Fields.stringField(StoreDefinition.AppMetadataStore.RUN_STATUS, TYPE_RUN_RECORD_ACTIVE)); + Collection> filterIndexes = + ImmutableList.of( + Fields.stringField(StoreDefinition.AppMetadataStore.FLOW_CONTROL_STATUS, + TYPE_FLOW_CONTROL_LAUNCHING)); + return (int) getRunRecordsTable().count(Collections.singletonList(Range.singleton(keyPrefix)), + filterIndexes, new StaleReadOption(readStalenessSeconds)); + } + + /** + * Count all records in running state. + * + * @return Count of records in running state. + */ + public int getFlowControlRunningCount(int readStalenessSeconds) throws IOException { + ImmutableList> keyPrefix = ImmutableList.of( + Fields.stringField(StoreDefinition.AppMetadataStore.RUN_STATUS, TYPE_RUN_RECORD_ACTIVE)); + Collection> filterIndexes = + ImmutableList.of( + Fields.stringField(StoreDefinition.AppMetadataStore.FLOW_CONTROL_STATUS, + TYPE_FLOW_CONTROL_RUNNING)); + return (int) getRunRecordsTable().count(Collections.singletonList(Range.singleton(keyPrefix)), + filterIndexes, new StaleReadOption(readStalenessSeconds)); + } + /** * Scans active runs, starting from the given cursor. * @@ -2519,6 +2685,7 @@ public void deleteAllAppMetadataTables() throws IOException { deleteTable(getProgramCountsTable(), StoreDefinition.AppMetadataStore.COUNT_TYPE); deleteTable(getSubscriberStateTable(), StoreDefinition.AppMetadataStore.SUBSCRIBER_TOPIC); deleteTable(getApplicationEditTable(), StoreDefinition.AppMetadataStore.NAMESPACE_FIELD); + deleteTable(getPluginDataTable(), StoreDefinition.ArtifactStore.PARENT_NAMESPACE_FIELD); } private void deleteTable(StructuredTable table, String firstKey) throws IOException { @@ -2703,9 +2870,12 @@ private ApplicationMeta decodeRow(StructuredRow row) { String changeSummary = row.getString(StoreDefinition.AppMetadataStore.CHANGE_SUMMARY_FIELD); Long creationTimeMillis = row.getLong(StoreDefinition.AppMetadataStore.CREATION_TIME_FIELD); Boolean latest = row.getBoolean(StoreDefinition.AppMetadataStore.LATEST_FIELD); - ApplicationMeta meta = GSON.fromJson( + String namespace = row.getString(StoreDefinition.AppMetadataStore.NAMESPACE_FIELD); + + ApplicationMeta meta = ApplicationMetaAdapter.fromJson( row.getString(StoreDefinition.AppMetadataStore.APPLICATION_DATA_FIELD), - ApplicationMeta.class); + ApplicationMeta.class, namespace, appSpecReductionEnabled, + getPluginDataTable(), getUniversalPluginDataTable()); SourceControlMeta sourceControl = GSON.fromJson( row.getString(StoreDefinition.AppMetadataStore.SOURCE_CONTROL_META), SourceControlMeta.class); @@ -2778,7 +2948,7 @@ private List> getRunRecordProgramPrefix(String status, @Nullable Progra return getRunRecordProgramPrefix(status, programId.getProgramReference(), programId.getVersion()); } - + private List> getRunRecordProgramPrefix(String status, @Nullable ProgramReference programRef, @Nullable String version) { List> fields = getRunRecordStatusPrefix(status); @@ -2920,6 +3090,33 @@ private static ApplicationId getApplicationIdFromRow(StructuredRow row) { row.getString(StoreDefinition.AppMetadataStore.VERSION_FIELD)); } + private String getFlowControlStatus(ProgramRunId programRunId, ProgramRunStatus runStatus, + Map systemArgs) { + if (!CONTROL_FLOW_PROGRAM_TYPES.contains(programRunId.getType())) { + return TYPE_FLOW_CONTROL_NONE; + } + if (NamespaceId.SYSTEM.getNamespace().equals(programRunId.getParent().getNamespace())) { + return TYPE_FLOW_CONTROL_NONE; + } + if (systemArgs.containsKey(ProgramOptionConstants.WORKFLOW_NAME)) { + return TYPE_FLOW_CONTROL_NONE; + } + if (runStatus == ProgramRunStatus.RUNNING) { + return TYPE_FLOW_CONTROL_RUNNING; + } + if (runStatus == ProgramRunStatus.PENDING || runStatus == ProgramRunStatus.STARTING) { + return TYPE_FLOW_CONTROL_LAUNCHING; + } + + return TYPE_FLOW_CONTROL_NONE; + } + + private void writeRunRecordWithPrimaryKeys(List> key, RunRecordDetail meta) throws IOException { + key.add(Fields.stringField(StoreDefinition.AppMetadataStore.FLOW_CONTROL_STATUS, meta.getFlowControlStatus())); + writeToStructuredTableWithPrimaryKeys( + key, meta, getRunRecordsTable(), StoreDefinition.AppMetadataStore.RUN_RECORD_DATA); + } + /** * Represents a position for scanning. */ @@ -2945,9 +3142,16 @@ private static final class AppScanEntry implements Map.Entry getAllAppVersionsAppIds(ApplicationReference appRef) { return TransactionRunners.run(transactionRunner, context -> { @@ -1179,6 +1191,14 @@ public void deleteAllStates(NamespaceId namespaceId, String appName) }, ApplicationNotFoundException.class); } + @Override + public long getApplicationCount(NamespaceId namespaceId) { + return TransactionRunners.run(transactionRunner, context -> { + return getAppMetadataStore(context).getNamespaceApplicationCount(namespaceId); + } + ); + } + private AppStateTable getAppStateTable(StructuredTableContext context) throws TableNotFoundException { return new AppStateTable(context); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/AppSpecDeserializationContext.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/AppSpecDeserializationContext.java new file mode 100644 index 000000000000..07972e077749 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/AppSpecDeserializationContext.java @@ -0,0 +1,236 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.store.adapters; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.util.concurrent.UncheckedExecutionException; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import io.cdap.cdap.api.artifact.ArtifactId; +import io.cdap.cdap.api.plugin.PluginClass; +import io.cdap.cdap.internal.app.runtime.plugin.PluginNotExistsException; +import io.cdap.cdap.spi.data.StructuredRow; +import io.cdap.cdap.spi.data.StructuredTable; +import io.cdap.cdap.spi.data.table.field.Field; +import io.cdap.cdap.spi.data.table.field.Fields; +import io.cdap.cdap.store.StoreDefinition.ArtifactStore; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import javax.annotation.Nonnull; + +/** + * Provides context for deserializing an {@code ApplicationSpecification}, primarily for resolving + * and loading {@link PluginClass} instances. It holds the current namespace, caches loaded plugins, + * and allows an outer deserializer to set parent artifact details (name and namespace) used for + * plugin resolution. + */ +public class AppSpecDeserializationContext { + + private static final Gson GSON = new Gson(); + private ArtifactId rootArtifact; + private String namespace; + private String appName; + private final StructuredTable pluginDataTable; + private final StructuredTable universalPluginDataTable; + private Set missingPlugins; + private final LoadingCache pluginCache; + + /** + * Constructs a context for deserializing an Application Specification. + * + * @param namespace The current application's namespace. + * @param pluginDataTable Table for loading plugin data. + * @param universalPluginDataTable Table for loading universal plugins. + */ + public AppSpecDeserializationContext(String namespace, StructuredTable pluginDataTable, + StructuredTable universalPluginDataTable) { + this.namespace = namespace; + this.pluginDataTable = pluginDataTable; + this.universalPluginDataTable = universalPluginDataTable; + this.missingPlugins = new HashSet<>(); + this.pluginCache = CacheBuilder.newBuilder().maximumSize(10) + .build(new CacheLoader() { + @Override + @Nonnull + public PluginClass load(@Nonnull PluginKey pluginKey) throws PluginNotExistsException { + return loadPlugin(pluginKey); + } + }); + } + + private PluginClass loadPlugin(PluginKey pluginKey) throws PluginNotExistsException { + PluginClass pluginClass = null; + Optional row = fetchFromPluginDataTable(pluginKey); + if (!row.isPresent()) { + row = fetchFromUniversalPluginDataTable(pluginKey); + if (!row.isPresent()) { + throw new PluginNotExistsException( + new io.cdap.cdap.proto.id.ArtifactId(pluginKey.getArtifactNamespace(), + pluginKey.getArtifactName(), pluginKey.getArtifactVersion()), + pluginKey.getPluginType(), pluginKey.getPluginName()); + } + } + String rawPluginData = row.get().getString(ArtifactStore.PLUGIN_DATA_FIELD); + if (rawPluginData != null) { + JsonObject pluginDataObject = new JsonParser().parse(rawPluginData).getAsJsonObject(); + if (pluginDataObject.has("pluginClass") && pluginDataObject.get("pluginClass") + .isJsonObject()) { + pluginClass = GSON.fromJson(pluginDataObject.getAsJsonObject("pluginClass"), + PluginClass.class); + } + } + if (pluginClass == null) { + throw new RuntimeException("Failed to deserialize PluginClass from data for key: " + pluginKey + + ". Raw data might be missing or malformed."); + } + return pluginClass; + } + + private Optional fetchFromPluginDataTable(PluginKey pluginKey) { + if (pluginKey.getParentName() == null || pluginKey.getParentName().isEmpty()) { + // For such scenarios, we need to fetch plugin data from universal plugin data table. + return Optional.empty(); + } + Collection> fields = new ArrayList<>(); + fields.add( + Fields.stringField(ArtifactStore.PARENT_NAMESPACE_FIELD, pluginKey.getParentNamespace())); + fields.add(Fields.stringField(ArtifactStore.PARENT_NAME_FIELD, pluginKey.getParentName())); + fields.add(Fields.stringField(ArtifactStore.PLUGIN_TYPE_FIELD, pluginKey.getPluginType())); + fields.add(Fields.stringField(ArtifactStore.PLUGIN_NAME_FIELD, pluginKey.getPluginName())); + fields.add(Fields.stringField(ArtifactStore.ARTIFACT_NAMESPACE_FIELD, + pluginKey.getArtifactNamespace())); + fields.add(Fields.stringField(ArtifactStore.ARTIFACT_NAME_FIELD, pluginKey.getArtifactName())); + fields.add( + Fields.stringField(ArtifactStore.ARTIFACT_VER_FIELD, pluginKey.getArtifactVersion())); + try { + return pluginDataTable.read(fields); + } catch (IOException e) { + throw new RuntimeException("Failed to read from plugin data table for key: " + pluginKey, e); + } + } + + private Optional fetchFromUniversalPluginDataTable(PluginKey pluginKey) { + Collection> fields = new ArrayList<>(); + fields.add(Fields.stringField(ArtifactStore.NAMESPACE_FIELD, namespace)); + fields.add(Fields.stringField(ArtifactStore.PLUGIN_TYPE_FIELD, pluginKey.getPluginType())); + fields.add(Fields.stringField(ArtifactStore.PLUGIN_NAME_FIELD, pluginKey.getPluginName())); + fields.add(Fields.stringField(ArtifactStore.ARTIFACT_NAMESPACE_FIELD, + pluginKey.getArtifactNamespace())); + fields.add(Fields.stringField(ArtifactStore.ARTIFACT_NAME_FIELD, pluginKey.getArtifactName())); + fields.add( + Fields.stringField(ArtifactStore.ARTIFACT_VER_FIELD, pluginKey.getArtifactVersion())); + try { + return universalPluginDataTable.read(fields); + } catch (IOException e) { + throw new RuntimeException( + "Failed to read from universal plugin data table for key: " + pluginKey, e); + } + } + + /** + * Gets the primary namespace for the current deserialization. + * + * @return The namespace string. + */ + public String getNamespace() { + return namespace; + } + + /** + * Sets the primary namespace for the current deserialization. + */ + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + /** + * Gets the root artifact, set externally. + */ + public ArtifactId getRootArtifact() { + return rootArtifact; + } + + /** + * Sets the root artifact for plugin resolution context. + */ + public void setRootArtifact(ArtifactId rootArtifact) { + this.rootArtifact = rootArtifact; + } + + /** + * Appends a plugin that could not be resolved during deserialization. + */ + public void appendMissingPlugin(String pluginInfo) { + if (pluginInfo != null) { + this.missingPlugins.add(pluginInfo); + } + } + + /** + * Returns a read-only view of the plugins that were missing during the deserialization process. + */ + public Set getMissingPlugins() { + return Collections.unmodifiableSet(this.missingPlugins); + } + + /** + * Sets the application name. + */ + public void setAppName(String appName) { + this.appName = appName; + } + + /** + * Gets the application name, set externally. + */ + public String getAppName() { + return appName; + } + + /** + * Retrieves the {@link PluginClass} for the given key, using an internal cache. + * + * @param pluginKey Key identifying the plugin. + * @return The resolved {@link PluginClass}. + * @throws RuntimeException if the plugin cannot be loaded. + */ + public PluginClass getPlugin(PluginKey pluginKey) throws PluginNotExistsException { + try { + return this.pluginCache.get(pluginKey); + } catch (ExecutionException | UncheckedExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof PluginNotExistsException) { + // cache.get() method wraps the PluginNotExistsException with Execution exception. + // Hence, we need to explicitly re-throw the PluginNotExistsException here. + throw (PluginNotExistsException) cause; + } + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + throw new RuntimeException("Failed to load plugin class for " + pluginKey, cause); + } + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/AppSpecDeserializationContextHolder.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/AppSpecDeserializationContextHolder.java new file mode 100644 index 000000000000..b017a35a72bf --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/AppSpecDeserializationContextHolder.java @@ -0,0 +1,67 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.store.adapters; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Holds the {@link AppSpecDeserializationContext} for the current thread's deserialization + * operation. Ensures context is cleared after use. + */ +public final class AppSpecDeserializationContextHolder { + + private static final Logger LOG = LoggerFactory.getLogger( + AppSpecDeserializationContextHolder.class); + private static final ThreadLocal CURRENT_CONTEXT = new ThreadLocal<>(); + + private AppSpecDeserializationContextHolder() { + } + + /** + * Sets the context for the current thread. Should be called before deserialization. + */ + public static void setContext(AppSpecDeserializationContext context) { + if (CURRENT_CONTEXT.get() != null) { + LOG.warn("Overwriting existing AppSpecDeserializationContext on thread {}. " + + "This might indicate a missing clearContext() call from a previous operation.", + Thread.currentThread().getName()); + } + CURRENT_CONTEXT.set(context); + } + + /** + * Gets the context for the current thread. + * + * @throws IllegalStateException if no context is set. + */ + public static AppSpecDeserializationContext getContext() { + AppSpecDeserializationContext context = CURRENT_CONTEXT.get(); + if (context == null) { + throw new IllegalStateException( + "AppSpecDeserializationContext has not been set for the current deserialization operation on this thread."); + } + return context; + } + + /** + * Clears the context for the current thread. Must be called in a finally block. + */ + public static void clearContext() { + CURRENT_CONTEXT.remove(); + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/ApplicationMetaAdapter.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/ApplicationMetaAdapter.java new file mode 100644 index 000000000000..003211c2076d --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/ApplicationMetaAdapter.java @@ -0,0 +1,111 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.store.adapters; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.app.ApplicationSpecification; +import io.cdap.cdap.api.plugin.Plugin; +import io.cdap.cdap.api.plugin.PluginClass; +import io.cdap.cdap.internal.app.ApplicationSpecificationAdapter; +import io.cdap.cdap.internal.app.ApplicationSpecificationCodec; +import io.cdap.cdap.internal.app.store.ApplicationMeta; +import io.cdap.cdap.spi.data.StructuredTable; +import java.util.Set; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages Gson instances specifically configured for ApplicationMeta serialization and + * deserialization, handling application specification reduction. This ensures thread-safe + * operations by managing the {@link AppSpecDeserializationContext} lifecycle per operation via + * {@link AppSpecDeserializationContextHolder}. It uses a static cache for Gson instances based on + * the {@code appSpecReductionEnabled} flag. + */ +public final class ApplicationMetaAdapter { + + private static final Gson GSON_INSTANCE_REDUCTION_ENABLED = buildGsonInternal(true); + private static final Gson GSON_INSTANCE_REDUCTION_DISABLED = buildGsonInternal(false); + private static final Logger LOG = LoggerFactory.getLogger(ApplicationMetaAdapter.class); + + private ApplicationMetaAdapter() { + } + + /** + * Deserializes a JSON string to an object of the specified class, managing the + * {@link AppSpecDeserializationContext} lifecycle via + * {@link AppSpecDeserializationContextHolder}. + */ + public static T fromJson(String jsonString, Class classOfT, String namespace, + boolean appSpecReductionEnabled, StructuredTable pluginDataTable, + @Nullable StructuredTable universalPluginDataTable) { + AppSpecDeserializationContext operationContext = new AppSpecDeserializationContext(namespace, + pluginDataTable, universalPluginDataTable); + AppSpecDeserializationContextHolder.setContext(operationContext); + try { + Gson gson = getGson(appSpecReductionEnabled); + T result = gson.fromJson(jsonString, classOfT); + Set missingPlugins = operationContext.getMissingPlugins(); + if (!missingPlugins.isEmpty()) { + LOG.trace("For application {} : {}", operationContext.getAppName(), + operationContext.getMissingPlugins()); + } + return result; + } finally { + AppSpecDeserializationContextHolder.clearContext(); + } + } + + /** + * Serializes an object to its JSON representation. + */ + public static String toJson(Object objectToSerialize, boolean appSpecReductionEnabled) { + Gson gson = getGson(appSpecReductionEnabled); + // No need to set / get the AppSpecDeserialization context in case of serialization. + return gson.toJson(objectToSerialize); + } + + private static Gson buildGsonInternal(boolean appSpecReductionEnabled) { + GsonBuilder gsonBuilder = new GsonBuilder(); + + if (appSpecReductionEnabled) { + gsonBuilder.registerTypeAdapter(ApplicationMeta.class, new ApplicationMetaCodec()); + gsonBuilder.registerTypeAdapter(PluginClass.class, new PluginClassSerializer()); + gsonBuilder.registerTypeAdapter(Plugin.class, new PluginDeserializer()); + gsonBuilder.registerTypeHierarchyAdapter(ApplicationSpecification.class, + new ApplicationSpecificationCodec()); + } + // This adds other necessary adapters for ApplicationSpecification internals (schemas, etc.) + ApplicationSpecificationAdapter.addTypeAdapters(gsonBuilder); + + return gsonBuilder.create(); + } + + /** + * Gets a pre-configured, cached, and thread-safe Gson instance. + * + * @param conf Configuration map, used to determine if app spec reduction is enabled. + * @return A shared Gson instance. + */ + private static Gson getGson(boolean appSpecReductionEnabled) { + if (appSpecReductionEnabled) { + return GSON_INSTANCE_REDUCTION_ENABLED; + } + return GSON_INSTANCE_REDUCTION_DISABLED; + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/ApplicationMetaCodec.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/ApplicationMetaCodec.java new file mode 100644 index 000000000000..5332cc8ff911 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/ApplicationMetaCodec.java @@ -0,0 +1,97 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.store.adapters; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; +import io.cdap.cdap.api.app.ApplicationSpecification; +import io.cdap.cdap.api.artifact.ArtifactId; +import io.cdap.cdap.internal.app.store.ApplicationMeta; +import java.lang.reflect.Type; + +/** + * Helper class to encode/decode {@link ApplicationMetaCodec} to/from json. + */ +public class ApplicationMetaCodec implements JsonSerializer, + JsonDeserializer { + + /** + * Serializes an {@link ApplicationMeta} object to its JSON representation. Includes the + * application ID and its specification. + */ + @Override + public JsonElement serialize(ApplicationMeta src, Type typeOfSrc, + JsonSerializationContext context) { + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("id", src.getId()); + + JsonElement specJson = context.serialize(src.getSpec()); + jsonObject.add("spec", specJson); + + return jsonObject; + } + + /** + * Deserializes a JSON element into an {@link ApplicationMeta} object. It extracts the + * application's {@link ArtifactId} from its specification JSON to set the parent name and + * namespace in the shared {@link AppSpecDeserializationContext}. This enables correct resolution + * of plugins within the application's scope. + * + * @throws JsonParseException If JSON is malformed or required fields are missing. + */ + @Override + public ApplicationMeta deserialize(JsonElement json, Type typeOfT, + JsonDeserializationContext context) throws JsonParseException { + AppSpecDeserializationContext appSpecDeserializationContext = AppSpecDeserializationContextHolder.getContext(); + JsonObject jsonObject = json.getAsJsonObject(); + + JsonElement idElement = jsonObject.get("id"); + if (idElement == null || idElement.isJsonNull()) { + throw new JsonParseException("ApplicationMeta 'id' field is missing or null"); + } + String id = idElement.getAsString(); + + JsonObject specJson = jsonObject.getAsJsonObject("spec"); + if (specJson == null || specJson.isJsonNull()) { + throw new JsonParseException("ApplicationMeta 'spec' field is missing or null for id: " + id); + } + + // Set parent context before fully deserializing the AppSpec, + // so plugins can be retrieved from DB using this parent information. + JsonElement artifactIdJson = specJson.get("artifactId"); + if (artifactIdJson != null && !artifactIdJson.isJsonNull()) { + ArtifactId artifactId = context.deserialize(artifactIdJson, ArtifactId.class); + if (artifactId != null) { + appSpecDeserializationContext.setRootArtifact(artifactId); + } else { + throw new JsonParseException( + "ArtifactId in ApplicationSpecification was null for app: " + id); + } + } + + JsonElement appNameJson = specJson.get("name"); + appSpecDeserializationContext.setAppName(appNameJson == null ? null : appNameJson.getAsString()); + + ApplicationSpecification spec = context.deserialize(specJson, ApplicationSpecification.class); + return new ApplicationMeta(id, spec, null, null); + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/PluginClassSerializer.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/PluginClassSerializer.java new file mode 100644 index 000000000000..40769daff4b3 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/PluginClassSerializer.java @@ -0,0 +1,44 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.store.adapters; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; +import io.cdap.cdap.api.plugin.PluginClass; +import java.lang.reflect.Type; + +/** + * Gson serializer for {@link PluginClass}. Serializes only essential identification fields: 'type' + * and 'name'. + */ +public class PluginClassSerializer implements JsonSerializer { + + /** + * Serializes a {@link PluginClass} to a JSON object. Only 'type' and 'name' fields are included + * in the serialized output, excluding other plugin information. + */ + @Override + public JsonElement serialize(PluginClass src, Type typeOfSrc, JsonSerializationContext context) { + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("type", src.getType()); + jsonObject.addProperty("name", src.getName()); + // Exclude other plugin information. + return jsonObject; + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/PluginDeserializer.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/PluginDeserializer.java new file mode 100644 index 000000000000..f440297fc034 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/PluginDeserializer.java @@ -0,0 +1,144 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.store.adapters; + +import com.google.common.collect.Iterables; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import io.cdap.cdap.api.artifact.ArtifactId; +import io.cdap.cdap.api.artifact.ArtifactScope; +import io.cdap.cdap.api.plugin.Plugin; +import io.cdap.cdap.api.plugin.PluginClass; +import io.cdap.cdap.api.plugin.PluginProperties; +import io.cdap.cdap.internal.app.runtime.plugin.PluginNotExistsException; +import io.cdap.cdap.internal.guava.reflect.TypeParameter; +import io.cdap.cdap.internal.guava.reflect.TypeToken; +import io.cdap.cdap.proto.id.NamespaceId; +import java.lang.reflect.Type; +import java.util.Collections; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Gson deserializer for {@link Plugin}s. Uses {@link AppSpecDeserializationContext} to enrich + * {@link PluginClass} data if initially minimal. + */ +public class PluginDeserializer implements JsonDeserializer { + + private static final Logger LOG = LoggerFactory.getLogger(PluginDeserializer.class); + + /** + * Deserializes JSON to {@link Plugin}. If {@link PluginClass#getClassName()} is empty, attempts + * to enrich it via {@link #resolvePluginClassData(List, ArtifactId, PluginClass)}. + * + * @param json The JSON element to deserialize. + * @param typeOfT The type of the object to deserialize to. + * @param context The Gson deserialization context. + * @return Deserialized {@link Plugin} object. + * @throws JsonParseException If JSON is malformed. + */ + @Override + public Plugin deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) + throws JsonParseException { + JsonObject jsonObject = json.getAsJsonObject(); + List parents = deserializeList(jsonObject.get("parents"), context, + ArtifactId.class); + ArtifactId artifactId = context.deserialize(jsonObject.get("artifactId"), ArtifactId.class); + PluginClass pluginClass = context.deserialize(jsonObject.get("pluginClass"), PluginClass.class); + + // Populate missing PluginClass data which was removed during serialization. + // This can be identified if the className field is empty. + if (pluginClass != null && (pluginClass.getClassName() == null || pluginClass.getClassName() + .isEmpty())) { + pluginClass = resolvePluginClassData(parents, artifactId, pluginClass); + } + + PluginProperties properties = context.deserialize(jsonObject.get("properties"), + PluginProperties.class); + + return new Plugin(parents, artifactId, pluginClass, properties); + } + + /** + * Attempts to load {@link PluginClass} details from {@link AppSpecDeserializationContext}. Parent + * context for lookup defaults to application context; for some plugins, it may be overridden by + * artifact in the {@code parents} list if specific conditions are met. Returns the result of + * {@code appSpecDeserializationContext.getPlugin(pluginKey)}. + * + * @param parents Declared parent {@link ArtifactId}s from JSON. + * @param artifactId The plugin's own {@link ArtifactId}. + * @param pluginClass Initial {@link PluginClass}, possibly incomplete. + * @return {@link PluginClass} from context, or the outcome of the lookup. + */ + private PluginClass resolvePluginClassData(List parents, ArtifactId artifactId, + PluginClass pluginClass) { + AppSpecDeserializationContext appSpecDeserializationContext = AppSpecDeserializationContextHolder.getContext(); + String artifactNamespace = resolveNamespaceByScope(artifactId.getScope(), + appSpecDeserializationContext.getNamespace()); + + Exception exception = null; + ArtifactId rootArtifact = appSpecDeserializationContext.getRootArtifact(); + for (ArtifactId parentId : Iterables.concat(parents, Collections.singleton(rootArtifact))) { + String parentName = parentId.getName(); + String parentNamespace = resolveNamespaceByScope(parentId.getScope(), + appSpecDeserializationContext.getNamespace()); + PluginKey pluginKey = new PluginKey(parentName, parentNamespace, artifactId.getName(), + artifactNamespace, artifactId.getVersion().getVersion(), pluginClass.getType(), + pluginClass.getName()); + try { + return appSpecDeserializationContext.getPlugin(pluginKey); + } catch (PluginNotExistsException e) { + exception = e; + } + } + if (exception != null) { + appSpecDeserializationContext.appendMissingPlugin(exception.getMessage()); + } + return pluginClass; + } + + /** + * Deserializes a JSON array into a {@link List} of {@code valueType}. Returns an empty list if + * JSON is null, not an array, or if deserialization results in null. + */ + private List deserializeList(JsonElement json, JsonDeserializationContext context, + Class valueType) { + if (json == null || json.isJsonNull() || !json.isJsonArray()) { + return Collections.emptyList(); + } + Type type = new TypeToken>() { + }.where(new TypeParameter() { + }, valueType).getType(); + List list = context.deserialize(json, type); + return list == null ? Collections.emptyList() : list; + } + + /** + * Resolves namespace string from {@link ArtifactScope}. Uses system namespace for + * {@link ArtifactScope#SYSTEM}, otherwise from context. + * + * @param scope The artifact scope. + * @return Corresponding namespace string. + */ + private String resolveNamespaceByScope(ArtifactScope scope, String namespace) { + return ArtifactScope.SYSTEM.equals(scope) ? NamespaceId.SYSTEM.getNamespace() : namespace; + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/PluginKey.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/PluginKey.java new file mode 100644 index 000000000000..70f53476f686 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/PluginKey.java @@ -0,0 +1,142 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.store.adapters; + +import com.google.common.base.Objects; + +/** + * Unique key for identifying a plugin. Used for look up and caching. + */ +class PluginKey { + + private final String parentName; + private final String parentNamespace; + private final String artifactName; + private final String artifactNamespace; + private final String artifactVersion; + private final String pluginType; + private final String pluginName; + + /** + * Creates a {@code PluginKey} with specified plugin identification details. + */ + PluginKey(String parentName, String parentNamespace, String artifactName, + String artifactNamespace, String artifactVersion, String pluginType, String pluginName) { + this.parentName = parentName; + this.parentNamespace = parentNamespace; + this.artifactName = artifactName; + this.artifactNamespace = artifactNamespace; + this.artifactVersion = artifactVersion; + this.pluginType = pluginType; + this.pluginName = pluginName; + } + + /** + * Returns the parent name. + */ + public String getParentName() { + return parentName; + } + + /** + * Returns the parent namespace. + */ + public String getParentNamespace() { + return parentNamespace; + } + + /** + * Returns the plugin's artifact name. + */ + public String getArtifactName() { + return artifactName; + } + + /** + * Returns the plugin's artifact namespace. + */ + public String getArtifactNamespace() { + return artifactNamespace; + } + + /** + * Returns the plugin's artifact version. + */ + public String getArtifactVersion() { + return artifactVersion; + } + + /** + * Returns the plugin type. + */ + public String getPluginType() { + return pluginType; + } + + /** + * Returns the plugin name. + */ + public String getPluginName() { + return pluginName; + } + + /** + * Checks equality with another object. + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof PluginKey)) { + return false; + } + PluginKey pluginKey = (PluginKey) o; + return Objects.equal(parentName, pluginKey.parentName) + && Objects.equal(parentNamespace, pluginKey.parentNamespace) + && Objects.equal(artifactName, pluginKey.artifactName) + && Objects.equal(artifactNamespace, pluginKey.artifactNamespace) + && Objects.equal(artifactVersion, pluginKey.artifactVersion) + && Objects.equal(pluginType, pluginKey.pluginType) + && Objects.equal(pluginName, pluginKey.pluginName); + } + + /** + * Returns a string representation of this key. + */ + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("parentName", parentName) + .add("parentNamespace", parentNamespace) + .add("artifactName", artifactName) + .add("artifactNamespace", artifactNamespace) + .add("artifactVersion", artifactVersion) + .add("pluginType", pluginType) + .add("pluginName", pluginName) + .toString(); + } + + /** + * Computes the hash code for this key. + */ + @Override + public int hashCode() { + return Objects.hashCode(parentName, parentNamespace, artifactName, artifactNamespace, + artifactVersion, pluginType, pluginName); + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/preview/DefaultPreviewStore.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/preview/DefaultPreviewStore.java index 2ae2e5f2074b..04c62cf3ab50 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/preview/DefaultPreviewStore.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/preview/DefaultPreviewStore.java @@ -13,6 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package io.cdap.cdap.internal.app.store.preview; import com.google.common.annotations.VisibleForTesting; @@ -54,17 +55,21 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Default implementation of the {@link PreviewStore} that stores data in a level db table. */ public class DefaultPreviewStore implements PreviewStore { + private static final Logger LOG = LoggerFactory.getLogger(DefaultPreviewStore.class); private static final DatasetId PREVIEW_TABLE_ID = NamespaceId.SYSTEM.dataset("preview.table"); private static final DatasetId PREVIEW_QUEUE_TABLE_ID = NamespaceId.SYSTEM.dataset( "preview.queue.table"); private static final byte[] DATA_ROW_KEY_PREFIX = Bytes.toBytes("dr"); private static final byte[] META_ROW_KEY_PREFIX = Bytes.toBytes("mr"); + private static final byte[] POLLER_INFO_TO_APP_ID_PREFIX = Bytes.toBytes("p2a"); private static final byte[] TRACER = Bytes.toBytes("t"); private static final byte[] PROPERTY = Bytes.toBytes("p"); private static final byte[] VALUE = Bytes.toBytes("v"); @@ -84,6 +89,10 @@ public class DefaultPreviewStore implements PreviewStore { private static final byte[] CONFIG = Bytes.toBytes("c"); private static final byte[] APPID = Bytes.toBytes("a"); private static final byte[] PRINCIPAL = Bytes.toBytes("p"); + private static final Gson ENTITY_GSON = new GsonBuilder().registerTypeAdapter(EntityId.class, + new EntityIdTypeAdapter()).create(); + private static final Gson SCHEMA_GSON = new GsonBuilder().registerTypeAdapter(Schema.class, + new SchemaTypeAdapter()).create(); private final AtomicLong counter = new AtomicLong(0L); @@ -127,9 +136,6 @@ public void put(ApplicationId applicationId, String tracerName, String propertyN @Override public Map> get(ApplicationId applicationId, String tracerName) { - // PreviewStore is a singleton and we have to create gson for each operation since gson is not thread safe. - Gson gson = new GsonBuilder().registerTypeAdapter(Schema.class, new SchemaTypeAdapter()) - .create(); byte[] startRowKey = getPreviewRowKeyBuilder(DATA_ROW_KEY_PREFIX, applicationId) .add(tracerName).build().getKey(); byte[] stopRowKey = new MDSKey(Bytes.stopKeyForPrefix(startRowKey)).getKey(); @@ -140,7 +146,7 @@ public Map> get(ApplicationId applicationId, String tr while ((indexRow = scanner.next()) != null) { Map columns = indexRow.getColumns(); String propertyName = Bytes.toString(columns.get(PROPERTY)); - JsonElement value = gson.fromJson(Bytes.toString(columns.get(VALUE)), JsonElement.class); + JsonElement value = SCHEMA_GSON.fromJson(Bytes.toString(columns.get(VALUE)), JsonElement.class); List values = result.computeIfAbsent(propertyName, k -> new ArrayList<>()); values.add(value); } @@ -167,6 +173,18 @@ private void removePreviewData(byte[] prefix, ApplicationId applicationId) { @Override public void remove(ApplicationId applicationId) { + byte[] pollerInfo = getPreviewRequestPollerInfo(applicationId); + if (pollerInfo != null) { + MDSKey indexKey = new MDSKey.Builder().add(POLLER_INFO_TO_APP_ID_PREFIX).add(pollerInfo) + .build(); + try { + previewTable.deleteDefaultVersion(indexKey.getKey(), APPID); + } catch (IOException e) { + // It is ok to not throw exception here, as the data will be eventually cleaned up by the ttl remover. + LOG.warn("Failed to delete poller info for application {}", applicationId, e); + } + } + removeFromWaitingState(applicationId); // remove actual preview user data removePreviewData(DATA_ROW_KEY_PREFIX, applicationId); @@ -176,14 +194,11 @@ public void remove(ApplicationId applicationId) { @Override public void setProgramId(ProgramRunId programRunId) { - // PreviewStore is a singleton and we have to create gson for each operation since gson is not thread safe. - Gson gson = new GsonBuilder().registerTypeAdapter(EntityId.class, new EntityIdTypeAdapter()) - .create(); MDSKey mdsKey = getPreviewRowKeyBuilder(META_ROW_KEY_PREFIX, programRunId.getParent().getParent()).build(); try { previewTable.putDefaultVersion(mdsKey.getKey(), RUN, - Bytes.toBytes(gson.toJson(programRunId))); + Bytes.toBytes(ENTITY_GSON.toJson(programRunId))); } catch (IOException e) { throw new RuntimeException(String.format("Failed to put %s into preview store", programRunId), e); @@ -192,9 +207,6 @@ public void setProgramId(ProgramRunId programRunId) { @Override public ProgramRunId getProgramRunId(ApplicationId applicationId) { - // PreviewStore is a singleton and we have to create gson for each operation since gson is not thread safe. - Gson gson = new GsonBuilder().registerTypeAdapter(EntityId.class, new EntityIdTypeAdapter()) - .create(); MDSKey mdsKey = getPreviewRowKeyBuilder(META_ROW_KEY_PREFIX, applicationId).build(); byte[] runId = null; @@ -206,7 +218,7 @@ public ProgramRunId getProgramRunId(ApplicationId applicationId) { String.format("Failed to get program run id for preview %s", applicationId), e); } if (runId != null) { - return gson.fromJson(Bytes.toString(runId), ProgramRunId.class); + return ENTITY_GSON.fromJson(Bytes.toString(runId), ProgramRunId.class); } return null; } @@ -253,9 +265,6 @@ public PreviewStatus getPreviewStatus(ApplicationId applicationId) { @Override public void add(ApplicationId applicationId, AppRequest appRequest, @Nullable Principal principal) { - // PreviewStore is a singleton and we have to create gson for each operation since gson is not thread safe. - Gson gson = new GsonBuilder().registerTypeAdapter(Schema.class, new SchemaTypeAdapter()) - .create(); long timeInSeconds = RunIds.getTime(applicationId.getApplication(), TimeUnit.SECONDS); MDSKey mdsKey = new MDSKey.Builder() .add(WAITING) @@ -266,11 +275,11 @@ public void add(ApplicationId applicationId, AppRequest appRequest, try { previewQueueTable.putDefaultVersion(mdsKey.getKey(), APPID, - Bytes.toBytes(gson.toJson(applicationId))); + Bytes.toBytes(SCHEMA_GSON.toJson(applicationId))); previewQueueTable.putDefaultVersion(mdsKey.getKey(), CONFIG, - Bytes.toBytes(gson.toJson(appRequest))); + Bytes.toBytes(SCHEMA_GSON.toJson(appRequest))); previewQueueTable.putDefaultVersion(mdsKey.getKey(), PRINCIPAL, - Bytes.toBytes(gson.toJson(principal))); + Bytes.toBytes(SCHEMA_GSON.toJson(principal))); long submitTimeInMillis = RunIds.getTime(applicationId.getApplication(), TimeUnit.MILLISECONDS); setPreviewStatus(applicationId, @@ -307,9 +316,6 @@ private void removeFromWaitingState(ApplicationId applicationId) { @Override public List getAllInWaitingState() { - // PreviewStore is a singleton and we have to create gson for each operation since gson is not thread safe. - Gson gson = new GsonBuilder().registerTypeAdapter(Schema.class, new SchemaTypeAdapter()) - .create(); byte[] startRowKey = new MDSKey.Builder().add(WAITING).build().getKey(); byte[] stopRowKey = new MDSKey(Bytes.stopKeyForPrefix(startRowKey)).getKey(); @@ -318,10 +324,10 @@ public List getAllInWaitingState() { Row indexRow; while ((indexRow = scanner.next()) != null) { Map columns = indexRow.getColumns(); - AppRequest request = gson.fromJson(Bytes.toString(columns.get(CONFIG)), AppRequest.class); - ApplicationId applicationId = gson.fromJson(Bytes.toString(columns.get(APPID)), + AppRequest request = SCHEMA_GSON.fromJson(Bytes.toString(columns.get(CONFIG)), AppRequest.class); + ApplicationId applicationId = SCHEMA_GSON.fromJson(Bytes.toString(columns.get(APPID)), ApplicationId.class); - Principal principal = gson.fromJson(Bytes.toString(columns.get(PRINCIPAL)), + Principal principal = SCHEMA_GSON.fromJson(Bytes.toString(columns.get(PRINCIPAL)), Principal.class); result.add(new PreviewRequest(applicationId, request, principal)); } @@ -355,9 +361,6 @@ public void setPreviewRequestPollerInfo(ApplicationId applicationId, @Nullable b } private void setPollerinfo(ApplicationId applicationId, byte[] pollerInfo) { - // PreviewStore is a singleton and we have to create gson for each operation since gson is not thread safe. - Gson gson = new GsonBuilder().registerTypeAdapter(Schema.class, new SchemaTypeAdapter()) - .create(); MDSKey mdsKey = getPreviewRowKeyBuilder(META_ROW_KEY_PREFIX, applicationId).build(); try { @@ -365,9 +368,19 @@ private void setPollerinfo(ApplicationId applicationId, byte[] pollerInfo) { } catch (IOException e) { String msg = String.format( "Error while setting the poller information %s for waiting preview application %s.", - gson.toJson(pollerInfo), applicationId); + SCHEMA_GSON.toJson(pollerInfo), applicationId); throw new RuntimeException(msg, e); } + + // Add an entry to the index: pollerInfo -> applicationId + MDSKey indexKey = new MDSKey.Builder().add(POLLER_INFO_TO_APP_ID_PREFIX).add(pollerInfo) + .build(); + try { + previewTable.putDefaultVersion(indexKey.getKey(), APPID, + Bytes.toBytes(SCHEMA_GSON.toJson(applicationId))); + } catch (IOException e) { + throw new RuntimeException("Error while creating poller info index.", e); + } } @Override @@ -381,17 +394,25 @@ public byte[] getPreviewRequestPollerInfo(ApplicationId applicationId) { throw new RuntimeException( String.format("Failed to get the poller info for preview %s", applicationId), e); } - if (pollerInfo != null) { - return pollerInfo; - } + return pollerInfo; + } - return null; + public ApplicationId getApplicationId(byte[] pollerInfo) { + MDSKey indexKey = new MDSKey.Builder().add(POLLER_INFO_TO_APP_ID_PREFIX).add(pollerInfo) + .build(); + try { + byte[] applicationIdBytes = previewTable.getDefaultVersion(indexKey.getKey(), APPID); + if (applicationIdBytes == null) { + return null; + } + return SCHEMA_GSON.fromJson(Bytes.toString(applicationIdBytes), ApplicationId.class); + } catch (IOException e) { + throw new RuntimeException("Error while getting application id for poller info.", e); + } } @Override public void deleteExpiredData(long ttlInSeconds) { - Gson gson = new GsonBuilder().registerTypeAdapter(EntityId.class, new EntityIdTypeAdapter()) - .create(); byte[] startRowKey = new MDSKey.Builder().add(META_ROW_KEY_PREFIX).build().getKey(); byte[] stopRowKey = new MDSKey(Bytes.stopKeyForPrefix(startRowKey)).getKey(); @@ -405,7 +426,7 @@ public void deleteExpiredData(long ttlInSeconds) { continue; } - ApplicationId applicationId = gson.fromJson(applicationIdGson, ApplicationId.class); + ApplicationId applicationId = ENTITY_GSON.fromJson(applicationIdGson, ApplicationId.class); long applicationSubmitTime = RunIds.getTime(applicationId.getApplication(), TimeUnit.SECONDS); if ((currentTimeInSeconds - applicationSubmitTime) > ttlInSeconds) { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerService.java index 1397263d7cf0..af0d98dd81c0 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerService.java @@ -30,11 +30,15 @@ import io.cdap.cdap.common.http.CommonNettyHttpServiceFactory; import io.cdap.cdap.common.internal.remote.TaskWorkerHttpHandlerInternal; import io.cdap.cdap.common.security.HttpsEnabler; +import io.cdap.cdap.gateway.handlers.PingHandler; import io.cdap.http.ChannelPipelineModifier; +import io.cdap.http.HttpHandler; import io.cdap.http.NettyHttpService; import io.netty.channel.ChannelPipeline; import io.netty.handler.codec.http.HttpContentDecompressor; import java.net.InetSocketAddress; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.twill.common.Cancellable; import org.apache.twill.discovery.DiscoveryService; @@ -70,6 +74,12 @@ public class TaskWorkerService extends AbstractIdleService { cConf.set(TaskWorker.WORK_DIR, workDir); } + List handlers = Arrays.asList( + new PingHandler(), + new TaskWorkerHttpHandlerInternal(cConf, discoveryService, + discoveryServiceClient, this::stopService, + metricsCollectionService)); + NettyHttpService.Builder builder = commonNettyHttpServiceFactory.builder( Constants.Service.TASK_WORKER) .setHost(cConf.get(Constants.TaskWorker.ADDRESS)) @@ -83,9 +93,7 @@ public void modify(ChannelPipeline pipeline) { pipeline.addAfter("compressor", "decompressor", new HttpContentDecompressor()); } }) - .setHttpHandlers(new TaskWorkerHttpHandlerInternal(cConf, discoveryService, - discoveryServiceClient, this::stopService, - metricsCollectionService)); + .setHttpHandlers(handlers); if (cConf.getBoolean(Constants.Security.SSL.INTERNAL_ENABLED)) { new HttpsEnabler().configureKeyStore(cConf, sConf).enable(builder); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerServiceLauncher.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerServiceLauncher.java index a776ee09f688..8f43c9697366 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerServiceLauncher.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerServiceLauncher.java @@ -269,6 +269,14 @@ public void run() { } } + if (cConf.getBoolean(TaskWorker.TASK_WORKER_PROBE_ENABLED)) { + if (twillPreparer instanceof ExtendedTwillPreparer) { + twillPreparer = ((ExtendedTwillPreparer) twillPreparer) + .addProbes(TaskWorkerTwillRunnable.class.getSimpleName(), + cConf.getPropsWithPrefix(TaskWorker.TASK_WORKER_PROBE_PREFIX)); + } + } + // Set JVM options for task worker and artifact localizer twillPreparer.setJVMOptions(TaskWorkerTwillRunnable.class.getSimpleName(), cConf.get(Constants.TaskWorker.CONTAINER_JVM_OPTS)); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerTwillRunnable.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerTwillRunnable.java index 8853bdd7cbd6..6ae74058e12e 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerTwillRunnable.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerTwillRunnable.java @@ -45,7 +45,7 @@ import io.cdap.cdap.logging.guice.RemoteLogAppenderModule; import io.cdap.cdap.master.environment.MasterEnvironments; import io.cdap.cdap.master.spi.environment.MasterEnvironment; -import io.cdap.cdap.messaging.guice.MessagingServiceModule; +import io.cdap.cdap.messaging.guice.client.TaskWorkerMessagingClientModule; import io.cdap.cdap.metrics.guice.MetricsClientRuntimeModule; import io.cdap.cdap.proto.id.NamespaceId; import io.cdap.cdap.security.auth.context.AuthenticationContextModules; @@ -95,7 +95,7 @@ static Injector createInjector(CConfiguration cConf, Configuration hConf) { modules.add(new IOModule()); modules.add(new AuthenticationContextModules().getMasterWorkerModule()); modules.add(coreSecurityModule); - modules.add(new MessagingServiceModule(cConf)); + modules.add(new TaskWorkerMessagingClientModule(cConf)); modules.add(new SystemAppModule()); modules.add(new MetricsClientRuntimeModule().getDistributedModules()); modules.add(new AuditLogWriterModule(cConf).getDistributedModules()); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerTwillRunnable.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerTwillRunnable.java index b8d0db7543e8..b45767ded47c 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerTwillRunnable.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerTwillRunnable.java @@ -33,6 +33,7 @@ import io.cdap.cdap.api.artifact.ArtifactManager; import io.cdap.cdap.api.metrics.MetricsCollectionService; import io.cdap.cdap.app.guice.AppFabricServiceRuntimeModule; +import io.cdap.cdap.app.guice.AppFabricServiceRuntimeModule.ServiceType; import io.cdap.cdap.app.guice.AuditLogWriterModule; import io.cdap.cdap.app.guice.AuthorizationModule; import io.cdap.cdap.app.guice.DistributedArtifactManagerModule; @@ -91,6 +92,7 @@ import java.io.File; import java.util.ArrayList; import java.util.Arrays; +import java.util.EnumSet; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -156,7 +158,8 @@ protected void bindKeyManager(Binder binder) { new AuthorizationModule(), new AuthorizationEnforcementModule().getMasterModule(), new AuditLogWriterModule(cConf).getDistributedModules(), - Modules.override(new AppFabricServiceRuntimeModule(cConf).getDistributedModules()) + Modules.override(new AppFabricServiceRuntimeModule(cConf, AppFabricServiceRuntimeModule.ALL_SERVICE_TYPES) + .getDistributedModules()) .with(new AbstractModule() { // To enable localisation of artifacts @Override diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/bootstrap/BootstrapService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/bootstrap/BootstrapService.java index c7f929ee0897..70f95fe93b77 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/bootstrap/BootstrapService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/bootstrap/BootstrapService.java @@ -82,6 +82,13 @@ public class BootstrapService extends AbstractIdleService { protected void startUp() { LOG.info("Starting {}", getClass().getSimpleName()); config = bootstrapConfigProvider.getConfig(); + + // Do not start other services if no bootstrap steps were executed. + if (config.getSteps().isEmpty()) { + LOG.info("Skipping execution of bootstrap steps as bootstrap config has no steps."); + return; + } + executorService = Executors.newSingleThreadExecutor( Threads.createDaemonThreadFactory("bootstrap-service")); executorService.execute(() -> { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/bootstrap/guice/BootstrapModules.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/bootstrap/guice/BootstrapModules.java index d2ca74ac73fe..36e4528eae02 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/bootstrap/guice/BootstrapModules.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/bootstrap/guice/BootstrapModules.java @@ -72,6 +72,23 @@ protected void configure() { }; } + /** + * Module with empty config to not perform any bootstrap steps. + * + * @return bootstrap module that do not need to execute bootstrap steps. + */ + public static Module getNoOpModule() { + return new BaseModule() { + @Override + protected void configure() { + super.configure(); + BootstrapConfigProvider inMemoryProvider = new InMemoryBootstrapConfigProvider( + BootstrapConfig.EMPTY); + bind(BootstrapConfigProvider.class).toInstance(inMemoryProvider); + } + }; + } + /** * Bindings common to all modules */ diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/capability/CapabilityManagementService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/capability/CapabilityManagementService.java index c8d5645345ad..f4df5e36e2f4 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/capability/CapabilityManagementService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/capability/CapabilityManagementService.java @@ -24,7 +24,6 @@ import io.cdap.cdap.common.service.AbstractRetryableScheduledService; import io.cdap.cdap.common.service.RetryStrategies; import io.cdap.cdap.common.utils.DirUtils; -import io.cdap.cdap.internal.app.services.SystemProgramManagementService; import java.io.File; import java.io.FileReader; import java.io.Reader; @@ -46,8 +45,7 @@ public class CapabilityManagementService extends AbstractRetryableScheduledServi private final CapabilityApplier capabilityApplier; @Inject - CapabilityManagementService(CConfiguration cConf, CapabilityApplier capabilityApplier, - SystemProgramManagementService systemProgramManagementService) { + CapabilityManagementService(CConfiguration cConf, CapabilityApplier capabilityApplier) { super(RetryStrategies .fixDelay(cConf.getLong(Constants.Capability.DIR_SCAN_INTERVAL_MINUTES), TimeUnit.MINUTES)); this.cConf = cConf; diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/events/StartProgramEventSubscriber.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/events/StartProgramEventSubscriber.java index e88502095182..2816dd10429c 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/events/StartProgramEventSubscriber.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/events/StartProgramEventSubscriber.java @@ -21,7 +21,7 @@ import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.internal.app.services.ProgramLifecycleService; -import io.cdap.cdap.internal.app.services.RunRecordMonitorService; +import io.cdap.cdap.internal.app.services.FlowControlService; import io.cdap.cdap.proto.ProgramType; import io.cdap.cdap.proto.id.ProgramReference; import io.cdap.cdap.proto.id.ProgramRunId; @@ -54,7 +54,7 @@ public class StartProgramEventSubscriber extends EventSubscriber { private final CConfiguration cConf; private final EventReaderProvider extensionProvider; private final ProgramLifecycleService lifecycleService; - private final RunRecordMonitorService runRecordMonitorService; + private final FlowControlService flowControlService; private ScheduledExecutorService executor; private Collection> readers; private ExecutorService threadPoolExecutor; @@ -66,17 +66,17 @@ public class StartProgramEventSubscriber extends EventSubscriber { * @param cConf CDAP configuration * @param extensionProvider eventReaderProvider for StartProgramEvent Readers * @param lifecycleService to publish start programs to TMS - * @param runRecordMonitorService basic flow-control + * @param flowControlService basic flow-control */ @Inject StartProgramEventSubscriber(CConfiguration cConf, EventReaderProvider extensionProvider, ProgramLifecycleService lifecycleService, - RunRecordMonitorService runRecordMonitorService) { + FlowControlService flowControlService) { this.cConf = cConf; this.extensionProvider = extensionProvider; this.lifecycleService = lifecycleService; - this.runRecordMonitorService = runRecordMonitorService; + this.flowControlService = flowControlService; maxConcurrentRuns = -1; } @@ -132,14 +132,14 @@ protected void runOneIteration() throws Exception { if (threadPoolExecutor != null) { for (EventReader reader : readers) { threadPoolExecutor.execute(() -> { - if (runRecordMonitorService.isRunning()) { + if (flowControlService.isRunning()) { // Only attempt to process event if there is no max or the current count is less than max if (hasNominalCapacity()) { processEvents(reader); } } else { - LOG.warn("RunRecordMonitorService not yet running, currently in state: {}." - + " Status will be checked again in next attempt.", runRecordMonitorService.state()); + LOG.warn("FlowControlService not yet running, currently in state: {}." + + " Status will be checked again in next attempt.", flowControlService.state()); } }); } @@ -153,7 +153,7 @@ protected void runOneIteration() throws Exception { */ @VisibleForTesting boolean hasNominalCapacity() { - RunRecordMonitorService.Counter counter = runRecordMonitorService.getCount(); + FlowControlService.Counter counter = flowControlService.getCounter(); // no limit if (maxConcurrentRuns <= 0) { return true; diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/OperationNotificationSubscriberService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/OperationNotificationSubscriberService.java index 04f9dd5eecf9..ae20897617f2 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/OperationNotificationSubscriberService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/OperationNotificationSubscriberService.java @@ -76,19 +76,15 @@ protected void startUp() throws Exception { RetryStrategy retryStrategy = RetryStrategies.fromConfiguration(cConf, Constants.Service.RUNTIME_MONITOR_RETRY_PREFIX); - TransactionRunners.run( - transactionRunner, context -> { - Retries.runWithRetries( - () -> { - processStartingOperations(context); - processRunningOperations(context); - processStoppingOperations(context); - }, - retryStrategy, - e -> true - ); - } - ); + Retries.runWithRetries(() -> + TransactionRunners.run(transactionRunner, context -> { + processStartingOperations(context); + processRunningOperations(context); + processStoppingOperations(context); + }), + retryStrategy, + e -> true); + List children = new ArrayList<>(); String topicPrefix = cConf.get(Constants.Operation.STATUS_EVENT_TOPIC); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/DefaultProvisionerContext.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/DefaultProvisionerContext.java index 182ba3f8da19..f77b9d39f4cb 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/DefaultProvisionerContext.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/DefaultProvisionerContext.java @@ -16,6 +16,8 @@ package io.cdap.cdap.internal.provision; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorCategory.ErrorCategoryEnum; import io.cdap.cdap.api.metrics.MetricsCollectionService; import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.common.logging.LoggingContext; @@ -62,6 +64,7 @@ public class DefaultProvisionerContext implements ProvisionerContext { private final String profileName; private final LoggingContext loggingContext; private final Executor executor; + private final ErrorCategory errorCategory; DefaultProvisionerContext(ProgramRunId programRunId, String provisionerName, Map properties, @@ -69,7 +72,7 @@ public class DefaultProvisionerContext implements ProvisionerContext { @Nullable VersionInfo appCDAPVersion, LocationFactory locationFactory, RuntimeMonitorType runtimeMonitorType, MetricsCollectionService metricsCollectionService, @Nullable String profileName, Executor executor, - LoggingContext loggingContext) { + LoggingContext loggingContext, ErrorCategory errorCategory) { this.programRun = new ProgramRun(programRunId.getNamespace(), programRunId.getApplication(), programRunId.getProgram(), programRunId.getRun()); this.programRunInfo = new ProgramRunInfo.Builder() @@ -92,6 +95,7 @@ public class DefaultProvisionerContext implements ProvisionerContext { this.metricsCollectionService = metricsCollectionService; this.provisionerName = provisionerName; this.executor = executor; + this.errorCategory = errorCategory; } @Override @@ -146,6 +150,11 @@ public String getProfileName() { return profileName; } + @Override + public ErrorCategory getErrorCategory() { + return errorCategory == null ? new ErrorCategory(ErrorCategoryEnum.OTHERS) : errorCategory; + } + @Override public ProvisionerMetrics getMetrics(Map context) { Map tags = new HashMap<>(context); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/ProvisionerStore.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/ProvisionerStore.java index 6212b2bdf7dc..ae269763a04d 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/ProvisionerStore.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/ProvisionerStore.java @@ -16,31 +16,42 @@ package io.cdap.cdap.internal.provision; +import com.google.common.collect.Lists; +import io.cdap.cdap.api.dataset.lib.CloseableIterator; +import io.cdap.cdap.internal.provision.adapters.ProvisioningTaskInfoAdapter; import io.cdap.cdap.proto.id.ProgramRunId; +import io.cdap.cdap.spi.data.StructuredRow; +import io.cdap.cdap.spi.data.StructuredTable; import io.cdap.cdap.spi.data.StructuredTableContext; import io.cdap.cdap.spi.data.TableNotFoundException; +import io.cdap.cdap.spi.data.table.field.Field; +import io.cdap.cdap.spi.data.table.field.Fields; +import io.cdap.cdap.spi.data.table.field.Range; import io.cdap.cdap.spi.data.transaction.TransactionRunner; import io.cdap.cdap.spi.data.transaction.TransactionRunners; +import io.cdap.cdap.store.StoreDefinition; import java.io.IOException; +import java.util.ArrayList; import java.util.List; +import java.util.Optional; import javax.annotation.Nullable; import javax.inject.Inject; /** * Stores information used for provisioning. * - * Stores subscriber offset information for TMS, cluster information for program runs, and state - * information for each provision and deprovision operation. + *

Stores subscriber offset information for TMS, cluster information for program runs, and state + * information for each provision and deprovision operation.

* - * Provisioner Store uses transactionRunners to perform underlying CRUD operations. + *

Provisioner Store uses transactionRunners to perform underlying CRUD operations.

*/ -final class ProvisionerStore { +public final class ProvisionerStore { private final TransactionRunner txRunner; - private ProvisionerTable getProvisionerTable(StructuredTableContext context) + private StructuredTable getProvisionerTable(StructuredTableContext context) throws TableNotFoundException { - return new ProvisionerTable(context); + return context.getTable(StoreDefinition.ProvisionerStore.PROVISIONER_TABLE); } @Inject @@ -48,49 +59,126 @@ private ProvisionerTable getProvisionerTable(StructuredTableContext context) this.txRunner = txRunner; } + /** + * @return List of {@link ProvisioningTaskInfo} + * @throws IOException if there is an error reading from underlying structured table. + */ List listTaskInfo() throws IOException { return TransactionRunners.run(txRunner, context -> { - return getProvisionerTable(context).listTaskInfo(); + List result = new ArrayList<>(); + try (CloseableIterator iterator = getProvisionerTable(context).scan( + Range.all(), Integer.MAX_VALUE)) { + while (iterator.hasNext()) { + StructuredRow row = iterator.next(); + result.add(ProvisioningTaskInfoAdapter.fromJson( + row.getString(StoreDefinition.ProvisionerStore.PROVISIONER_TASK_INFO_FIELD), + row.getString(StoreDefinition.ProvisionerStore.NAMESPACE_FIELD), context)); + } + } + return result; }, IOException.class); } + /** + * Fetch Provisioning Task Information. + * + * @param key ProvisioningTaskKey for the corresponding task info. + * @return instance of {@link ProvisioningTaskInfo}. + * @throws IOException if there is an issue reading from underlying structured table. + */ @Nullable - ProvisioningTaskInfo getTaskInfo(final ProvisioningTaskKey key) throws IOException { + public ProvisioningTaskInfo getTaskInfo(final ProvisioningTaskKey key) throws IOException { return TransactionRunners.run(txRunner, context -> { - return getProvisionerTable(context).getTaskInfo(key); + return fetchTaskInfo(context, key); }, IOException.class); } - void putTaskInfo(final ProvisioningTaskInfo taskInfo) throws IOException { + /** + * Persist the provisioning taskInfo. + * + * @param taskInfo {@link ProvisioningTaskInfo}to be persisted. + * @throws IOException if there is an issue writing to the underlying structured table. + */ + public void putTaskInfo(final ProvisioningTaskInfo taskInfo) throws IOException { TransactionRunners.run(txRunner, context -> { - getProvisionerTable(context).putTaskInfo(taskInfo); + persistTaskInfo(context, taskInfo); }, IOException.class); } + /** + * Delete provisioning task info for the corresponding program run id. + * + * @param runId to delete. + * @throws IOException if there is an issue deleting from the underlying structured table. + */ void deleteTaskInfo(ProgramRunId programRunId) throws IOException { TransactionRunners.run(txRunner, context -> { - getProvisionerTable(context).deleteTaskInfo(programRunId); + // Delete the keys with Provision and Deprovision, type is set to null to delete provision and deprovision types + getProvisionerTable(context).deleteAll(Range.singleton(createPrimaryKey(programRunId, null))); }, IOException.class); } @Nullable ProvisioningTaskInfo getExistingAndCancel(final ProvisioningTaskKey taskKey) throws IOException { return TransactionRunners.run(txRunner, context -> { - ProvisionerTable table = getProvisionerTable(context); - ProvisioningTaskInfo currentTaskInfo = table.getTaskInfo(taskKey); + ProvisioningTaskInfo currentTaskInfo = fetchTaskInfo(context, taskKey); if (currentTaskInfo == null) { return null; } // write that the state has been cancelled. This is in case CDAP dies or is killed before the cluster can // be deprovisioned and the task state cleaned up. When CDAP starts back up, it will see that the task is // cancelled and will not resume the task. - ProvisioningOp newOp = - new ProvisioningOp(currentTaskInfo.getProvisioningOp().getType(), - ProvisioningOp.Status.CANCELLED); + ProvisioningOp newOp = new ProvisioningOp(currentTaskInfo.getProvisioningOp().getType(), + ProvisioningOp.Status.CANCELLED); ProvisioningTaskInfo newTaskInfo = new ProvisioningTaskInfo(currentTaskInfo, newOp, currentTaskInfo.getCluster()); - table.putTaskInfo(newTaskInfo); + persistTaskInfo(context, newTaskInfo); return currentTaskInfo; }, IOException.class); } + + private List> createPrimaryKey(ProgramRunId runId, @Nullable ProvisioningOp.Type type) { + List> fields = Lists.newArrayList( + Fields.stringField(StoreDefinition.ProvisionerStore.NAMESPACE_FIELD, runId.getNamespace()), + Fields.stringField(StoreDefinition.ProvisionerStore.APPLICATION_FIELD, + runId.getApplication()), + Fields.stringField(StoreDefinition.ProvisionerStore.VERSION_FIELD, runId.getVersion()), + Fields.stringField(StoreDefinition.ProvisionerStore.PROGRAM_TYPE_FIELD, + runId.getType().name()), + Fields.stringField(StoreDefinition.ProvisionerStore.PROGRAM_FIELD, runId.getProgram()), + Fields.stringField(StoreDefinition.ProvisionerStore.RUN_FIELD, runId.getRun())); + + if (null != type) { + fields.add(Fields.stringField(StoreDefinition.ProvisionerStore.KEY_TYPE, type.name())); + } + return fields; + } + + /** + * Persists {@link ProvisioningTaskInfo} in the provisioner table. + */ + private void persistTaskInfo(StructuredTableContext context, ProvisioningTaskInfo taskInfo) + throws IOException { + String serializedTaskInfo = ProvisioningTaskInfoAdapter.toJson(taskInfo, context); + List> fields = createPrimaryKey(taskInfo.getTaskKey().getProgramRunId(), + taskInfo.getTaskKey().getType()); + fields.add(Fields.stringField(StoreDefinition.ProvisionerStore.PROVISIONER_TASK_INFO_FIELD, + serializedTaskInfo)); + getProvisionerTable(context).upsert(fields); + } + + /** + * Fetches {@link ProvisioningTaskInfo} from the provisioner table. + */ + private ProvisioningTaskInfo fetchTaskInfo(StructuredTableContext context, + ProvisioningTaskKey key) throws IOException { + Optional row = getProvisionerTable(context).read( + createPrimaryKey(key.getProgramRunId(), key.getType())); + String taskInfoJson = row.map(structuredRow -> structuredRow.getString( + StoreDefinition.ProvisionerStore.PROVISIONER_TASK_INFO_FIELD)).orElse(null); + String namespace = row.map( + structuredRow -> structuredRow.getString(StoreDefinition.ProvisionerStore.NAMESPACE_FIELD)) + .orElse(null); + return ProvisioningTaskInfoAdapter.fromJson(taskInfoJson, namespace, context); + } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/ProvisionerTable.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/ProvisionerTable.java deleted file mode 100644 index aa311c6e362c..000000000000 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/ProvisionerTable.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.cdap.internal.provision; - -import com.google.common.collect.Lists; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import io.cdap.cdap.api.dataset.lib.CloseableIterator; -import io.cdap.cdap.app.runtime.Arguments; -import io.cdap.cdap.app.runtime.ProgramOptions; -import io.cdap.cdap.internal.app.ApplicationSpecificationAdapter; -import io.cdap.cdap.internal.app.runtime.codec.ArgumentsCodec; -import io.cdap.cdap.internal.app.runtime.codec.ProgramOptionsCodec; -import io.cdap.cdap.proto.id.ProgramRunId; -import io.cdap.cdap.spi.data.StructuredRow; -import io.cdap.cdap.spi.data.StructuredTable; -import io.cdap.cdap.spi.data.StructuredTableContext; -import io.cdap.cdap.spi.data.TableNotFoundException; -import io.cdap.cdap.spi.data.table.field.Field; -import io.cdap.cdap.spi.data.table.field.Fields; -import io.cdap.cdap.spi.data.table.field.Range; -import io.cdap.cdap.store.StoreDefinition; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import javax.annotation.Nullable; - -/** - * Operations on top of StructuredTable for Provisioning related CRUD operations - */ -public class ProvisionerTable { - - private final StructuredTable table; - private static final Gson GSON = ApplicationSpecificationAdapter.addTypeAdapters( - new GsonBuilder()) - .registerTypeAdapter(ProgramOptions.class, new ProgramOptionsCodec()) - .registerTypeAdapter(Arguments.class, new ArgumentsCodec()) - .create(); - - public ProvisionerTable(StructuredTableContext context) throws TableNotFoundException { - this.table = context.getTable(StoreDefinition.ProvisionerStore.PROVISIONER_TABLE); - } - - /** - * @return List of {@link ProvisioningTaskInfo} - * @throws IOException if there is an error reading from underlying structured table. - */ - public List listTaskInfo() throws IOException { - List result; - try (CloseableIterator iterator = table.scan(Range.all(), Integer.MAX_VALUE)) { - result = new ArrayList<>(); - while (iterator.hasNext()) { - result.add( - deserialize(iterator.next() - .getString(StoreDefinition.ProvisionerStore.PROVISIONER_TASK_INFO_FIELD)) - ); - } - } - return result; - } - - /** - * Fetch Provisioning Task Information - * - * @param key ProvisioningTaskKey for the corresponding task info. - * @return instance of {@link ProvisioningTaskInfo}. - * @throws IOException if there is an issue reading from underlying structured table. - */ - @Nullable - public ProvisioningTaskInfo getTaskInfo(ProvisioningTaskKey key) throws IOException { - Optional row = table.read( - createPrimaryKey(key.getProgramRunId(), key.getType())); - return row.isPresent() - ? deserialize( - row.get().getString(StoreDefinition.ProvisionerStore.PROVISIONER_TASK_INFO_FIELD)) : - null; - } - - /** - * Persist the provisioning taskInfo. - * - * @param taskInfo {@link ProvisioningTaskInfo}to be persisted. - * @throws IOException if there is an issue writing to the underlying structured table. - */ - public void putTaskInfo(ProvisioningTaskInfo taskInfo) throws IOException { - List> fields = createPrimaryKey(taskInfo.getTaskKey().getProgramRunId(), - taskInfo.getTaskKey().getType()); - fields.add(Fields.stringField(StoreDefinition.ProvisionerStore.PROVISIONER_TASK_INFO_FIELD, - serialize(taskInfo))); - table.upsert(fields); - } - - /** - * Delete provisioning task info for the corresponding program run id. - * - * @param runId to delete. - * @throws IOException if there is an issue deleting from the underlying structured table. - */ - public void deleteTaskInfo(ProgramRunId runId) throws IOException { - // Delete the keys with Provision and Deprovision, type is set to null to delete provision and deprovision types - table.deleteAll(Range.singleton(createPrimaryKey(runId, null))); - } - - private List> createPrimaryKey(ProgramRunId runId, @Nullable ProvisioningOp.Type type) { - List> fields = Lists.newArrayList( - Fields.stringField(StoreDefinition.ProvisionerStore.NAMESPACE_FIELD, runId.getNamespace()), - Fields.stringField(StoreDefinition.ProvisionerStore.APPLICATION_FIELD, - runId.getApplication()), - Fields.stringField(StoreDefinition.ProvisionerStore.VERSION_FIELD, runId.getVersion()), - Fields.stringField(StoreDefinition.ProvisionerStore.PROGRAM_TYPE_FIELD, - runId.getType().name()), - Fields.stringField(StoreDefinition.ProvisionerStore.PROGRAM_FIELD, runId.getProgram()), - Fields.stringField(StoreDefinition.ProvisionerStore.RUN_FIELD, runId.getRun())); - - if (null != type) { - fields.add(Fields.stringField(StoreDefinition.ProvisionerStore.KEY_TYPE, type.name())); - } - return fields; - } - - private ProvisioningTaskInfo deserialize(String provisioningTaskInfo) { - return GSON.fromJson(provisioningTaskInfo, ProvisioningTaskInfo.class); - } - - private String serialize(ProvisioningTaskInfo taskInfo) { - return GSON.toJson(taskInfo, ProvisioningTaskInfo.class); - } -} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/ProvisioningService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/ProvisioningService.java index 8fbed80b5bbb..a23e15788f96 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/ProvisioningService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/ProvisioningService.java @@ -23,6 +23,11 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import com.google.inject.Inject; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorCategory.ErrorCategoryEnum; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; +import io.cdap.cdap.api.exception.ProgramFailureException; import io.cdap.cdap.api.macro.InvalidMacroException; import io.cdap.cdap.api.macro.MacroEvaluator; import io.cdap.cdap.api.macro.MacroParserOptions; @@ -112,7 +117,8 @@ /** * Service for provisioning related operations. * - * TODO (CDAP-21111): Split ProvisioningService for Appfabric Server and Appfabric Processor. + *

TODO (CDAP-21111): Split ProvisioningService for Appfabric Server and Appfabric + * Processor.

*/ public class ProvisioningService extends AbstractIdleService { @@ -197,10 +203,10 @@ protected void shutDown() throws Exception { /** * Returns the {@link ClusterStatus} for the cluster being used to execute the given program run. * - * @param programRunId the program run id for checking the cluster status + * @param programRunId the program run id for checking the cluster status * @param programOptions the program options for the given run - * @param cluster the {@link Cluster} information for the given run - * @param userId the user id to use for {@link SecureStore} operation. + * @param cluster the {@link Cluster} information for the given run + * @param userId the user id to use for {@link SecureStore} operation. * @return the {@link ClusterStatus} * @throws Exception if non-retryable exception is encountered when querying cluster status */ @@ -226,7 +232,7 @@ public ClusterStatus getClusterStatus(ProgramRunId programRunId, ProgramOptions Networks.getAddress(cConf, Constants.NETWORK_PROXY_ADDRESS), null, null); } context = createContext(cConf, programOptions, programRunId, userId, properties, - defaultSSHContext); + defaultSSHContext, new ErrorCategory(ErrorCategoryEnum.OTHERS)); } catch (InvalidMacroException e) { // This shouldn't happen runWithProgramLogging(programRunId, systemArgs, @@ -351,7 +357,7 @@ Optional cancelDeprovisionTask(ProgramRunId programRunId) * using their own executor. * * @param provisionRequest the provision request - * @param context context for the transaction + * @param context context for the transaction * @return runnable that will actually execute the cluster provisioning */ public Runnable provision(ProvisionRequest provisionRequest, StructuredTableContext context) @@ -363,15 +369,18 @@ public Runnable provision(ProvisionRequest provisionRequest, StructuredTableCont Map args = programOptions.getArguments().asMap(); String name = SystemArguments.getProfileProvisioner(args); Provisioner provisioner = provisionerInfo.get().provisioners.get(name); + ErrorCategory errorCategory = new ErrorCategory(ErrorCategoryEnum.PROVISIONING); // any errors seen here will transition the state straight to deprovisioned since no cluster create was attempted if (provisioner == null) { + String errorMessage = String.format("Could not provision cluster for the run because " + + "provisioner %s does not exist.", name); + String errorReason = String.format("Provisioner %s does not exist.", name); + ProgramFailureException ex = ErrorUtils.getProgramFailureException(errorCategory, errorReason, + errorMessage, ErrorType.SYSTEM, false, null); runWithProgramLogging( programRunId, args, - () -> LOG.error( - "Could not provision cluster for the run because provisioner {} does not exist.", - name)); - programStateWriter.error(programRunId, - new IllegalStateException("Provisioner does not exist.")); + () -> LOG.error(errorMessage, ex)); + programStateWriter.error(programRunId, ex); provisionerNotifier.deprovisioned(programRunId); return () -> { }; @@ -385,15 +394,17 @@ public Runnable provision(ProvisionRequest provisionRequest, StructuredTableCont Set unfulfilledRequirements = getUnfulfilledRequirements(provisioner.getCapabilities(), requirements); if (!unfulfilledRequirements.isEmpty()) { + String errorMessage = String.format("'%s' cannot be run using profile '%s' because " + + "the profile does not met all plugin requirements. Following requirements " + + "were not meet by the listed plugins: '%s'", programRunId.getProgram(), name, + groupByRequirement(unfulfilledRequirements)); + String errorReason = String.format("Provisioner %s does not meet all the requirements for " + + "the program %s to run.", name, programRunId.getProgram()); + ProgramFailureException ex = ErrorUtils.getProgramFailureException(errorCategory, errorReason, + errorMessage, ErrorType.SYSTEM, false, null); runWithProgramLogging(programRunId, args, () -> - LOG.error(String.format( - "'%s' cannot be run using profile '%s' because the profile does not met all " - + "plugin requirements. Following requirements were not meet by the listed " - + "plugins: '%s'", programRunId.getProgram(), name, - groupByRequirement(unfulfilledRequirements)))); - programStateWriter.error(programRunId, - new IllegalArgumentException("Provisioner does not meet all the " - + "requirements for the program to run.")); + LOG.error(errorMessage, ex)); + programStateWriter.error(programRunId, ex); provisionerNotifier.deprovisioned(programRunId); return () -> { }; @@ -408,8 +419,7 @@ public Runnable provision(ProvisionRequest provisionRequest, StructuredTableCont programOptions, properties, name, provisionRequest.getUser(), provisioningOp, createKeysDirectory(programRunId).toURI(), null); - ProvisionerTable provisionerTable = new ProvisionerTable(context); - provisionerTable.putTaskInfo(provisioningTaskInfo); + provisionerStore.putTaskInfo(provisioningTaskInfo); return createProvisionTask(provisioningTaskInfo, provisioner); } @@ -428,7 +438,7 @@ public Optional getRuntimeJobManager(ProgramRunId programRunI String user = programOptions.getArguments().getOption(ProgramOptionConstants.USER_ID); Map properties = SystemArguments.getProfileProperties(systemArgs); ProvisionerContext context = createContext(cConf, programOptions, programRunId, user, - properties, null); + properties, null, new ErrorCategory(ErrorCategoryEnum.OTHERS)); return provisioner.getRuntimeJobManager(context) .map(manager -> new RuntimeJobManagerCallWrapper(provisioner.getClass().getClassLoader(), manager)); @@ -496,8 +506,7 @@ Runnable deprovision(ProgramRunId programRunId, StructuredTableContext context, ProvisioningOp.Status.REQUESTING_DELETE); ProvisioningTaskInfo provisioningTaskInfo = new ProvisioningTaskInfo(existing, provisioningOp, existing.getCluster()); - ProvisionerTable provisionerTable = new ProvisionerTable(context); - provisionerTable.putTaskInfo(provisioningTaskInfo); + provisionerStore.putTaskInfo(provisioningTaskInfo); return createDeprovisionTask(provisioningTaskInfo, provisioner, taskCleanup); } @@ -665,6 +674,7 @@ private Runnable createProvisionTask(ProvisioningTaskInfo taskInfo, Provisioner ProgramRunId programRunId = taskInfo.getProgramRunId(); ProgramOptions programOptions = taskInfo.getProgramOptions(); Map systemArgs = programOptions.getArguments().asMap(); + ErrorCategory errorCategory = new ErrorCategory(ErrorCategoryEnum.PROVISIONING); ProvisionerContext context; try { SSHContext sshContext = new DefaultSSHContext( @@ -672,21 +682,29 @@ private Runnable createProvisionTask(ProvisioningTaskInfo taskInfo, Provisioner locationFactory.create(taskInfo.getSecureKeysDir()), createSSHKeyPair(taskInfo)); context = createContext(cConf, programOptions, programRunId, taskInfo.getUser(), - taskInfo.getProvisionerProperties(), sshContext); + taskInfo.getProvisionerProperties(), sshContext, errorCategory); } catch (IOException e) { + String errorReason = "Failed to load ssh key."; + String errorMessage = + String.format("Failed to load ssh key with message: %s", e.getMessage()); + Exception ex = ErrorUtils.getProgramFailureException(errorCategory, errorReason, + errorMessage, ErrorType.SYSTEM, false, e); runWithProgramLogging(taskInfo.getProgramRunId(), systemArgs, - () -> LOG.error("Failed to load ssh key. The run will be marked as failed.", e)); - programStateWriter.error(programRunId, - new IllegalStateException("Failed to load ssh key.", e)); + () -> LOG.error("The run will be marked as failed.", ex)); + programStateWriter.error(programRunId, ex); provisionerNotifier.deprovisioning(taskInfo.getProgramRunId()); return () -> { }; } catch (InvalidMacroException e) { - runWithProgramLogging(taskInfo.getProgramRunId(), systemArgs, - () -> LOG.error("Could not evaluate macros while provisoning. " - + "The run will be marked as failed.", e)); - programStateWriter.error(programRunId, - new IllegalStateException("Could not evaluate macros while provisioning", e)); + String errorReason = "Could not evaluate macros while provisioning."; + String errorMessage = String.format("Could not evaluate macros with message: %s", + e.getMessage()); + ProgramFailureException ex = ErrorUtils.getProgramFailureException(errorCategory, errorReason, + errorMessage, ErrorType.USER, false, null); + ex.addSuppressed(e); + runWithProgramLogging(programRunId, systemArgs, + () -> LOG.error("The run will be marked as failed.", ex)); + programStateWriter.error(programRunId, ex); provisionerNotifier.deprovisioning(taskInfo.getProgramRunId()); return () -> { }; @@ -694,7 +712,7 @@ private Runnable createProvisionTask(ProvisioningTaskInfo taskInfo, Provisioner // TODO: (CDAP-13246) pick up timeout from profile instead of hardcoding ProvisioningTask task = new ProvisionTask(taskInfo, transactionRunner, provisioner, context, - provisionerNotifier, programStateWriter, 300); + provisionerNotifier, programStateWriter, provisionerStore, 300); ProvisioningTaskKey taskKey = new ProvisioningTaskKey(programRunId, ProvisioningOp.Type.PROVISION); @@ -715,13 +733,20 @@ private Runnable createProvisionTask(ProvisioningTaskInfo taskInfo, Provisioner private Runnable createDeprovisionTask(ProvisioningTaskInfo taskInfo, Provisioner provisioner, Consumer taskCleanup) { Map properties = taskInfo.getProvisionerProperties(); + ErrorCategory errorCategory = new ErrorCategory(ErrorCategoryEnum.DEPROVISIONING); ProvisionerContext context; SSHKeyPair sshKeyPair = null; try { sshKeyPair = createSSHKeyPair(taskInfo); } catch (IOException e) { - LOG.warn("Failed to load ssh key. No SSH key will be available for the deprovision task", e); + String errorReason = "Failed to load ssh key."; + String errorMessage = + String.format("Failed to load ssh key with message: %s", e.getMessage()); + Exception ex = ErrorUtils.getProgramFailureException(errorCategory, errorReason, + errorMessage, ErrorType.SYSTEM, false, e); + LOG.warn("Failed to load ssh key. No SSH key will be available for the deprovision task", + ex); } ProgramRunId programRunId = taskInfo.getProgramRunId(); @@ -732,18 +757,22 @@ private Runnable createDeprovisionTask(ProvisioningTaskInfo taskInfo, Provisione Networks.getAddress(cConf, Constants.NETWORK_PROXY_ADDRESS), null, sshKeyPair); context = createContext(cConf, taskInfo.getProgramOptions(), programRunId, taskInfo.getUser(), - properties, - sshContext); + properties, sshContext, errorCategory); } catch (InvalidMacroException e) { + String errorReason = "Could not evaluate macros while deprovisioning."; + String errorMessage = String.format("Could not evaluate macros with message: %s", + e.getMessage()); + ProgramFailureException ex = ErrorUtils.getProgramFailureException(errorCategory, errorReason, + errorMessage, ErrorType.USER, false, null); + ex.addSuppressed(e); runWithProgramLogging(programRunId, systemArgs, - () -> LOG.error("Could not evaluate macros while deprovisoning. " - + "The cluster will be marked as orphaned.", e)); + () -> LOG.error("The cluster will be marked as orphaned.", ex)); provisionerNotifier.orphaned(programRunId); return () -> { }; } DeprovisionTask task = new DeprovisionTask(taskInfo, transactionRunner, 300, - provisioner, context, provisionerNotifier, locationFactory); + provisioner, provisionerStore, context, provisionerNotifier, locationFactory); ProvisioningTaskKey taskKey = new ProvisioningTaskKey(programRunId, ProvisioningOp.Type.DEPROVISION); @@ -850,9 +879,8 @@ private Location createKeysDirectory(ProgramRunId programRunId) { } private ProvisionerContext createContext(CConfiguration cConf, ProgramOptions programOptions, - ProgramRunId programRunId, String userId, - Map properties, - @Nullable SSHContext sshContext) { + ProgramRunId programRunId, String userId, Map properties, + @Nullable SSHContext sshContext, ErrorCategory errorCategory) { RuntimeMonitorType runtimeMonitorType = SystemArguments.getRuntimeMonitorType(cConf, programOptions); Map systemArgs = programOptions.getArguments().asMap(); @@ -866,9 +894,8 @@ private ProvisionerContext createContext(CConfiguration cConf, ProgramOptions pr LoggingContext loggingContext = LoggingContextHelper.getLoggingContextWithRunId(programRunId, systemArgs); return new DefaultProvisionerContext(programRunId, provisionerName, evaluated, sparkCompat, - sshContext, - appCDAPVersion, locationFactory, runtimeMonitorType, - metricsCollectionService, profileName, contextExecutor, loggingContext); + sshContext, appCDAPVersion, locationFactory, runtimeMonitorType, metricsCollectionService, + profileName, contextExecutor, loggingContext, errorCategory); } /** diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/adapters/ProgramDescriptorDeserializer.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/adapters/ProgramDescriptorDeserializer.java new file mode 100644 index 000000000000..3681aa4adde4 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/adapters/ProgramDescriptorDeserializer.java @@ -0,0 +1,75 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.provision.adapters; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import io.cdap.cdap.api.app.ApplicationSpecification; +import io.cdap.cdap.api.artifact.ArtifactId; +import io.cdap.cdap.app.program.ProgramDescriptor; +import io.cdap.cdap.internal.app.store.adapters.AppSpecDeserializationContext; +import io.cdap.cdap.internal.app.store.adapters.AppSpecDeserializationContextHolder; +import io.cdap.cdap.proto.id.ProgramId; +import java.lang.reflect.Type; + +/** + * Custom Gson deserializer for {@link ProgramDescriptor}. This deserializer is responsible for + * reconstructing a {@link ProgramDescriptor} from its JSON representation, particularly by setting + * the root artifact in the {@link AppSpecDeserializationContext} to enable correct plugin + * resolution during the deserialization of the nested {@link ApplicationSpecification}. + */ +public class ProgramDescriptorDeserializer implements JsonDeserializer { + + @Override + public ProgramDescriptor deserialize(JsonElement json, Type typeOfT, + JsonDeserializationContext context) throws JsonParseException { + AppSpecDeserializationContext appSpecDeserializationContext = AppSpecDeserializationContextHolder.getContext(); + JsonObject jsonObject = json.getAsJsonObject(); + + JsonElement programIdElement = jsonObject.get("programId"); + if (programIdElement == null || programIdElement.isJsonNull()) { + throw new JsonParseException("ApplicationMeta 'id' field is missing or null"); + } + ProgramId programId = context.deserialize(programIdElement, ProgramId.class); + appSpecDeserializationContext.setNamespace(programId.getNamespaceId().getNamespace()); + + JsonObject specJson = jsonObject.getAsJsonObject("appSpec"); + if (specJson == null || specJson.isJsonNull()) { + throw new JsonParseException( + "ApplicationMeta 'spec' field is missing or null for id: " + programId.getApplication()); + } + // Set parent context before fully deserializing the AppSpec, + // so plugins can be retrieved from DB using this parent information. + JsonElement artifactIdJson = specJson.get("artifactId"); + if (artifactIdJson != null && !artifactIdJson.isJsonNull()) { + ArtifactId artifactId = context.deserialize(artifactIdJson, ArtifactId.class); + if (artifactId != null) { + appSpecDeserializationContext.setRootArtifact(artifactId); + } else { + throw new JsonParseException("ArtifactId in ApplicationSpecification was null for app: " + + programId.getApplication()); + } + } + JsonElement appNameJson = specJson.get("name"); + appSpecDeserializationContext.setAppName(appNameJson != null ? appNameJson.getAsString() : null); + ApplicationSpecification spec = context.deserialize(specJson, ApplicationSpecification.class); + return new ProgramDescriptor(programId, spec); + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/adapters/ProvisioningTaskInfoAdapter.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/adapters/ProvisioningTaskInfoAdapter.java new file mode 100644 index 000000000000..da4a7f4bf002 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/adapters/ProvisioningTaskInfoAdapter.java @@ -0,0 +1,140 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.provision.adapters; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.app.ApplicationSpecification; +import io.cdap.cdap.api.plugin.Plugin; +import io.cdap.cdap.api.plugin.PluginClass; +import io.cdap.cdap.app.program.ProgramDescriptor; +import io.cdap.cdap.app.runtime.Arguments; +import io.cdap.cdap.app.runtime.ProgramOptions; +import io.cdap.cdap.common.conf.Constants.AppMetaStore; +import io.cdap.cdap.internal.app.ApplicationSpecificationAdapter; +import io.cdap.cdap.internal.app.ApplicationSpecificationCodec; +import io.cdap.cdap.internal.app.runtime.codec.ArgumentsCodec; +import io.cdap.cdap.internal.app.runtime.codec.ProgramOptionsCodec; +import io.cdap.cdap.internal.app.store.adapters.AppSpecDeserializationContext; +import io.cdap.cdap.internal.app.store.adapters.AppSpecDeserializationContextHolder; +import io.cdap.cdap.internal.app.store.adapters.PluginClassSerializer; +import io.cdap.cdap.internal.app.store.adapters.PluginDeserializer; +import io.cdap.cdap.internal.provision.ProvisioningTaskInfo; +import io.cdap.cdap.spi.data.StructuredTable; +import io.cdap.cdap.spi.data.StructuredTableContext; +import io.cdap.cdap.spi.data.TableNotFoundException; +import io.cdap.cdap.store.StoreDefinition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages Gson instances specifically configured for {@link ProvisioningTaskInfo} serialization and + * deserialization, handling application specification reduction. This ensures thread-safe + * operations by managing the {@link AppSpecDeserializationContext} lifecycle per operation via + * {@link AppSpecDeserializationContextHolder}. It uses a static cache for Gson instances based on + * the {@code appSpecReductionEnabled} flag. + */ +public final class ProvisioningTaskInfoAdapter { + + private static final Logger LOG = LoggerFactory.getLogger(ProvisioningTaskInfo.class); + private static final Gson GSON_INSTANCE_REDUCTION_ENABLED = buildGsonInternal(true); + private static final Gson GSON_INSTANCE_REDUCTION_DISABLED = buildGsonInternal(false); + + private ProvisioningTaskInfoAdapter() { + } + + private static StructuredTable getPluginDataTable(StructuredTableContext context) { + try { + return context.getTable(StoreDefinition.ArtifactStore.PLUGIN_DATA_TABLE); + } catch (TableNotFoundException e) { + throw new RuntimeException(e); + } + } + + private static StructuredTable getUniversalPluginDataTable(StructuredTableContext context) { + try { + return context.getTable(StoreDefinition.ArtifactStore.UNIV_PLUGIN_DATA_TABLE); + } catch (TableNotFoundException e) { + throw new RuntimeException(e); + } + } + + private static boolean isAppSpecReductionEnabled(StructuredTableContext context) { + return AppMetaStore.APPSPEC_REDUCTION_SUPPORTED_STORAGE_PROVIDERS.contains( + context.getStorageProvider()); + } + + /** + * Deserializes a JSON string to an object of the specified class, managing the + * {@link AppSpecDeserializationContext} lifecycle via + * {@link AppSpecDeserializationContextHolder}. + */ + public static ProvisioningTaskInfo fromJson(String jsonString, String namespace, + StructuredTableContext context) { + boolean reductionEnabled = isAppSpecReductionEnabled(context); + AppSpecDeserializationContext opContext = + reductionEnabled ? new AppSpecDeserializationContext(namespace, getPluginDataTable(context), + getUniversalPluginDataTable(context)) : null; + if (opContext != null) { + AppSpecDeserializationContextHolder.setContext(opContext); + } + try { + Gson gson = getGson(reductionEnabled); + ProvisioningTaskInfo taskInfo = gson.fromJson(jsonString, ProvisioningTaskInfo.class); + if (opContext != null && !opContext.getMissingPlugins().isEmpty()) { + LOG.trace("Missing plugins for application {}: {}", opContext.getAppName(), + opContext.getMissingPlugins()); + } + return taskInfo; + } finally { + AppSpecDeserializationContextHolder.clearContext(); + } + } + + /** + * Serializes an object to its JSON representation. + */ + public static String toJson(Object objectToSerialize, StructuredTableContext context) { + Gson gson = getGson(isAppSpecReductionEnabled(context)); + // No need to set / get the AppSpecDeserialization context in case of serialization. + return gson.toJson(objectToSerialize, ProvisioningTaskInfo.class); + } + + private static Gson buildGsonInternal(boolean appSpecReductionEnabled) { + GsonBuilder gsonBuilder = new GsonBuilder(); + ApplicationSpecificationAdapter.addTypeAdapters(gsonBuilder); + gsonBuilder.registerTypeAdapter(ProgramOptions.class, new ProgramOptionsCodec()); + gsonBuilder.registerTypeAdapter(Arguments.class, new ArgumentsCodec()); + + if (appSpecReductionEnabled) { + gsonBuilder.registerTypeAdapter(ProgramDescriptor.class, new ProgramDescriptorDeserializer()); + gsonBuilder.registerTypeAdapter(PluginClass.class, new PluginClassSerializer()); + gsonBuilder.registerTypeAdapter(Plugin.class, new PluginDeserializer()); + gsonBuilder.registerTypeHierarchyAdapter(ApplicationSpecification.class, + new ApplicationSpecificationCodec()); + } + + return gsonBuilder.create(); + } + + private static Gson getGson(boolean appSpecReductionEnabled) { + if (appSpecReductionEnabled) { + return GSON_INSTANCE_REDUCTION_ENABLED; + } + return GSON_INSTANCE_REDUCTION_DISABLED; + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/task/DeprovisionTask.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/task/DeprovisionTask.java index 50520064be85..c63ec1a02c55 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/task/DeprovisionTask.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/task/DeprovisionTask.java @@ -19,6 +19,7 @@ import io.cdap.cdap.common.io.Locations; import io.cdap.cdap.internal.provision.ProvisionerNotifier; +import io.cdap.cdap.internal.provision.ProvisionerStore; import io.cdap.cdap.internal.provision.ProvisioningOp; import io.cdap.cdap.internal.provision.ProvisioningTaskInfo; import io.cdap.cdap.runtime.spi.provisioner.ClusterStatus; @@ -59,10 +60,11 @@ public class DeprovisionTask extends ProvisioningTask { private final Location keysDir; public DeprovisionTask(ProvisioningTaskInfo initialTaskInfo, TransactionRunner transactionRunner, - int retryTimeLimitSecs, Provisioner provisioner, + int retryTimeLimitSecs, Provisioner provisioner, ProvisionerStore provisionerStore, ProvisionerContext provisionerContext, ProvisionerNotifier provisionerNotifier, LocationFactory locationFactory) { - super(provisioner, provisionerContext, initialTaskInfo, transactionRunner, retryTimeLimitSecs); + super(provisioner, provisionerContext, initialTaskInfo, transactionRunner, provisionerStore, + retryTimeLimitSecs); this.provisionerNotifier = provisionerNotifier; this.keysDir = locationFactory.create(initialTaskInfo.getSecureKeysDir()); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/task/ProvisionTask.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/task/ProvisionTask.java index 9b545edb736a..e0172b877236 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/task/ProvisionTask.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/task/ProvisionTask.java @@ -23,6 +23,7 @@ import io.cdap.cdap.common.service.Retries; import io.cdap.cdap.common.service.RetryStrategies; import io.cdap.cdap.internal.provision.ProvisionerNotifier; +import io.cdap.cdap.internal.provision.ProvisionerStore; import io.cdap.cdap.internal.provision.ProvisioningOp; import io.cdap.cdap.internal.provision.ProvisioningTaskInfo; import io.cdap.cdap.runtime.spi.provisioner.ClusterStatus; @@ -77,8 +78,9 @@ public class ProvisionTask extends ProvisioningTask { public ProvisionTask(ProvisioningTaskInfo initialTaskInfo, TransactionRunner transactionRunner, Provisioner provisioner, ProvisionerContext provisionerContext, ProvisionerNotifier provisionerNotifier, ProgramStateWriter programStateWriter, - int retryTimeLimitSecs) { - super(provisioner, provisionerContext, initialTaskInfo, transactionRunner, retryTimeLimitSecs); + ProvisionerStore provisionerStore, int retryTimeLimitSecs) { + super(provisioner, provisionerContext, initialTaskInfo, transactionRunner, provisionerStore, + retryTimeLimitSecs); this.provisionerNotifier = provisionerNotifier; this.programStateWriter = programStateWriter; } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/task/ProvisioningTask.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/task/ProvisioningTask.java index 4cd655642c12..2ada9d9b53a5 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/task/ProvisioningTask.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/provision/task/ProvisioningTask.java @@ -17,6 +17,10 @@ package io.cdap.cdap.internal.provision.task; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; +import io.cdap.cdap.api.exception.FailureDetailsProvider; +import io.cdap.cdap.api.exception.ProgramFailureException; import io.cdap.cdap.common.async.RepeatedTask; import io.cdap.cdap.common.lang.Exceptions; import io.cdap.cdap.common.logging.LogSamplers; @@ -24,7 +28,7 @@ import io.cdap.cdap.common.service.Retries; import io.cdap.cdap.common.service.RetryStrategies; import io.cdap.cdap.common.service.RetryStrategy; -import io.cdap.cdap.internal.provision.ProvisionerTable; +import io.cdap.cdap.internal.provision.ProvisionerStore; import io.cdap.cdap.internal.provision.ProvisioningOp; import io.cdap.cdap.internal.provision.ProvisioningTaskInfo; import io.cdap.cdap.internal.provision.ProvisioningTaskKey; @@ -62,6 +66,7 @@ public abstract class ProvisioningTask implements RepeatedTask { private final TransactionRunner transactionRunner; private final ProvisioningTaskKey taskKey; private final ProvisioningTaskInfo initialTaskInfo; + private final ProvisionerStore provisionerStore; private ProvisioningTaskInfo taskInfo; private RetryStrategy retryStrategy; @@ -72,10 +77,11 @@ public abstract class ProvisioningTask implements RepeatedTask { protected ProvisioningTask(Provisioner provisioner, ProvisionerContext provisionerContext, ProvisioningTaskInfo initialTaskInfo, TransactionRunner transactionRunner, - int retryTimeLimitSecs) { + ProvisionerStore provisionerStore, int retryTimeLimitSecs) { this.provisioner = provisioner; this.provisionerContext = provisionerContext; this.initialTaskInfo = initialTaskInfo; + this.provisionerStore = provisionerStore; this.taskKey = new ProvisioningTaskKey(initialTaskInfo.getProgramRunId(), initialTaskInfo.getProvisioningOp().getType()); this.programRunId = initialTaskInfo.getProgramRunId(); @@ -105,14 +111,16 @@ public final long executeOnce() throws Exception { } // Get the sub-task to execute + ProvisioningOp.Type type = currentTaskInfo.getProvisioningOp().getType(); ProvisioningSubtask subtask = subTasks.get(state); if (subtask == null) { // should never happen - throw new IllegalStateException( - String.format("Invalid state '%s' in provisioning task for program run '%s'. " - + "This means there is a bug in provisioning state machine. " - + "Please reach out to the development team.", - state, programRunId)); + String errorReason = String.format("Invalid state '%s' in provisioning task for " + + "program run '%s'.", state, programRunId); + String errorMessage = String.format("%s This means there is a bug in provisioning state" + + "machine. Please reach out to the development team.", errorReason); + throw ErrorUtils.getProgramFailureException(provisionerContext.getErrorCategory(), + errorReason, errorMessage, ErrorType.SYSTEM, false, null); } if (subtask == EndSubtask.INSTANCE) { LOG.debug("Completed {} task for program run {}.", @@ -158,10 +166,16 @@ public final long executeOnce() throws Exception { } catch (InterruptedException e) { throw e; } catch (Throwable e) { - LOG.error("{} task failed in {} state for program run {} due to {}.", - currentTaskInfo.getProvisioningOp().getType(), state, programRunId, - Exceptions.condenseThrowableMessage(e), e); - handleSubtaskFailure(currentTaskInfo, e); + String errorReason = String.format("'%s' task failed in '%s' state for program run '%s'", + currentTaskInfo.getProvisioningOp().getType(), state, programRunId); + ProgramFailureException ex = null; + if (!(e instanceof FailureDetailsProvider)) { + ex = ErrorUtils.getProgramFailureException(provisionerContext.getErrorCategory(), + errorReason, Exceptions.condenseThrowableMessage(e), ErrorType.UNKNOWN, false, e); + } + LOG.error("{} due to {}.", errorReason, + Exceptions.condenseThrowableMessage(ex == null ? e : ex), ex == null ? e : ex); + handleSubtaskFailure(currentTaskInfo, ex); ProvisioningOp failureOp = new ProvisioningOp(currentTaskInfo.getProvisioningOp().getType(), ProvisioningOp.Status.FAILED); ProvisioningTaskInfo failureInfo = new ProvisioningTaskInfo(currentTaskInfo, failureOp, @@ -174,7 +188,7 @@ public final long executeOnce() throws Exception { } /** - * Write the task state to the {@link ProvisionerTable}, retrying if any exception is caught. + * Write the task state to the {@link ProvisionerStore}, retrying if any exception is caught. * Before persisting the state, the current state will be checked. If the current state is * cancelled, it will not be overwritten. * @@ -192,14 +206,13 @@ private ProvisioningTaskInfo persistTaskInfo(ProvisioningTaskInfo taskInfo, // Stop retrying if we are interrupted. Otherwise, retry on every exception, up to the retry limit return Retries.callWithInterruptibleRetries( () -> TransactionRunners.run(transactionRunner, context -> { - ProvisionerTable provisionerTable = new ProvisionerTable(context); - ProvisioningTaskInfo currentState = provisionerTable.getTaskInfo(taskKey); + ProvisioningTaskInfo currentState = provisionerStore.getTaskInfo(taskKey); // if the state is cancelled, don't write anything and transition to the end subtask. if (currentState != null && currentState.getProvisioningOp().getStatus() == ProvisioningOp.Status.CANCELLED) { return currentState; } - provisionerTable.putTaskInfo(taskInfo); + provisionerStore.putTaskInfo(taskInfo); return taskInfo; }), retryStrategy, t -> true); } catch (RuntimeException e) { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/CoreSchedulerService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/CoreSchedulerService.java index d60b815c03ea..51507d596861 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/CoreSchedulerService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/CoreSchedulerService.java @@ -25,6 +25,7 @@ import com.google.inject.Inject; import io.cdap.cdap.api.dataset.lib.CloseableIterator; import io.cdap.cdap.api.security.AccessException; +import io.cdap.cdap.api.service.ServiceUnavailableException; import io.cdap.cdap.app.program.ProgramDescriptor; import io.cdap.cdap.app.store.Store; import io.cdap.cdap.common.AlreadyExistsException; @@ -32,7 +33,6 @@ import io.cdap.cdap.common.ConflictException; import io.cdap.cdap.common.NotFoundException; import io.cdap.cdap.common.ProfileConflictException; -import io.cdap.cdap.api.service.ServiceUnavailableException; import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.service.RetryOnStartFailureService; import io.cdap.cdap.internal.app.runtime.ProgramOptionConstants; @@ -48,8 +48,8 @@ import io.cdap.cdap.internal.app.runtime.schedule.store.Schedulers; import io.cdap.cdap.internal.app.store.profile.ProfileStore; import io.cdap.cdap.internal.profile.AdminEventPublisher; -import io.cdap.cdap.messaging.spi.MessagingService; import io.cdap.cdap.messaging.context.MultiThreadMessagingContext; +import io.cdap.cdap.messaging.spi.MessagingService; import io.cdap.cdap.proto.ProgramType; import io.cdap.cdap.proto.id.ApplicationId; import io.cdap.cdap.proto.id.NamespaceId; @@ -98,6 +98,7 @@ public class CoreSchedulerService extends AbstractIdleService implements Schedul @Inject CoreSchedulerService(TimeSchedulerService timeSchedulerService, + ScheduleNotificationSubscriberService scheduleNotificationSubscriberService, ConstraintCheckerService constraintCheckerService, MessagingService messagingService, CConfiguration cConf, Store store, Impersonator impersonator, @@ -123,12 +124,14 @@ protected void startUp() { timeSchedulerService.startAndWait(); cleanupJobs(); constraintCheckerService.startAndWait(); + scheduleNotificationSubscriberService.startAndWait(); startedLatch.countDown(); LOG.info("Started core scheduler service."); } @Override protected void shutDown() { + scheduleNotificationSubscriberService.stopAndWait(); constraintCheckerService.stopAndWait(); timeSchedulerService.stopAndWait(); LOG.info("Stopped core scheduler service."); @@ -567,8 +570,7 @@ public void reEnableSchedules(NamespaceId namespaceId, long startTimeMillis, lon } /** - * Gets a copy of the given {@link ProgramSchedule} and add user and artifact ID in the schedule - * properties + * Gets a copy of the given {@link ProgramSchedule} and add user and artifact ID in the schedule properties. * TODO CDAP-13662 - move logic to find artifactId and userId to dashboard service and remove this method */ private ProgramSchedule getProgramScheduleWithUserAndArtifactId(ProgramSchedule schedule) { diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/app/runtime/AbstractProgramRuntimeServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/app/runtime/AbstractProgramRuntimeServiceTest.java index 1c6cd896fe6b..4b1922a33ca6 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/app/runtime/AbstractProgramRuntimeServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/app/runtime/AbstractProgramRuntimeServiceTest.java @@ -287,7 +287,8 @@ public void testTetheredRun() throws IOException, ExecutionException, Interrupte InMemoryProgramRunDispatcher launchDispatcher = new TestProgramRunDispatcher(cConf, runnerFactory, program, locationFactory, remoteClientFactory, true); - ProgramRuntimeService runtimeService = new TestProgramRuntimeService(cConf, runnerFactory, null, launchDispatcher); + ProgramRuntimeService runtimeService = new TestProgramRuntimeService(cConf, runnerFactory, null, + launchDispatcher); runtimeService.startAndWait(); try { ProgramDescriptor descriptor = new ProgramDescriptor(program.getId(), null, diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/AppFabricClient.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/AppFabricClient.java index 2c4e0fde2c8a..f9e02e62d815 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/AppFabricClient.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/AppFabricClient.java @@ -33,6 +33,8 @@ import io.cdap.cdap.gateway.handlers.AppLifecycleHttpHandler; import io.cdap.cdap.gateway.handlers.NamespaceHttpHandler; import io.cdap.cdap.gateway.handlers.ProgramLifecycleHttpHandler; +import io.cdap.cdap.gateway.handlers.ProgramRuntimeHttpHandler; +import io.cdap.cdap.gateway.handlers.ProgramScheduleHttpHandler; import io.cdap.cdap.gateway.handlers.WorkflowHttpHandler; import io.cdap.cdap.gateway.handlers.util.AbstractAppFabricHttpHandler; import io.cdap.cdap.internal.app.BufferFileInputStream; @@ -106,6 +108,8 @@ public class AppFabricClient { private final LocationFactory locationFactory; private final AppLifecycleHttpHandler appLifecycleHttpHandler; private final ProgramLifecycleHttpHandler programLifecycleHttpHandler; + private final ProgramRuntimeHttpHandler programRuntimeHttpHandler; + private final ProgramScheduleHttpHandler programScheduleHttpHandler; private final WorkflowHttpHandler workflowHttpHandler; private final NamespaceHttpHandler namespaceHttpHandler; private final NamespaceQueryAdmin namespaceQueryAdmin; @@ -114,12 +118,16 @@ public class AppFabricClient { public AppFabricClient(LocationFactory locationFactory, AppLifecycleHttpHandler appLifecycleHttpHandler, ProgramLifecycleHttpHandler programLifecycleHttpHandler, + ProgramRuntimeHttpHandler programRuntimeHttpHandler, + ProgramScheduleHttpHandler programScheduleHttpHandler, NamespaceHttpHandler namespaceHttpHandler, NamespaceQueryAdmin namespaceQueryAdmin, WorkflowHttpHandler workflowHttpHandler) { this.locationFactory = locationFactory; this.appLifecycleHttpHandler = appLifecycleHttpHandler; this.programLifecycleHttpHandler = programLifecycleHttpHandler; + this.programRuntimeHttpHandler = programRuntimeHttpHandler; + this.programScheduleHttpHandler = programScheduleHttpHandler; this.namespaceHttpHandler = namespaceHttpHandler; this.namespaceQueryAdmin = namespaceQueryAdmin; this.workflowHttpHandler = workflowHttpHandler; @@ -241,7 +249,7 @@ public void setWorkerInstances(String namespaceId, String appId, String workerId json.addProperty("instances", instances); request.content().writeCharSequence(json.toString(), StandardCharsets.UTF_8); HttpUtil.setContentLength(request, request.content().readableBytes()); - programLifecycleHttpHandler.setWorkerInstances(request, responder, namespaceId, appId, workerId); + programRuntimeHttpHandler.setWorkerInstances(request, responder, namespaceId, appId, workerId); verifyResponse(HttpResponseStatus.OK, responder.getStatus(), "Set worker instances failed"); } @@ -249,7 +257,7 @@ public Instances getWorkerInstances(String namespaceId, String appId, String wor MockResponder responder = new MockResponder(); String uri = String.format("%s/apps/%s/worker/%s/instances", getNamespacePath(namespaceId), appId, workerId); HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri); - programLifecycleHttpHandler.getWorkerInstances(request, responder, namespaceId, appId, workerId); + programRuntimeHttpHandler.getWorkerInstances(request, responder, namespaceId, appId, workerId); verifyResponse(HttpResponseStatus.OK, responder.getStatus(), "Get worker instances failed"); return responder.decodeResponseContent(Instances.class); } @@ -264,7 +272,7 @@ public void setServiceInstances(String namespaceId, String applicationId, String json.addProperty("instances", instances); request.content().writeCharSequence(json.toString(), StandardCharsets.UTF_8); HttpUtil.setContentLength(request, request.content().readableBytes()); - programLifecycleHttpHandler.setServiceInstances(request, responder, namespaceId, applicationId, serviceName); + programRuntimeHttpHandler.setServiceInstances(request, responder, namespaceId, applicationId, serviceName); verifyResponse(HttpResponseStatus.OK, responder.getStatus(), "Set service instances failed"); } @@ -274,7 +282,7 @@ public ServiceInstances getServiceInstances(String namespaceId, String applicati String uri = String.format("%s/apps/%s/services/%s/instances", getNamespacePath(namespaceId), applicationId, serviceName); HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri); - programLifecycleHttpHandler.getServiceInstances(request, responder, namespaceId, applicationId, serviceName); + programRuntimeHttpHandler.getServiceInstances(request, responder, namespaceId, applicationId, serviceName); verifyResponse(HttpResponseStatus.OK, responder.getStatus(), "Get service instances failed"); return responder.decodeResponseContent(ServiceInstances.class); } @@ -286,7 +294,7 @@ public List getProgramSchedules(String namespace, String app, St getNamespacePath(namespace), app, workflow); HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri); try { - programLifecycleHttpHandler.getProgramSchedules(request, responder, namespace, app, workflow, null, null, null); + programScheduleHttpHandler.getProgramSchedules(request, responder, namespace, app, workflow, null, null, null); } catch (Exception e) { // cannot happen throw Throwables.propagate(e); @@ -389,8 +397,7 @@ public void suspend(String namespaceId, String appId, String scheduleName) throw String uri = String.format("%s/apps/%s/schedules/%s/suspend", getNamespacePath(namespaceId), appId, scheduleName); FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri); HttpUtil.setContentLength(request, request.content().readableBytes()); - programLifecycleHttpHandler.performAction(request, responder, namespaceId, appId, - "schedules", scheduleName, "suspend"); + programScheduleHttpHandler.performAction(request, responder, namespaceId, appId, scheduleName, "suspend"); verifyResponse(HttpResponseStatus.OK, responder.getStatus(), "Suspend workflow schedules failed"); } @@ -399,8 +406,7 @@ public void resume(String namespaceId, String appId, String schedName) throws Ex String uri = String.format("%s/apps/%s/schedules/%s/resume", getNamespacePath(namespaceId), appId, schedName); FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri); HttpUtil.setContentLength(request, request.content().readableBytes()); - programLifecycleHttpHandler.performAction(request, responder, namespaceId, appId, - "schedules", schedName, "resume"); + programScheduleHttpHandler.performAction(request, responder, namespaceId, appId, schedName, "resume"); verifyResponse(HttpResponseStatus.OK, responder.getStatus(), "Resume workflow schedules failed"); } @@ -410,7 +416,7 @@ public String scheduleStatus(String namespaceId, String appId, String schedId, i String uri = String.format("%s/apps/%s/schedules/%s/status", getNamespacePath(namespaceId), appId, schedId); HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri); try { - programLifecycleHttpHandler.getStatus(request, responder, namespaceId, appId, "schedules", schedId); + programScheduleHttpHandler.getStatus(request, responder, namespaceId, appId, schedId); } catch (NotFoundException e) { return "NOT_FOUND"; } @@ -614,7 +620,7 @@ public void addSchedule(ApplicationId application, ScheduleDetail scheduleDetail FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.PUT, uri); request.content().writeCharSequence(GSON.toJson(scheduleDetail), StandardCharsets.UTF_8); HttpUtil.setContentLength(request, request.content().readableBytes()); - programLifecycleHttpHandler.addScheduleVersioned(request, responder, application.getNamespace(), + programScheduleHttpHandler.addScheduleVersioned(request, responder, application.getNamespace(), application.getApplication(), application.getVersion(), scheduleDetail.getName()); verifyResponse(HttpResponseStatus.OK, responder.getStatus(), "Add schedule failed"); @@ -627,9 +633,8 @@ public void enableSchedule(ScheduleId scheduleId) throws Exception { scheduleId.getSchedule()); FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri); HttpUtil.setContentLength(request, 0); - programLifecycleHttpHandler.performAction(request, responder, scheduleId.getNamespace(), - scheduleId.getApplication(), "schedules", - scheduleId.getSchedule(), "enable"); + programScheduleHttpHandler.performAction(request, responder, scheduleId.getNamespace(), + scheduleId.getApplication(), scheduleId.getSchedule(), "enable"); verifyResponse(HttpResponseStatus.OK, responder.getStatus(), "Enable schedule failed"); } @@ -642,7 +647,7 @@ public void updateSchedule(ScheduleId scheduleId, ScheduleDetail scheduleDetail) FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri); request.content().writeCharSequence(GSON.toJson(scheduleDetail), StandardCharsets.UTF_8); HttpUtil.setContentLength(request, request.content().readableBytes()); - programLifecycleHttpHandler.updateScheduleVersioned(request, responder, application.getNamespace(), + programScheduleHttpHandler.updateScheduleVersioned(request, responder, application.getNamespace(), application.getApplication(), application.getVersion(), scheduleId.getSchedule()); verifyResponse(HttpResponseStatus.OK, responder.getStatus(), "Update schedule failed"); @@ -654,7 +659,7 @@ public void deleteSchedule(ScheduleId scheduleId) throws Exception { String uri = String.format("%s/apps/%s/versions/%s/schedules/%s", getNamespacePath(application.getNamespace()), application.getApplication(), application.getVersion(), scheduleId.getSchedule()); HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.DELETE, uri); - programLifecycleHttpHandler.deleteScheduleVersioned(request, responder, application.getNamespace(), + programScheduleHttpHandler.deleteScheduleVersioned(request, responder, application.getNamespace(), application.getApplication(), application.getVersion(), scheduleId.getSchedule()); verifyResponse(HttpResponseStatus.OK, responder.getStatus(), "Delete schedule failed"); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/preview/DefaultPreviewRequestQueueTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/preview/DefaultPreviewRequestQueueTest.java index da9641eef21a..9821561394eb 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/preview/DefaultPreviewRequestQueueTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/preview/DefaultPreviewRequestQueueTest.java @@ -142,6 +142,12 @@ public byte[] getPreviewRequestPollerInfo(ApplicationId applicationId) { return new byte[0]; } + @Nullable + @Override + public ApplicationId getApplicationId(byte[] pollerInfo) { + return null; + } + @Override public void deleteExpiredData(long ttlInSeconds) { diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/preview/PreviewRunnerTwillRunnableTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/preview/PreviewRunnerTwillRunnableTest.java index dbb29302229d..554c6cc538ca 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/preview/PreviewRunnerTwillRunnableTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/preview/PreviewRunnerTwillRunnableTest.java @@ -36,7 +36,7 @@ public void testNoSQLInjector() { CConfiguration cConf = CConfiguration.create(); cConf.set(Constants.Dataset.DATA_STORAGE_IMPLEMENTATION, Constants.Dataset.DATA_STORAGE_NOSQL); Injector injector = PreviewRunnerTwillRunnable.createInjector(cConf, new Configuration(), - new PreviewRequestPollerInfo(0, "testuid")); + new PreviewRequestPollerInfo(0, "testuid", null)); DefaultPreviewRunnerManager defaultPreviewRunnerManager = (DefaultPreviewRunnerManager) injector .getInstance(PreviewRunnerManager.class); Injector previewInjector = defaultPreviewRunnerManager.createPreviewInjector(); @@ -48,7 +48,7 @@ public void testPostgresQLInjector() { CConfiguration cConf = CConfiguration.create(); cConf.set(Constants.Dataset.DATA_STORAGE_IMPLEMENTATION, Constants.Dataset.DATA_STORAGE_SQL); Injector injector = PreviewRunnerTwillRunnable.createInjector(cConf, new Configuration(), - new PreviewRequestPollerInfo(0, "testuid")); + new PreviewRequestPollerInfo(0, "testuid", null)); DefaultPreviewRunnerManager defaultPreviewRunnerManager = (DefaultPreviewRunnerManager) injector .getInstance(PreviewRunnerManager.class); Injector previewInjector = defaultPreviewRunnerManager.createPreviewInjector(); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/program/MessagingProgramStatePublisherTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/program/MessagingProgramStatePublisherTest.java index 9ffa8a7a77eb..a6c1e25b98bf 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/program/MessagingProgramStatePublisherTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/program/MessagingProgramStatePublisherTest.java @@ -29,6 +29,7 @@ import io.cdap.cdap.proto.id.ApplicationId; import io.cdap.cdap.proto.id.ProgramRunId; import java.io.IOException; +import java.net.SocketTimeoutException; import java.util.Map; import org.junit.Assert; import org.junit.Rule; @@ -95,4 +96,65 @@ public void testMultipleTopics_noRun() throws TopicNotFoundException, IOExceptio StoreRequest storeRequest = storeRequestCaptor.getValue(); Assert.assertEquals("programstatusevent0", storeRequest.getTopicId().getTopic()); } + + @Test + public void testPublishSuccessOnRetryableException() throws TopicNotFoundException, IOException { + CConfiguration cConf = CConfiguration.create(); + cConf.setInt(Constants.AppFabric.PROGRAM_STATUS_EVENT_NUM_PARTITIONS, 1); + cConf.setInt("system.program.state.retry.policy.max.retries", 3); + + MessagingService messagingService = Mockito.mock(MessagingService.class); + MessagingProgramStatePublisher publisher = new MessagingProgramStatePublisher(cConf, messagingService); + Mockito.when(messagingService.publish(Mockito.any())) + .thenThrow(new SocketTimeoutException()) + .thenReturn(null); + + publisher.publish(Notification.Type.PROGRAM_STATUS, ImmutableMap.of()); + + Mockito.verify(messagingService, Mockito.times(2)).publish(storeRequestCaptor.capture()); + StoreRequest storeRequest = storeRequestCaptor.getValue(); + Assert.assertEquals("programstatusevent", storeRequest.getTopicId().getTopic()); + } + + @Test + public void testPublishThrowsOnRetryExhausted() throws TopicNotFoundException, IOException { + CConfiguration cConf = CConfiguration.create(); + cConf.setInt(Constants.AppFabric.PROGRAM_STATUS_EVENT_NUM_PARTITIONS, 1); + cConf.setInt("system.program.state.retry.policy.max.retries", 3); + + MessagingService messagingService = Mockito.mock(MessagingService.class); + MessagingProgramStatePublisher publisher = new MessagingProgramStatePublisher(cConf, messagingService); + Mockito.when(messagingService.publish(Mockito.any())) + .thenThrow(new SocketTimeoutException()); + + RuntimeException outerException = Assert.assertThrows(RuntimeException.class, + () -> publisher.publish(Notification.Type.PROGRAM_STATUS, ImmutableMap.of())); + Assert.assertNotNull(outerException.getCause()); + Assert.assertTrue(outerException.getCause() instanceof SocketTimeoutException); + + Mockito.verify(messagingService, Mockito.times(4)).publish(storeRequestCaptor.capture()); + StoreRequest storeRequest = storeRequestCaptor.getValue(); + Assert.assertEquals("programstatusevent", storeRequest.getTopicId().getTopic()); + } + + @Test + public void testPublishThrowsForNonRetryableException() throws TopicNotFoundException, IOException { + CConfiguration cConf = CConfiguration.create(); + cConf.setInt(Constants.AppFabric.PROGRAM_STATUS_EVENT_NUM_PARTITIONS, 1); + + MessagingService messagingService = Mockito.mock(MessagingService.class); + MessagingProgramStatePublisher publisher = new MessagingProgramStatePublisher(cConf, + messagingService); + Mockito.when(messagingService.publish(Mockito.any())) + .thenThrow(new IOException()) + .thenReturn(null); + + RuntimeException outerException = Assert.assertThrows(RuntimeException.class, + () -> publisher.publish(Notification.Type.PROGRAM_STATUS, ImmutableMap.of())); + Assert.assertNotNull(outerException.getCause()); + Assert.assertTrue(outerException.getCause() instanceof IOException); + Mockito.verify(messagingService).publish(storeRequestCaptor.capture()); + StoreRequest storeRequest = storeRequestCaptor.getValue(); + Assert.assertEquals("programstatusevent", storeRequest.getTopicId().getTopic()); + } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleServiceTest.java index c411940e7230..d008739fe573 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleServiceTest.java @@ -34,6 +34,7 @@ import io.cdap.cdap.common.ArtifactNotFoundException; import io.cdap.cdap.common.BadRequestException; import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.conf.Constants.Gateway; import io.cdap.cdap.common.id.Id; import io.cdap.cdap.common.id.Id.Namespace; import io.cdap.cdap.common.io.Locations; @@ -843,6 +844,24 @@ public void testUpdateSourceControlMetaWithEmptyGitFileHash() throws Exception { ); } + @Test + public void testGetApplicationCount() throws Exception { + createNamespace("customNS"); + deploy(AllProgramsApp.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, "customNS"); + deploy(ConfigTestApp.class, 200, Gateway.API_VERSION_3_TOKEN, "customNS"); + + long count = applicationLifecycleService.getApplicationsCount(new NamespaceId("customNS")); + + Assert.assertEquals(2, count); + deleteNamespace("customNS"); + } + + @Test(expected = IllegalStateException.class) + public void testGetApplicationCountNullNamespace() { + applicationLifecycleService.getApplicationsCount(null); + } + + private void waitForRuns(int expected, final ProgramId programId, final ProgramRunStatus status) throws Exception { Tasks.waitFor(expected, () -> getProgramRuns(Id.Program.fromEntityId(programId), status).size(), 5, TimeUnit.SECONDS); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramLifecycleServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramLifecycleServiceTest.java index 546198425f35..12f5700ff401 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramLifecycleServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramLifecycleServiceTest.java @@ -87,7 +87,7 @@ public static void shutdown() { @Test public void testEmptyRunsIsStopped() { - Assert.assertEquals(ProgramStatus.STOPPED, ProgramLifecycleService.getProgramStatus(Collections.emptyList())); + Assert.assertEquals(ProgramStatus.STOPPED, programLifecycleService.getProgramStatus(Collections.emptyList())); } @Test @@ -101,33 +101,33 @@ public void testProgramStatusFromSingleRun() { .build(); // pending or starting -> starting - ProgramStatus status = ProgramLifecycleService.getProgramStatus(Collections.singleton(record)); + ProgramStatus status = programLifecycleService.getProgramStatus(Collections.singleton(record)); Assert.assertEquals(ProgramStatus.STARTING, status); record = RunRecordDetail.builder(record).setStatus(ProgramRunStatus.STARTING).build(); - status = ProgramLifecycleService.getProgramStatus(Collections.singleton(record)); + status = programLifecycleService.getProgramStatus(Collections.singleton(record)); Assert.assertEquals(ProgramStatus.STARTING, status); // running, suspended, resuming -> running record = RunRecordDetail.builder(record).setStatus(ProgramRunStatus.RUNNING).build(); - status = ProgramLifecycleService.getProgramStatus(Collections.singleton(record)); + status = programLifecycleService.getProgramStatus(Collections.singleton(record)); Assert.assertEquals(ProgramStatus.RUNNING, status); record = RunRecordDetail.builder(record).setStatus(ProgramRunStatus.SUSPENDED).build(); - status = ProgramLifecycleService.getProgramStatus(Collections.singleton(record)); + status = programLifecycleService.getProgramStatus(Collections.singleton(record)); Assert.assertEquals(ProgramStatus.RUNNING, status); // failed, killed, completed -> stopped record = RunRecordDetail.builder(record).setStatus(ProgramRunStatus.FAILED).build(); - status = ProgramLifecycleService.getProgramStatus(Collections.singleton(record)); + status = programLifecycleService.getProgramStatus(Collections.singleton(record)); Assert.assertEquals(ProgramStatus.STOPPED, status); record = RunRecordDetail.builder(record).setStatus(ProgramRunStatus.KILLED).build(); - status = ProgramLifecycleService.getProgramStatus(Collections.singleton(record)); + status = programLifecycleService.getProgramStatus(Collections.singleton(record)); Assert.assertEquals(ProgramStatus.STOPPED, status); record = RunRecordDetail.builder(record).setStatus(ProgramRunStatus.COMPLETED).build(); - status = ProgramLifecycleService.getProgramStatus(Collections.singleton(record)); + status = programLifecycleService.getProgramStatus(Collections.singleton(record)); Assert.assertEquals(ProgramStatus.STOPPED, status); } @@ -158,18 +158,18 @@ public void testProgramStatusFromMultipleRuns() { .setStatus(ProgramRunStatus.COMPLETED).build(); // running takes precedence over others - ProgramStatus status = ProgramLifecycleService.getProgramStatus( + ProgramStatus status = programLifecycleService.getProgramStatus( Arrays.asList(pending, starting, running, killed, failed, completed)); Assert.assertEquals(ProgramStatus.RUNNING, status); // starting takes precedence over stopped - status = ProgramLifecycleService.getProgramStatus(Arrays.asList(pending, killed, failed, completed)); + status = programLifecycleService.getProgramStatus(Arrays.asList(pending, killed, failed, completed)); Assert.assertEquals(ProgramStatus.STARTING, status); - status = ProgramLifecycleService.getProgramStatus(Arrays.asList(starting, killed, failed, completed)); + status = programLifecycleService.getProgramStatus(Arrays.asList(starting, killed, failed, completed)); Assert.assertEquals(ProgramStatus.STARTING, status); // end states are stopped - status = ProgramLifecycleService.getProgramStatus(Arrays.asList(killed, failed, completed)); + status = programLifecycleService.getProgramStatus(Arrays.asList(killed, failed, completed)); Assert.assertEquals(ProgramStatus.STOPPED, status); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramNotificationSubscriberServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramNotificationSubscriberServiceTest.java index 6c2a9cfd6f5b..51a004cf8848 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramNotificationSubscriberServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramNotificationSubscriberServiceTest.java @@ -361,7 +361,7 @@ public void testLaunchingCountMetricsOnRestart() throws Exception { // terminate the main service. notificationService.shutDown(); notificationService.startUp(); - // Running counts are not based on metadata store in RunRecordMonitorService so not asserting it + // Running counts are not based on metadata store in FlowControlService so not asserting it // here. Tasks.waitFor(0L, () -> queryMetrics(metricStore, SYSTEM_METRIC_PREFIX + FlowControl.LAUNCHING_COUNT, new HashMap<>()), 10, TimeUnit.SECONDS); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/SystemProgramManagementServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/SystemProgramManagementServiceTest.java index 054de2984f41..620e3225942f 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/SystemProgramManagementServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/SystemProgramManagementServiceTest.java @@ -26,6 +26,7 @@ import io.cdap.cdap.common.test.AppJarHelper; import io.cdap.cdap.internal.AppFabricTestHelper; import io.cdap.cdap.internal.app.runtime.BasicArguments; +import io.cdap.cdap.internal.app.runtime.ProgramStartRequest; import io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository; import io.cdap.cdap.internal.app.services.http.AppFabricTestBase; import io.cdap.cdap.proto.ProgramRunStatus; @@ -53,9 +54,9 @@ public class SystemProgramManagementServiceTest extends AppFabricTestBase { private static SystemProgramManagementService progmMgmtSvc; private static ProgramLifecycleService programLifecycleService; private static ApplicationLifecycleService applicationLifecycleService; + private static ProgramRuntimeService runtimeService; private static LocationFactory locationFactory; private static ArtifactRepository artifactRepository; - private static final String VERSION = "1.0.0"; private static final String APP_NAME = SystemProgramManagementTestApp.NAME; private static final String NAMESPACE = "system"; @@ -65,6 +66,7 @@ public class SystemProgramManagementServiceTest extends AppFabricTestBase { @BeforeClass public static void setup() { programLifecycleService = getInjector().getInstance(ProgramLifecycleService.class); + runtimeService = getInjector().getInstance(ProgramRuntimeService.class); applicationLifecycleService = getInjector().getInstance(ApplicationLifecycleService.class); locationFactory = getInjector().getInstance(LocationFactory.class); artifactRepository = getInjector().getInstance(ArtifactRepository.class); @@ -100,8 +102,8 @@ public void testIteration() throws Exception { Assert.assertEquals(ProgramStatus.STOPPED.name(), getProgramStatus(programId)); assertProgramRuns(programId, ProgramRunStatus.RUNNING, 0); //Run the program manually twice to test pruning. One run should be killed - programLifecycleService.start(programId, new HashMap<>(), false, false); - programLifecycleService.start(programId, new HashMap<>(), false, false); + startProgram(programId, new HashMap<>()); + startProgram(programId, new HashMap<>()); assertProgramRuns(programId, ProgramRunStatus.RUNNING, 2); progmMgmtSvc.setProgramsEnabled(enabledServices); progmMgmtSvc.runTask(); @@ -125,4 +127,12 @@ private void deployTestApp() throws Exception { // no-op }, null, false, false, false, Collections.emptyMap()); } + + private void startProgram(ProgramId programId, Map overrides) + throws Exception { + ProgramStartRequest startRequest = programLifecycleService.prepareStart( + programId, overrides, false, false); + runtimeService.run(startRequest.getProgramDescriptor(), startRequest.getProgramOptions(), startRequest.getRunId()) + .getController(); + } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java index ef407808268b..09843dfae79c 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java @@ -646,17 +646,13 @@ protected List getAppList(String namespace) throws Exception { return readResponse(response, LIST_JSON_OBJECT_TYPE); } - protected JsonObject getAppListForPaginatedApi(String namespace, int pageSize, String token, - String filter) throws Exception { - return getAppListForPaginatedApi(namespace, pageSize, token, null, filter, null, null, null); + protected long getAppCount(String namespace) throws Exception { + HttpResponse response = doGet(getVersionedApiPath("apps/count", + Constants.Gateway.API_VERSION_3_TOKEN, namespace)); + assertResponseCode(200, response); + return Long.parseLong(response.getResponseBodyAsString()); } - protected JsonObject getAppListForPaginatedApi(String namespace, int pageSize, String token, - String filter, String nameFilterType, - Boolean latestOnly, Boolean sortCreationTime) throws Exception { - return getAppListForPaginatedApi(namespace, pageSize, token, null, filter, nameFilterType, latestOnly, - sortCreationTime); - } protected List getAppListForNegativePaginatedApi(String namespace, int pageSize) throws Exception { String uri = "apps/?pageSize=" + pageSize; @@ -667,10 +663,38 @@ protected List getAppListForNegativePaginatedApi(String namespace, i return readResponse(response, LIST_JSON_OBJECT_TYPE); } - protected JsonObject getAppListForPaginatedApi(String namespace, int pageSize, String token, + protected List getAppListForDefaultPaginationDisabledApi(String namespace, + Boolean enabledDefaultPagination) throws Exception { + String uri = "apps/?enabledDefaultPagination=" + enabledDefaultPagination; + + HttpResponse response = doGet(getVersionedApiPath(uri, + Constants.Gateway.API_VERSION_3_TOKEN, namespace)); + assertResponseCode(200, response); + return readResponse(response, LIST_JSON_OBJECT_TYPE); + } + + protected JsonObject getAppListForPaginatedApi(String namespace, Integer pageSize, String token, + String filter) throws Exception { + return getAppListForPaginatedApi(namespace, pageSize, token, null, filter, null, null, null, false); + } + + protected JsonObject getAppListForPaginatedApi(String namespace, Integer pageSize, String token, + String filter, String nameFilterType, + Boolean latestOnly, Boolean sortCreationTime, + boolean enableDefaultPagination) throws Exception { + return getAppListForPaginatedApi(namespace, pageSize, token, null, filter, nameFilterType, latestOnly, + sortCreationTime, enableDefaultPagination); + } + + protected JsonObject getAppListForPaginatedApi(String namespace, Integer pageSize, String token, String orderBy, String filter, String nameFilterType, - Boolean latestOnly, Boolean sortCreationTime) throws Exception { - String uri = "apps/?pageSize=" + pageSize; + Boolean latestOnly, Boolean sortCreationTime, + boolean enableDefaultPagination) throws Exception { + String uri = "apps/?"; + + if (pageSize != null) { + uri += ("&pageSize=" + pageSize); + } if (token != null) { uri += ("&pageToken=" + token); @@ -696,6 +720,11 @@ protected JsonObject getAppListForPaginatedApi(String namespace, int pageSize, S uri += ("&sortCreationTime=" + sortCreationTime); } + if (enableDefaultPagination) { + uri += ("&enableDefaultPagination=" + enableDefaultPagination); + } + + uri = uri.replace("?&", "?"); HttpResponse response = doGet(getVersionedApiPath(uri, Constants.Gateway.API_VERSION_3_TOKEN, namespace)); assertResponseCode(200, response); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/handlers/AppLifecycleHttpHandlerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/handlers/AppLifecycleHttpHandlerTest.java index 54d01986ee59..4e69de6b8491 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/handlers/AppLifecycleHttpHandlerTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/handlers/AppLifecycleHttpHandlerTest.java @@ -53,6 +53,7 @@ import io.cdap.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms; import io.cdap.cdap.internal.app.runtime.SystemArguments; import io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository; +import io.cdap.cdap.internal.app.runtime.schedule.ScheduleManager; import io.cdap.cdap.internal.app.services.ApplicationLifecycleService; import io.cdap.cdap.internal.app.services.http.AppFabricTestBase; import io.cdap.cdap.internal.app.store.state.AppStateKey; @@ -75,7 +76,6 @@ import io.cdap.cdap.proto.id.ProgramId; import io.cdap.cdap.proto.id.ProgramReference; import io.cdap.cdap.proto.profile.Profile; -import io.cdap.cdap.scheduler.Scheduler; import io.cdap.cdap.security.impersonation.CurrentUGIProvider; import io.cdap.cdap.security.impersonation.Impersonator; import io.cdap.cdap.security.impersonation.OwnerAdmin; @@ -112,6 +112,7 @@ public class AppLifecycleHttpHandlerTest extends AppFabricTestBase { @BeforeClass public static void beforeClass() throws Throwable { cConf = createBasicCconf(); + cConf.set(Constants.GET_APPS_DEFAULT_PAGE_SIZE, "4"); initializeAndStartServices(cConf); } @@ -135,7 +136,7 @@ protected void configure() { @Provides @Singleton public ApplicationLifecycleService createLifeCycleService(CConfiguration cConf, - Store store, Scheduler scheduler, UsageRegistry usageRegistry, + Store store, ScheduleManager scheduleManager, UsageRegistry usageRegistry, PreferencesService preferencesService, MetricsSystemClient metricsSystemClient, OwnerAdmin ownerAdmin, ArtifactRepository artifactRepository, ManagerFactory managerFactory, @@ -144,7 +145,7 @@ public ApplicationLifecycleService createLifeCycleService(CConfiguration cConf, MessagingService messagingService, Impersonator impersonator, CapabilityReader capabilityReader) { - return Mockito.spy(new ApplicationLifecycleService(cConf, store, scheduler, + return Mockito.spy(new ApplicationLifecycleService(cConf, store, scheduleManager, usageRegistry, preferencesService, metricsSystemClient, ownerAdmin, artifactRepository, managerFactory, metadataServiceClient, accessEnforcer, authenticationContext, messagingService, impersonator, capabilityReader, new NoOpMetricsCollectionService())); @@ -686,6 +687,79 @@ public void testListAndGetForPaginatedAPI() throws Exception { Assert.assertEquals(0, apps.size()); } + @Test + public void testListAppsWithDefaultPaginationEnabled() throws Exception { + for (int i = 0; i < 10; i++) { + // deploy with name to testnamespace1 + String ns1AppName = AllProgramsApp.NAME + i; + Id.Namespace ns1 = Id.Namespace.from(TEST_NAMESPACE1); + Id.Artifact ns1ArtifactId = Id.Artifact.from(ns1, AllProgramsApp.class.getSimpleName(), + "1.0.0-SNAPSHOT"); + + HttpResponse response = addAppArtifact(ns1ArtifactId, AllProgramsApp.class); + Assert.assertEquals(200, response.getResponseCode()); + Id.Application appId = Id.Application.from(ns1, ns1AppName); + response = deploy(appId, + new AppRequest<>(ArtifactSummary.from(ns1ArtifactId.toArtifactId()))); + Assert.assertEquals(200, response.getResponseCode()); + } + + int count = 0; + String token = null; + boolean isLastPage = false; + boolean emptyListReceived = false; + + while (!isLastPage) { + JsonObject result = getAppListForPaginatedApi(TEST_NAMESPACE1, null, token, null, null, null, null, true); + int currentResultSize = result.get("applications").getAsJsonArray().size(); + count += currentResultSize; + emptyListReceived = (currentResultSize == 0); + token = result.get("nextPageToken") == null ? null : result.get("nextPageToken").getAsString(); + isLastPage = (token == null); + if (isLastPage) { + Assert.assertEquals(2, currentResultSize); + } + else { + Assert.assertEquals(4, currentResultSize); + } + } + Assert.assertEquals(10, count); + Assert.assertFalse(emptyListReceived); + + HttpResponse response = doDelete(getVersionedApiPath("apps/", + Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1)); + Assert.assertEquals(200, response.getResponseCode()); + List apps = getAppList(TEST_NAMESPACE1); + Assert.assertEquals(0, apps.size()); + } + + @Test + public void testListAppsWithDefaultPaginationDisabled() throws Exception { + for (int i = 0; i < 10; i++) { + String ns1AppName = AllProgramsApp.NAME + i; + Id.Namespace ns1 = Id.Namespace.from(TEST_NAMESPACE1); + Id.Artifact ns1ArtifactId = Id.Artifact.from(ns1, AllProgramsApp.class.getSimpleName(), + "1.0.0-SNAPSHOT"); + + HttpResponse response = addAppArtifact(ns1ArtifactId, AllProgramsApp.class); + Assert.assertEquals(200, response.getResponseCode()); + Id.Application appId = Id.Application.from(ns1, ns1AppName); + response = deploy(appId, + new AppRequest<>(ArtifactSummary.from(ns1ArtifactId.toArtifactId()))); + Assert.assertEquals(200, response.getResponseCode()); + } + + List result = getAppListForDefaultPaginationDisabledApi(TEST_NAMESPACE1, false); + int resultSize = result.size(); + Assert.assertEquals(10, resultSize); + + HttpResponse response = doDelete(getVersionedApiPath("apps/", + Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1)); + Assert.assertEquals(200, response.getResponseCode()); + List apps = getAppList(TEST_NAMESPACE1); + Assert.assertEquals(0, apps.size()); + } + @Test public void testListAppsWithNegativePageSize() throws Exception { for (int i = 0; i < 10; i++) { @@ -821,17 +895,18 @@ public void testListAndGetForPaginatedAPIWithNameFilterType() throws Exception { String token = null; JsonObject result = getAppListForPaginatedApi(TEST_NAMESPACE1, 3, token, AllProgramsApp.NAME.toUpperCase(Locale.ROOT), - "EQUALS", null, null); + "EQUALS", null, null, false); int currentResultSize = result.get("applications").getAsJsonArray().size(); Assert.assertEquals(0, currentResultSize); JsonObject result1 = getAppListForPaginatedApi(TEST_NAMESPACE1, 3, token, AllProgramsApp.NAME.toUpperCase(Locale.ROOT), - "EQUALS_IGNORE_CASE", null, null); + "EQUALS_IGNORE_CASE", null, null, false); int currentResultSize1 = result1.get("applications").getAsJsonArray().size(); Assert.assertEquals(1, currentResultSize1); - JsonObject result2 = getAppListForPaginatedApi(TEST_NAMESPACE1, 3, token, AllProgramsApp.NAME, null, null, null); + JsonObject result2 = getAppListForPaginatedApi(TEST_NAMESPACE1, 3, token, AllProgramsApp.NAME, + null, null, null, false); int currentResultSize2 = result2.get("applications").getAsJsonArray().size(); Assert.assertEquals(2, currentResultSize2); @@ -879,7 +954,8 @@ public void testListAndGetForPaginatedAPIWithLatestOnlyLCMFlagEnabled() throws E boolean isLastPage = false; int currentResultSize = 0; while (!isLastPage) { - JsonObject result = getAppListForPaginatedApi(TEST_NAMESPACE1, 3, token, AllProgramsApp.NAME, null, true, null); + JsonObject result = getAppListForPaginatedApi(TEST_NAMESPACE1, 3, token, + AllProgramsApp.NAME, null, true, null, false); currentResultSize = result.get("applications").getAsJsonArray().size(); count += currentResultSize; token = result.get("nextPageToken") == null ? null : result.get("nextPageToken").getAsString(); @@ -893,7 +969,8 @@ public void testListAndGetForPaginatedAPIWithLatestOnlyLCMFlagEnabled() throws E isLastPage = false; currentResultSize = 0; while (!isLastPage) { - JsonObject result = getAppListForPaginatedApi(TEST_NAMESPACE1, 3, token, AllProgramsApp.NAME, null, false, null); + JsonObject result = getAppListForPaginatedApi(TEST_NAMESPACE1, 3, token, + AllProgramsApp.NAME, null, false, null, false); currentResultSize = result.get("applications").getAsJsonArray().size(); count += currentResultSize; token = result.get("nextPageToken") == null ? null : result.get("nextPageToken").getAsString(); @@ -931,7 +1008,8 @@ public void testListAndGetLCMFlagEnabledForVersionHistory() throws Exception { boolean isLastPage = false; int currentResultSize = 0; while (!isLastPage) { - JsonObject result = getAppListForPaginatedApi(TEST_NAMESPACE1, 3, token, AllProgramsApp.NAME, null, true, null); + JsonObject result = getAppListForPaginatedApi(TEST_NAMESPACE1, 3, token, + AllProgramsApp.NAME, null, true, null, false); currentResultSize = result.get("applications").getAsJsonArray().size(); count += currentResultSize; token = result.get("nextPageToken") == null ? null : result.get("nextPageToken").getAsString(); @@ -947,7 +1025,7 @@ public void testListAndGetLCMFlagEnabledForVersionHistory() throws Exception { List creationTimeList = new ArrayList<>(); while (!isLastPage) { JsonObject result = getAppListForPaginatedApi(TEST_NAMESPACE1, 3, token, - "DESC", AllProgramsApp.NAME, "EQUALS", false, true); + "DESC", AllProgramsApp.NAME, "EQUALS", false, true, false); currentResultSize = result.get("applications").getAsJsonArray().size(); count += currentResultSize; token = result.get("nextPageToken") == null ? null : result.get("nextPageToken").getAsString(); @@ -1516,6 +1594,33 @@ public void testDeployAppWithDisabledProfileInSchedule() throws Exception { deleteArtifact(artifactId, 200); } + @Test + public void testGetApplicationCount() throws Exception { + for (int i = 0; i < 10; i++) { + String ns1AppName = AllProgramsApp.NAME + i; + Id.Namespace ns1 = Id.Namespace.from(TEST_NAMESPACE1); + Id.Artifact ns1ArtifactId = Id.Artifact.from(ns1, AllProgramsApp.class.getSimpleName(), + "1.0.0-SNAPSHOT"); + + HttpResponse response = addAppArtifact(ns1ArtifactId, AllProgramsApp.class); + Assert.assertEquals(200, response.getResponseCode()); + Id.Application appId = Id.Application.from(ns1, ns1AppName); + response = deploy(appId, + new AppRequest<>(ArtifactSummary.from(ns1ArtifactId.toArtifactId()))); + Assert.assertEquals(200, response.getResponseCode()); + } + + long result = getAppCount(TEST_NAMESPACE1); + Assert.assertEquals(10, result); + + HttpResponse response = doDelete(getVersionedApiPath("apps/", + Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1)); + Assert.assertEquals(200, response.getResponseCode()); + + result = getAppCount(TEST_NAMESPACE1); + Assert.assertEquals(0, result); + } + @After public void cleanup() throws Exception { setLCMFlag(false); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/handlers/MonitorHandlerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/handlers/MonitorHandlerTest.java index f3587dc7360a..9e875adb30ef 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/handlers/MonitorHandlerTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/handlers/MonitorHandlerTest.java @@ -66,7 +66,7 @@ public void testSystemServices() throws Exception { List actual = GSON.fromJson(new String(ByteStreams.toByteArray(urlConn.getInputStream()), Charsets.UTF_8), token); - Assert.assertEquals(10, actual.size()); + Assert.assertEquals(9, actual.size()); urlConn.disconnect(); } @@ -81,7 +81,7 @@ public void testSystemServicesStatus() throws Exception { Map result = GSON.fromJson(new String(ByteStreams.toByteArray(urlConn.getInputStream()), Charsets.UTF_8), token); - Assert.assertEquals(10, result.size()); + Assert.assertEquals(9, result.size()); urlConn.disconnect(); Assert.assertEquals("OK", result.get(Constants.Service.APP_FABRIC_HTTP)); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/AppMetadataStoreTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/AppMetadataStoreTest.java index ab80eccd8eb8..81a8a2bc940d 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/AppMetadataStoreTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/AppMetadataStoreTest.java @@ -24,6 +24,12 @@ import io.cdap.cdap.AllProgramsApp; import io.cdap.cdap.api.app.ApplicationSpecification; import io.cdap.cdap.api.artifact.ArtifactId; +import io.cdap.cdap.api.plugin.Plugin; +import io.cdap.cdap.api.schedule.SchedulableProgramType; +import io.cdap.cdap.api.workflow.ScheduleProgramInfo; +import io.cdap.cdap.api.workflow.WorkflowActionNode; +import io.cdap.cdap.api.workflow.WorkflowNode; +import io.cdap.cdap.api.workflow.WorkflowSpecification; import io.cdap.cdap.app.store.ScanApplicationsRequest; import io.cdap.cdap.common.app.RunIds; import io.cdap.cdap.common.utils.ProjectInfo; @@ -32,6 +38,7 @@ import io.cdap.cdap.internal.app.DefaultApplicationSpecification; import io.cdap.cdap.internal.app.deploy.Specifications; import io.cdap.cdap.internal.app.runtime.SystemArguments; +import io.cdap.cdap.internal.app.store.plugin.Plugins; import io.cdap.cdap.proto.ProgramRunStatus; import io.cdap.cdap.proto.ProgramType; import io.cdap.cdap.proto.artifact.ChangeDetail; @@ -60,6 +67,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.TreeSet; import java.util.UUID; @@ -861,7 +869,7 @@ public void testDuplicateWritesIgnored() throws Exception { byte[] sourceId = new byte[] { 0 }; TransactionRunners.run(transactionRunner, context -> { AppMetadataStore store = AppMetadataStore.create(context); - assertSecondCallIsNull(() -> store.recordProgramProvisioning(runId, null, SINGLETON_PROFILE_MAP, + Assert.assertNotNull(store.recordProgramProvisioning(runId, null, SINGLETON_PROFILE_MAP, sourceId, ARTIFACT_ID)); assertSecondCallIsNull(() -> store.recordProgramProvisioned(runId, 0, sourceId)); assertSecondCallIsNull(() -> store.recordProgramStart(runId, null, Collections.emptyMap(), sourceId)); @@ -1101,6 +1109,49 @@ public void testBatchApplications() { } } + @Test + public void testGetApplicationCount() { + ApplicationSpecification appSpec = Specifications.from(new AllProgramsApp()); + NamespaceId customNamespace = new NamespaceId("custom"); + createMultipleApplications(NamespaceId.DEFAULT, "test-default", 20, appSpec); + createMultipleApplications(NamespaceId.SYSTEM, "test-system", 5, appSpec); + createMultipleApplications(customNamespace, "test-custom", 15, appSpec); + + TransactionRunners.run(transactionRunner, context -> { + AppMetadataStore store = AppMetadataStore.create(context); + long count = store.getApplicationCount(); + // System apps are not included in the count. Default(20) and custom(15) are included. + Assert.assertEquals(35, count); + }); + + TransactionRunners.run(transactionRunner, context -> { + AppMetadataStore store = AppMetadataStore.create(context); + // System namespace has 20 applications. + long count = store.getNamespaceApplicationCount(NamespaceId.DEFAULT); + Assert.assertEquals(20, count); + // System namespace has 5 applications. + count = store.getNamespaceApplicationCount(NamespaceId.SYSTEM); + Assert.assertEquals(5, count); + // Custom namespace has 15 applications. + count = store.getNamespaceApplicationCount(customNamespace); + Assert.assertEquals(15, count); + }); + + } + + private void createMultipleApplications(NamespaceId namespaceId, String appPrefix, int count, + ApplicationSpecification appSpec) { + for (int i = 0; i < count; i++) { + String appName = appPrefix + i; + TransactionRunners.run(transactionRunner, context -> { + AppMetadataStore store = AppMetadataStore.create(context); + store.writeApplication(namespaceId.getNamespace(), appName, ApplicationId.DEFAULT_VERSION, + appSpec, + new ChangeDetail(null, null, null, creationTimeMillis), null); + }); + } + } + @Test public void testScanApplications() { ApplicationSpecification appSpec = Specifications.from(new AllProgramsApp()); @@ -1617,6 +1668,64 @@ public void testConcurrentCreateAppAfterTheFirstVersion() throws Exception { Assert.assertEquals(1 + numThreads, appEditNumber.get()); } + @Test + public void testCreateAndGetApplication() { + String appName = "application1"; + ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId(); + ApplicationReference appRef = new ApplicationReference(NamespaceId.DEFAULT, appName); + ApplicationId appId = appRef.app(appName + "_version_" + 1); + ApplicationSpecification spec = createDummyAppSpecWithWorkflow(appId, artifactId, false); + ApplicationMeta meta = new ApplicationMeta(spec.getName(), spec, + new ChangeDetail(null, null, null, + creationTimeMillis + 1, true)); + + // Deploy the first version + TransactionRunners.run(transactionRunner, context -> { + AppMetadataStore metaStore = AppMetadataStore.create(context); + StructuredTable pluginDataTable = metaStore.getPluginDataTable(); + Plugins.addWranglerPluginToTable(pluginDataTable); + Plugins.addNowDirectivePluginToTable(pluginDataTable); + Plugins.addFormatTextPluginToTable(pluginDataTable); + metaStore.createLatestApplicationVersion(appId, meta); + }); + + // Verify latest version + final ApplicationMeta[] gotMeta = {null}; + TransactionRunners.run(transactionRunner, context -> { + AppMetadataStore metaStore = AppMetadataStore.create(context); + gotMeta[0] = metaStore.getApplication(appId); + }); + + Assert.assertEquals(meta.toString(), gotMeta[0].toString()); + } + + @Test + public void testCreateAndGetApplicationWithMissingPlugins() { + String appName = "application1"; + ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId(); + ApplicationReference appRef = new ApplicationReference(NamespaceId.DEFAULT, appName); + ApplicationId appId = appRef.app(appName + "_version_" + 1); + ApplicationSpecification spec = createDummyAppSpecWithWorkflow(appId, artifactId, true); + ApplicationMeta meta = new ApplicationMeta(spec.getName(), spec, + new ChangeDetail(null, null, null, + creationTimeMillis + 1, true)); + + // Deploy the first version + TransactionRunners.run(transactionRunner, context -> { + AppMetadataStore metaStore = AppMetadataStore.create(context); + StructuredTable pluginDataTable = metaStore.getPluginDataTable(); + Plugins.addFormatTextPluginToTable(pluginDataTable); + metaStore.createLatestApplicationVersion(appId, meta); + }); + + // Verify latest version + TransactionRunners.run(transactionRunner, context -> { + AppMetadataStore metaStore = AppMetadataStore.create(context); + ApplicationMeta gotMeta = metaStore.getApplication(appId); + Assert.assertEquals(meta.toString(), Objects.requireNonNull(gotMeta).toString()); + }); + } + @Test public void testDeleteCompletedRunsStartedBefore() throws Exception { // Map an iterator to one of 15 different program+workflow permutations. Used to ensure @@ -1757,6 +1866,31 @@ private ApplicationSpecification createDummyAppSpec(String appName, String appVe Collections.emptyMap()); } + private ApplicationSpecification createDummyAppSpecWithWorkflow(ApplicationId appId, + ArtifactId artifactId, boolean isAppSpecReductionEnabled) { + Map plugins = + isAppSpecReductionEnabled ? Plugins.createDummyPluginsForReducedAppSpec() + : Plugins.createDummyPlugins(); + ImmutableList nodes = ImmutableList.of( + new WorkflowActionNode("mr1", + new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "mr1")), + new WorkflowActionNode("spark1", + new ScheduleProgramInfo(SchedulableProgramType.SPARK, "spark1"))); + WorkflowSpecification wfSpec = + new WorkflowSpecification("test", "wf1", "", Collections.emptyMap(), + nodes, + Collections.emptyMap(), plugins); + return new DefaultApplicationSpecification( + appId.getApplication(), appId.getVersion(), ProjectInfo.getVersion().toString(), "desc", + null, artifactId, + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap(), + ImmutableMap.of(appId.workflow("wf1").getProgram(), wfSpec), Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + plugins); + } + private void runConcurrentOperation(String name, int numThreads, Runnable runnable) throws Exception { ExecutorService executorService = Executors.newFixedThreadPool(numThreads); CountDownLatch startLatch = new CountDownLatch(1); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/DefaultStoreTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/DefaultStoreTest.java index 3173d0dee397..29054c9b0c62 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/DefaultStoreTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/DefaultStoreTest.java @@ -435,7 +435,9 @@ public void testLogProgramRunHistory() { .setProperties(noRuntimeArgsProps) .setCluster(emptyCluster) .setArtifactId(artifactId) - .setSourceId(AppFabricTestHelper.createSourceId(sourceId)).build(); + .setSourceId(AppFabricTestHelper.createSourceId(sourceId)) + .setFlowControlStatus("") + .build(); RunRecordDetail actualRecord7 = store.getRun(programId.run(run7.getId())); Assert.assertEquals(expectedRunRecord7, actualRecord7); @@ -450,7 +452,9 @@ public void testLogProgramRunHistory() { .setProperties(noRuntimeArgsProps) .setCluster(emptyCluster) .setArtifactId(artifactId) - .setSourceId(AppFabricTestHelper.createSourceId(sourceId)).build(); + .setSourceId(AppFabricTestHelper.createSourceId(sourceId)) + .setFlowControlStatus("") + .build(); RunRecordDetail actualRecord8 = store.getRun(programId.run(run8.getId())); Assert.assertEquals(expectedRunRecord8, actualRecord8); @@ -470,7 +474,9 @@ public void testLogProgramRunHistory() { .setProperties(noRuntimeArgsProps) .setCluster(emptyCluster) .setArtifactId(artifactId) - .setSourceId(AppFabricTestHelper.createSourceId(sourceId)).build(); + .setSourceId(AppFabricTestHelper.createSourceId(sourceId)) + .setFlowControlStatus("") + .build(); RunRecordDetail actualRecord9 = store.getRun(programId.run(run9.getId())); Assert.assertEquals(expectedRunRecord9, actualRecord9); @@ -1446,6 +1452,23 @@ public void testListRunsWithLegacyRows() throws ConflictException { Assert.assertEquals(latestAppId.get(), actualApps.get(0)); } + @Test + public void testGetApplicationCount() throws ConflictException { + ApplicationSpecification spec = Specifications.from(new AllProgramsApp()); + NamespaceId namespaceId = new NamespaceId("custom"); + ApplicationMeta appMeta = new ApplicationMeta(spec.getName(), spec, + new ChangeDetail(null, null, null, + System.currentTimeMillis())); + for (int i = 0; i < 5; i++) { + ApplicationId appId = namespaceId.app("application" + i); + store.addLatestApplication(appId, appMeta); + Assert.assertNotNull(store.getApplication(appId)); + } + long result = store.getApplicationCount(namespaceId); + + Assert.assertEquals(5, result); + } + private void writeStartRecord(ProgramRunId run, ArtifactId artifactId) { setStartAndRunning(run, artifactId); Assert.assertNotNull(store.getRun(run)); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/SqlAppMetadataStoreTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/SqlAppMetadataStoreTest.java index 8593b7ba59ad..b85e75d94486 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/SqlAppMetadataStoreTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/SqlAppMetadataStoreTest.java @@ -68,6 +68,7 @@ protected void configure() { transactionRunner = injector.getInstance(TransactionRunner.class); StoreDefinition.AppMetadataStore.create(injector.getInstance(StructuredTableAdmin.class)); + StoreDefinition.ArtifactStore.create(injector.getInstance(StructuredTableAdmin.class)); } @AfterClass diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/plugin/Plugins.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/plugin/Plugins.java new file mode 100644 index 000000000000..2f0125b257b9 --- /dev/null +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/plugin/Plugins.java @@ -0,0 +1,209 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.store.plugin; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import io.cdap.cdap.api.artifact.ArtifactId; +import io.cdap.cdap.api.plugin.Plugin; +import io.cdap.cdap.api.plugin.PluginClass; +import io.cdap.cdap.api.plugin.PluginProperties; +import io.cdap.cdap.api.plugin.PluginPropertyField; +import io.cdap.cdap.api.plugin.Requirements; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.spi.data.StructuredTable; +import io.cdap.cdap.spi.data.table.field.Field; +import io.cdap.cdap.spi.data.table.field.Fields; +import io.cdap.cdap.store.StoreDefinition.ArtifactStore; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Util for building plugins for tests. + */ +public final class Plugins { + + private Plugins() { + } + + /** + * Creates dummy plugins. + */ + public static Map createDummyPlugins() { + Map plugins = new HashMap<>(); + + Gson gson = new Gson(); + Map childProperties = ImmutableMap.of("child1", "childVal1", "child2", + "${secure(acc)}", "child3", "val3"); + Map properties = ImmutableMap.of("key2", gson.toJson(childProperties), "key1", + "val1"); + + PluginClass wranglerPluginClass = PluginClass.builder() + .setClassName("io.cdap.wrangler.Wrangler").setName("Wrangler").setConfigFieldName("config") + .setRequirements(Requirements.EMPTY).setType("transform") + .setDescription("Wrangler - A interactive tool for data cleansing and transformation.") + .add("schema", + new PluginPropertyField("schema", "Specifies the schema that has to be output.", + "string", true, true)).add("preconditionSQL", + new PluginPropertyField("preconditionSQL", + "SQL Precondition expression specifying filtering before applying directives (false to filter)", + "string", false, true, false)).build(); + Plugin wranglerPlugin = new Plugin(Collections.emptyList(), + NamespaceId.DEFAULT.artifact("wrangler-transform", "1.0").toApiArtifactId(), + wranglerPluginClass, PluginProperties.builder().addAll(properties).build()); + plugins.put("p1", wranglerPlugin); + + PluginClass directivePluginClass = PluginClass.builder() + .setClassName("com.google.cloud.datafusion.directives.Now").setName("now") + .setRequirements(Requirements.EMPTY).setType("directive") + .setDescription("Populates a column with the current date and time").build(); + ArtifactId parent = NamespaceId.DEFAULT.artifact("Wrangler", "1.0").toApiArtifactId(); + Plugin directivePlugin = new Plugin(Collections.singleton(parent), + NamespaceId.DEFAULT.artifact("wrangler", "1.0").toApiArtifactId(), directivePluginClass, + PluginProperties.builder().addAll(properties).build()); + plugins.put("p2", directivePlugin); + + PluginClass gCloudFormatTextPluginClass = PluginClass.builder() + .setClassName("io.cdap.plugin.format.text.input.TextInputFormatProvider").setName("text") + .setRequirements(Requirements.EMPTY).setType("validatingInputFormat") + .setDescription("Plugin for reading files in text format.").setConfigFieldName("conf") + .build(); + parent = NamespaceId.DEFAULT.artifact("google-cloud", "1.0").toApiArtifactId(); + Plugin gCloudFormatTextPlugin = new Plugin(Collections.singleton(parent), + NamespaceId.DEFAULT.artifact("format-text", "1.0").toApiArtifactId(), + gCloudFormatTextPluginClass, PluginProperties.builder().addAll(properties).build()); + plugins.put("p3", gCloudFormatTextPlugin); + + return plugins; + } + + public static Map createDummyPluginsForReducedAppSpec() { + Map plugins = new HashMap<>(); + + Gson gson = new Gson(); + Map childProperties = ImmutableMap.of("child1", "childVal1", "child2", + "${secure(acc)}", "child3", "val3"); + Map properties = ImmutableMap.of("key2", gson.toJson(childProperties), "key1", + "val1"); + + PluginClass wranglerPluginClass = new PluginClass("transform", "Wrangler", null, null, + Collections.emptyMap(), new Requirements(Collections.emptySet()), ""); + Plugin wranglerPlugin = new Plugin(Collections.emptyList(), + NamespaceId.DEFAULT.artifact("wrangler-transform", "1.0").toApiArtifactId(), + wranglerPluginClass, PluginProperties.builder().addAll(properties).build()); + plugins.put("p1", wranglerPlugin); + + PluginClass directivePluginClass = new PluginClass("directive", "now", null, null, + Collections.emptyMap(), new Requirements(Collections.emptySet()), ""); + ArtifactId parent = NamespaceId.DEFAULT.artifact("Wrangler", "1.0").toApiArtifactId(); + Plugin directivePlugin = new Plugin(Collections.singleton(parent), + NamespaceId.DEFAULT.artifact("wrangler", "1.0").toApiArtifactId(), directivePluginClass, + PluginProperties.builder().addAll(properties).build()); + plugins.put("p2", directivePlugin); + + PluginClass gCloudFormatTextPluginClass = PluginClass.builder() + .setClassName("io.cdap.plugin.format.text.input.TextInputFormatProvider").setName("text") + .setRequirements(Requirements.EMPTY).setType("validatingInputFormat") + .setDescription("Plugin for reading files in text format.").setConfigFieldName("conf") + .build(); + parent = NamespaceId.DEFAULT.artifact("google-cloud", "1.0").toApiArtifactId(); + Plugin gCloudFormatTextPlugin = new Plugin(Collections.singleton(parent), + NamespaceId.DEFAULT.artifact("format-text", "1.0").toApiArtifactId(), + gCloudFormatTextPluginClass, PluginProperties.builder().addAll(properties).build()); + plugins.put("p3", gCloudFormatTextPlugin); + + return plugins; + } + + private static void addPluginEntryToTable(StructuredTable pluginDataTable, + String parentContextName, String pluginType, String pluginName, String artifactName, + String pluginDataJson) throws IOException { + Collection> fields = new ArrayList<>(); + fields.add(Fields.stringField(ArtifactStore.PARENT_NAMESPACE_FIELD, "default")); + fields.add(Fields.stringField(ArtifactStore.PARENT_NAME_FIELD, parentContextName)); + fields.add(Fields.stringField(ArtifactStore.PLUGIN_TYPE_FIELD, pluginType)); + fields.add(Fields.stringField(ArtifactStore.PLUGIN_NAME_FIELD, pluginName)); + fields.add(Fields.stringField(ArtifactStore.ARTIFACT_NAMESPACE_FIELD, "default")); + fields.add(Fields.stringField(ArtifactStore.ARTIFACT_NAME_FIELD, artifactName)); + fields.add(Fields.stringField(ArtifactStore.ARTIFACT_VER_FIELD, "1.0")); + fields.add(Fields.stringField(ArtifactStore.PLUGIN_DATA_FIELD, pluginDataJson)); + pluginDataTable.upsert(fields); + } + + /** + * Initializes a {@link StructuredTable} with a dummy entry for a Wrangler plugin. + */ + public static void addWranglerPluginToTable(StructuredTable pluginDataTable) throws IOException { + String wranglerPluginJsonData = "{\"pluginClass\":{\"type\":" + + "\"transform\",\"name\":\"Wrangler\",\"description\":\"Wrangler - A interactive tool " + + "for data cleansing and transformation.\",\"className\":\"io.cdap.wrangler.Wrangler\"," + + "\"configFieldName\":\"config\",\"properties\":{\"schema\":{\"name\":\"schema\",\"description" + + "\":\"Specifies the schema that has to be output.\",\"type\":\"string\",\"required\":true," + + "\"macroSupported\":true,\"macroEscapingEnabled\":false,\"children\":[]},\"preconditionSQL\"" + + ":{\"name\":\"preconditionSQL\",\"description\":\"SQL Precondition expression specifying" + + " filtering before applying directives (false to filter)\",\"type\":\"string\",\"required" + + "\":false,\"macroSupported\":true,\"macroEscapingEnabled\":false,\"children\":[]}}," + + "\"requirements\":{\"datasetTypes\":[],\"" + + "capabilities\":[]}},\"artifactLocationPath\":\"/cdap/namespaces/system/artifacts/wrangler-transform" + + "/4.11.0-SNAPSHOT.d26c4ac8-600a-4bf0-8280-0561037036d8.jar\",\"usableBy\":\"" + + "system:cdap-data-pipeline[6.10.0,7.0.0-SNAPSHOT)\"}"; // Example path/usableBy + + addPluginEntryToTable(pluginDataTable, "testArtifact", "transform", "Wrangler", + "wrangler-transform", wranglerPluginJsonData); + } + + /** + * Initializes a {@link StructuredTable} with a dummy entry for a "now" directive plugin. + */ + public static void addNowDirectivePluginToTable(StructuredTable pluginDataTable) + throws IOException { + + ArtifactId parent = NamespaceId.DEFAULT.artifact("Wrangler", "1.0").toApiArtifactId(); + String nowDirectiveJsonData = "{\"pluginClass\":{\"type\":\"directive\",\"name\":\"now\"," + + "\"description\":\"Populates a column with the current date and time\"," + + "\"className\":\"com.google.cloud.datafusion.directives.Now\",\"properties\":{}," + + "\"requirements\":{\"datasetTypes\":[],\"capabilities\":[]}}," + + "\"artifactLocationPath\":\"/cdap/namespaces/default/artifacts/now-directive/1.0.0-SNAPSHOT.jar\"," + + "\"usableBy\":\"system:wrangler-transform[4.0.0,5.0.0)\"}"; + + addPluginEntryToTable(pluginDataTable, parent.getName(), "directive", "now", "wrangler", + nowDirectiveJsonData); + } + + /** + * Initializes a {@link StructuredTable} with a dummy entry for a "format-text" plugin. + */ + public static void addFormatTextPluginToTable(StructuredTable pluginDataTable) + throws IOException { + + String formatTextJsonData = + "{\"pluginClass\":{\"type\":\"validatingInputFormat\",\"name\":\"text\"," + + "\"description\":\"Plugin for reading files in text format.\"," + + "\"className\":\"io.cdap.plugin.format.text.input.TextInputFormatProvider\"," + + "\"configFieldName\":\"conf\",\"requirements\":{\"datasetTypes\":[]," + + "\"capabilities\":[]}},\"artifactLocationPath\":\"/cdap/namespaces/system/artifacts/format" + + "-text/2.14.0-SNAPSHOT.4fed59ac-ef3f-4b76-ac29-d5aa8bc38061.jar\"," + + "\"usableBy\":\"system:cdap-data-pipeline[6.8.0-SNAPSHOT,7.0.0-SNAPSHOT)\"}"; + + addPluginEntryToTable(pluginDataTable, "testArtifact", "validatingInputFormat", "text", + "format-text", formatTextJsonData); + } +} diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/preview/DefaultPreviewStoreTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/preview/DefaultPreviewStoreTest.java index 59a8c171372f..8cdfe8e1a048 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/preview/DefaultPreviewStoreTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/preview/DefaultPreviewStoreTest.java @@ -69,7 +69,7 @@ public static void beforeClass() throws IOException { cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath()); Injector injector = Guice.createInjector( - new PreviewConfigModule(cConf, new Configuration(), SConfiguration.create()) + new PreviewConfigModule(cConf, new Configuration(), SConfiguration.create()) ); store = injector.getInstance(DefaultPreviewStore.class); } @@ -87,10 +87,12 @@ public void before() throws Exception { @Test public void testPreviewStore() { String firstApplication = RunIds.generate().getId(); - ApplicationId firstApplicationId = new ApplicationId(NamespaceMeta.DEFAULT.getName(), firstApplication); + ApplicationId firstApplicationId = new ApplicationId(NamespaceMeta.DEFAULT.getName(), + firstApplication); String secondApplication = RunIds.generate().getId(); - ApplicationId secondApplicationId = new ApplicationId(NamespaceMeta.DEFAULT.getName(), secondApplication); + ApplicationId secondApplicationId = new ApplicationId(NamespaceMeta.DEFAULT.getName(), + secondApplication); // put data for the first application store.put(firstApplicationId, "mytracer", "key1", "value1"); @@ -113,10 +115,12 @@ public void testPreviewStore() { Assert.assertEquals(2, firstApplicationData.get("key1").get(1).getAsInt()); Assert.assertEquals(3, firstApplicationData.get("key2").get(0).getAsInt()); Assert.assertEquals(propertyMap, GSON.fromJson(firstApplicationData.get("key2").get(1), - new TypeToken>() { }.getType())); + new TypeToken>() { + }.getType())); // get the data for second application and logger name "mytracer" - Map> secondApplicationData = store.get(secondApplicationId, "mytracer"); + Map> secondApplicationData = store.get(secondApplicationId, + "mytracer"); Assert.assertEquals(1, secondApplicationData.size()); Assert.assertEquals("value1", secondApplicationData.get("key1").get(0).getAsString()); @@ -136,9 +140,10 @@ public void testPreviewInfo() throws IOException { // test put and get ApplicationId applicationId = new ApplicationId("ns1", "app1"); ProgramRunId runId = new ProgramRunId("ns1", "app1", ProgramType.WORKFLOW, "test", - RunIds.generate().getId()); - PreviewStatus status = new PreviewStatus(PreviewStatus.Status.COMPLETED, System.currentTimeMillis(), null, 0L, - System.currentTimeMillis()); + RunIds.generate().getId()); + PreviewStatus status = new PreviewStatus(PreviewStatus.Status.COMPLETED, + System.currentTimeMillis(), null, 0L, + System.currentTimeMillis()); store.setProgramId(runId); store.setPreviewStatus(applicationId, status); @@ -151,7 +156,8 @@ public void testPreviewWaitingRequests() throws Exception { byte[] pollerInfo = Bytes.toBytes("runner-1"); PreviewConfig previewConfig = new PreviewConfig("WordCount", ProgramType.WORKFLOW, null, null); - AppRequest testRequest = new AppRequest<>(new ArtifactSummary("test", "1.0"), null, previewConfig); + AppRequest testRequest = new AppRequest<>(new ArtifactSummary("test", "1.0"), null, + previewConfig); Assert.assertEquals(0, store.getAllInWaitingState().size()); RunId id1 = RunIds.generate(); @@ -191,9 +197,9 @@ public void testPreviewWaitingRequests() throws Exception { // Add a preview request that has a principle associated with it ApplicationId applicationId4 = new ApplicationId("ns1", RunIds.generate().getId()); Principal principal = new Principal("userForApplicationId4", - Principal.PrincipalType.USER, - new Credential("userForApplicationId4Credential", - Credential.CredentialType.EXTERNAL)); + Principal.PrincipalType.USER, + new Credential("userForApplicationId4Credential", + Credential.CredentialType.EXTERNAL)); store.add(applicationId4, testRequest, principal); allWaiting = store.getAllInWaitingState(); Assert.assertEquals(1, allWaiting.size()); @@ -207,16 +213,20 @@ public void testPreviewWaitingRequests() throws Exception { @Test public void testPreviewTTL() throws Exception { PreviewConfig previewConfig = new PreviewConfig("WordCount", ProgramType.WORKFLOW, null, null); - AppRequest testRequest = new AppRequest<>(new ArtifactSummary("test", "1.0"), null, previewConfig); + AppRequest testRequest = new AppRequest<>(new ArtifactSummary("test", "1.0"), null, + previewConfig); String firstApplication = RunIds.generate(System.currentTimeMillis() - 5000).getId(); - ApplicationId firstApplicationId = new ApplicationId(NamespaceMeta.DEFAULT.getName(), firstApplication); + ApplicationId firstApplicationId = new ApplicationId(NamespaceMeta.DEFAULT.getName(), + firstApplication); String secondApplication = RunIds.generate(System.currentTimeMillis() - 4000).getId(); - ApplicationId secondApplicationId = new ApplicationId(NamespaceMeta.DEFAULT.getName(), secondApplication); + ApplicationId secondApplicationId = new ApplicationId(NamespaceMeta.DEFAULT.getName(), + secondApplication); String thirdApplication = RunIds.generate(System.currentTimeMillis()).getId(); - ApplicationId thirdApplicationId = new ApplicationId(NamespaceMeta.DEFAULT.getName(), thirdApplication); + ApplicationId thirdApplicationId = new ApplicationId(NamespaceMeta.DEFAULT.getName(), + thirdApplication); store.add(firstApplicationId, testRequest, null); store.add(secondApplicationId, testRequest, null); @@ -246,6 +256,39 @@ public void testPreviewTTL() throws Exception { // only thirdApplication should be in waiting now Assert.assertEquals(1, store.getAllInWaitingState().size()); - Assert.assertEquals(thirdApplicationId, store.getAllInWaitingState().iterator().next().getProgram().getParent()); + Assert.assertEquals(thirdApplicationId, + store.getAllInWaitingState().iterator().next().getProgram().getParent()); + } + + @Test + public void testGetApplicationIdForPollerInfo() throws Exception { + byte[] pollerInfo1 = Bytes.toBytes("runner-1"); + byte[] pollerInfo2 = Bytes.toBytes("runner-2"); + + PreviewConfig previewConfig = new PreviewConfig("WordCount", ProgramType.WORKFLOW, null, null); + AppRequest testRequest = new AppRequest<>(new ArtifactSummary("test", "1.0"), null, + previewConfig); + + // 1. Test success case: associate pollerInfo1 with applicationId1 + ApplicationId applicationId1 = new ApplicationId("ns1", RunIds.generate().getId()); + store.add(applicationId1, testRequest, null); + store.setPreviewRequestPollerInfo(applicationId1, pollerInfo1); + + // Verify that the mapping is correct + Assert.assertEquals(applicationId1, store.getApplicationId(pollerInfo1)); + + // 2. Test failure case: pollerInfo2 is not associated with any application + Assert.assertNull(store.getApplicationId(pollerInfo2)); + + // 3. Test re-association: associate pollerInfo1 with a new application + // This simulates a scenario where a runner with the same info is reused, overwriting the index. + ApplicationId applicationId2 = new ApplicationId("ns1", RunIds.generate().getId()); + store.add(applicationId2, testRequest, null); + store.setPreviewRequestPollerInfo(applicationId2, pollerInfo1); + Assert.assertEquals(applicationId2, store.getApplicationId(pollerInfo1)); + + // 4. Test removal: remove the application and verify the mapping is gone + store.remove(applicationId2); + Assert.assertNull(store.getApplicationId(pollerInfo1)); } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/TaskWorkerServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/TaskWorkerServiceTest.java index 0f757e5e4ea7..a701045a37fe 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/TaskWorkerServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/TaskWorkerServiceTest.java @@ -72,6 +72,8 @@ public class TaskWorkerServiceTest { private TaskWorkerService taskWorkerService; private CompletableFuture serviceCompletionFuture; + private SecurityManager securityManager; + private CConfiguration createCConf() { CConfiguration cConf = CConfiguration.create(); cConf.set(Constants.TaskWorker.ADDRESS, "localhost"); @@ -88,7 +90,7 @@ private SConfiguration createSConf() { } @Before - public void beforeTest() { + public void beforeTest() throws Exception { CConfiguration cConf = createCConf(); SConfiguration sConf = createSConf(); @@ -102,6 +104,8 @@ public void beforeTest() { // start the service taskWorkerService.startAndWait(); this.taskWorkerService = taskWorkerService; + securityManager = System.getSecurityManager(); + System.setSecurityManager(new NoExitSecurityManager()); } @After @@ -110,6 +114,7 @@ public void afterTest() { taskWorkerService.stopAndWait(); taskWorkerService = null; } + System.setSecurityManager(securityManager); } @Test @@ -496,4 +501,17 @@ public void run(RunnableTaskContext context) throws Exception { context.writeResult(context.getParam().getBytes(StandardCharsets.UTF_8)); } } + + public class NoExitSecurityManager extends SecurityManager { + + @Override + public void checkExit(int status) { + throw new SecurityException("System.exit attempted, status=" + status); + } + + @Override + public void checkPermission(java.security.Permission perm) { + // allow everything else + } + } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/capability/CapabilityManagementServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/capability/CapabilityManagementServiceTest.java index 041b66a63395..2f24fb2d850a 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/capability/CapabilityManagementServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/capability/CapabilityManagementServiceTest.java @@ -30,6 +30,8 @@ import io.cdap.cdap.api.artifact.ArtifactSummary; import io.cdap.cdap.app.program.ManifestFields; import io.cdap.cdap.app.program.ProgramDescriptor; +import io.cdap.cdap.app.runtime.ProgramRuntimeService; +import io.cdap.cdap.app.runtime.ProgramStateWriter; import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.common.id.Id; @@ -38,6 +40,7 @@ import io.cdap.cdap.common.test.PluginJarHelper; import io.cdap.cdap.internal.AppFabricTestHelper; import io.cdap.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms; +import io.cdap.cdap.internal.app.runtime.ProgramStartRequest; import io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository; import io.cdap.cdap.internal.app.services.ApplicationLifecycleService; import io.cdap.cdap.internal.app.services.ProgramLifecycleService; @@ -82,6 +85,8 @@ public class CapabilityManagementServiceTest extends AppFabricTestBase { private static ApplicationLifecycleService applicationLifecycleService; private static ProgramLifecycleService programLifecycleService; private static CapabilityStatusStore capabilityStatusStore; + private static ProgramStateWriter programStateWriter; + private static ProgramRuntimeService runtimeService; private static final Gson GSON = new Gson(); @BeforeClass @@ -93,6 +98,8 @@ public static void setup() { capabilityStatusStore = new CapabilityStatusStore(getInjector().getInstance(TransactionRunner.class)); applicationLifecycleService = getInjector().getInstance(ApplicationLifecycleService.class); programLifecycleService = getInjector().getInstance(ProgramLifecycleService.class); + programStateWriter = getInjector().getInstance(ProgramStateWriter.class); + runtimeService = getInjector().getInstance(ProgramRuntimeService.class); capabilityManagementService.stopAndWait(); } @@ -561,7 +568,7 @@ public void testProgramStart() throws Exception { }); Iterable programs = applicationWithPrograms.getPrograms(); for (ProgramDescriptor program : programs) { - programLifecycleService.start(program.getProgramId(), new HashMap<>(), false, false); + startProgram(program.getProgramId(), new HashMap<>()); } ProgramId programId = new ProgramId(applicationId, ProgramType.WORKFLOW, CapabilitySleepingWorkflowApp.SleepWorkflow.class.getSimpleName()); @@ -588,7 +595,7 @@ public void testProgramStart() throws Exception { //try starting programs for (ProgramDescriptor program : programs) { try { - programLifecycleService.start(program.getProgramId(), new HashMap<>(), false, false); + startProgram(program.getProgramId(), new HashMap<>()); Assert.fail("expecting exception"); } catch (CapabilityNotAvailableException ex) { //expecting exception @@ -641,7 +648,7 @@ public void testProgramWithPluginStart() throws Exception { }); Iterable programs = applicationWithPrograms.getPrograms(); for (ProgramDescriptor program : programs) { - programLifecycleService.start(program.getProgramId(), new HashMap<>(), false, false); + startProgram(program.getProgramId(), new HashMap<>()); } ProgramId programId = new ProgramId(applicationId, ProgramType.WORKFLOW, CapabilitySleepingWorkflowPluginApp.SleepWorkflow.class.getSimpleName()); @@ -668,7 +675,7 @@ public void testProgramWithPluginStart() throws Exception { //try starting programs for (ProgramDescriptor program : programs) { try { - programLifecycleService.start(program.getProgramId(), new HashMap<>(), false, false); + startProgram(program.getProgramId(), new HashMap<>()); Assert.fail("expecting exception"); } catch (CapabilityNotAvailableException ex) { //expecting exception @@ -819,4 +826,12 @@ private void deployArtifactAndApp(Class applicationClass, String appName, Str null, programId -> { }); } + + private void startProgram(ProgramId programId, Map overrides) + throws Exception { + ProgramStartRequest startRequest = programLifecycleService.prepareStart( + programId, overrides, false, false); + runtimeService.run(startRequest.getProgramDescriptor(), startRequest.getProgramOptions(), startRequest.getRunId()) + .getController(); + } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/events/StartProgramEventSubscriberTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/events/StartProgramEventSubscriberTest.java index 83f59b7e16f6..b36b6a0cb356 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/events/StartProgramEventSubscriberTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/events/StartProgramEventSubscriberTest.java @@ -27,7 +27,7 @@ import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.internal.app.services.ProgramLifecycleService; -import io.cdap.cdap.internal.app.services.RunRecordMonitorService; +import io.cdap.cdap.internal.app.services.FlowControlService; import io.cdap.cdap.internal.app.services.http.AppFabricTestBase; import io.cdap.cdap.internal.events.dummy.DummyEventReader; import io.cdap.cdap.internal.events.dummy.DummyEventReaderExtensionProvider; @@ -38,7 +38,6 @@ import java.util.Collection; import org.junit.Before; import org.junit.Test; -import org.mockito.Mock; import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,8 +48,8 @@ public class StartProgramEventSubscriberTest extends AppFabricTestBase { private static final Logger LOG = LoggerFactory.getLogger(StartProgramEventSubscriberTest.class); private ProgramLifecycleService lifecycleService; - private RunRecordMonitorService runRecordMonitorService; - private RunRecordMonitorService.Counter mockCounter; + private FlowControlService flowControlService; + private FlowControlService.Counter mockCounter; private CConfiguration cConf; private DummyEventReader eventReader; private Injector injector; @@ -59,16 +58,16 @@ public class StartProgramEventSubscriberTest extends AppFabricTestBase { @Before public void setup() { lifecycleService = Mockito.mock(ProgramLifecycleService.class); - runRecordMonitorService = Mockito.mock(RunRecordMonitorService.class); - mockCounter = Mockito.mock(RunRecordMonitorService.Counter.class); - Mockito.doReturn(mockCounter).when(runRecordMonitorService).getCount(); + flowControlService = Mockito.mock(FlowControlService.class); + mockCounter = Mockito.mock(FlowControlService.Counter.class); + Mockito.doReturn(mockCounter).when(flowControlService).getCounter(); cConf = CConfiguration.create(); eventReader = new DummyEventReader<>(mockedEvents()); injector = Guice.createInjector(new AbstractModule() { @Override protected void configure() { bind(ProgramLifecycleService.class).toInstance(lifecycleService); - bind(RunRecordMonitorService.class).toInstance(runRecordMonitorService); + bind(FlowControlService.class).toInstance(flowControlService); bind(CConfiguration.class).toInstance(cConf); bind(new TypeLiteral>() { }) diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/guice/AppFabricTestModule.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/guice/AppFabricTestModule.java index 45e75f1d9646..05231a57bb8d 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/guice/AppFabricTestModule.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/guice/AppFabricTestModule.java @@ -91,8 +91,9 @@ protected void configure() { install(RemoteAuthenticatorModules.getNoOpModule()); install(new IOModule()); install(new InMemoryDiscoveryModule()); - install(new AppFabricServiceRuntimeModule(cConf).getInMemoryModules()); - install(new MonitorHandlerModule(false)); + install(new AppFabricServiceRuntimeModule(cConf, AppFabricServiceRuntimeModule.ALL_SERVICE_TYPES) + .getInMemoryModules()); + install(new MonitorHandlerModule(false, cConf)); install(new ProgramRunnerRuntimeModule().getInMemoryModules()); install(new NonCustomLocationUnitTestModule()); install(new LocalLogAppenderModule()); diff --git a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/oauth/OAuthAccessToken.java b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/oauth/OAuthAccessToken.java new file mode 100644 index 000000000000..7afff882947b --- /dev/null +++ b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/oauth/OAuthAccessToken.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ + +package io.cdap.cdap.datapipeline.oauth; + +import com.google.common.base.Preconditions; + +/** + * OAuth access token, with related metadata required to retrieve a long-lived access token. + */ +public class OAuthAccessToken { + private final String accessToken; + + public OAuthAccessToken(String accessToken) { + this.accessToken = accessToken; + } + + public String getAccessToken() { + return accessToken; + } + + public static Builder newBuilder() { + return new Builder(); + } + + /** + * Builder class for {@link OAuthAccessToken}. + */ + public static class Builder { + private String accessToken; + + public Builder() {} + + public Builder withAccessToken(String accessToken) { + this.accessToken = accessToken; + return this; + } + + public OAuthAccessToken build() { + Preconditions.checkNotNull(accessToken, "OAuth access token missing"); + return new OAuthAccessToken(accessToken); + } + } +} diff --git a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/oauth/OAuthProvider.java b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/oauth/OAuthProvider.java index 9ba6b59e9358..0aeea2777325 100644 --- a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/oauth/OAuthProvider.java +++ b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/oauth/OAuthProvider.java @@ -26,18 +26,25 @@ public class OAuthProvider { private final String tokenRefreshURL; @Nullable private final OAuthClientCredentials clientCreds; + @Nullable private final CredentialEncodingStrategy strategy; + // Optional string to send as a USER_AGENT header + @Nullable + private final String userAgent; + public OAuthProvider(String name, String loginURL, String tokenRefreshURL, @Nullable OAuthClientCredentials clientCreds, - @Nullable CredentialEncodingStrategy strategy) { + @Nullable CredentialEncodingStrategy strategy, + @Nullable String userAgent) { this.name = name; this.loginURL = loginURL; this.tokenRefreshURL = tokenRefreshURL; this.clientCreds = clientCreds; this.strategy = strategy; + this.userAgent = userAgent; } public String getName() { @@ -57,12 +64,18 @@ public OAuthClientCredentials getClientCredentials() { return clientCreds; } + @Nullable public CredentialEncodingStrategy getCredentialEncodingStrategy() { return strategy; } + @Nullable + public String getUserAgent() { + return userAgent; + } + public enum CredentialEncodingStrategy { - // (default) Sends client ID & secret as part of the POST request body + // (default) Sends client ID & secret as part of the POST request body FORM_BODY, // Sends client ID & secret as part of a HTTP Basic Auth header BASIC_AUTH, @@ -81,6 +94,7 @@ public static class Builder { private String tokenRefreshURL; private OAuthClientCredentials clientCreds; private CredentialEncodingStrategy strategy; + private String userAgent; public Builder() {} @@ -109,6 +123,11 @@ public Builder withCredentialEncodingStrategy(@Nullable CredentialEncodingStrate return this; } + public Builder withUserAgent(@Nullable String userAgent) { + this.userAgent = userAgent; + return this; + } + public OAuthProvider build() { Preconditions.checkNotNull(name, "OAuth provider name missing"); Preconditions.checkNotNull(loginURL, "Login URL missing"); @@ -117,7 +136,7 @@ public OAuthProvider build() { if (strategy == null) { this.strategy = CredentialEncodingStrategy.FORM_BODY; } - return new OAuthProvider(name, loginURL, tokenRefreshURL, clientCreds, strategy); + return new OAuthProvider(name, loginURL, tokenRefreshURL, clientCreds, strategy, userAgent); } } } diff --git a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/oauth/OAuthStore.java b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/oauth/OAuthStore.java index a1a9a9718090..27bd024fbdbd 100644 --- a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/oauth/OAuthStore.java +++ b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/oauth/OAuthStore.java @@ -44,7 +44,10 @@ public class OAuthStore { private static final String OAUTH_PROVIDER_COL = "oauthprovider"; private static final String LOGIN_URL_COL = "loginurl"; private static final String TOKEN_REFRESH_URL_COL = "tokenrefreshurl"; + private static final String CREDENTIAL_ENCODING_STRATEGY_COL = "credentialencodingstrategy"; + private static final String USER_AGENT_COL = "useragent"; private static final String CLIENT_CREDS_KEY_PREFIX = "oauthclientcreds"; + private static final String ACCESS_TOKEN_KEY_PREFIX = "oauthaccesstoken"; private static final String REFRESH_TOKEN_KEY_PREFIX = "oauthrefreshtoken"; private static final Gson GSON = new Gson(); private final TransactionRunner transactionRunner; @@ -56,7 +59,9 @@ public class OAuthStore { .withId(TABLE_ID) .withFields(Fields.stringType(OAUTH_PROVIDER_COL), Fields.stringType(LOGIN_URL_COL), - Fields.stringType(TOKEN_REFRESH_URL_COL)) + Fields.stringType(TOKEN_REFRESH_URL_COL), + Fields.stringType(CREDENTIAL_ENCODING_STRATEGY_COL), + Fields.stringType(USER_AGENT_COL)) .withPrimaryKeys(OAUTH_PROVIDER_COL) .build(); @@ -151,6 +156,35 @@ public Optional getProvider(String name) throws OAuthStoreExcepti } } + /** + * Remove an OAuth provider. + * + * @param name name of {@link OAuthProvider} to read + * @throws OAuthStoreException if the read fails + */ + public void deleteProvider(String name) throws OAuthStoreException { + // Delete associated Client Credentials with given provider. + try { + secureStoreManager.delete(NamespaceId.SYSTEM.getNamespace(), getClientCredsKey(name)); + } catch (Exception e) { + // If key is not found, then we can safely delete provider. For any other exception, throw it. + if (!e.getClass().getName().contains("NotFoundException")) { + throw new OAuthStoreException("Failed to delete client credential from OAuth provider secure storage", e); + } + } + + try { + TransactionRunners.run(transactionRunner, context -> { + StructuredTable table = context.getTable(TABLE_ID); + table.delete(getKey(name)); + }, TableNotFoundException.class, InvalidFieldException.class); + } catch (TableNotFoundException e) { + throw new OAuthStoreException("OAuth provider table not found", e); + } catch (InvalidFieldException e) { + throw new OAuthStoreException("Failed to delete OAuth provider, object fields do not match table", e); + } + } + /** * Write an OAuth refresh token for the given provider and credential. * @@ -199,6 +233,57 @@ public Optional getRefreshToken(String oauthProvider, String } } + /** + * Write an OAuth access token for the given provider and credential. This is used for providers which do not provide + * a refresh token and instead opt for a permanent access token. + * + * @param oauthProvider name of OAuth provider the access token is sourced from + * @param credentialId ID used to identify this credential + * @param token the {@link OAuthAccessToken} to write + * @throws OAuthStoreException if the write fails + */ + public void writeAccessToken( + String oauthProvider, + String credentialId, + OAuthAccessToken token) throws OAuthStoreException { + String namespace = NamespaceId.SYSTEM.getNamespace(); + try { + secureStoreManager.put( + namespace, + getAccessTokenKey(oauthProvider, credentialId), + GSON.toJson(token), + "OAuth access token", + Collections.emptyMap()); + } catch (IOException e) { + throw new OAuthStoreException("Failed to write OAuth access token", e); + } catch (Exception e) { + throw new OAuthStoreException("Namespace \"" + namespace + "\" does not exist", e); + } + } + + /** + * Retrieve the {@link OAuthAccessToken} associated with the given OAuth provider and credential + * + * @param oauthProvider name of the OAuth provider the access token is sourced from + * @param credentialId ID used to identify this credential + * @throws OAuthStoreException if the read fails + */ + public Optional getAccessToken(String oauthProvider, String credentialId) + throws OAuthStoreException { + try { + String tokenJson = new String( + secureStore.getData(NamespaceId.SYSTEM.getNamespace(), getAccessTokenKey(oauthProvider, credentialId)), + StandardCharsets.UTF_8); + return Optional.of(GSON.fromJson(tokenJson, OAuthAccessToken.class)); + } catch (IOException e) { + throw new OAuthStoreException("Failed to read from OAuth access token secure storage", e); + } catch (JsonSyntaxException e) { + throw new OAuthStoreException("Invalid JSON for OAuth access token", e); + } catch (Exception e) { + return Optional.empty(); + } + } + private static String getClientCredsKey(String oauthProvider) { return String.format("%s-%s", CLIENT_CREDS_KEY_PREFIX, oauthProvider.toLowerCase()); } @@ -207,6 +292,10 @@ private static String getRefreshTokenKey(String oauthProvider, String credential return String.format("%s-%s-%s", REFRESH_TOKEN_KEY_PREFIX, oauthProvider.toLowerCase(), credentialId.toLowerCase()); } + private static String getAccessTokenKey(String oauthProvider, String credentialId) { + return String.format("%s-%s-%s", ACCESS_TOKEN_KEY_PREFIX, oauthProvider.toLowerCase(), credentialId.toLowerCase()); + } + private static List> getKey(String name) { List> keyFields = new ArrayList<>(1); keyFields.add(Fields.stringField(OAUTH_PROVIDER_COL, name)); @@ -217,12 +306,18 @@ private static OAuthProvider fromRow(StructuredRow row, OAuthClientCredentials c String name = row.getString(OAUTH_PROVIDER_COL); String loginURL = row.getString(LOGIN_URL_COL); String tokenRefreshURL = row.getString(TOKEN_REFRESH_URL_COL); + String credentialEncodingStrategy = row.getString(CREDENTIAL_ENCODING_STRATEGY_COL); + String userAgent = row.getString(USER_AGENT_COL); return OAuthProvider.newBuilder() .withName(name) .withLoginURL(loginURL) .withTokenRefreshURL(tokenRefreshURL) .withClientCredentials(clientCreds) + .withCredentialEncodingStrategy( + Optional.ofNullable(credentialEncodingStrategy) + .map(OAuthProvider.CredentialEncodingStrategy::valueOf).orElse(null)) + .withUserAgent(userAgent) .build(); } @@ -231,6 +326,10 @@ private static List> getRow(OAuthProvider oauthProvider) { fields.add(Fields.stringField(OAUTH_PROVIDER_COL, oauthProvider.getName())); fields.add(Fields.stringField(LOGIN_URL_COL, oauthProvider.getLoginURL())); fields.add(Fields.stringField(TOKEN_REFRESH_URL_COL, oauthProvider.getTokenRefreshURL())); + fields.add(Fields.stringField( + CREDENTIAL_ENCODING_STRATEGY_COL, + oauthProvider.getCredentialEncodingStrategy().toString())); + fields.add(Fields.stringField(USER_AGENT_COL, oauthProvider.getUserAgent())); return fields; } } diff --git a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/oauth/PutOAuthProviderRequest.java b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/oauth/PutOAuthProviderRequest.java index 72616896706b..b872f881ae56 100644 --- a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/oauth/PutOAuthProviderRequest.java +++ b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/oauth/PutOAuthProviderRequest.java @@ -26,22 +26,21 @@ public class PutOAuthProviderRequest { private final String clientId; private final String clientSecret; private final OAuthProvider.CredentialEncodingStrategy strategy; - - public PutOAuthProviderRequest(String loginURL, String tokenRefreshURL, String clientId, String clientSecret) { - this(loginURL, tokenRefreshURL, clientId, clientSecret, OAuthProvider.CredentialEncodingStrategy.FORM_BODY); - } + private final String userAgent; public PutOAuthProviderRequest( String loginURL, String tokenRefreshURL, String clientId, String clientSecret, - OAuthProvider.CredentialEncodingStrategy strategy) { + OAuthProvider.CredentialEncodingStrategy strategy, + String userAgent) { this.loginURL = loginURL; this.tokenRefreshURL = tokenRefreshURL; this.clientId = clientId; this.clientSecret = clientSecret; this.strategy = strategy; + this.userAgent = userAgent; } public String getLoginURL() { @@ -63,4 +62,8 @@ public String getClientSecret() { public OAuthProvider.CredentialEncodingStrategy getCredentialEncodingStrategy() { return strategy; } + + public String getUserAgent() { + return userAgent; + } } diff --git a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/service/ErrorHandlingGsonTypeAdapterFactory.java b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/service/ErrorHandlingGsonTypeAdapterFactory.java new file mode 100644 index 000000000000..5efc2595442e --- /dev/null +++ b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/service/ErrorHandlingGsonTypeAdapterFactory.java @@ -0,0 +1,52 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.datapipeline.service; + +import com.google.gson.Gson; +import com.google.gson.TypeAdapter; +import com.google.gson.TypeAdapterFactory; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; + +import java.io.IOException; + +/* + * TypeAdapter factory for GSON which will skip any fields which have a parsing error + */ +public class ErrorHandlingGsonTypeAdapterFactory implements TypeAdapterFactory { + public final TypeAdapter create(Gson gson, TypeToken type) { + TypeAdapter delegate = gson.getDelegateAdapter(this, type); + + return new TypeAdapter() { + @Override + public void write(JsonWriter writer, T value) throws IOException { + delegate.write(writer, value); + } + + @Override + public T read(JsonReader reader) throws IOException { + try { + return delegate.read(reader); + } catch (Exception e) { + reader.skipValue(); + return null; + } + } + }; + } +} diff --git a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/service/OAuthHandler.java b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/service/OAuthHandler.java index e61d89594077..9f8b064c1c04 100644 --- a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/service/OAuthHandler.java +++ b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/service/OAuthHandler.java @@ -26,6 +26,7 @@ import io.cdap.cdap.api.service.http.SystemHttpServiceContext; import io.cdap.cdap.datapipeline.oauth.CredentialIsValidResponse; import io.cdap.cdap.datapipeline.oauth.GetAccessTokenResponse; +import io.cdap.cdap.datapipeline.oauth.OAuthAccessToken; import io.cdap.cdap.datapipeline.oauth.OAuthClientCredentials; import io.cdap.cdap.datapipeline.oauth.OAuthProvider; import io.cdap.cdap.datapipeline.oauth.OAuthProvider.CredentialEncodingStrategy; @@ -46,6 +47,7 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Optional; +import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.PUT; @@ -65,6 +67,7 @@ public class OAuthHandler extends AbstractSystemHttpServiceHandler { private static final String API_VERSION = "v1"; private static final Gson GSON = new GsonBuilder() .setPrettyPrinting() + .registerTypeAdapterFactory(new ErrorHandlingGsonTypeAdapterFactory()) .create(); private OAuthStore oauthStore; @@ -118,6 +121,7 @@ public void putOAuthProvider(HttpServiceRequest request, HttpServiceResponder re StandardCharsets.UTF_8.decode(request.getContent()).toString(), PutOAuthProviderRequest.class); CredentialEncodingStrategy strategy = putOAuthProviderRequest.getCredentialEncodingStrategy(); + String userAgent = putOAuthProviderRequest.getUserAgent(); // Validate URLs URL loginURL = new URL(putOAuthProviderRequest.getLoginURL()); URL tokenRefreshURL = new URL(putOAuthProviderRequest.getTokenRefreshURL()); @@ -136,6 +140,7 @@ public void putOAuthProvider(HttpServiceRequest request, HttpServiceResponder re .withTokenRefreshURL(tokenRefreshURL.toString()) .withClientCredentials(clientCredentials) .withCredentialEncodingStrategy(strategy) + .withUserAgent(userAgent) .build(); oauthStore.writeProvider(provider, reuseClientCredentials); responder.sendStatus(HttpURLConnection.HTTP_OK); @@ -153,6 +158,24 @@ public void putOAuthProvider(HttpServiceRequest request, HttpServiceResponder re } } + @DELETE + @Path(API_VERSION + "/oauth/provider/{provider}") + public void deleteOAuthProvider(HttpServiceRequest request, HttpServiceResponder responder, + @PathParam("provider") String oauthProvider) { + try { + try { + oauthStore.deleteProvider(oauthProvider); + responder.sendStatus(HttpURLConnection.HTTP_OK); + } catch (NullPointerException e) { + throw new OAuthServiceException(HttpURLConnection.HTTP_BAD_REQUEST, "Invalid provider: " + e.getMessage(), e); + } catch (OAuthStoreException e) { + throw new OAuthServiceException(HttpURLConnection.HTTP_INTERNAL_ERROR, "Failed to delete OAuth provider.", e); + } + } catch (OAuthServiceException e) { + e.respond(responder); + } + } + @PUT @Path(API_VERSION + "/oauth/provider/{provider}/credential/{credential}") public void putOAuthCredential(HttpServiceRequest request, HttpServiceResponder responder, @@ -206,21 +229,46 @@ public void putOAuthCredential(HttpServiceRequest request, HttpServiceResponder HttpURLConnection.HTTP_INTERNAL_ERROR, "Failed to parse JSON: " + e.getMessage(), e); } - if (refreshTokenResponse.getRefreshToken() == null || refreshTokenResponse.getRefreshToken().isEmpty()) { + boolean hasRefreshToken = refreshTokenResponse.getRefreshToken() != null + && !refreshTokenResponse.getRefreshToken().isEmpty(); + boolean hasAccessToken = refreshTokenResponse.getAccessToken() != null + && !refreshTokenResponse.getAccessToken().isEmpty(); + + if (!hasAccessToken && !hasRefreshToken) { throw new OAuthServiceException( - HttpURLConnection.HTTP_INTERNAL_ERROR, "Refresh token response body did not contain refresh token"); + HttpURLConnection.HTTP_BAD_REQUEST, + String.format( + "Refresh token response is missing the required access token or refresh token. " + + "The actual response received: %s", + response.getResponseBodyAsString())); } - try { - OAuthRefreshToken refreshToken = OAuthRefreshToken.newBuilder() - .withRefreshToken(refreshTokenResponse.getRefreshToken()) - .withRedirectURI(putOAuthCredentialRequest.getRedirectURI()) - .build(); - oauthStore.writeRefreshToken(provider, credentialId, refreshToken); - } catch (NullPointerException e) { - throw new OAuthServiceException(HttpURLConnection.HTTP_INTERNAL_ERROR, e.getMessage(), e); - } catch (OAuthStoreException e) { - throw new OAuthServiceException(HttpURLConnection.HTTP_INTERNAL_ERROR, "Failed to write refresh token", e); + if (hasRefreshToken) { + try { + OAuthRefreshToken refreshToken = OAuthRefreshToken.newBuilder() + .withRefreshToken(refreshTokenResponse.getRefreshToken()) + .withRedirectURI(putOAuthCredentialRequest.getRedirectURI()) + .build(); + oauthStore.writeRefreshToken(provider, credentialId, refreshToken); + } catch (NullPointerException e) { + throw new OAuthServiceException(HttpURLConnection.HTTP_BAD_REQUEST, e.getMessage(), e); + } catch (OAuthStoreException e) { + throw new OAuthServiceException(HttpURLConnection.HTTP_BAD_REQUEST, "Failed to write refresh token", e); + } + } else { + // Refresh token call gave us an access token without a refresh token. + // Store the access token instead. + + try { + OAuthAccessToken accessToken = OAuthAccessToken.newBuilder() + .withAccessToken(refreshTokenResponse.getAccessToken()) + .build(); + oauthStore.writeAccessToken(provider, credentialId, accessToken); + } catch (NullPointerException e) { + throw new OAuthServiceException(HttpURLConnection.HTTP_BAD_REQUEST, e.getMessage(), e); + } catch (OAuthStoreException e) { + throw new OAuthServiceException(HttpURLConnection.HTTP_BAD_REQUEST, "Failed to write access token", e); + } } responder.sendStatus(HttpURLConnection.HTTP_OK); @@ -229,6 +277,14 @@ public void putOAuthCredential(HttpServiceRequest request, HttpServiceResponder } } + /** + * If a refresh token is stored, use it to request a short-lived access token from the 3rd-party OAuth API. + * If a long-lived access token is stored, return it. + * @param request + * @param responder + * @param provider ID of OAuth provider + * @param credentialId ID of stored credential + */ @GET @Path(API_VERSION + "/oauth/provider/{provider}/credential/{credential}") public void getOAuthCredential(HttpServiceRequest request, HttpServiceResponder responder, @@ -236,6 +292,17 @@ public void getOAuthCredential(HttpServiceRequest request, HttpServiceResponder @PathParam("credential") String credentialId) { try { OAuthProvider oauthProvider = getProvider(provider); + Optional oAuthAccessToken = getAccessToken(provider, credentialId); + + // If found, send the long-lived access token + if (oAuthAccessToken.isPresent()) { + responder.sendString(GSON.toJson( + new GetAccessTokenResponse(oAuthAccessToken.get().getAccessToken(), ""))); + return; + } + + // If no long-lived access token was found, request a short-lived access token from the 3rd-party API using the + // stored refresh token OAuthRefreshToken refreshToken = getRefreshToken(provider, credentialId); HttpResponse response; @@ -247,7 +314,7 @@ public void getOAuthCredential(HttpServiceRequest request, HttpServiceResponder if (response.getResponseCode() != 200) { throw new OAuthServiceException( - response.getResponseCode(), + response.getResponseCode(), "Request for refresh token did not return 200. Response code: " + response.getResponseCode() + " , response message: " @@ -262,9 +329,33 @@ public void getOAuthCredential(HttpServiceRequest request, HttpServiceResponder } catch (JsonSyntaxException e) { throw new OAuthServiceException(HttpURLConnection.HTTP_INTERNAL_ERROR, "Error parsing JSON response", e); } - if (refreshTokenResponse.getAccessToken() == null || refreshTokenResponse.getAccessToken().isEmpty()) { + + boolean hasRefreshToken = refreshTokenResponse.getRefreshToken() != null + && !refreshTokenResponse.getRefreshToken().isEmpty(); + boolean hasAccessToken = refreshTokenResponse.getAccessToken() != null + && !refreshTokenResponse.getAccessToken().isEmpty(); + + if (!hasAccessToken) { throw new OAuthServiceException( - HttpURLConnection.HTTP_INTERNAL_ERROR, "Refresh token response body does not have refresh token"); + HttpURLConnection.HTTP_BAD_REQUEST, + String.format( + "Access token response body does not have access token. The actual response received : %s", + response.getResponseBodyAsString())); + } + + // API has given us a new refresh token + if (hasRefreshToken && !refreshToken.getRefreshToken().equals(refreshTokenResponse.getRefreshToken())) { + OAuthRefreshToken newRefreshToken = OAuthRefreshToken.newBuilder() + .withRefreshToken(refreshTokenResponse.getRefreshToken()) + .withRedirectURI(refreshToken.getRedirectURI()) + .build(); + + try { + oauthStore.writeRefreshToken(provider, credentialId, newRefreshToken); + } catch (OAuthStoreException e) { + throw new OAuthServiceException( + HttpURLConnection.HTTP_INTERNAL_ERROR, "An error occurred while writing the new refresh token"); + } } responder.sendString(GSON.toJson( @@ -281,6 +372,13 @@ public void getOAuthCredentialValidity(HttpServiceRequest request, HttpServiceRe @PathParam("credential") String credentialId) { try { OAuthProvider oauthProvider = getProvider(provider); + Optional oAuthAccessToken = getAccessToken(provider, credentialId); + + if (oAuthAccessToken.isPresent()) { + responder.sendString(GSON.toJson(new CredentialIsValidResponse(true))); + return; + } + OAuthRefreshToken refreshToken = getRefreshToken(provider, credentialId); HttpResponse response; @@ -348,7 +446,8 @@ private HttpRequest.Builder buildHttpRequest(String body, CredentialEncodingStrategy strategy, OAuthClientCredentials clientCreds, String refreshTokenURL, - boolean addContentType) throws MalformedURLException { + boolean addContentType, + String userAgent) throws MalformedURLException { HttpRequest.Builder requestBuilder = HttpRequest.post(new URL(refreshTokenURL)) .withBody(body); @@ -360,6 +459,10 @@ private HttpRequest.Builder buildHttpRequest(String body, requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, getBasicAuthHeader(clientCreds)); } + if (userAgent != null) { + requestBuilder.addHeader(HttpHeaders.USER_AGENT, userAgent); + } + return requestBuilder; } @@ -375,9 +478,10 @@ private HttpRequest createGetRefreshTokenRequest(OAuthProvider provider, String CredentialEncodingStrategy strategy = provider.getCredentialEncodingStrategy(); String tokenRefreshURL = provider.getTokenRefreshURL(); String body = buildRequestBody(strategy, "authorization_code", code, redirectURI, null, clientCreds); + String userAgent = provider.getUserAgent(); try { - return buildHttpRequest(body, strategy, clientCreds, tokenRefreshURL, true).build(); + return buildHttpRequest(body, strategy, clientCreds, tokenRefreshURL, true, userAgent).build(); } catch (MalformedURLException e) { throw new OAuthServiceException(HttpURLConnection.HTTP_INTERNAL_ERROR, "Malformed URL", e); } @@ -394,9 +498,10 @@ private HttpRequest createGetAccessTokenRequest(OAuthProvider provider, String r CredentialEncodingStrategy strategy = provider.getCredentialEncodingStrategy(); String tokenRefreshURL = provider.getTokenRefreshURL(); String body = buildRequestBody(strategy, "refresh_token", null, null, refreshToken, clientCreds); + String userAgent = provider.getUserAgent(); try { - return buildHttpRequest(body, strategy, clientCreds, tokenRefreshURL, false).build(); + return buildHttpRequest(body, strategy, clientCreds, tokenRefreshURL, false, userAgent).build(); } catch (MalformedURLException e) { throw new OAuthServiceException(HttpURLConnection.HTTP_INTERNAL_ERROR, "Malformed URL", e); } @@ -404,7 +509,7 @@ private HttpRequest createGetAccessTokenRequest(OAuthProvider provider, String r private String getBasicAuthHeader(OAuthClientCredentials clientCreds) { String authInfo = String.format("%s:%s", clientCreds.getClientId(), clientCreds.getClientSecret()); - return String.format("Basic %s", Base64.getEncoder().encode(authInfo.getBytes())); + return String.format("Basic %s", Base64.getEncoder().encodeToString(authInfo.getBytes())); } private OAuthProvider getProvider(String provider) throws OAuthServiceException { @@ -420,6 +525,13 @@ private OAuthProvider getProvider(String provider) throws OAuthServiceException } } + /** + * Fetch a refresh token from the secure store + * @param provider + * @param credentialId + * @return a long-lived refresh token stored in the secure store + * @throws OAuthServiceException + */ private OAuthRefreshToken getRefreshToken(String provider, String credentialId) throws OAuthServiceException { try { Optional refreshTokenOptional = oauthStore.getRefreshToken(provider, credentialId); @@ -433,6 +545,23 @@ private OAuthRefreshToken getRefreshToken(String provider, String credentialId) } } + /** + * Fetch a long-lived access token from the secure store. + * @param provider + * @param credentialId + * @return a long-lived access token stored in the secure store + * @throws OAuthServiceException + */ + private Optional getAccessToken(String provider, String credentialId) + throws OAuthServiceException { + try { + return oauthStore.getAccessToken(provider, credentialId); + } catch (OAuthStoreException e) { + throw new OAuthServiceException( + HttpURLConnection.HTTP_INTERNAL_ERROR, "Failed to read OAuth access token from secure store", e); + } + } + private static class OAuthServiceException extends Exception { private final int status; @@ -446,6 +575,10 @@ private static class OAuthServiceException extends Exception { this.status = status; } + int getStatus() { + return this.status; + } + void respond(HttpServiceResponder responder) { if (status == HttpURLConnection.HTTP_INTERNAL_ERROR) { LOG.error("An internal error has occurred", this); diff --git a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/test/java/io/cdap/cdap/datapipeline/OAuthServiceTest.java b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/test/java/io/cdap/cdap/datapipeline/OAuthServiceTest.java index 0be3fc1f6928..b99e8f77dedd 100644 --- a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/test/java/io/cdap/cdap/datapipeline/OAuthServiceTest.java +++ b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/test/java/io/cdap/cdap/datapipeline/OAuthServiceTest.java @@ -44,7 +44,13 @@ public void testCreateProvider() throws IOException { String tokenRefreshURL = "http://www.example.com/token"; String clientId = "clientid"; String clientSecret = "clientsecret"; - PutOAuthProviderRequest request = new PutOAuthProviderRequest(loginURL, tokenRefreshURL, clientId, clientSecret); + PutOAuthProviderRequest request = new PutOAuthProviderRequest( + loginURL, + tokenRefreshURL, + clientId, + clientSecret, + OAuthProvider.CredentialEncodingStrategy.FORM_BODY, + null); HttpResponse createResponse = makePutCall("provider/testprovider", request); Assert.assertEquals(200, createResponse.getResponseCode()); @@ -60,7 +66,13 @@ public void testCreateProviderWithClientCredentialsMissing() throws IOException // Attempt to create provider with missing client credentials should fail with 400 status code. String loginURL = "http://www.example.com/login"; String tokenRefreshURL = "http://www.example.com/token"; - PutOAuthProviderRequest request = new PutOAuthProviderRequest(loginURL, tokenRefreshURL, null, null); + PutOAuthProviderRequest request = new PutOAuthProviderRequest( + loginURL, + tokenRefreshURL, + null, + null, + OAuthProvider.CredentialEncodingStrategy.FORM_BODY, + null); HttpResponse createResponse = makePutCall("provider/testprovider", request); Assert.assertEquals(400, createResponse.getResponseCode()); } @@ -71,7 +83,13 @@ public void testCreateProviderWithReuseClientCredentialsTrue() throws IOExceptio // param 'true' should succeed with 200 status code. String loginURL = "http://www.example.com/login"; String tokenRefreshURL = "http://www.example.com/token"; - PutOAuthProviderRequest request = new PutOAuthProviderRequest(loginURL, tokenRefreshURL, null, null); + PutOAuthProviderRequest request = new PutOAuthProviderRequest( + loginURL, + tokenRefreshURL, + null, + null, + OAuthProvider.CredentialEncodingStrategy.FORM_BODY, + null); HttpResponse createResponse = makePutCall("provider/testprovider10?reuse_client_credentials=true", request); Assert.assertEquals(500, createResponse.getResponseCode()); } @@ -83,7 +101,13 @@ public void testCreateProviderReuseCredentialsWithReuseClientCredentialsTrue() t String tokenRefreshURL = "http://www.example.com/token20"; String clientId = "clientid"; String clientSecret = "clientsecret"; - PutOAuthProviderRequest request = new PutOAuthProviderRequest(loginURL, tokenRefreshURL, clientId, clientSecret); + PutOAuthProviderRequest request = new PutOAuthProviderRequest( + loginURL, + tokenRefreshURL, + clientId, + clientSecret, + OAuthProvider.CredentialEncodingStrategy.FORM_BODY, + null); HttpResponse createResponse = makePutCall("provider/testprovider20", request); Assert.assertEquals(200, createResponse.getResponseCode()); @@ -91,7 +115,13 @@ public void testCreateProviderReuseCredentialsWithReuseClientCredentialsTrue() t // param 'true' should succeed with 200 status code. loginURL = "http://www.example.com/login21"; tokenRefreshURL = "http://www.example.com/token21"; - request = new PutOAuthProviderRequest(loginURL, tokenRefreshURL, null, null); + request = new PutOAuthProviderRequest( + loginURL, + tokenRefreshURL, + null, + null, + OAuthProvider.CredentialEncodingStrategy.FORM_BODY, + null); createResponse = makePutCall("provider/testprovider20?reuse_client_credentials=true", request); Assert.assertEquals(200, createResponse.getResponseCode()); } @@ -102,7 +132,13 @@ public void testCreateProviderWithReuseClientCredentialsFalse() throws IOExcepti // query param 'false' should fail with 400 status code. String loginURL = "http://www.example.com/login30"; String tokenRefreshURL = "http://www.example.com/token30"; - PutOAuthProviderRequest request = new PutOAuthProviderRequest(loginURL, tokenRefreshURL, null, null); + PutOAuthProviderRequest request = new PutOAuthProviderRequest( + loginURL, + tokenRefreshURL, + null, + null, + OAuthProvider.CredentialEncodingStrategy.FORM_BODY, + null); HttpResponse createResponse = makePutCall("provider/testprovider30?reuse_client_credentials=false", request); Assert.assertEquals(400, createResponse.getResponseCode()); } @@ -119,7 +155,8 @@ public void testCreateProviderWithBasicAuth() throws IOException { tokenRefreshURL, clientId, clientSecret, - OAuthProvider.CredentialEncodingStrategy.BASIC_AUTH); + OAuthProvider.CredentialEncodingStrategy.BASIC_AUTH, + null); HttpResponse createOauthProviderResponse = makePutCall("provider/testprovider31", request); Assert.assertEquals(200, createOauthProviderResponse.getResponseCode()); @@ -130,13 +167,43 @@ public void testCreateProviderWithBasicAuth() throws IOException { Assert.assertEquals("http://www.example.com/login31?client_id=clientid&redirect_uri=null", authURL); } + @Test + public void testCreateProviderWithBasicAuthAndUserAgent() throws IOException { + // Attempt to create provider + String loginURL = "http://www.example.com/login32"; + String tokenRefreshURL = "http://www.example.com/token32"; + String clientId = "clientid"; + String clientSecret = "clientsecret"; + PutOAuthProviderRequest request = new PutOAuthProviderRequest( + loginURL, + tokenRefreshURL, + clientId, + clientSecret, + OAuthProvider.CredentialEncodingStrategy.BASIC_AUTH, + "cdap-test"); + HttpResponse createOauthProviderResponse = makePutCall("provider/testprovider32", request); + Assert.assertEquals(200, createOauthProviderResponse.getResponseCode()); + + // Grab OAuth login URL to verify write succeeded + HttpResponse getAuthUrlResponse = makeGetCall("provider/testprovider32/authurl"); + Assert.assertEquals(200, getAuthUrlResponse.getResponseCode()); + String authURL = getAuthUrlResponse.getResponseBodyAsString(); + Assert.assertEquals("http://www.example.com/login32?client_id=clientid&redirect_uri=null", authURL); + } + @Test public void testGetAuthURLForMissingClientCredentials() throws IOException { // Attempt to create provider with missing client credentials and 'reuse_client_credentials' // query param 'true'. String loginURL = "http://www.example.com/login40"; String tokenRefreshURL = "http://www.example.com/token40"; - PutOAuthProviderRequest request = new PutOAuthProviderRequest(loginURL, tokenRefreshURL, null, null); + PutOAuthProviderRequest request = new PutOAuthProviderRequest( + loginURL, + tokenRefreshURL, + null, + null, + OAuthProvider.CredentialEncodingStrategy.FORM_BODY, + null); HttpResponse createResponse = makePutCall("provider/testprovider40?reuse_client_credentials=false", request); Assert.assertEquals(400, createResponse.getResponseCode()); @@ -152,7 +219,13 @@ public void testGetAuthURLForReusedClientCredentials() throws IOException { String tokenRefreshURL = "http://www.example.com/token50"; String clientId = "clientid"; String clientSecret = "clientsecret"; - PutOAuthProviderRequest request = new PutOAuthProviderRequest(loginURL, tokenRefreshURL, clientId, clientSecret); + PutOAuthProviderRequest request = new PutOAuthProviderRequest( + loginURL, + tokenRefreshURL, + clientId, + clientSecret, + OAuthProvider.CredentialEncodingStrategy.FORM_BODY, + null); HttpResponse createResponse = makePutCall("provider/testprovider50", request); Assert.assertEquals(200, createResponse.getResponseCode()); @@ -166,7 +239,13 @@ public void testGetAuthURLForReusedClientCredentials() throws IOException { // param 'true' should succeed with 200 status code. loginURL = "http://www.example.com/login51"; tokenRefreshURL = "http://www.example.com/token51"; - request = new PutOAuthProviderRequest(loginURL, tokenRefreshURL, null, null); + request = new PutOAuthProviderRequest( + loginURL, + tokenRefreshURL, + null, + null, + OAuthProvider.CredentialEncodingStrategy.FORM_BODY, + null); createResponse = makePutCall("provider/testprovider50?reuse_client_credentials=true", request); Assert.assertEquals(200, createResponse.getResponseCode()); @@ -184,7 +263,13 @@ public void testCreateProviderBadLoginURL() throws IOException { String tokenRefreshURL = "http://www.example.com/token"; String clientId = "clientid"; String clientSecret = "clientsecret"; - PutOAuthProviderRequest request = new PutOAuthProviderRequest(loginURL, tokenRefreshURL, clientId, clientSecret); + PutOAuthProviderRequest request = new PutOAuthProviderRequest( + loginURL, + tokenRefreshURL, + clientId, + clientSecret, + OAuthProvider.CredentialEncodingStrategy.FORM_BODY, + null); HttpResponse createResponse = makePutCall("provider/testprovider", request); Assert.assertEquals(400, createResponse.getResponseCode()); } @@ -196,7 +281,13 @@ public void testCreateProviderBadTokenRefreshURL() throws IOException { String tokenRefreshURL = "badurl"; String clientId = "clientid"; String clientSecret = "clientsecret"; - PutOAuthProviderRequest request = new PutOAuthProviderRequest(loginURL, tokenRefreshURL, clientId, clientSecret); + PutOAuthProviderRequest request = new PutOAuthProviderRequest( + loginURL, + tokenRefreshURL, + clientId, + clientSecret, + OAuthProvider.CredentialEncodingStrategy.FORM_BODY, + null); HttpResponse createResponse = makePutCall("provider/testprovider", request); Assert.assertEquals(400, createResponse.getResponseCode()); } diff --git a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/test/java/io/cdap/cdap/datapipeline/OAuthStoreTest.java b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/test/java/io/cdap/cdap/datapipeline/OAuthStoreTest.java new file mode 100644 index 000000000000..1e5ed3f34b1f --- /dev/null +++ b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/test/java/io/cdap/cdap/datapipeline/OAuthStoreTest.java @@ -0,0 +1,180 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ + +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +import org.junit.Before; +import org.junit.Test; + +import io.cdap.cdap.api.security.store.SecureStore; +import io.cdap.cdap.api.security.store.SecureStoreManager; +import io.cdap.cdap.datapipeline.oauth.OAuthAccessToken; +import io.cdap.cdap.datapipeline.oauth.OAuthProvider; +import io.cdap.cdap.datapipeline.oauth.OAuthRefreshToken; +import io.cdap.cdap.datapipeline.oauth.OAuthStore; +import io.cdap.cdap.datapipeline.oauth.OAuthStoreException; +import io.cdap.cdap.spi.data.StructuredRow; +import io.cdap.cdap.spi.data.StructuredTable; +import io.cdap.cdap.spi.data.StructuredTableContext; +import io.cdap.cdap.spi.data.transaction.TransactionRunner; +import io.cdap.cdap.spi.data.transaction.TxRunnable; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class OAuthStoreTest { + private OAuthStore oauthStore; + private TransactionRunner mockTransactionRunner; + private SecureStore mockSecureStore; + private SecureStoreManager mockSecureStoreManager; + private StructuredTable mockTable; + private StructuredRow mockRow; + + private static final String PROVIDER_NAME = "test-provider"; + private static final String TOKEN_REFRESH_URL = "http://token.example.com"; + private static final String LOGIN_URL = "http://login.example.com"; + private static final String USER_AGENT = "test-Agent"; + + @Before + public void setUp() { + mockTransactionRunner = mock(TransactionRunner.class); + mockSecureStore = mock(SecureStore.class); + mockSecureStoreManager = mock(SecureStoreManager.class); + mockTable = mock(StructuredTable.class); + mockRow = mock(StructuredRow.class); + + oauthStore = new OAuthStore(mockTransactionRunner, mockSecureStore, mockSecureStoreManager); + } + + @Test + public void testGetProviderWithNullCredentialStrategy() throws Exception { + String clientCredsJson = "{\"clientId\":\"test-client\",\"clientSecret\":\"test-secret\"}"; + when(mockSecureStore.getData(any(), any())).thenReturn( + clientCredsJson.getBytes(StandardCharsets.UTF_8)); + + doAnswer(invocation -> { + TxRunnable runnable = invocation.getArgument(0); + StructuredTableContext mockContext = mock(StructuredTableContext.class); + when(mockContext.getTable(any())).thenReturn(mockTable); + runnable.run(mockContext); + return null; + }).when(mockTransactionRunner).run(any(TxRunnable.class)); + + when(mockRow.getString("oauthprovider")).thenReturn(PROVIDER_NAME); + when(mockRow.getString("loginurl")).thenReturn(LOGIN_URL); + when(mockRow.getString("tokenrefreshurl")).thenReturn(TOKEN_REFRESH_URL); + when(mockRow.getString("credentialencodingstrategy")).thenReturn(null); + when(mockRow.getString("useragent")).thenReturn(USER_AGENT); + + when(mockTable.read(any())).thenReturn(Optional.of(mockRow)); + + Optional provider = oauthStore.getProvider(PROVIDER_NAME); + + assertTrue(provider.isPresent()); + assertEquals(provider.get().getCredentialEncodingStrategy(), + OAuthProvider.CredentialEncodingStrategy.FORM_BODY); + } + + @Test + public void testWriteRefreshToken() throws Exception { + doNothing().when(mockSecureStoreManager).put(anyString(), anyString(), any(), anyString(), any()); + + OAuthRefreshToken token = OAuthRefreshToken.newBuilder() + .withRefreshToken("muhtoken") + .withRedirectURI("uri") + .build(); + oauthStore.writeRefreshToken("Provider", "ID0", token); + + verify(mockSecureStoreManager, times(1)) + .put(eq("system"), eq("oauthrefreshtoken-provider-id0"), any(), eq("OAuth refresh token"), any()); + } + + @Test + public void testWriteAccessToken() throws Exception { + doNothing().when(mockSecureStoreManager).put(anyString(), anyString(), any(), anyString(), any()); + + OAuthAccessToken token = OAuthAccessToken.newBuilder() + .withAccessToken("muhtoken") + .build(); + oauthStore.writeAccessToken("Provider", "ID0", token); + + verify(mockSecureStoreManager, times(1)) + .put(eq("system"), eq("oauthaccesstoken-provider-id0"), any(), eq("OAuth access token"), any()); + } + + @Test + public void testDeleteProvider() throws Exception { + doNothing().when(mockSecureStoreManager).delete(any(), any()); + + doAnswer(invocation -> { + TxRunnable runnable = invocation.getArgument(0); + StructuredTableContext mockContext = mock(StructuredTableContext.class); + when(mockContext.getTable(any())).thenReturn(mockTable); + runnable.run(mockContext); + return null; + }).when(mockTransactionRunner).run(any(TxRunnable.class)); + + doNothing().when(mockTable).delete(any()); + + oauthStore.deleteProvider(PROVIDER_NAME); + verify(mockSecureStoreManager, times(1)).delete(any(), any()); + } + + @Test + public void testDeleteProviderWhenSecureKeyNotFound() throws Exception { + class NotFoundException extends Exception { + public NotFoundException(String message) { + super(message); + } + } + + doAnswer(invocation -> { + TxRunnable runnable = invocation.getArgument(0); + StructuredTableContext mockContext = mock(StructuredTableContext.class); + when(mockContext.getTable(any())).thenReturn(mockTable); + runnable.run(mockContext); + return null; + }).when(mockTransactionRunner).run(any(TxRunnable.class)); + doNothing().when(mockTable).delete(any()); + + // CASE 1 : When Secure keys not found, the provider should be deleted. + doThrow(new NotFoundException("Keys not found.")).when(mockSecureStoreManager).delete(any(), any()); + oauthStore.deleteProvider(PROVIDER_NAME); + verify(mockSecureStoreManager, times(1)).delete(any(), any()); + verify(mockTable, times(1)).delete(any()); + + // CASE 2 : When secure keys were not deleted because of any reason, the provider should NOT be deleted. + org.mockito.Mockito.clearInvocations(mockSecureStoreManager); + org.mockito.Mockito.clearInvocations(mockTable); + doThrow(new Exception("Unable to delete secure key")).when(mockSecureStoreManager).delete(any(), any()); + try { + oauthStore.deleteProvider(PROVIDER_NAME); + } catch (Exception e) { + assertEquals(e.getClass(), OAuthStoreException.class); + } + verify(mockSecureStoreManager, times(1)).delete(any(), any()); + verify(mockTable, times(0)).delete(any()); + } +} diff --git a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/test/java/io/cdap/cdap/datapipeline/service/ErrorHandlingGsonTypeAdapterFactoryTest.java b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/test/java/io/cdap/cdap/datapipeline/service/ErrorHandlingGsonTypeAdapterFactoryTest.java new file mode 100644 index 000000000000..9b95b3b3f771 --- /dev/null +++ b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/test/java/io/cdap/cdap/datapipeline/service/ErrorHandlingGsonTypeAdapterFactoryTest.java @@ -0,0 +1,47 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.datapipeline.service; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.datapipeline.oauth.RefreshTokenResponse; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Tests for ErrorHandlingGsonTypeAdapterFactory + */ +public class ErrorHandlingGsonTypeAdapterFactoryTest { + private static final Gson GSON = new GsonBuilder() + .setPrettyPrinting() + .registerTypeAdapterFactory(new ErrorHandlingGsonTypeAdapterFactory()) + .create(); + + // Ensure parsing doesn't throw when one of the items is not a String + @Test + public void refreshTokenParseTest() { + String json = "{" + + " \"access_token\": \"asdf1234\"," + + " \"scope\": [1, 2, 3]" + + "}"; + RefreshTokenResponse refreshTokenResponse = GSON.fromJson(json, RefreshTokenResponse.class); + + Assert.assertEquals(refreshTokenResponse.getAccessToken(), "asdf1234"); + Assert.assertNull(refreshTokenResponse.getScope()); + } +} diff --git a/cdap-app-templates/cdap-etl/cdap-data-pipeline3_2.12/pom.xml b/cdap-app-templates/cdap-etl/cdap-data-pipeline3_2.12/pom.xml index 443a1ab7e17b..3b1ebbe3455b 100644 --- a/cdap-app-templates/cdap-etl/cdap-data-pipeline3_2.12/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-data-pipeline3_2.12/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap-etl - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-data-pipeline3_2.12 @@ -99,6 +99,11 @@ ${project.version} test + + org.mockito + mockito-core + test + io.cdap.common common-http diff --git a/cdap-app-templates/cdap-etl/cdap-data-streams-base/src/main/java/io/cdap/cdap/datastreams/SparkStreamingPipelineRunner.java b/cdap-app-templates/cdap-etl/cdap-data-streams-base/src/main/java/io/cdap/cdap/datastreams/SparkStreamingPipelineRunner.java index 762444b403d4..0cb2ebd272ab 100644 --- a/cdap-app-templates/cdap-etl/cdap-data-streams-base/src/main/java/io/cdap/cdap/datastreams/SparkStreamingPipelineRunner.java +++ b/cdap-app-templates/cdap-etl/cdap-data-streams-base/src/main/java/io/cdap/cdap/datastreams/SparkStreamingPipelineRunner.java @@ -16,8 +16,10 @@ package io.cdap.cdap.datastreams; +import com.google.common.base.Throwables; import io.cdap.cdap.api.Transactionals; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.exception.WrappedStageException; import io.cdap.cdap.api.macro.MacroEvaluator; import io.cdap.cdap.api.plugin.PluginContext; import io.cdap.cdap.api.preview.DataTracer; @@ -313,10 +315,24 @@ protected void processDag(PhaseSpec phaseSpec, String sourcePluginType, JavaSpar //process the remaining stages for (int i = 1; i < topologicalOrder.size(); i++) { String stageName = topologicalOrder.get(i); - processStage(phaseSpec, sourcePluginType, sec, stagePartitions, pluginContext, collectors, - pipelinePhase, functionCacheFactory, macroEvaluator, emittedRecords, groupedDag, - groups, branchers, shufflers, sinkRunnables, stageName, time.milliseconds(), context, - sinkRunnableProvider); + try { + processStage(phaseSpec, sourcePluginType, sec, stagePartitions, pluginContext, + collectors, + pipelinePhase, functionCacheFactory, macroEvaluator, emittedRecords, groupedDag, + groups, branchers, shufflers, sinkRunnables, stageName, time.milliseconds(), + context, + sinkRunnableProvider); + } catch (Exception e) { + List causalChain = Throwables.getCausalChain(e); + for (Throwable t : causalChain) { + if (t instanceof WrappedStageException) { + // avoid double wrapping + throw e; + } + } + // this can occur in cases like `joins` where we do `SparkCollection#join` + throw new WrappedStageException(e, stageName); + } } }, Exception.class); diff --git a/cdap-app-templates/cdap-etl/cdap-data-streams3_2.12/pom.xml b/cdap-app-templates/cdap-etl/cdap-data-streams3_2.12/pom.xml index c5f62a61fdc4..bc961c9f8e14 100644 --- a/cdap-app-templates/cdap-etl/cdap-data-streams3_2.12/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-data-streams3_2.12/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap-etl - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-data-streams3_2.12 diff --git a/cdap-app-templates/cdap-etl/cdap-etl-api-spark/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-api-spark/pom.xml index bc7e32a758f3..6b0fde959c21 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-api-spark/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-api-spark/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap-etl - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-etl-api-spark diff --git a/cdap-app-templates/cdap-etl/cdap-etl-api/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-api/pom.xml index 6707ace3e0b8..213b81acd1da 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-api/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-api/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap-etl - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-etl-api diff --git a/cdap-app-templates/cdap-etl/cdap-etl-archetypes/cdap-data-pipeline-plugins-archetype/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-archetypes/cdap-data-pipeline-plugins-archetype/pom.xml index a827776d380e..7b98416028a8 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-archetypes/cdap-data-pipeline-plugins-archetype/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-archetypes/cdap-data-pipeline-plugins-archetype/pom.xml @@ -21,7 +21,7 @@ io.cdap.cdap cdap-etl-archetypes - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-data-pipeline-plugins-archetype diff --git a/cdap-app-templates/cdap-etl/cdap-etl-archetypes/cdap-data-pipeline-plugins-archetype/src/main/resources/archetype-resources/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-archetypes/cdap-data-pipeline-plugins-archetype/src/main/resources/archetype-resources/pom.xml index 64b63e113356..89fce2789f1f 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-archetypes/cdap-data-pipeline-plugins-archetype/src/main/resources/archetype-resources/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-archetypes/cdap-data-pipeline-plugins-archetype/src/main/resources/archetype-resources/pom.xml @@ -28,7 +28,7 @@ UTF-8 3.1.1 - 6.11.0-SNAPSHOT + 6.11.0 2.3.0 widgets @@ -40,11 +40,11 @@ sonatype - https://oss.sonatype.org/content/groups/public + https://ossrh-staging-api.central.sonatype.com/content/groups/public sonatype-snapshots - https://oss.sonatype.org/content/repositories/snapshots + https://central.sonatype.com/repository/maven-snapshots diff --git a/cdap-app-templates/cdap-etl/cdap-etl-archetypes/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-archetypes/pom.xml index c35709665e8e..e4c4efc311c9 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-archetypes/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-archetypes/pom.xml @@ -22,7 +22,7 @@ io.cdap.cdap cdap-etl - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-etl-archetypes diff --git a/cdap-app-templates/cdap-etl/cdap-etl-batch/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-batch/pom.xml index e4a97210b2a5..67a176a89d2d 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-batch/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-batch/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap-etl - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-etl-batch diff --git a/cdap-app-templates/cdap-etl/cdap-etl-core/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-core/pom.xml index da1f851c45a0..a50b1d0b792b 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-core/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-core/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap-etl - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-etl-core @@ -61,6 +61,11 @@ ${project.version} test + + org.mockito + mockito-core + test + diff --git a/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/batch/connector/ConnectorSource.java b/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/batch/connector/ConnectorSource.java index 9ddbb1f9d708..306c01e8f201 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/batch/connector/ConnectorSource.java +++ b/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/batch/connector/ConnectorSource.java @@ -1,5 +1,5 @@ /* - * Copyright © 2016 Cask Data, Inc. + * Copyright © 2025 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -24,6 +24,7 @@ import io.cdap.cdap.etl.api.batch.BatchSource; import io.cdap.cdap.etl.api.batch.BatchSourceContext; import io.cdap.cdap.etl.common.Constants; +import java.io.IOException; import java.util.HashMap; import java.util.Map; import org.apache.hadoop.io.LongWritable; @@ -31,6 +32,9 @@ import org.apache.hadoop.mapreduce.lib.input.CombineTextInputFormat; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.twill.filesystem.Location; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Internal batch source used as a connector between pipeline phases. Though this extends @@ -48,6 +52,7 @@ */ public class ConnectorSource extends BatchSource { + private static final Logger LOG = LoggerFactory.getLogger(ConnectorSource.class); // you can't read from the basedir of a FileSet so adding an arbitrary directory where data will be stored/read. static final String DATA_DIR = "data"; private final String datasetName; @@ -74,4 +79,32 @@ public void prepareRun(BatchSourceContext context) { context.setInput(Input.ofDataset(datasetName, arguments)); } + @Override + public void onRunFinish(boolean succeeded, BatchSourceContext context) { + super.onRunFinish(succeeded, context); + try { + cleanupDataset(context); + } catch (Exception e) { + LOG.warn("Failed to clean up source dataset '{}'.", datasetName, e); + } + } + + /** + * Deletes the underlying FileSet data. + */ + private void cleanupDataset(BatchSourceContext context) throws IOException { + FileSet fileSet = context.getDataset(datasetName); + Location baseLocation = fileSet.getBaseLocation(); + if (baseLocation == null || !baseLocation.exists()) { + LOG.info("Base location does not exist for dataset '{}'. Skipping cleanup.", datasetName); + return; + } + + if (!baseLocation.delete(true)) { + throw new IOException( + String.format("Failed to delete file(s) at location '%s'.", baseLocation)); + } + + LOG.info("Successfully cleaned up file(s) at location '{}'.", baseLocation); + } } diff --git a/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/AbstractServiceRetryableMacroEvaluator.java b/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/AbstractServiceRetryableMacroEvaluator.java index f1248e45df86..684d62972945 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/AbstractServiceRetryableMacroEvaluator.java +++ b/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/AbstractServiceRetryableMacroEvaluator.java @@ -20,6 +20,11 @@ import com.google.common.base.Stopwatch; import com.google.common.io.ByteStreams; import com.google.common.io.CharStreams; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorCategory.ErrorCategoryEnum; +import io.cdap.cdap.api.exception.ErrorCodeType; +import io.cdap.cdap.api.exception.ErrorUtils; +import io.cdap.cdap.api.exception.ErrorUtils.ActionErrorPair; import io.cdap.cdap.api.macro.InvalidMacroException; import io.cdap.cdap.api.macro.MacroEvaluator; import io.cdap.cdap.api.retry.RetryableException; @@ -45,8 +50,10 @@ abstract class AbstractServiceRetryableMacroEvaluator implements MacroEvaluator private static final double RETRY_RANDOMIZE_FACTOR = 0.1d; private final String functionName; + private final String serviceName; - AbstractServiceRetryableMacroEvaluator(String functionName) { + AbstractServiceRetryableMacroEvaluator(String serviceName, String functionName) { + this.serviceName = serviceName; this.functionName = functionName; } @@ -54,15 +61,18 @@ abstract class AbstractServiceRetryableMacroEvaluator implements MacroEvaluator public String lookup(String property) throws InvalidMacroException { throw new InvalidMacroException("The '" + functionName + "' macro function doesn't support direct property lookup for property '" - + property + "'"); + + property + "'", new ErrorCategory(ErrorCategoryEnum.MACROS, + String.format("%s-%s", serviceName, functionName))); } @Override public String evaluate(String macroFunction, String... args) throws InvalidMacroException { if (!functionName.equals(macroFunction)) { // This shouldn't happen - throw new IllegalArgumentException( - "Invalid function name " + macroFunction + ". Expecting " + functionName); + throw new InvalidMacroException( + "Invalid function name " + macroFunction + ". Expecting " + functionName, + new ErrorCategory(ErrorCategoryEnum.MACROS, + String.format("%s-%s", serviceName, functionName))); } long delay = RETRY_BASE_DELAY_MILLIS; @@ -85,8 +95,9 @@ public String evaluate(String macroFunction, String... args) throws InvalidMacro (long) (delay * (minMultiplier + Math.random() * (maxMultiplier - minMultiplier + 1))); delay = Math.min(delay, RETRY_MAX_DELAY_MILLIS); - } catch (IOException e) { - throw new InvalidMacroException(e); + } catch (IOException | HttpResponseException e) { + throw new InvalidMacroException(e, new ErrorCategory(ErrorCategoryEnum.MACROS, + String.format("%s-%s", serviceName, functionName))); } } } catch (InterruptedException e) { @@ -129,8 +140,9 @@ public Map evaluateMap(String macroFunction, String... args) throws InvalidMacroException { if (!functionName.equals(macroFunction)) { // This shouldn't happen - throw new IllegalArgumentException("Invalid function name " + macroFunction - + ". Expecting " + functionName); + throw new InvalidMacroException("Invalid function name " + macroFunction + + ". Expecting " + functionName, new ErrorCategory(ErrorCategoryEnum.MACROS, + String.format("%s-%s", serviceName, functionName))); } // Make call with exponential delay on failure retry. @@ -149,6 +161,15 @@ public Map evaluateMap(String macroFunction, String... args) delay = (long) (delay * (minMultiplier + Math.random() * (maxMultiplier - minMultiplier + 1))); delay = Math.min(delay, RETRY_MAX_DELAY_MILLIS); + } catch (HttpResponseException e) { + int responseCode = e.getResponseCode(); + ActionErrorPair pair = ErrorUtils.getActionErrorByStatusCode(responseCode); + String errorReason = String.format("Failed to call %s service with status '%s'. %s", + serviceName, responseCode, pair.getCorrectiveAction()); + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.MACROS, + String.format("%s-%s", serviceName, functionName)), errorReason, e.getMessage(), + pair.getErrorType(), false, ErrorCodeType.HTTP, String.valueOf(responseCode), null, + e); } catch (IOException e) { throw new RuntimeException("Failed to evaluate the macro function '" + functionName + "' with args " + Arrays.asList(args), e); @@ -166,7 +187,7 @@ public Map evaluateMap(String macroFunction, String... args) } protected String validateAndRetrieveContent(String serviceName, - HttpURLConnection urlConn) throws IOException { + HttpURLConnection urlConn) throws HttpResponseException { if (urlConn == null) { throw new RetryableException(serviceName + " service is not available"); } @@ -182,7 +203,7 @@ protected String validateAndRetrieveContent(String serviceName, } private void validateResponseCode(String serviceName, HttpURLConnection urlConn) - throws IOException { + throws HttpResponseException { int responseCode; try { responseCode = urlConn.getResponseCode(); @@ -195,15 +216,15 @@ private void validateResponseCode(String serviceName, HttpURLConnection urlConn) throw new RetryableException( serviceName + " service is not available with status " + responseCode); } - throw new IOException( + throw new HttpResponseException( "Failed to call " + serviceName + " service with status " + responseCode + ": " - + getError(urlConn)); + + getError(urlConn), responseCode); } } abstract Map evaluateMacroMap( String macroFunction, String... args) - throws InvalidMacroException, IOException, RetryableException; + throws InvalidMacroException, IOException, RetryableException, HttpResponseException; abstract String evaluateMacro( String macroFunction, String... args) @@ -222,4 +243,16 @@ private String getError(HttpURLConnection urlConn) { return "Unknown error due to failure to read from error output: " + e.getMessage(); } } + + private static class HttpResponseException extends RuntimeException { + private final int responseCode; + + private HttpResponseException(String message, int responseCode) { + super(message); + this.responseCode = responseCode; + } + + int getResponseCode() { + return responseCode;} + } } diff --git a/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/ConnectionMacroEvaluator.java b/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/ConnectionMacroEvaluator.java index 2ac7bc121f9d..008f67794e4a 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/ConnectionMacroEvaluator.java +++ b/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/ConnectionMacroEvaluator.java @@ -19,6 +19,8 @@ import com.google.gson.Gson; import io.cdap.cdap.api.ServiceDiscoverer; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorCategory.ErrorCategoryEnum; import io.cdap.cdap.api.macro.InvalidMacroException; import io.cdap.cdap.api.macro.MacroEvaluator; import io.cdap.cdap.api.retry.RetryableException; @@ -44,7 +46,7 @@ public class ConnectionMacroEvaluator extends AbstractServiceRetryableMacroEvalu private final Gson gson; public ConnectionMacroEvaluator(String namespace, ServiceDiscoverer serviceDiscoverer) { - super(FUNCTION_NAME); + super(SERVICE_NAME, FUNCTION_NAME); this.namespace = namespace; this.serviceDiscoverer = serviceDiscoverer; this.gson = new Gson(); @@ -63,7 +65,9 @@ Map evaluateMacroMap(String macroFunction, String... args) throws InvalidMacroException, IOException, RetryableException { if (args.length != 1) { throw new InvalidMacroException( - "Macro '" + FUNCTION_NAME + "' should have exactly 1 arguments"); + "Macro '" + FUNCTION_NAME + "' should have exactly 1 arguments", + new ErrorCategory(ErrorCategoryEnum.MACROS, String.format("%s-%s", SERVICE_NAME, + FUNCTION_NAME))); } // only encode the connection name here since / will get encoded to %2f and some router cannot recognize it diff --git a/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/OAuthAccessTokenMacroEvaluator.java b/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/OAuthAccessTokenMacroEvaluator.java index 7aaaf7a4627f..bdf3a950da9a 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/OAuthAccessTokenMacroEvaluator.java +++ b/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/OAuthAccessTokenMacroEvaluator.java @@ -18,6 +18,8 @@ import com.google.gson.Gson; import io.cdap.cdap.api.ServiceDiscoverer; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorCategory.ErrorCategoryEnum; import io.cdap.cdap.api.macro.InvalidMacroException; import io.cdap.cdap.api.macro.MacroEvaluator; import io.cdap.cdap.api.retry.RetryableException; @@ -39,7 +41,7 @@ public class OAuthAccessTokenMacroEvaluator extends AbstractServiceRetryableMacr private final Gson gson; public OAuthAccessTokenMacroEvaluator(ServiceDiscoverer serviceDiscoverer) { - super(FUNCTION_NAME); + super(SERVICE_NAME, FUNCTION_NAME); this.serviceDiscoverer = serviceDiscoverer; this.gson = new Gson(); } @@ -66,7 +68,8 @@ public String evaluateMacro(String macroFunction, String... args) RetryableException { if (args.length != 2) { throw new InvalidMacroException( - "Macro '" + FUNCTION_NAME + "' should have exactly 2 arguments"); + "Macro '" + FUNCTION_NAME + "' should have exactly 2 arguments", new ErrorCategory( + ErrorCategoryEnum.MACROS, String.format("%s-%s", SERVICE_NAME, FUNCTION_NAME))); } return getAccessToken(args[0], args[1]); diff --git a/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/OAuthMacroEvaluator.java b/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/OAuthMacroEvaluator.java index 18501433f72c..e3cf99d62369 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/OAuthMacroEvaluator.java +++ b/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/OAuthMacroEvaluator.java @@ -43,7 +43,7 @@ public class OAuthMacroEvaluator extends AbstractServiceRetryableMacroEvaluator private final Gson gson; public OAuthMacroEvaluator(ServiceDiscoverer serviceDiscoverer) { - super(FUNCTION_NAME); + super(SERVICE_NAME, FUNCTION_NAME); this.serviceDiscoverer = serviceDiscoverer; this.gson = new Gson(); } diff --git a/cdap-app-templates/cdap-etl/cdap-etl-core/src/test/java/io/cdap/cdap/etl/batch/connector/ConnectorSourceTest.java b/cdap-app-templates/cdap-etl/cdap-etl-core/src/test/java/io/cdap/cdap/etl/batch/connector/ConnectorSourceTest.java new file mode 100644 index 000000000000..e5bbdf0d5aea --- /dev/null +++ b/cdap-app-templates/cdap-etl/cdap-etl-core/src/test/java/io/cdap/cdap/etl/batch/connector/ConnectorSourceTest.java @@ -0,0 +1,98 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.etl.batch.connector; + +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.cdap.cdap.api.dataset.lib.FileSet; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import java.io.IOException; +import org.apache.twill.filesystem.Location; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +/** + * Unit tests for the {@link ConnectorSource} class. + */ +@RunWith(MockitoJUnitRunner.class) +public class ConnectorSourceTest { + + private static final String TEST_DATASET_NAME = "testConnectorData"; + + @Mock + private BatchSourceContext mockContext; + + @Mock + private FileSet mockFileSet; + + @Mock + private Location mockLocation; + + private ConnectorSource connectorSource; + + @Before + public void setUp() { + connectorSource = new ConnectorSource<>(TEST_DATASET_NAME); + } + + @Test + public void testOnRunFinish_whenRunSucceeds_shouldDeleteLocation() throws IOException { + when(mockContext.getDataset(TEST_DATASET_NAME)).thenReturn(mockFileSet); + when(mockFileSet.getBaseLocation()).thenReturn(mockLocation); + when(mockLocation.exists()).thenReturn(true); + when(mockLocation.delete(true)).thenReturn(true); + connectorSource.onRunFinish(true, mockContext); + + verify(mockContext).getDataset(TEST_DATASET_NAME); + verify(mockLocation).delete(true); + } + + @Test + public void testOnRunFinish_whenBaseLocationDoesNotExist_shouldSkipDelete() throws IOException { + when(mockContext.getDataset(TEST_DATASET_NAME)).thenReturn(mockFileSet); + when(mockFileSet.getBaseLocation()).thenReturn(mockLocation); + when(mockLocation.exists()).thenReturn(false); + connectorSource.onRunFinish(true, mockContext); + + verify(mockLocation, never()).delete(true); + } + + @Test + public void testOnRunFinish_whenBaseLocationIsNull_shouldSkipDelete() { + when(mockContext.getDataset(TEST_DATASET_NAME)).thenReturn(mockFileSet); + when(mockFileSet.getBaseLocation()).thenReturn(null); + connectorSource.onRunFinish(true, mockContext); + + verify(mockFileSet).getBaseLocation(); + } + + @Test + public void testOnRunFinish_whenDeletionFails_shouldLogWarningAndNotThrow() throws IOException { + when(mockContext.getDataset(TEST_DATASET_NAME)).thenReturn(mockFileSet); + when(mockFileSet.getBaseLocation()).thenReturn(mockLocation); + when(mockLocation.exists()).thenReturn(true); + when(mockLocation.delete(true)).thenReturn(false); + connectorSource.onRunFinish(true, mockContext); + + verify(mockLocation).delete(true); + } +} diff --git a/cdap-app-templates/cdap-etl/cdap-etl-proto/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-proto/pom.xml index b3177da560fb..b61e009b29e4 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-proto/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-proto/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap-etl - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-etl-proto diff --git a/cdap-app-templates/cdap-etl/cdap-etl-tools/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-tools/pom.xml index cdec7a8030f0..1142cf019672 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-tools/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-tools/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap-etl - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-etl-tools diff --git a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/OpaqueDatasetCollection.java b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/OpaqueDatasetCollection.java index 371bbef134e2..df9c84996a62 100644 --- a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/OpaqueDatasetCollection.java +++ b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/OpaqueDatasetCollection.java @@ -16,6 +16,9 @@ package io.cdap.cdap.etl.spark.batch; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.util.concurrent.UncheckedExecutionException; import io.cdap.cdap.api.data.DatasetContext; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; @@ -26,6 +29,10 @@ import io.cdap.cdap.etl.spark.function.FunctionCache; import io.cdap.cdap.etl.spark.join.JoinExpressionRequest; import io.cdap.cdap.etl.spark.join.JoinRequest; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.concurrent.ExecutionException; + import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.MapFunction; @@ -47,6 +54,10 @@ */ public class OpaqueDatasetCollection extends DatasetCollection { + private static final Cache> ENCODER_CACHE = CacheBuilder.newBuilder() + .maximumSize(128) + .build(); + private final Dataset dataset; private OpaqueDatasetCollection(Dataset dataset, @@ -108,12 +119,46 @@ public SparkCollection join(JoinExpressionRequest joinRequest) { @Override public DataframeCollection toDataframeCollection(Schema schema) { StructType sparkSchema = DataFrames.toDataType(schema); - ExpressionEncoder encoder = RowEncoder.apply(sparkSchema); + ExpressionEncoder encoder = getRowEncoder(sparkSchema); Dataset ds = (Dataset) getDataset(); MapFunction converter = r -> DataFrames.toRow(r, sparkSchema); return new DataframeCollection(schema, ds.map(converter, encoder), sec, jsc, sqlContext, datasetContext, sinkFactory, functionCacheFactory); } + /** + * This is required to handle breaking changes between spark 3.3.2 (Dataproc 2.1) to 3.5.1 (Dataproc 2.2). + * And we need to support both. + * Here we are trying to check if the new method introduced in 3.5.1 exists or not, and based on that we + * invoke the new method or the old one. + */ + public static ExpressionEncoder getRowEncoder(StructType sparkSchema) { + + try { + return ENCODER_CACHE.get(sparkSchema, () -> { + StringBuilder errorStrBuilder = new StringBuilder("Failed to load a suitable Encoder dynamically. Errors : "); + try { + Method encoderForMethod = RowEncoder.class.getMethod("encoderFor", StructType.class); + Object agnosticEncoderObj = encoderForMethod.invoke(null, sparkSchema); + Method applyMethod = ExpressionEncoder.class.getMethod("apply", + Class.forName("org.apache.spark.sql.catalyst.encoders.AgnosticEncoder")); + return (ExpressionEncoder)applyMethod.invoke(null, agnosticEncoderObj); + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | + ClassNotFoundException e) { + errorStrBuilder.append(System.lineSeparator()).append(e.getMessage()); + } + + // If code reaches here, meaning it should be spark 3.3.2 or lower. + try { + return RowEncoder.apply(sparkSchema); + } catch (Exception e) { + errorStrBuilder.append(System.lineSeparator()).append(e.getMessage()); + } + throw new RuntimeException(errorStrBuilder.toString()); + }); + } catch (ExecutionException e) { + throw new UncheckedExecutionException(e); + } + } } diff --git a/cdap-app-templates/cdap-etl/hydrator-spark-core3_2.12/pom.xml b/cdap-app-templates/cdap-etl/hydrator-spark-core3_2.12/pom.xml index 567cfa22638b..386a45ece583 100644 --- a/cdap-app-templates/cdap-etl/hydrator-spark-core3_2.12/pom.xml +++ b/cdap-app-templates/cdap-etl/hydrator-spark-core3_2.12/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap-etl - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT hydrator-spark-core3_2.12 diff --git a/cdap-app-templates/cdap-etl/hydrator-test/pom.xml b/cdap-app-templates/cdap-etl/hydrator-test/pom.xml index 0e83f22c790f..26f5c3cb8ad0 100644 --- a/cdap-app-templates/cdap-etl/hydrator-test/pom.xml +++ b/cdap-app-templates/cdap-etl/hydrator-test/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap-etl - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT hydrator-test diff --git a/cdap-app-templates/cdap-etl/pom.xml b/cdap-app-templates/cdap-etl/pom.xml index aa9f149084ea..13af5ef69bbd 100644 --- a/cdap-app-templates/cdap-etl/pom.xml +++ b/cdap-app-templates/cdap-etl/pom.xml @@ -21,7 +21,7 @@ io.cdap.cdap cdap-app-templates - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT 4.0.0 diff --git a/cdap-app-templates/cdap-program-report/pom.xml b/cdap-app-templates/cdap-program-report/pom.xml index d7bf14b5592f..db631db9dea8 100644 --- a/cdap-app-templates/cdap-program-report/pom.xml +++ b/cdap-app-templates/cdap-program-report/pom.xml @@ -22,7 +22,7 @@ io.cdap.cdap cdap-app-templates - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT 4.0.0 diff --git a/cdap-app-templates/pom.xml b/cdap-app-templates/pom.xml index 2782e0f52f4c..10f58d71316b 100644 --- a/cdap-app-templates/pom.xml +++ b/cdap-app-templates/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-app-templates diff --git a/cdap-authenticator-ext-gcp/pom.xml b/cdap-authenticator-ext-gcp/pom.xml index 5c6752c264cf..7e4170883962 100644 --- a/cdap-authenticator-ext-gcp/pom.xml +++ b/cdap-authenticator-ext-gcp/pom.xml @@ -21,7 +21,7 @@ cdap io.cdap.cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT 4.0.0 diff --git a/cdap-cli-tests/pom.xml b/cdap-cli-tests/pom.xml index 05c00d0a834d..65e4d7feb603 100644 --- a/cdap-cli-tests/pom.xml +++ b/cdap-cli-tests/pom.xml @@ -20,7 +20,7 @@ cdap io.cdap.cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT 4.0.0 @@ -140,11 +140,14 @@ - org.apache.maven.plugins - maven-deploy-plugin - 2.8 + org.sonatype.central + central-publishing-maven-plugin + true - true + sonatype.release + false + true + true diff --git a/cdap-cli/pom.xml b/cdap-cli/pom.xml index 86d725311fdf..1ece19aa4416 100644 --- a/cdap-cli/pom.xml +++ b/cdap-cli/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-cli @@ -288,34 +288,6 @@ exec-maven-plugin 1.3.1 - - - - - org.apache.maven.plugins - maven-deploy-plugin - 2.8 - - - deploy-rpm - deploy - - deploy-file - - - ${project.version} - ${dist.deploy.groupId} - ${project.artifactId} - noarch.rpm - false - ${project.build.directory}/${project.artifactId}-${package.version}-1.noarch.rpm - 1 - continuuity - ${deploy.url} - - - - @@ -329,33 +301,6 @@ exec-maven-plugin 1.3.1 - - - - - org.apache.maven.plugins - maven-deploy-plugin - 2.8 - - - deploy-deb - deploy - - deploy-file - - - ${project.version} - ${dist.deploy.groupId} - ${project.artifactId} - deb - false - ${project.build.directory}/${project.artifactId}_${package.version}-1_all.deb - continuuity - ${deploy.url} - - - - @@ -369,33 +314,6 @@ maven-assembly-plugin 2.4 - - - - - org.apache.maven.plugins - maven-deploy-plugin - 2.8 - - - deploy-tgz - deploy - - deploy-file - - - ${project.version} - ${dist.deploy.groupId} - ${project.artifactId} - tar.gz - false - ${project.build.directory}/${project.artifactId}-${package.version}.tar.gz - continuuity - ${deploy.url} - - - - diff --git a/cdap-client-tests/pom.xml b/cdap-client-tests/pom.xml index 249cfe5a580f..03a2da8e18a9 100644 --- a/cdap-client-tests/pom.xml +++ b/cdap-client-tests/pom.xml @@ -22,7 +22,7 @@ the License. cdap io.cdap.cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-client-tests @@ -149,11 +149,14 @@ the License. - org.apache.maven.plugins - maven-deploy-plugin - 2.8 + org.sonatype.central + central-publishing-maven-plugin + true - true + sonatype.release + false + true + true diff --git a/cdap-client-tests/src/test/java/io/cdap/cdap/client/AbstractClientTest.java b/cdap-client-tests/src/test/java/io/cdap/cdap/client/AbstractClientTest.java index e9e9ca6d3975..15e29b687e9c 100644 --- a/cdap-client-tests/src/test/java/io/cdap/cdap/client/AbstractClientTest.java +++ b/cdap-client-tests/src/test/java/io/cdap/cdap/client/AbstractClientTest.java @@ -102,7 +102,7 @@ public void setUp() throws Throwable { .setVerifySSLCert(false) .setDefaultReadTimeout(60 * 1000) .setUploadReadTimeout(120 * 1000) - .setConnectionConfig(connectionConfig).build(); + .setConnectionConfig(connectionConfig).setAppListPageSize(25).build(); } protected ClientConfig getClientConfig() { diff --git a/cdap-client-tests/src/test/java/io/cdap/cdap/client/ApplicationClientTestRun.java b/cdap-client-tests/src/test/java/io/cdap/cdap/client/ApplicationClientTestRun.java index f7e728be2bc9..649b56310023 100644 --- a/cdap-client-tests/src/test/java/io/cdap/cdap/client/ApplicationClientTestRun.java +++ b/cdap-client-tests/src/test/java/io/cdap/cdap/client/ApplicationClientTestRun.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import com.google.gson.JsonObject; import io.cdap.cdap.ConfigTestApp; import io.cdap.cdap.api.Config; import io.cdap.cdap.api.artifact.ArtifactSummary; @@ -163,6 +164,42 @@ public void testAppConfig() throws Exception { } } + @Test + public void testPaginatedList() throws Exception { + ApplicationId app = NamespaceId.DEFAULT.app(FakeApp.NAME); + for (int i = 0; i < 30; i++) { + appClient.deploy(NamespaceId.DEFAULT, createAppJarFile(FakeApp.class, + FakeApp.NAME, "1.0.0-SNAPSHOT")); + ApplicationDetail appDetail = appClient.get(app); + app = new ApplicationId(app.getNamespace(), app.getApplication(), appDetail.getAppVersion()); + appClient.waitForDeployed(app, 30, TimeUnit.SECONDS); + } + Assert.assertEquals(30, appClient.list(NamespaceId.DEFAULT).size()); + + int count = 0; + String token = null; + boolean isLastPage = false; + int currentResultSize = 0; + while (!isLastPage) { + JsonObject result = appClient.paginatedList(NamespaceId.DEFAULT, token); + currentResultSize = result.get("applications").getAsJsonArray().size(); + count += currentResultSize; + token = + result.get("nextPageToken") == null ? null : result.get("nextPageToken").getAsString(); + isLastPage = (token == null); + if (!isLastPage) { + Assert.assertEquals(25, currentResultSize); + } + } + + Assert.assertEquals(5, currentResultSize); + Assert.assertEquals(30, count); + + appClient.deleteAll(NamespaceId.DEFAULT); + appClient.waitForDeleted(app, 30, TimeUnit.SECONDS); + Assert.assertEquals(0, appClient.list(NamespaceId.DEFAULT).size()); + } + @Test public void testAppUpdate() throws Exception { String artifactName = "cfg-programs"; diff --git a/cdap-client/pom.xml b/cdap-client/pom.xml index 902a01866d67..0406acbf3d72 100644 --- a/cdap-client/pom.xml +++ b/cdap-client/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-client diff --git a/cdap-client/src/main/java/io/cdap/cdap/client/ApplicationClient.java b/cdap-client/src/main/java/io/cdap/cdap/client/ApplicationClient.java index 74cd061fe5b9..06b14843320b 100644 --- a/cdap-client/src/main/java/io/cdap/cdap/client/ApplicationClient.java +++ b/cdap-client/src/main/java/io/cdap/cdap/client/ApplicationClient.java @@ -24,6 +24,7 @@ import com.google.common.collect.Maps; import com.google.common.reflect.TypeToken; import com.google.gson.Gson; +import com.google.gson.JsonObject; import io.cdap.cdap.api.Config; import io.cdap.cdap.api.annotation.Beta; import io.cdap.cdap.api.security.AccessException; @@ -108,6 +109,34 @@ public List list(NamespaceId namespace) }).getResponseObject(); } + /** + * Retrieves a paginated list of applications within the specified namespace. + * + * @param namespace The {@link NamespaceId} representing the namespace from which to list + * applications. + * @param nextPageToken The token for fetching the next page of results. + * @return A {@link JsonObject} containing the paginated list of applications and the next page + * token if available. + * @throws IOException If a network error occurred. + * @throws UnauthenticatedException If the request is not authorized successfully in th gateway + * server + * @throws UnauthorizedException If the caller lacks sufficient permissions. + */ + public JsonObject paginatedList(NamespaceId namespace, String nextPageToken) + throws IOException, UnauthenticatedException, UnauthorizedException { + StringBuilder pathBuilder = new StringBuilder("apps?latestOnly=false&pageSize=").append( + config.getAppListPageSize()); + + if (nextPageToken != null && !nextPageToken.isEmpty()) { + pathBuilder.append("&pageToken=").append(nextPageToken); + } + HttpResponse response = restClient.execute(HttpMethod.GET, + config.resolveNamespacedURLV3(namespace, pathBuilder.toString()), + config.getAccessToken()); + return ObjectResponse.fromJsonBody(response, new TypeToken() { + }).getResponseObject(); + } + /** * Lists all applications currently deployed, optionally filtering to only include applications * that use the specified artifact name and version. diff --git a/cdap-client/src/main/java/io/cdap/cdap/client/ProgramClient.java b/cdap-client/src/main/java/io/cdap/cdap/client/ProgramClient.java index 9aeaeaa0f55a..68d4617170bb 100644 --- a/cdap-client/src/main/java/io/cdap/cdap/client/ProgramClient.java +++ b/cdap-client/src/main/java/io/cdap/cdap/client/ProgramClient.java @@ -21,6 +21,7 @@ import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; import io.cdap.cdap.api.annotation.Beta; import io.cdap.cdap.api.customaction.CustomActionSpecification; import io.cdap.cdap.api.workflow.ConditionSpecification; @@ -275,36 +276,53 @@ public void stopAll(NamespaceId namespace) throws IOException, UnauthenticatedException, InterruptedException, TimeoutException, UnauthorizedException, ApplicationNotFoundException, BadRequestException { - List allApps = applicationClient.list(namespace); - for (ApplicationRecord applicationRecord : allApps) { - ApplicationId appId = new ApplicationId(namespace.getNamespace(), applicationRecord.getName(), - applicationRecord.getAppVersion()); - List programRecords = applicationClient.listPrograms(appId); - for (ProgramRecord programRecord : programRecords) { - try { - ProgramId program = appId.program(programRecord.getType(), programRecord.getName()); - String status = this.getStatus(program); - if (!status.equals("STOPPED")) { + String token = null; + boolean isLastPage = false; + while (!isLastPage) { + JsonObject paginatedListResponse = applicationClient.paginatedList(namespace, token); + token = paginatedListResponse.get("nextPageToken") == null ? null + : paginatedListResponse.get("nextPageToken").getAsString(); + LOG.debug("Called paginated list API to stop programs and got token: {}", token); + if (paginatedListResponse.get("applications").getAsJsonArray().size() != 0) { + Type appListType = new TypeToken>() { + }.getType(); + List records = GSON.fromJson( + paginatedListResponse.get("applications").getAsJsonArray(), appListType); + for (ApplicationRecord applicationRecord : records) { + ApplicationId appId = new ApplicationId(namespace.getNamespace(), + applicationRecord.getName(), + applicationRecord.getAppVersion()); + List programRecords = applicationClient.listPrograms(appId); + for (ProgramRecord programRecord : programRecords) { try { - this.stop(program); - } catch (IOException ioe) { - // ProgramClient#stop calls RestClient, which throws an IOException if the HTTP response code is 400, - // which can be due to the program already being stopped when calling stop on it. - // Most likely, there was a race condition that the program stopped between the time we checked its - // status and calling the stop method. - LOG.warn( - "Program {} is already stopped, proceeding even though the following exception is raised.", - program, ioe); + ProgramId program = appId.program(programRecord.getType(), programRecord.getName()); + String status = this.getStatus(program); + if (!status.equals("STOPPED")) { + try { + this.stop(program); + } catch (IOException ioe) { + // ProgramClient#stop calls RestClient, which throws an IOException if the + // HTTP response code is 400, which can be due to the program already being + // stopped when calling stop on it.Most likely, there was a race condition that + // the program stopped between the time we checked its status and calling + // the stop method. + LOG.warn( + "Program {} is already stopped, proceeding even though the following exception is raised.", + program, ioe); + } + // YarnTwillController has a timeout of 60 seconds after sending a stop signal + // using ZK. If this fails, it kills the app usin Yarn API. In cases where there + // is a failure to send the message via ZK, it waits for 60 seconds. + // So a wait of 60 seconds here is not enough. + this.waitForStatus(program, ProgramStatus.STOPPED, 120, TimeUnit.SECONDS); + } + } catch (ProgramNotFoundException e) { + // IGNORE } - // YarnTwillController has a timeout of 60 seconds after sending a stop signal using ZK. - // If this fails, it kills the app usin Yarn API. In cases where there is a failure to send the message - // via ZK, it waits for 60 seconds. So a wait of 60 seconds here is not enough. - this.waitForStatus(program, ProgramStatus.STOPPED, 120, TimeUnit.SECONDS); } - } catch (ProgramNotFoundException e) { - // IGNORE } } + isLastPage = (token == null); } } diff --git a/cdap-client/src/main/java/io/cdap/cdap/client/config/ClientConfig.java b/cdap-client/src/main/java/io/cdap/cdap/client/config/ClientConfig.java index 50c0953c6863..6674a6fc0c13 100644 --- a/cdap-client/src/main/java/io/cdap/cdap/client/config/ClientConfig.java +++ b/cdap-client/src/main/java/io/cdap/cdap/client/config/ClientConfig.java @@ -44,6 +44,8 @@ public class ClientConfig { private static final int DEFAULT_READ_TIMEOUT = 15000; private static final int DEFAULT_CONNECT_TIMEOUT = 15000; + private static final int DEFAULT_APP_LIST_PAGE_SIZE = 25; + private static final String DEFAULT_VERSION = Constants.Gateway.API_VERSION_3_TOKEN; @Nullable @@ -56,6 +58,7 @@ public class ClientConfig { private int uploadConnectTimeout; private int unavailableRetryLimit; + private int appListPageSize; private String apiVersion; private Supplier accessToken; private Map additionalHeaders; @@ -65,7 +68,7 @@ private ClientConfig(@Nullable ConnectionConfig connectionConfig, String apiVersion, Supplier accessToken, int defaultReadTimeout, int defaultConnectTimeout, int uploadReadTimeout, int uploadConnectTimeout, - Map additionalHeaders) { + Map additionalHeaders, int appListPageSize) { this.connectionConfig = connectionConfig; this.verifySSLCert = verifySSLCert; this.apiVersion = apiVersion; @@ -76,6 +79,7 @@ private ClientConfig(@Nullable ConnectionConfig connectionConfig, this.uploadReadTimeout = uploadReadTimeout; this.uploadConnectTimeout = uploadConnectTimeout; this.additionalHeaders = additionalHeaders; + this.appListPageSize = appListPageSize; } public static ClientConfig getDefault() { @@ -167,6 +171,8 @@ public int getUploadConnectTimeout() { return uploadConnectTimeout; } + public int getAppListPageSize() { return appListPageSize; } + public Map getAdditionalHeaders() { return additionalHeaders; } @@ -198,6 +204,10 @@ public void setDefaultConnectTimeout(int defaultConnectTimeout) { this.defaultConnectTimeout = defaultConnectTimeout; } + public void setAppListPageSize(int appListPageSize) { + this.appListPageSize = appListPageSize; + } + public void setUploadReadTimeout(int uploadReadTimeout) { this.uploadReadTimeout = uploadReadTimeout; } @@ -264,6 +274,7 @@ public static final class Builder { private int uploadConnectTimeout = DEFAULT_UPLOAD_CONNECT_TIMEOUT; private int defaultReadTimeout = DEFAULT_READ_TIMEOUT; private int defaultConnectTimeout = DEFAULT_CONNECT_TIMEOUT; + private int appListPageSize = DEFAULT_APP_LIST_PAGE_SIZE; private int unavailableRetryLimit = DEFAULT_SERVICE_UNAVAILABLE_RETRY_LIMIT; private Map additionalHeaders = new HashMap<>(); @@ -281,6 +292,7 @@ public Builder(ClientConfig clientConfig) { this.defaultReadTimeout = clientConfig.defaultReadTimeout; this.defaultConnectTimeout = clientConfig.defaultConnectTimeout; this.unavailableRetryLimit = clientConfig.unavailableRetryLimit; + this.appListPageSize = clientConfig.appListPageSize; } public Builder setConnectionConfig(ConnectionConfig connectionConfig) { @@ -313,6 +325,11 @@ public Builder setDefaultConnectTimeout(int defaultConnectTimeout) { return this; } + public Builder setAppListPageSize(int appListPageSize) { + this.appListPageSize = appListPageSize; + return this; + } + public Builder setAccessToken(Supplier accessToken) { this.accessToken = accessToken; return this; @@ -342,7 +359,8 @@ public ClientConfig build() { return new ClientConfig(connectionConfig, verifySSLCert, unavailableRetryLimit, apiVersion, accessToken, defaultReadTimeout, defaultConnectTimeout, - uploadReadTimeout, uploadConnectTimeout, ImmutableMap.copyOf(additionalHeaders)); + uploadReadTimeout, uploadConnectTimeout, ImmutableMap.copyOf(additionalHeaders), + appListPageSize); } } diff --git a/cdap-common-unit-test/pom.xml b/cdap-common-unit-test/pom.xml index cc87c196e229..d56167c8d62b 100644 --- a/cdap-common-unit-test/pom.xml +++ b/cdap-common-unit-test/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-common-unit-test diff --git a/cdap-common/bin/functions.sh b/cdap-common/bin/functions.sh index 0a812042b889..381c8d955f22 100644 --- a/cdap-common/bin/functions.sh +++ b/cdap-common/bin/functions.sh @@ -345,12 +345,12 @@ cdap_set_java () { die "JAVA_HOME is not set and 'java' was not found in your PATH. Please set JAVA_HOME to the location of your Java install" fi fi - __java_version=$("${__java}" -version 2>&1 | grep version | awk '{print $3}' | awk -F '.' '{print $2}') + __java_version_raw=$("${__java}" -version 2>&1 | awk -F '"' '/version/ {print $2}') + __java_version=$(echo "$__java_version_raw" | awk -F. '{if ($1 == "1") print $2; else print $1}') if [[ -z ${__java_version} ]]; then die "Could not detect Java version. Aborting..." - elif [[ ${__java_version} -lt 8 ]]; then - die "Java version not supported. Please install Java 8 - other versions of Java are not supported." fi + export JAVA_VERSION=${__java_version} export JAVA=${__java} return 0 } @@ -645,7 +645,7 @@ cdap_start_java() { # Setup classpaths. cdap_set_classpath "${CDAP_HOME}"/${__comp_home} "${CDAP_CONF}" # Setup Java - cdap_set_java || return 1 + cdap_set_java || die "Unable to locate JAVA or JAVA_HOME" # Set JAVA_HEAPMAX from variable defined in JAVA_HEAP_VAR, unless defined already JAVA_HEAPMAX=${JAVA_HEAPMAX:-${!JAVA_HEAP_VAR}} export JAVA_HEAPMAX @@ -657,7 +657,12 @@ cdap_start_java() { if [[ ${HEAPDUMP_ON_OOM} == true ]]; then __defines+=" -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${__gc_log_and_heapdump_dir}" fi - __defines+=" -verbose:gc -Xloggc:${__gc_log_and_heapdump_dir}/gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1M" + + if [ "$JAVA_VERSION" -le 8 ]; then + __defines+=" -verbose:gc -Xloggc:${__gc_log_and_heapdump_dir}/gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1M" + else + __defines+="-Xlog:gc*:file=${__gc_log_and_heapdump_dir}/gc.log:time,uptime,level,tags:filecount=10,filesize=1M" + fi logecho "$(date) Starting CDAP ${__name} service on ${HOSTNAME}" echo if [[ ${CDAP_SERVICE} == master ]]; then @@ -731,9 +736,12 @@ cdap_run_class() { # Setup Java cdap_set_java || return 1 cdap_set_spark || logecho "$(date) [WARN] Could not determine SPARK_HOME! Spark support unavailable!" - cdap_create_local_dir || die "Could not create local directory" + cdap_create_local_dir || die "Could not create local directory : ${LOCAL_DIR}" + # Replace the with local directory if present in JVM OPTS. + OPTS="${OPTS///$LOCAL_DIR}" + echo "$(date) Major Java version: $JAVA_VERSION" if [[ -n ${__args} ]] && [[ ${__args} != '' ]]; then - echo "$(date) Running class ${__class} with arguments: ${__args}" + echo "$(date) Running class ${__class} with arguments: ${__args} and JVM OPTS : ${OPTS}" else echo "$(date) Running class ${__class}" fi @@ -760,9 +768,12 @@ cdap_exec_class() { # Setup Java cdap_set_java || return 1 cdap_set_spark || logecho "$(date) [WARN] Could not determine SPARK_HOME! Spark support unavailable!" - cdap_create_local_dir || die "Could not create local directory" + cdap_create_local_dir || die "Could not create local directory : ${LOCAL_DIR}" + # Replace the with local directory if present in JVM OPTS. + OPTS="${OPTS///$LOCAL_DIR}" + echo "$(date) Major Java version: $JAVA_VERSION" if [[ -n ${__args} ]] && [[ ${__args} != '' ]]; then - echo "$(date) Running class ${__class} with arguments: ${__args}" + echo "$(date) Running class ${__class} with arguments: ${__args} and JVM OPTS : ${OPTS}" else echo "$(date) Running class ${__class}" fi @@ -927,7 +938,13 @@ cdap_sdk_start() { CDAP_SDK_DEFAULT_JVM_OPTS="-Xmx2048m" fi - SDK_GC_OPTS="-verbose:gc -Xloggc:${LOG_DIR}/gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1M" + cdap_set_java || die "Unable to locate JAVA or JAVA_HOME" + + if [ "$JAVA_VERSION" -le 8 ]; then + SDK_GC_OPTS="-verbose:gc -Xloggc:${LOG_DIR}/gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1M" + else + SDK_GC_OPTS="-Xlog:gc*:file=${LOG_DIR}/gc.log:time,uptime,level,tags:filecount=10,filesize=1M" + fi if [[ ${HEAPDUMP_ON_OOM} == true ]]; then CDAP_SDK_OPTS+=" -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOG_DIR}" fi @@ -952,8 +969,6 @@ cdap_sdk_start() { ROUTER_OPTS="-Drouter.address=$(hostname -I | awk '{print $1}')" # -I is safe since we know we're Linux fi - cdap_set_java || die "Unable to locate JAVA or JAVA_HOME" - # In order to ensure that we can do hacks, need to make sure classpath is sorted # so that cdap jars are placed earlier in the classpath than twill or hadoop jars CLASSPATH=$(find "${CDAP_HOME}/lib" -type f | sort | tr '\n' ':') diff --git a/cdap-common/pom.xml b/cdap-common/pom.xml index 04a4d8b5c4eb..b3b9a7ce1dda 100644 --- a/cdap-common/pom.xml +++ b/cdap-common/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-common diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/conf/Constants.java b/cdap-common/src/main/java/io/cdap/cdap/common/conf/Constants.java index 5cf9f9c2dc95..0fdc04b391e1 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/conf/Constants.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/conf/Constants.java @@ -24,7 +24,10 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import java.util.Collections; +import java.util.HashSet; import java.util.Map; +import java.util.Set; /** * Constants used by different systems are all defined here. @@ -67,6 +70,7 @@ public final class Constants { // Directory path for caching remove location content public static final String LOCATION_CACHE_PATH = "location.cache.path"; public static final String LOCATION_CACHE_EXPIRATION_MS = "location.cache.expiration.ms"; + public static final String LOCATION_CACHE_FORCE_CLEANUP_MS = "location.cache.force.cleanup.ms"; public static final String CLUSTER_NAME = "cluster.name"; @@ -229,11 +233,18 @@ public static final class Dangerous { */ public static final class AppFabric { + /** + * App Fabric Processor. + */ + + public static final String PROCESSOR_PORT = "appfabric.processor.bind.port"; + public static final String PROCESSOR_ANNOUNCE_PORT = "appfabric.processor.announce.port"; + /** * App Fabric Server. */ - public static final String SERVER_PORT = "app.bind.port"; - public static final String SERVER_ANNOUNCE_PORT = "app.announce.port"; + public static final String SERVER_PORT = "appfabric.bind.port"; + public static final String SERVER_ANNOUNCE_PORT = "appfabric.announce.port"; public static final String OUTPUT_DIR = "app.output.dir"; public static final String TEMP_DIR = "app.temp.dir"; public static final String REST_PORT = "app.rest.port"; @@ -284,10 +295,6 @@ public static final class AppFabric { public static final String PROGRAM_TRANSACTION_CONTROL = "app.program.transaction.control"; public static final String MAX_CONCURRENT_RUNS = "app.max.concurrent.runs"; public static final String MAX_CONCURRENT_LAUNCHING = "app.max.concurrent.launching"; - public static final String MONITOR_RECORD_AGE_THRESHOLD_SECONDS = - "run.record.monitor.record.age.threshold.seconds"; - public static final String MONITOR_CLEANUP_INTERVAL_SECONDS = - "run.record.monitor.cleanup.interval.seconds"; public static final String PROGRAM_LAUNCH_THREADS = "app.program.launch.threads"; public static final String PROGRAM_KILL_THREADS = "app.program.kill.threads"; public static final String RUN_DATA_CLEANUP_TTL_DAYS = "app.run.records.ttl.days"; @@ -469,6 +476,7 @@ public static final class Preview { public static final String CONTAINER_JVM_OPTS = "preview.runner.container.jvm.opts"; public static final String GCE_METADATA_HOST_ENV_VAR = "GCE_METADATA_HOST"; public static final String INTERNAL_ROUTER_ENABLED = "preview.runner.internal.router.enabled"; + public static final String HTTP_COMPRESS_PAYLOAD = "preview.http.compress.payload"; } /** @@ -546,8 +554,14 @@ public static final class TaskWorker { public static final String METADATA_SERVICE_END_POINT = "task.worker.metadata.service.endpoint"; public static final String METRIC_PREFIX = "task.worker."; public static final String GCE_METADATA_HOST_ENV_VAR = "GCE_METADATA_HOST"; - } + /** + * Task worker container probe configurations. + * For usage for k8s probes refer : `io.cdap.cdap.k8s.util.ProbeFactory` + */ + public static final String TASK_WORKER_PROBE_ENABLED = "task.worker.probe.enabled"; + public static final String TASK_WORKER_PROBE_PREFIX = "task.worker.probe."; + } /** * System pods. @@ -635,6 +649,8 @@ public static final class Scheduler { public static final class AppMetaStore { public static final String TABLE = "app.meta"; + public static final Set APPSPEC_REDUCTION_SUPPORTED_STORAGE_PROVIDERS = new HashSet<>( + Collections.singleton(Dataset.DATA_STORAGE_SPANNER)); } /** @@ -727,6 +743,7 @@ public static final class Dataset { public static final String DATA_STORAGE_IMPLEMENTATION = "data.storage.implementation"; public static final String DATA_STORAGE_NOSQL = "nosql"; public static final String DATA_STORAGE_SQL = "postgresql"; + public static final String DATA_STORAGE_SPANNER = "gcp-spanner"; public static final String DATA_STORAGE_SQL_DRIVER_EXTERNAL = "data.storage.sql.jdbc.driver.external"; public static final String DATA_STORAGE_SQL_DRIVER_DIRECTORY = "data.storage.sql.jdbc.driver.directory"; public static final String DATA_STORAGE_SQL_JDBC_DRIVER_NAME = "data.storage.sql.jdbc.driver.name"; @@ -972,7 +989,7 @@ public static final class Metrics { ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, NamespaceId.SYSTEM.getNamespace(), Constants.Metrics.Tag.COMPONENT, - Constants.Service.METRICS_PROCESSOR); + Constants.Service.METRICS); public static final Map TRANSACTION_MANAGER_CONTEXT = ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, @@ -1102,6 +1119,14 @@ public static final class Tag { // For operations public static final String OPERATION_RUN = "operation"; + + // For error classification + public static final String ERROR_CATEGORY = "ectgry"; + public static final String ERROR_TYPE = "etpe"; + public static final String DEPENDENCY = "edep"; + public static final String ERROR_CODE_TYPE = "ecdtpe"; + public static final String ERROR_CODE = "ecd"; + public static final String FROM_PLUGIN = "frmplg"; } /** @@ -1157,6 +1182,7 @@ public static final class FlowControl { public static final String LAUNCHING_COUNT = "flowcontrol.launching.count"; public static final String RUNNING_COUNT = "flowcontrol.running.count"; + public static final String READ_STALENESS_SECONDS = "flowcontrol.read.staleness.seconds"; } /** @@ -1177,6 +1203,8 @@ public static final class Program { public static final String APPLICATION_COUNT = "application.count"; public static final String NAMESPACE_COUNT = "namespace.count"; public static final String APPLICATION_PLUGIN_COUNT = "application.plugin.count"; + public static final String FAILED_RUNS_CLASSIFICATION_COUNT = + "program.failed.runs.classified.count"; } /** @@ -1389,6 +1417,7 @@ public static final class LogSaver { public static final String LOG_SAVER_HANDLER = "log.saver.handler"; public static final String ADDRESS = "log.saver.status.bind.address"; + public static final String PORT = "log.saver.status.bind.port"; public static final String SERVICE_DESCRIPTION = "Service to collect and store logs."; @@ -2121,6 +2150,7 @@ public static final class MessagingSystem { public static final String EXTENSIONS_DIR = "messaging.service.extensions.dir"; public static final String MESSAGING_SERVICE_NAME = "messaging.service.name"; + public static final String MESSAGING_SERVICE_ENABLED = "messaging.service.enabled"; public static final String CACHE_SIZE_MB = "messaging.cache.size.mb"; public static final String HBASE_MAX_SCAN_THREADS = "messaging.hbase.max.scan.threads"; diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/guice/DFSLocationModule.java b/cdap-common/src/main/java/io/cdap/cdap/common/guice/DFSLocationModule.java index e1b458f4f119..ee28b237f9af 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/guice/DFSLocationModule.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/guice/DFSLocationModule.java @@ -95,8 +95,10 @@ public LocationFactory get() { Path cachePath = Paths.get(locationCachePath).toAbsolutePath(); long expiry = cConf.getLong(Constants.LOCATION_CACHE_EXPIRATION_MS); + long forceCleanupInterval = cConf.getLong(Constants.LOCATION_CACHE_FORCE_CLEANUP_MS); return new CachingLocationFactory(lf, - new DefaultCachingPathProvider(cachePath, expiry, TimeUnit.MILLISECONDS)); + new DefaultCachingPathProvider(cachePath, expiry, TimeUnit.MILLISECONDS, + forceCleanupInterval)); } } } diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/http/AuthenticationChannelHandler.java b/cdap-common/src/main/java/io/cdap/cdap/common/http/AuthenticationChannelHandler.java index 9c5fb79bbb6f..38547203a811 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/http/AuthenticationChannelHandler.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/http/AuthenticationChannelHandler.java @@ -56,6 +56,8 @@ public class AuthenticationChannelHandler extends ChannelDuplexHandler { private static final String EMPTY_USER_IP = "CDAP-empty-user-ip"; static final String AUDIT_LOG_REQ_BUILDER_ATTR = "AUDIT_LOG_REQ_BUILDER"; static final String AUDIT_LOG_USER_IP_ATTR = "AUDIT_LOG_USER_IP"; + static final String CDAP_USER_ID_ATTR = "CDAP_USER_ID"; + static final String CDAP_USER_CREDENTIAL_ATTR = "CDAP_USER_CREDENTIAL"; static final String AUDIT_LOG_CONTEXT_QUEUE_ATTR = "AUDIT_LOG_CONTEXT_QUEUE"; private final boolean internalAuthEnabled; @@ -77,9 +79,7 @@ public AuthenticationChannelHandler(boolean internalAuthEnabled, boolean auditLo public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { SecurityRequestContext.reset(); - // Only set SecurityRequestContext for the HttpRequest but not for subsequence chunks. - // We cannot set a default/placeholder value until CDAP-18773 - // is fixed since we may perform auth checks in the thread processing the last chunk. + // TODO: CDAP-21121 ensure request is authorized before sending response if (msg instanceof HttpRequest) { String currentUserId = null; @@ -135,6 +135,20 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception SecurityRequestContext.setUserIp(currentUserIp); //Also set userIp in ATTR , to be used in audit logging incase it was replaced at a later stage ctx.channel().attr(AttributeKey.valueOf(AUDIT_LOG_USER_IP_ATTR)).set(currentUserIp); + ctx.channel().attr(AttributeKey.valueOf(CDAP_USER_ID_ATTR)).set(currentUserId); + ctx.channel().attr(AttributeKey.valueOf(CDAP_USER_CREDENTIAL_ATTR)).set(currentUserCredential); + } else { + Object userIpObj = ctx.channel().attr(AttributeKey.valueOf(AUDIT_LOG_USER_IP_ATTR)).get(); + Object userIdObj = ctx.channel().attr(AttributeKey.valueOf(CDAP_USER_ID_ATTR)).get(); + Object userCredentialObj = + ctx.channel().attr(AttributeKey.valueOf(CDAP_USER_CREDENTIAL_ATTR)).get(); + if (userIpObj != null) { + SecurityRequestContext.setUserIp((String) userIpObj); + } + if (userIdObj != null && userCredentialObj != null) { + SecurityRequestContext.setUserId((String) userIdObj); + SecurityRequestContext.setUserCredential((Credential) userCredentialObj); + } } try { diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/internal/remote/RemoteTaskExecutor.java b/cdap-common/src/main/java/io/cdap/cdap/common/internal/remote/RemoteTaskExecutor.java index 7ae94e2e965c..ad9cce353c29 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/internal/remote/RemoteTaskExecutor.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/internal/remote/RemoteTaskExecutor.java @@ -46,6 +46,7 @@ import java.io.Writer; import java.net.HttpURLConnection; import java.net.NoRouteToHostException; +import java.net.SocketTimeoutException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.HashMap; @@ -64,7 +65,8 @@ public class RemoteTaskExecutor { private static final String TASK_WORKER_URL = "/worker/run"; private static final String SYSTEM_WORKER_URL = "/system/run"; private static final Predicate RETRYABLE_PREDICATE_SYSTEM_WORKER = throwable -> - (throwable instanceof RetryableException) || (throwable instanceof ServiceException); + (throwable instanceof RetryableException) || (throwable instanceof ServiceException) + || (throwable instanceof SocketTimeoutException); private static final Predicate RETRYABLE_PREDICATE_TASK_WORKER = throwable -> (throwable instanceof RetryableException); private final boolean compression; diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/internal/remote/TaskWorkerHttpHandlerInternal.java b/cdap-common/src/main/java/io/cdap/cdap/common/internal/remote/TaskWorkerHttpHandlerInternal.java index a098e752dad1..c2b2b52757cf 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/internal/remote/TaskWorkerHttpHandlerInternal.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/internal/remote/TaskWorkerHttpHandlerInternal.java @@ -183,9 +183,7 @@ private void enablePeriodicRestart(CConfiguration cConf, "Task worker service is about to restart in {} seconds, no new tasks will be accepted.", finalTaskDeadlineSeconds); if (runningRequestCount.get() == 0) { - stopper.accept(""); - executorService.shutdown(); - return; + stopAndShutdown(executorService, stopper); } try { Thread.sleep(TimeUnit.SECONDS.toMillis(finalTaskDeadlineSeconds)); @@ -194,11 +192,16 @@ private void enablePeriodicRestart(CConfiguration cConf, "Interrupted while waiting for task completion. Stopping immediately", e); } - stopper.accept(""); - executorService.shutdown(); + stopAndShutdown(executorService, stopper); }, waitTime, finalTaskDeadlineSeconds, TimeUnit.SECONDS); } + private void stopAndShutdown(ScheduledExecutorService executorService, Consumer stopper) { + stopper.accept(""); + executorService.shutdown(); + System.exit(0); + } + /** * Run a new {@link io.cdap.cdap.api.service.worker.RunnableTask}. * diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/io/DefaultCachingPathProvider.java b/cdap-common/src/main/java/io/cdap/cdap/common/io/DefaultCachingPathProvider.java index 2d394713eaa2..15e0c8988c43 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/io/DefaultCachingPathProvider.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/io/DefaultCachingPathProvider.java @@ -31,7 +31,10 @@ import java.util.Collection; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import org.apache.twill.common.Threads; import org.apache.twill.filesystem.Location; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,8 +51,10 @@ public class DefaultCachingPathProvider implements CachingPathProvider { private final LoadingCache cache; private final Path cacheDir; + private final ScheduledExecutorService cleanupExecutor; - public DefaultCachingPathProvider(Path cacheDir, long cacheExpiry, TimeUnit cacheExpiryUnit) { + public DefaultCachingPathProvider(Path cacheDir, long cacheExpiry, TimeUnit cacheExpiryUnit, + long forceCleanupIntervalMillis) { this.cacheDir = cacheDir; this.cache = CacheBuilder.newBuilder() .expireAfterAccess(cacheExpiry, cacheExpiryUnit) @@ -81,6 +86,20 @@ public Path load(CacheKey key) { }); populateCache(cacheDir, cache); + // Cache is not automatically cleaned up and for low throughput caches this needs to be + // maintained: https://github.com/google/guava/wiki/CachesExplained#when-does-cleanup-happen + if (forceCleanupIntervalMillis > 0) { + // A non-positive value indicates that the cache should not be periodically cleaned up. + cleanupExecutor = Executors.newSingleThreadScheduledExecutor( + Threads.createDaemonThreadFactory("location-cache-cleanup")); + cleanupExecutor.scheduleAtFixedRate(() -> { + LOG.trace("Cleaning up location cache"); + cache.cleanUp(); + }, 0, forceCleanupIntervalMillis, TimeUnit.MILLISECONDS); + } else { + // If cleanup interval is <= 0 the executor is not initialized. + cleanupExecutor = null; + } } @Override diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/lang/jar/BundleJarUtil.java b/cdap-common/src/main/java/io/cdap/cdap/common/lang/jar/BundleJarUtil.java index 8aadb85c52e3..33617a9dddb5 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/lang/jar/BundleJarUtil.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/lang/jar/BundleJarUtil.java @@ -34,6 +34,7 @@ import java.nio.file.SimpleFileVisitor; import java.nio.file.StandardCopyOption; import java.nio.file.attribute.BasicFileAttributes; +import java.util.Comparator; import java.util.EnumSet; import java.util.function.Predicate; import java.util.jar.JarEntry; @@ -41,6 +42,7 @@ import java.util.jar.JarInputStream; import java.util.jar.JarOutputStream; import java.util.jar.Manifest; +import java.util.stream.Stream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; @@ -168,13 +170,13 @@ public static void addToArchive(final File input, final boolean includeDirName, public static void addToArchive(final File input, final boolean includeDirName, final ZipOutputStream output, final Predicate fileNameFilter) throws IOException { - final URI baseURI = input.toURI(); + final URI baseUri = input.toURI(); Files.walkFileTree(input.toPath(), EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, new SimpleFileVisitor() { @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - URI uri = baseURI.relativize(dir.toUri()); + URI uri = baseUri.relativize(dir.toUri()); String entryName = includeDirName ? input.getName() + "/" + uri.getPath() : uri.getPath(); @@ -191,7 +193,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) if (fileNameFilter.test(file.getFileName().toString())) { return FileVisitResult.CONTINUE; } - URI uri = baseURI.relativize(file.toUri()); + URI uri = baseUri.relativize(file.toUri()); if (uri.getPath().isEmpty()) { // Only happen if the given "input" is a file. output.putNextEntry(new ZipEntry(file.toFile().getName())); @@ -210,7 +212,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) * Takes a jar or a local directory and prepares a folder to be loaded by classloader. If a jar is * provided it unpacks a manifest and any nested jars and links original jar into the destination * folder, so that it would be picked up by classloader to load any classes or resources. - * + *

* If a directory is provided, it assumes that this directory already contains the unpacked jar * contents (ie. this directory was used as the destinationFolder in a previous call to this * method). In this case, no unpacking is needed. The {@link ClassLoaderFolder} returned will be @@ -314,6 +316,58 @@ private static File unJar(File jarFile, File destinationFolder, Predicate nameFilter, + CopyOption... copyOptions) + throws IOException { + + Path targetPath = targetDirectory.toPath().normalize(); + boolean isSuccess = false; + boolean isTargetDirectoryNewlyCreated = false; + + try { + // Check if the directory exists before trying to create it + if (!Files.exists(targetPath)) { + Files.createDirectories(targetPath); + isTargetDirectoryNewlyCreated = true; + } + + ZipEntry entry; + while ((entry = input.getNextEntry()) != null) { + String entryName = entry.getName(); + + if (!nameFilter.test(entryName)) { + continue; + } + + Path output = targetPath.resolve(entryName).normalize(); + + if (!output.startsWith(targetPath)) { + throw new IllegalArgumentException("Illegal path detected for Jar Entry : " + entryName + ". This will try to" + + " write outside the target directory, which is not allowed."); + } + + if (entry.isDirectory()) { + Files.createDirectories(output); + } else { + Files.createDirectories(output.getParent()); + Files.copy(input, output, copyOptions); + } + input.closeEntry(); + } + isSuccess = true; + } finally { + // Cleanup logic: Only delete if not successful AND we created the directory + if (!isSuccess && isTargetDirectoryNewlyCreated && Files.exists(targetPath)) { + try (Stream paths = Files.walk(targetPath)) { + paths.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + } + } + } + /** * Search for {@link Manifest} from the given {@link JarInputStream}. * @@ -341,28 +395,6 @@ private static Manifest getManifest(JarInputStream jarInput) throws IOException return null; } - private static void unJar(ZipInputStream input, File targetDirectory, - Predicate nameFilter, - CopyOption... copyOptions) - throws IOException { - Path targetPath = targetDirectory.toPath(); - Files.createDirectories(targetPath); - - ZipEntry entry; - while ((entry = input.getNextEntry()) != null) { - if (nameFilter.test(entry.getName())) { - Path output = targetPath.resolve(entry.getName()); - - if (entry.isDirectory()) { - Files.createDirectories(output); - } else { - Files.createDirectories(output.getParent()); - Files.copy(input, output, copyOptions); - } - } - } - } - private BundleJarUtil() { } } diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/metrics/MetricsReporterHook.java b/cdap-common/src/main/java/io/cdap/cdap/common/metrics/MetricsReporterHook.java index a724978e190c..65a6b9bc8603 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/metrics/MetricsReporterHook.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/metrics/MetricsReporterHook.java @@ -28,8 +28,6 @@ import io.cdap.cdap.common.feature.DefaultFeatureFlagsProvider; import io.cdap.cdap.common.http.HttpHeaderNames; import io.cdap.cdap.proto.id.NamespaceId; -import io.cdap.cdap.security.spi.authentication.SecurityRequestContext; -import io.cdap.cdap.security.spi.authorization.AuditLogRequest; import io.cdap.http.AbstractHandlerHook; import io.cdap.http.HttpResponder; import io.cdap.http.internal.HandlerInfo; @@ -37,6 +35,9 @@ import io.netty.handler.codec.http.HttpResponseStatus; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,6 +47,7 @@ public class MetricsReporterHook extends AbstractHandlerHook { private static final Logger LOG = LoggerFactory.getLogger(MetricsReporterHook.class); + private static final Pattern NAMESPACE_PATTERN = Pattern.compile("/namespaces/([^/]+)"); private static final String LATENCY_METRIC_NAME = "response.latency"; private final String serviceName; @@ -78,7 +80,7 @@ public boolean preCall(HttpRequest request, HttpResponder responder, HandlerInfo return true; } try { - MetricsContext collector = collectorCache.get(createContext(handlerInfo)); + MetricsContext collector = collectorCache.get(createContext(request, handlerInfo)); collector.increment("request.received", 1); request.headers().add(HttpHeaderNames.CDAP_REQ_TIMESTAMP_HDR, System.nanoTime()); } catch (Throwable e) { @@ -93,7 +95,7 @@ public void postCall(HttpRequest request, HttpResponseStatus status, HandlerInfo return; } try { - MetricsContext collector = collectorCache.get(createContext(handlerInfo)); + MetricsContext collector = collectorCache.get(createContext(request, handlerInfo)); String name; int code = status.code(); if (code < 100) { @@ -128,10 +130,10 @@ public void postCall(HttpRequest request, HttpResponseStatus status, HandlerInfo } } - private Map createContext(HandlerInfo handlerInfo) { + private Map createContext(HttpRequest request, HandlerInfo handlerInfo) { // todo: really inefficient to call this on the intense data flow path return ImmutableMap.of( - Constants.Metrics.Tag.NAMESPACE, NamespaceId.SYSTEM.getEntityName(), + Constants.Metrics.Tag.NAMESPACE, getNamespaceFromUriIfPresent(request.uri()), Constants.Metrics.Tag.COMPONENT, serviceName, Constants.Metrics.Tag.HANDLER, getSimpleName(handlerInfo.getHandlerName()), Constants.Metrics.Tag.METHOD, handlerInfo.getMethodName()); @@ -141,4 +143,17 @@ private String getSimpleName(String className) { int ind = className.lastIndexOf('.'); return className.substring(ind + 1); } + + private static String getNamespaceFromUriIfPresent(@Nullable final String uri) { + if (uri == null || uri.isEmpty()) { + return NamespaceId.SYSTEM.getEntityName(); + } + + Matcher matcher = NAMESPACE_PATTERN.matcher(uri); + if (matcher.find()) { + return matcher.group(1); + } + + return NamespaceId.SYSTEM.getEntityName(); + } } diff --git a/cdap-common/src/main/java/io/cdap/cdap/extension/AbstractExtensionLoader.java b/cdap-common/src/main/java/io/cdap/cdap/extension/AbstractExtensionLoader.java index 77b929554ffa..e37d4229b634 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/extension/AbstractExtensionLoader.java +++ b/cdap-common/src/main/java/io/cdap/cdap/extension/AbstractExtensionLoader.java @@ -340,12 +340,16 @@ private ClassLoader getExtensionParentClassLoader() { return new FilterClassLoader(getClass().getClassLoader(), new FilterClassLoader.Filter() { @Override public boolean acceptResource(String resource) { - return resource.startsWith("org/slf4j") || filter.acceptResource(resource); + return resource.startsWith("org/slf4j") + || resource.startsWith("io/cdap/cdap/api/exception") + || filter.acceptResource(resource); } @Override public boolean acceptPackage(String packageName) { - return packageName.startsWith("org.slf4j") || filter.acceptPackage(packageName); + return packageName.startsWith("org.slf4j") + || packageName.startsWith("io.cdap.cdap.api.exception") + || filter.acceptPackage(packageName); } }); } diff --git a/cdap-common/src/main/java/io/cdap/cdap/internal/app/store/RunRecordDetail.java b/cdap-common/src/main/java/io/cdap/cdap/internal/app/store/RunRecordDetail.java index 4d70f44d2b5d..a2b5d7e945a9 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/internal/app/store/RunRecordDetail.java +++ b/cdap-common/src/main/java/io/cdap/cdap/internal/app/store/RunRecordDetail.java @@ -26,6 +26,7 @@ import io.cdap.cdap.proto.RunRecord; import io.cdap.cdap.proto.id.ProfileId; import io.cdap.cdap.proto.id.ProgramRunId; +import io.cdap.cdap.runtime.spi.provisioner.Cluster; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -62,6 +63,10 @@ public class RunRecordDetail extends RunRecord { @Nullable private final String principal; + @SerializedName("flowcontrolstatus") + @Nullable + private final String flowControlStatus; + // carries the user arguments decoded from properties. No need to serialize since it is from the properties. private transient volatile Map userArgs; @@ -72,7 +77,7 @@ protected RunRecordDetail(ProgramRunId programRunId, long startTs, @Nullable Lon @Nullable Map properties, @Nullable Map systemArgs, @Nullable String twillRunId, ProgramRunCluster cluster, ProfileId profileId, @Nullable String peerName, byte[] sourceId, @Nullable ArtifactId artifactId, - @Nullable String principal) { + @Nullable String principal, @Nullable String flowControlStatus) { super(programRunId.getRun(), startTs, runTs, stopTs, suspendTs, resumeTs, stoppingTs, terminateTs, status, properties, cluster, profileId, peerName, programRunId.getVersion()); @@ -82,6 +87,7 @@ protected RunRecordDetail(ProgramRunId programRunId, long startTs, @Nullable Lon this.sourceId = sourceId; this.artifactId = artifactId; this.principal = principal; + this.flowControlStatus = flowControlStatus; } @Nullable @@ -137,6 +143,11 @@ public String getPrincipal() { return principal; } + @Nullable + public String getFlowControlStatus() { + return flowControlStatus; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -161,15 +172,16 @@ public boolean equals(Object o) { && Objects.equal(this.getTwillRunId(), that.getTwillRunId()) && Arrays.equals(this.getSourceId(), that.getSourceId()) && Objects.equal(this.getArtifactId(), that.getArtifactId()) - && Objects.equal(this.getPrincipal(), that.getPrincipal()); + && Objects.equal(this.getPrincipal(), that.getPrincipal()) + && Objects.equal(this.getFlowControlStatus(), that.getFlowControlStatus()); } @Override public int hashCode() { return Objects.hashCode(getProgramRunId(), getStartTs(), getRunTs(), getStopTs(), - getSuspendTs(), getResumeTs(), - getStoppingTs(), getTerminateTs(), getStatus(), getProperties(), getPeerName(), - getTwillRunId(), Arrays.hashCode(getSourceId()), getArtifactId(), getPrincipal()); + getSuspendTs(), getResumeTs(), getStoppingTs(), getTerminateTs(), getStatus(), + getProperties(), getPeerName(), getTwillRunId(), Arrays.hashCode(getSourceId()), + getArtifactId(), getPrincipal(), getFlowControlStatus()); } @Override @@ -193,6 +205,7 @@ public String toString() { .add("sourceId", getSourceId() == null ? null : Bytes.toHexString(getSourceId())) .add("artifactId", getArtifactId()) .add("principal", getPrincipal()) + .add("flowcontrolstatus", getFlowControlStatus()) .toString(); } @@ -239,6 +252,7 @@ public abstract static class ABuilder extends RunRecord.Buil protected byte[] sourceId; protected String principal; protected ArtifactId artifactId; + protected String flowControlStatus; protected ABuilder() { systemArgs = new HashMap<>(); @@ -252,6 +266,7 @@ protected ABuilder(RunRecordDetail record) { sourceId = record.getSourceId(); principal = record.getPrincipal(); artifactId = record.getArtifactId(); + flowControlStatus = record.getFlowControlStatus(); } public T setProgramRunId(ProgramRunId programRunId) { @@ -286,12 +301,16 @@ public T setArtifactId(ArtifactId artifactId) { this.artifactId = artifactId; return (T) this; } + public T setFlowControlStatus(String flowControlStatus) { + this.flowControlStatus = flowControlStatus; + return (T) this; + } public RunRecordDetail build() { if (programRunId == null) { throw new IllegalArgumentException("Run record run id must be specified."); } - if (sourceId == null) { + if (status != ProgramRunStatus.PENDING && sourceId == null) { throw new IllegalArgumentException("Run record source id must be specified."); } // we are not validating artifactId for null, @@ -300,7 +319,7 @@ public RunRecordDetail build() { return new RunRecordDetail(programRunId, startTs, runTs, stopTs, suspendTs, resumeTs, stoppingTs, terminateTs, status, properties, systemArgs, twillRunId, cluster, - profileId, peerName, sourceId, artifactId, principal); + profileId, peerName, sourceId, artifactId, principal, flowControlStatus); } } } diff --git a/cdap-common/src/main/java/io/cdap/cdap/internal/app/store/RunRecordDetailWithExistingStatus.java b/cdap-common/src/main/java/io/cdap/cdap/internal/app/store/RunRecordDetailWithExistingStatus.java index 9283757b8068..292152a1d23f 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/internal/app/store/RunRecordDetailWithExistingStatus.java +++ b/cdap-common/src/main/java/io/cdap/cdap/internal/app/store/RunRecordDetailWithExistingStatus.java @@ -35,15 +35,15 @@ public final class RunRecordDetailWithExistingStatus extends RunRecordDetail { private RunRecordDetailWithExistingStatus(ProgramRunId programRunId, long startTs, @Nullable Long runTs, @Nullable Long stopTs, @Nullable Long suspendTs, @Nullable Long resumeTs, - @Nullable Long stoppingTs, @Nullable Long terminateTs, - ProgramRunStatus status, @Nullable Map properties, - @Nullable Map systemArgs, @Nullable String twillRunId, - ProgramRunCluster cluster, ProfileId profileId, @Nullable String peerName, - byte[] sourceId, @Nullable ArtifactId artifactId, - @Nullable String principal, @Nullable ProgramRunStatus existingStatus) { + @Nullable Long stoppingTs, @Nullable Long terminateTs, ProgramRunStatus status, + @Nullable Map properties, @Nullable Map systemArgs, + @Nullable String twillRunId, ProgramRunCluster cluster, ProfileId profileId, + @Nullable String peerName, byte[] sourceId, @Nullable ArtifactId artifactId, + @Nullable String principal, @Nullable ProgramRunStatus existingStatus, + @Nullable String flowControlStatus) { super(programRunId, startTs, runTs, stopTs, suspendTs, resumeTs, stoppingTs, terminateTs, - status, properties, - systemArgs, twillRunId, cluster, profileId, peerName, sourceId, artifactId, principal); + status, properties, systemArgs, twillRunId, cluster, profileId, peerName, sourceId, + artifactId, principal, flowControlStatus); this.existingStatus = existingStatus; } @@ -81,10 +81,9 @@ public RunRecordDetailWithExistingStatus build() { // artifactId could be null for program starts that were recorded pre 5.0 but weren't processed // we don't want to throw exception while processing them return new RunRecordDetailWithExistingStatus(programRunId, startTs, runTs, stopTs, suspendTs, - resumeTs, - stoppingTs, terminateTs, status, properties, systemArgs, + resumeTs, stoppingTs, terminateTs, status, properties, systemArgs, twillRunId, cluster, profileId, peerName, sourceId, artifactId, - principal, existingStatus); + principal, existingStatus, flowControlStatus); } } } diff --git a/cdap-common/src/main/resources/cdap-default.xml b/cdap-common/src/main/resources/cdap-default.xml index ecd3b4cb884a..85a5ece16834 100644 --- a/cdap-common/src/main/resources/cdap-default.xml +++ b/cdap-common/src/main/resources/cdap-default.xml @@ -213,6 +213,15 @@ + + location.cache.force.cleanup.ms + 3600000 + + Time duration after which the location cache is forced cleaned up. A + non-positive value indicates that the cache is never forced cleaned up. + + + thrift.max.read.buffer 16777216 @@ -514,10 +523,18 @@ - app.bind.port + appfabric.processor.bind.port + 0 + + App Fabric processor bind port; if 0, binds to a random port + + + + + appfabric.bind.port 0 - App Fabric service bind port; if 0, binds to a random port + App Fabric server bind port; if 0, binds to a random port @@ -575,31 +592,6 @@ - - run.record.monitor.record.age.threshold.seconds - 3600 - - Maximum amount of time (in seconds) in which a run record is retained in - run record monitor. - This is to safe guard launch requests flow-control such that if a request - is somehow stuck in PENDING/STARTING - state, it will be dropped from after the threshold. - Note that run.record.monitor.cleanup.interval.seconds might be needed to - changed if this config changes. - - - - - run.record.monitor.cleanup.interval.seconds - 60 - - Cleanup interval (in seconds) in which run record monitor service cleanup - logic runs to delete old entries. - Note that run.record.monitor.record.age.threshold.seconds might be needed - to changed if this config changes. - - - app.program.launch.threads 20 @@ -1277,6 +1269,33 @@ + + data.storage.properties.gcp-spanner.tx.runner.max.retries + 12 + + The max number of retries for the spanner transaction runner on retryable + failures. + + + + + data.storage.properties.gcp-spanner.compression.config + application_specs:application_data:SNAPPY,provisioner_data:provisioner_task_info:SNAPPY,destination_fields_table:destination_data:SNAPPY,summary_fields_table:destination_data:SNAPPY,run_records:run_record_data:SNAPPY,workflows:workflow_data:SNAPPY + + Represents table_name:column_name:compressor_type which possibly can cross spanner cell limit + and will need compression to reduce size. + WARNING: Removing fields from here is a backward incompatible operation. + + + + + data.storage.properties.gcp-spanner.tx.runner.initial.delay.ms + 500 + + The initial delay between retries due to spanner transaction failures in milliseconds. + + + data.tx.enabled true @@ -2245,6 +2264,13 @@ + + log.saver.status.bind.port + 0 + + Log saver HTTP service bind port + + @@ -2405,6 +2431,11 @@ + + messaging.service.enabled + true + + messaging.service.extensions.dir /opt/cdap/master/ext/messagingproviders @@ -3815,6 +3846,14 @@ + + preview.http.compress.payload + true + + Compress payload for HTTP calls in the preview http handler. + + + service.retry.policy.base.delay.ms 100 @@ -4981,6 +5020,26 @@ + + security.store.system.properties.gcp-cloudkms.key.rotation.enabled + true + + Enable key rotation for KMS keys used to encrypt and decrypt secure keys. + When true, it will enable rotation for newly created keys as well as + existing keys (after upgrade). + + + + + security.store.system.properties.gcp-cloudkms.key.rotation.period.days + 90 + + Specifies the rotation period applied to KMS keys when + security.store.system.properties.gcp-cloudkms.key.rotation.enabled is true. + The rotation period must be at least 1 day. + + + security.token.digest.algorithm HmacSHA256 @@ -5546,6 +5605,95 @@ + + task.worker.probe.enabled + false + + Whether to configure a probe for the task worker. + + + + + task.worker.probe.k8s.names + liveness + + A Comma separated value of kubernetes probe names. Ex : "liveness","readiness" + + + + + task.worker.probe.k8s.liveness.type + httpget + + Which type of liveness probe to be used. + + + + + task.worker.probe.k8s.liveness.http.port + ${task.worker.bind.port} + + The port number on the container where the API server is listening. + + + + + task.worker.probe.k8s.liveness.http.path + /ping + + The specific URL path to access on the container. + + + + + task.worker.probe.k8s.liveness.init.delay.seconds + 20 + + Seconds to wait after container start before the first probe for a task worker. + + + + + task.worker.probe.k8s.liveness.timeout.seconds + 5 + + Seconds after which the probe is considered failed due to no response. + + + + + task.worker.probe.k8s.liveness.failure.threshold + 12 + + Number of consecutive failures before k8s restarts the task worker container. + + + + + task.worker.probe.k8s.liveness.period.seconds + 10 + + The time interval (in seconds) between consecutive probe executions. + + + + + task.worker.probe.k8s.liveness.success.threshold + 1 + + Minimum consecutive successes for the probe to be considered successful after having failed. + Keeping same as K8s default. + + + + + task.worker.probe.k8s.liveness.http.scheme + HTTPS + + The protocol used to connect to the host. + + + system.worker.program.twill.controller.start.seconds @@ -6090,6 +6238,14 @@ + + feature.wrangler.workspace.auth.check.enabled + false + + Enables authorization enforcement on workspace actions in Wrangler. If disabled, no checks will be performed. + + + feature.namespaced.service.accounts.enabled false @@ -6563,4 +6719,12 @@ + + ui.default.poll.interval.millis + 10000 + + The default polling interval in milliseconds for poll calls from cdap-ui. + + + diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/http/AuthenticationChannelHandlerTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/http/AuthenticationChannelHandlerTest.java index 86124e582d18..e0f1ddd42b94 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/http/AuthenticationChannelHandlerTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/http/AuthenticationChannelHandlerTest.java @@ -133,18 +133,26 @@ public void testCallOrderCreateNamespaceForAuditLog() throws Exception { handler.channelRead(ctx, req); //ENUSRE THAT ATTR WAS SET in ACH's Finally Block. - verify(ctx.channel().attr(AttributeKey.valueOf(AuthenticationChannelHandler.AUDIT_LOG_CONTEXT_QUEUE_ATTR)), - times(1)).set(any()); - verify(ctx.channel().attr(AttributeKey.valueOf(AuthenticationChannelHandler.AUDIT_LOG_REQ_BUILDER_ATTR)), - times(1)).set(any()); + verify(ctx.channel().attr(AttributeKey.valueOf( + AuthenticationChannelHandler.AUDIT_LOG_CONTEXT_QUEUE_ATTR)), times(1)).set(any()); + verify(ctx.channel().attr(AttributeKey.valueOf( + AuthenticationChannelHandler.AUDIT_LOG_REQ_BUILDER_ATTR)), times(1)).set(any()); + verify(ctx.channel().attr(AttributeKey.valueOf( + AuthenticationChannelHandler.AUDIT_LOG_USER_IP_ATTR)), times(1)).set(any()); + verify(ctx.channel().attr(AttributeKey.valueOf( + AuthenticationChannelHandler.CDAP_USER_ID_ATTR)), times(1)).set(any()); + verify(ctx.channel().attr(AttributeKey.valueOf( + AuthenticationChannelHandler.CDAP_USER_CREDENTIAL_ATTR)), times(1)).set(any()); // Now in Write and getAuditLogRequest , should create AuditLogRequest properly from ATTRs - Mockito.when(ctx.channel().attr(AttributeKey.valueOf(AuthenticationChannelHandler.AUDIT_LOG_CONTEXT_QUEUE_ATTR)) - .get()).thenReturn(getAuditLogContexts()); - Mockito.when(ctx.channel().attr(AttributeKey.valueOf(AuthenticationChannelHandler.AUDIT_LOG_USER_IP_ATTR)).get()) - .thenReturn("testuserIp"); - Mockito.when(ctx.channel().attr(AttributeKey.valueOf(AuthenticationChannelHandler.AUDIT_LOG_REQ_BUILDER_ATTR)) - .get()).thenReturn(getAuditLogRequestBuilder()); + Mockito.when(ctx.channel().attr(AttributeKey.valueOf( + AuthenticationChannelHandler.AUDIT_LOG_CONTEXT_QUEUE_ATTR)).get()).thenReturn( + getAuditLogContexts()); + Mockito.when(ctx.channel().attr(AttributeKey.valueOf( + AuthenticationChannelHandler.AUDIT_LOG_USER_IP_ATTR)).get()).thenReturn("testuserIp"); + Mockito.when(ctx.channel().attr(AttributeKey.valueOf( + AuthenticationChannelHandler.AUDIT_LOG_REQ_BUILDER_ATTR)).get()).thenReturn( + getAuditLogRequestBuilder()); handler.write(ctx, "msg", new DefaultChannelPromise(ctx.channel())); verify(auditLogWriterMock, times(1)).publish(any()); diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/io/CachingLocationTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/io/CachingLocationTest.java index 9f21c9a83c64..35546f5e3663 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/io/CachingLocationTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/io/CachingLocationTest.java @@ -90,8 +90,10 @@ public void testCachingPathProvider() throws IOException, InterruptedException { Path cachePath = TEMP_FOLDER.newFolder().toPath(); String message = "Testing message"; - DefaultCachingPathProvider cacheProvider = new DefaultCachingPathProvider(cachePath, 1, TimeUnit.HOURS); - LocationFactory lf = new CachingLocationFactory(new LocalLocationFactory(TEMP_FOLDER.newFolder()), cacheProvider); + DefaultCachingPathProvider cacheProvider = new DefaultCachingPathProvider(cachePath, 1, + TimeUnit.HOURS, 0); + LocationFactory lf = new CachingLocationFactory( + new LocalLocationFactory(TEMP_FOLDER.newFolder()), cacheProvider); // Write out a location Location location = lf.create("test"); @@ -143,13 +145,54 @@ public void testCachingPathProvider() throws IOException, InterruptedException { Assert.assertFalse(Files.exists(cacheFileDir)); } + @Test + public void testForceCacheCleanup() throws IOException, InterruptedException { + Path cachePath = TEMP_FOLDER.newFolder().toPath(); + String message = "Testing message force cleanup"; + + DefaultCachingPathProvider cacheProvider = new DefaultCachingPathProvider(cachePath, 1, + TimeUnit.SECONDS, 100); + LocationFactory lf = new CachingLocationFactory( + new LocalLocationFactory(TEMP_FOLDER.newFolder()), cacheProvider); + + // Write out a location + Location location = lf.create("test-cleanup"); + try (OutputStream os = location.getOutputStream()) { + os.write(message.getBytes(StandardCharsets.UTF_8)); + } + try (InputStream is = location.getInputStream()) { + Assert.assertEquals(message, new String(ByteStreams.toByteArray(is), StandardCharsets.UTF_8)); + } + + // Check if the cache file is there + long oldLastModified = location.lastModified(); + Path cachedFile = cacheProvider.getCachePath(cacheProvider.getCacheName(location), + oldLastModified); + Assert.assertTrue(Files.exists(cachedFile)); + try (InputStream is = Files.newInputStream(cachedFile)) { + Assert.assertEquals(message, new String(ByteStreams.toByteArray(is), StandardCharsets.UTF_8)); + } + + // Sleep 3 seconds to force expiry of cache(1 second) and force cleanup of cache(100ms) with + // some buffer for operations. + TimeUnit.SECONDS.sleep(3); + + Path cacheFileDir = cacheProvider.getCachePath(cacheProvider.getCacheName(location), + location.lastModified()).getParent(); + Assert.assertFalse(Files.exists(cacheFileDir)); + // Entries should be cleaned up automatically. + Assert.assertEquals(0, cacheProvider.getCacheEntries().size()); + } + @Test public void testCachePopulate() throws IOException { Path cachePath = TEMP_FOLDER.newFolder().toPath(); String message = "Testing message"; - DefaultCachingPathProvider cacheProvider = new DefaultCachingPathProvider(cachePath, 1, TimeUnit.HOURS); - LocationFactory lf = new CachingLocationFactory(new LocalLocationFactory(TEMP_FOLDER.newFolder()), cacheProvider); + DefaultCachingPathProvider cacheProvider = new DefaultCachingPathProvider(cachePath, 1, + TimeUnit.HOURS, 0); + LocationFactory lf = new CachingLocationFactory( + new LocalLocationFactory(TEMP_FOLDER.newFolder()), cacheProvider); // Write out a location Location location = lf.create("test"); @@ -163,7 +206,8 @@ public void testCachePopulate() throws IOException { // Create a new preview cache provider from the same directory, it should // populate the cache. - Collection cacheEntries = new DefaultCachingPathProvider(cachePath, 1, TimeUnit.HOURS).getCacheEntries(); + Collection cacheEntries = new DefaultCachingPathProvider(cachePath, 1, TimeUnit.HOURS, + 0).getCacheEntries(); Assert.assertTrue(cacheEntries.contains(cacheProvider.getCachePath(cacheProvider.getCacheName(location), location.lastModified()))); } @@ -175,7 +219,8 @@ public void testNoDuplicateCacheEntries() throws IOException, InterruptedExcepti // Create a new cache provider from the same directory, it should // populate the cache. - DefaultCachingPathProvider cachingPathProvider = new DefaultCachingPathProvider(cachePath, 1, TimeUnit.HOURS); + DefaultCachingPathProvider cachingPathProvider = new DefaultCachingPathProvider(cachePath, 1, + TimeUnit.HOURS, 0); LocationFactory lf = new CachingLocationFactory( new LocalLocationFactory(TEMP_FOLDER.newFolder()), cachingPathProvider); @@ -220,8 +265,10 @@ public void testDelete() throws IOException { Path cachePath = TEMP_FOLDER.newFolder().toPath(); String message = "Testing message"; - DefaultCachingPathProvider cacheProvider = new DefaultCachingPathProvider(cachePath, 1, TimeUnit.HOURS); - LocationFactory lf = new CachingLocationFactory(new LocalLocationFactory(TEMP_FOLDER.newFolder()), cacheProvider); + DefaultCachingPathProvider cacheProvider = new DefaultCachingPathProvider(cachePath, 1, + TimeUnit.HOURS, 0); + LocationFactory lf = new CachingLocationFactory( + new LocalLocationFactory(TEMP_FOLDER.newFolder()), cacheProvider); // Write out a location Location location = lf.create("test"); diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/lang/jar/BundleJarUtilTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/lang/jar/BundleJarUtilTest.java index 32a7d0e38bc8..2e5ee64fb504 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/lang/jar/BundleJarUtilTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/lang/jar/BundleJarUtilTest.java @@ -21,6 +21,7 @@ import com.google.common.io.CharStreams; import com.google.common.io.Files; import io.cdap.cdap.common.io.Locations; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -32,9 +33,13 @@ import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.jar.JarOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import org.junit.Assert; import org.junit.ClassRule; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; /** @@ -42,6 +47,9 @@ */ public class BundleJarUtilTest { + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + @ClassRule public static final TemporaryFolder TEMP_FOLDER = new TemporaryFolder(); @@ -159,6 +167,35 @@ public boolean test(String name) { } } + + @Test + public void testZipSlipValidation() throws IOException { + + expectedEx.expect(IllegalArgumentException.class); + expectedEx.expectMessage("Illegal path detected for Jar Entry : ../../../malicious.jar. This will try to " + + "write outside the target directory, which is not allowed."); + + // Create a file inside a sub-dir. + File inputJarDirectory = TEMP_FOLDER.newFolder(); + + String maliciousEntryName = "../../../malicious.jar"; + + // 1. Create malicious JAR content in memory + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(bos)) { + ZipEntry maliciousEntry = new ZipEntry(maliciousEntryName); + zos.putNextEntry(maliciousEntry); + zos.write("some random message!".getBytes()); + zos.closeEntry(); + } + byte[] maliciousJarContent = bos.toByteArray(); + File maliciousJarFile = new File(inputJarDirectory, "malicious.jar"); + java.nio.file.Files.createDirectories(inputJarDirectory.toPath()); + java.nio.file.Files.write(maliciousJarFile.toPath(), maliciousJarContent); + + BundleJarUtil.unJar(maliciousJarFile, TEMP_FOLDER.newFolder()); + } + /** * Helper method to recursively check if two directories are equal */ diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/metrics/MetricsReporterHookTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/metrics/MetricsReporterHookTest.java index b23d1dd828b8..40f2d29dc19e 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/metrics/MetricsReporterHookTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/metrics/MetricsReporterHookTest.java @@ -23,6 +23,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.common.collect.ImmutableMap; import io.cdap.cdap.api.metrics.MetricsCollectionService; import io.cdap.cdap.api.metrics.MetricsContext; import io.cdap.cdap.common.conf.CConfiguration; @@ -32,6 +33,7 @@ import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpVersion; +import java.util.Map; import org.junit.Test; public class MetricsReporterHookTest { @@ -43,7 +45,13 @@ public class MetricsReporterHookTest { public void testReponseTimeCollection() throws InterruptedException { MetricsContext mockCollector = mock(MetricsContext.class); MetricsCollectionService mockCollectionService = mock(MetricsCollectionService.class); - when(mockCollectionService.getContext(anyMap())).thenReturn(mockCollector); + Map tags = ImmutableMap.of( + "ns", "system", + "cmp", TESTSERVICENAME, + "hnd", "handler", + "mtd", TESTMETHODNAME + ); + when(mockCollectionService.getContext(tags)).thenReturn(mockCollector); HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "http://ignore"); HandlerInfo handlerInfo = new HandlerInfo(TESTHANDLERNAME, TESTMETHODNAME); @@ -55,4 +63,56 @@ public void testReponseTimeCollection() throws InterruptedException { verify(mockCollector).event(eq("response.latency"), anyLong()); } + + @Test + public void testNamespacesExtraction() throws InterruptedException { + MetricsContext mockCollector = mock(MetricsContext.class); + MetricsCollectionService mockCollectionService = mock(MetricsCollectionService.class); + HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, + "http://ignore/namespaces/testNamespace"); + HandlerInfo handlerInfo = new HandlerInfo(TESTHANDLERNAME, TESTMETHODNAME); + MetricsReporterHook hook = new MetricsReporterHook(CConfiguration.create(), + mockCollectionService, TESTSERVICENAME); + + Map tags = ImmutableMap.of( + "ns", "testNamespace", + "cmp", TESTSERVICENAME, + "hnd", "handler", + "mtd", TESTMETHODNAME + ); + when(mockCollectionService.getContext(tags)).thenReturn(mockCollector); + + hook.preCall(request, null, handlerInfo); + verify(mockCollector).increment("request.received", 1); + + hook.postCall(request, HttpResponseStatus.OK, handlerInfo); + verify(mockCollector).increment("response.successful", 1); + verify(mockCollector).event(eq("response.latency"), anyLong()); + } + + @Test + public void testNoNamespaceToExtract() throws InterruptedException { + MetricsContext mockCollector = mock(MetricsContext.class); + MetricsCollectionService mockCollectionService = mock(MetricsCollectionService.class); + HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, + "http://ignore/namespaces"); + HandlerInfo handlerInfo = new HandlerInfo(TESTHANDLERNAME, TESTMETHODNAME); + MetricsReporterHook hook = new MetricsReporterHook(CConfiguration.create(), + mockCollectionService, TESTSERVICENAME); + + Map tags = ImmutableMap.of( + "ns", "system", + "cmp", TESTSERVICENAME, + "hnd", "handler", + "mtd", TESTMETHODNAME + ); + when(mockCollectionService.getContext(tags)).thenReturn(mockCollector); + + hook.preCall(request, null, handlerInfo); + verify(mockCollector).increment("request.received", 1); + + hook.postCall(request, HttpResponseStatus.OK, handlerInfo); + verify(mockCollector).increment("response.successful", 1); + verify(mockCollector).event(eq("response.latency"), anyLong()); + } } diff --git a/cdap-coverage/pom.xml b/cdap-coverage/pom.xml index d0c3b7e2763b..01af7484e648 100644 --- a/cdap-coverage/pom.xml +++ b/cdap-coverage/pom.xml @@ -20,7 +20,7 @@ cdap io.cdap.cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT 4.0.0 @@ -108,16 +108,6 @@ cdap-security ${project.version} - - io.cdap.cdap - cdap-hbase-spi - ${project.version} - - - io.cdap.cdap - cdap-hbase-compat-base - ${project.version} - io.cdap.cdap cdap-tms @@ -233,11 +223,6 @@ cdap-program-report ${project.version} - - io.cdap.cdap - cdap-hbase-compat-1.0 - ${project.version} - io.cdap.cdap cdap-data-fabric-tests @@ -283,26 +268,6 @@ cdap-app-fabric-tests ${project.version} - - io.cdap.cdap - cdap-hbase-compat-1.0-cdh5.5.0 - ${project.version} - - - io.cdap.cdap - cdap-hbase-compat-1.0-cdh - ${project.version} - - - io.cdap.cdap - cdap-hbase-compat-1.1 - ${project.version} - - - io.cdap.cdap - cdap-hbase-compat-1.2-cdh5.7.0 - ${project.version} - io.cdap.cdap cdap-kubernetes diff --git a/cdap-credential-ext-gcp-wi/pom.xml b/cdap-credential-ext-gcp-wi/pom.xml index b73cd7d0317a..2c047c5bf026 100644 --- a/cdap-credential-ext-gcp-wi/pom.xml +++ b/cdap-credential-ext-gcp-wi/pom.xml @@ -21,7 +21,7 @@ cdap io.cdap.cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT 4.0.0 diff --git a/cdap-data-fabric-tests/pom.xml b/cdap-data-fabric-tests/pom.xml index 6812782b6265..18b54fc9e86a 100644 --- a/cdap-data-fabric-tests/pom.xml +++ b/cdap-data-fabric-tests/pom.xml @@ -24,7 +24,7 @@ io.cdap.cdap cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-data-fabric-tests @@ -139,11 +139,14 @@ - org.apache.maven.plugins - maven-deploy-plugin - 2.8 + org.sonatype.central + central-publishing-maven-plugin + true - true + sonatype.release + false + true + true diff --git a/cdap-data-fabric/pom.xml b/cdap-data-fabric/pom.xml index 308d13bbb1d1..571bff3353e2 100644 --- a/cdap-data-fabric/pom.xml +++ b/cdap-data-fabric/pom.xml @@ -23,7 +23,7 @@ io.cdap.cdap cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-data-fabric diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableContext.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableContext.java index eb48f0851d1c..661dda851026 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableContext.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableContext.java @@ -21,6 +21,7 @@ import io.cdap.cdap.api.data.DatasetInstantiationException; import io.cdap.cdap.api.dataset.lib.IndexedTable; import io.cdap.cdap.api.metrics.MetricsCollector; +import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.spi.data.StructuredTable; import io.cdap.cdap.spi.data.StructuredTableContext; import io.cdap.cdap.spi.data.StructuredTableInstantiationException; @@ -75,4 +76,9 @@ public StructuredTable getTable(StructuredTableId tableId) tableId, String.format("Error instantiating table %s", tableId), e); } } + + @Override + public String getStorageProvider() { + return Constants.Dataset.DATA_STORAGE_NOSQL; + } } diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/spi/data/sql/PostgreSqlStorageProvider.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/spi/data/sql/PostgreSqlStorageProvider.java index a507e0fc2c98..66dc246db295 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/spi/data/sql/PostgreSqlStorageProvider.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/spi/data/sql/PostgreSqlStorageProvider.java @@ -186,7 +186,9 @@ private static void loadJDBCDriver(CConfiguration cConf, String storageImpl) { } // Create a separate classloader for the JDBC driver, which doesn't have any CDAP dependencies in it. - ClassLoader driverClassLoader = new DirectoryClassLoader(driverExtensionDir, null); + // Passing java.sql.Driver's parent classloader for java 11 + as it doesn't load them if we pass null. + ClassLoader driverClassLoader = new DirectoryClassLoader(driverExtensionDir, + java.sql.Driver.class.getClassLoader()); try { Driver driver = (Driver) Class.forName(driverName, true, driverClassLoader).newInstance(); diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/spi/data/sql/SqlStructuredTableContext.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/spi/data/sql/SqlStructuredTableContext.java index ef60352afaa4..11e39536a83d 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/spi/data/sql/SqlStructuredTableContext.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/spi/data/sql/SqlStructuredTableContext.java @@ -17,6 +17,7 @@ package io.cdap.cdap.spi.data.sql; import io.cdap.cdap.api.metrics.MetricsCollector; +import io.cdap.cdap.common.conf.Constants.Dataset; import io.cdap.cdap.spi.data.StructuredTable; import io.cdap.cdap.spi.data.StructuredTableAdmin; import io.cdap.cdap.spi.data.StructuredTableContext; @@ -59,4 +60,9 @@ public StructuredTable getTable(StructuredTableId tableId) throw new StructuredTableInstantiationException(tableId, "Failed to get the table schema", e); } } + + @Override + public String getStorageProvider() { + return Dataset.DATA_STORAGE_SQL; + } } diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/store/StoreDefinition.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/store/StoreDefinition.java index ac5d2c121ac3..6975bf0768b8 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/store/StoreDefinition.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/store/StoreDefinition.java @@ -504,6 +504,7 @@ public static final class AppMetadataStore { public static final String RUN_STATUS = "run_status"; public static final String RUN_START_TIME = "run_start_time"; public static final String RUN_RECORD_DATA = "run_record_data"; + public static final String FLOW_CONTROL_STATUS = "flow_control_status"; public static final String WORKFLOW_DATA = "workflow_data"; public static final String COUNT_TYPE = "count_type"; public static final String COUNTS = "counts"; @@ -567,10 +568,11 @@ public static final class AppMetadataStore { Fields.stringType(PROGRAM_FIELD), Fields.longType(RUN_START_TIME), Fields.stringType(RUN_FIELD), + Fields.stringType(FLOW_CONTROL_STATUS), Fields.stringType(RUN_RECORD_DATA)) .withPrimaryKeys(RUN_STATUS, NAMESPACE_FIELD, APPLICATION_FIELD, VERSION_FIELD, - PROGRAM_TYPE_FIELD, - PROGRAM_FIELD, RUN_START_TIME, RUN_FIELD) + PROGRAM_TYPE_FIELD, PROGRAM_FIELD, RUN_START_TIME, RUN_FIELD) + .withIndexes(FLOW_CONTROL_STATUS) .build(); public static final StructuredTableSpecification WORKFLOWS_SPEC = diff --git a/cdap-distributions/pom.xml b/cdap-distributions/pom.xml index a32f4e943560..246bb4774b92 100644 --- a/cdap-distributions/pom.xml +++ b/cdap-distributions/pom.xml @@ -24,7 +24,7 @@ io.cdap.cdap cdap - 6.11.0-SNAPSHOT + 6.11.2-SNAPSHOT cdap-distributions @@ -127,35 +127,6 @@ - - - - - org.apache.maven.plugins - maven-deploy-plugin - 2.8 - - - deploy-rpm - deploy - - deploy-file - - - ${project.version} - ${dist.deploy.groupId} - cdap - noarch.rpm - false - ${project.build.directory}/cdap-${package.version}-1.noarch.rpm - 1 - continuuity - ${deploy.url} - - - - - @@ -207,33 +178,6 @@ - - - - - org.apache.maven.plugins - maven-deploy-plugin - 2.8 - - - deploy-deb - deploy - - deploy-file - - - ${project.version} - ${dist.deploy.groupId} - cdap - deb - false - ${project.build.directory}/cdap_${package.version}-1_all.deb - continuuity - ${deploy.url} - - - - diff --git a/cdap-distributions/src/COPYRIGHT/@ajainarayanan-react-pan-zoom@0.0.4/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/@ajainarayanan-react-pan-zoom@0.0.4/COPYRIGHT new file mode 100644 index 000000000000..bd16440d4274 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/@ajainarayanan-react-pan-zoom@0.0.4/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Ajai + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/@apollo-protobufjs@1.0.5/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/@apollo-protobufjs@1.0.5/COPYRIGHT new file mode 100644 index 000000000000..57b7e30978f8 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/@apollo-protobufjs@1.0.5/COPYRIGHT @@ -0,0 +1,39 @@ +This license applies to all parts of protobuf.js except those files +either explicitly including or referencing a different license or +located in a directory containing a different LICENSE file. + +--- + +Copyright (c) 2016, Daniel Wirtz All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +* Neither the name of its author, nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +Code generated by the command line utilities is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/cdap-distributions/src/COPYRIGHT/@apollo-react-common@3.1.4/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/@apollo-react-common@3.1.4/COPYRIGHT new file mode 100644 index 000000000000..8c09781ba7e1 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/@apollo-react-common@3.1.4/COPYRIGHT @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2019 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/cdap-distributions/src/COPYRIGHT/@apollo-react-components@3.1.5/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/@apollo-react-components@3.1.5/COPYRIGHT new file mode 100644 index 000000000000..8c09781ba7e1 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/@apollo-react-components@3.1.5/COPYRIGHT @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2019 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/cdap-distributions/src/COPYRIGHT/@apollo-react-hoc@3.1.5/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/@apollo-react-hoc@3.1.5/COPYRIGHT new file mode 100644 index 000000000000..8c09781ba7e1 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/@apollo-react-hoc@3.1.5/COPYRIGHT @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2019 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/cdap-distributions/src/COPYRIGHT/@apollo-react-hooks@3.1.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/@apollo-react-hooks@3.1.3/COPYRIGHT new file mode 100644 index 000000000000..8c09781ba7e1 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/@apollo-react-hooks@3.1.3/COPYRIGHT @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2019 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/cdap-distributions/src/COPYRIGHT/@apollo-react-hooks@3.1.5/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/@apollo-react-hooks@3.1.5/COPYRIGHT new file mode 100644 index 000000000000..8c09781ba7e1 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/@apollo-react-hooks@3.1.5/COPYRIGHT @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2019 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/cdap-distributions/src/COPYRIGHT/@apollo-react-ssr@3.1.5/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/@apollo-react-ssr@3.1.5/COPYRIGHT new file mode 100644 index 000000000000..8c09781ba7e1 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/@apollo-react-ssr@3.1.5/COPYRIGHT @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2019 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/cdap-distributions/src/COPYRIGHT/@apollographql-apollo-tools@0.4.9/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/@apollographql-apollo-tools@0.4.9/COPYRIGHT new file mode 100644 index 000000000000..1558a68a8a8e --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/@apollographql-apollo-tools@0.4.9/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/@apollographql-graphql-playground-html@1.6.24/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/@apollographql-graphql-playground-html@1.6.24/COPYRIGHT new file mode 100644 index 000000000000..66d3d35b6eda --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/@apollographql-graphql-playground-html@1.6.24/COPYRIGHT @@ -0,0 +1,6 @@ +# graphql-playground-html + +This package is being used by the GraphQL Playground middlewares. + +For local development, you can `yarn link` this package, then use `yarn link graphql-playground-html` in the +middleware you want to develop. diff --git a/cdap-distributions/src/COPYRIGHT/@apollographql-graphql-playground-html@1.6.26/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/@apollographql-graphql-playground-html@1.6.26/COPYRIGHT new file mode 100644 index 000000000000..3ccf3fcdcda1 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/@apollographql-graphql-playground-html@1.6.26/COPYRIGHT @@ -0,0 +1,5 @@ +# @apollographql/graphql-playground-html + +**NOTE:** This is a fork of [`graphql-playground-html`](https://npm.im/graphql-playground-html) which is meant to be used by Apollo Server and only by Apollo Server. It is not intended to be used directly. Those looking to use GraphQL Playground directly can refer to [the upstream repository](https://github.com/prisma-labs/graphql-playground) for usage instructions. + +> **SECURITY WARNING:** Via the upstream fork, this package had a severe XSS Reflection attack vulnerability until version `1.6.25` of this package. **While we have published a fix, users were only affected if they were using `@apollographql/graphql-playground-html` directly as their own custom middleware.** The direct usage of this package was never recommended as it provided no advantage over the upstream package in that regard. Users of Apollo Server who leverage this package automatically by the dependency declared within Apollo Sever were not affected since Apollo Server never provided dynamic facilities to customize playground options per request. Users of Apollo Server would have had to statically embedded very explicit vulnerabilities (e.g., using malicious, unescaped code, ` +``` + +Then add `ngAnimate` as a dependency for your app: + +```javascript +angular.module('myApp', ['ngAnimate']); +``` + +## Documentation + +Documentation is available on the +[AngularJS docs site](http://docs.angularjs.org/api/ngAnimate). + +## License + +The MIT License + +Copyright (c) 2010-2015 Google, Inc. http://angularjs.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/angular-bootstrap@1.1.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-bootstrap@1.1.2/COPYRIGHT deleted file mode 100644 index 8472327ffd21..000000000000 --- a/cdap-distributions/src/COPYRIGHT/angular-bootstrap@1.1.2/COPYRIGHT +++ /dev/null @@ -1 +0,0 @@ -The MIT License (MIT) \ No newline at end of file diff --git a/cdap-distributions/src/COPYRIGHT/angular-breadcrumb@0.3.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-breadcrumb@0.3.3/COPYRIGHT index c84b1df7e2ce..2634550878ad 100644 --- a/cdap-distributions/src/COPYRIGHT/angular-breadcrumb@0.3.3/COPYRIGHT +++ b/cdap-distributions/src/COPYRIGHT/angular-breadcrumb@0.3.3/COPYRIGHT @@ -1,5 +1,3 @@ -The MIT License - Copyright (c) 2013 Nicolas Cuillery Permission is hereby granted, free of charge, to any person @@ -21,4 +19,4 @@ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +OTHER DEALINGS IN THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/angular-cookies@1.4.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-cookies@1.4.3/COPYRIGHT index 18967694ac24..7b190d34615e 100644 --- a/cdap-distributions/src/COPYRIGHT/angular-cookies@1.4.3/COPYRIGHT +++ b/cdap-distributions/src/COPYRIGHT/angular-cookies@1.4.3/COPYRIGHT @@ -1,9 +1,68 @@ -The MIT License (MIT) +# packaged angular-cookies -Copyright (c) 2016 Angular +This repo is for distribution on `npm` and `bower`. The source for this module is in the +[main AngularJS repo](https://github.com/angular/angular.js/tree/master/src/ngCookies). +Please file issues and pull requests against that repo. -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +## Install -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +You can install this package either with `npm` or with `bower`. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +### npm + +```shell +npm install angular-cookies +``` + +Then add `ngCookies` as a dependency for your app: + +```javascript +angular.module('myApp', [require('angular-cookies')]); +``` + +### bower + +```shell +bower install angular-cookies +``` + +Add a ` +``` + +Then add `ngCookies` as a dependency for your app: + +```javascript +angular.module('myApp', ['ngCookies']); +``` + +## Documentation + +Documentation is available on the +[AngularJS docs site](http://docs.angularjs.org/api/ngCookies). + +## License + +The MIT License + +Copyright (c) 2010-2015 Google, Inc. http://angularjs.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/angular-cron-jobs@1.4.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-cron-jobs@1.4.1/COPYRIGHT index be2dd32592e3..e3e7a93e24ea 100644 --- a/cdap-distributions/src/COPYRIGHT/angular-cron-jobs@1.4.1/COPYRIGHT +++ b/cdap-distributions/src/COPYRIGHT/angular-cron-jobs@1.4.1/COPYRIGHT @@ -18,4 +18,5 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. + diff --git a/cdap-distributions/src/COPYRIGHT/angular-file-saver@1.1.4/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-file-saver@1.1.4/COPYRIGHT index 26fc737da0f5..d40152a5bbb8 100644 --- a/cdap-distributions/src/COPYRIGHT/angular-file-saver@1.1.4/COPYRIGHT +++ b/cdap-distributions/src/COPYRIGHT/angular-file-saver@1.1.4/COPYRIGHT @@ -4,4 +4,4 @@ Permission is hereby granted, free of charge, to any person obtaining a copy of The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/angular-gridster@0.13.5/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-gridster@0.13.5/COPYRIGHT index ee86cc462344..445c4681f8fb 100644 --- a/cdap-distributions/src/COPYRIGHT/angular-gridster@0.13.5/COPYRIGHT +++ b/cdap-distributions/src/COPYRIGHT/angular-gridster@0.13.5/COPYRIGHT @@ -17,4 +17,4 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/angular-inview@1.5.7/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-inview@1.5.7/COPYRIGHT index bb385649a1ca..7190e3a94632 100644 --- a/cdap-distributions/src/COPYRIGHT/angular-inview@1.5.7/COPYRIGHT +++ b/cdap-distributions/src/COPYRIGHT/angular-inview@1.5.7/COPYRIGHT @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/angular-loading-bar@0.8.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-loading-bar@0.8.0/COPYRIGHT index 5fca9070cd42..252c23aa19cc 100644 --- a/cdap-distributions/src/COPYRIGHT/angular-loading-bar@0.8.0/COPYRIGHT +++ b/cdap-distributions/src/COPYRIGHT/angular-loading-bar@0.8.0/COPYRIGHT @@ -17,4 +17,4 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/angular-marked@1.0.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-marked@1.0.1/COPYRIGHT index 71970ae7ad09..84307daf7575 100644 --- a/cdap-distributions/src/COPYRIGHT/angular-marked@1.0.1/COPYRIGHT +++ b/cdap-distributions/src/COPYRIGHT/angular-marked@1.0.1/COPYRIGHT @@ -1,3 +1,170 @@ -The MIT License (MIT) - -Copyright (c) 2013-2015 Jayson Harshbarger \ No newline at end of file +# angular-marked + +[![NPM version][npm-badge]][npm] +[![Downloads][download-badge]][npm] +![Downloads][bower-badge] + +[![Build Status][travis-image]][travis-url] +[![Codacy Badge][codacy-badge]][Codacy] + +[![js-semistandard-style][standard-badge]][semistandard] +[![License][license-badge]][MIT License] + +AngularJS Markdown using [marked](https://github.com/chjj/marked). + +**Please note:** neither this directive nor marked (by default) implement sanitization. As always, sanitizing is necessary for user-generated content. + +## Install + +`bower install angular-marked` + +or + +`npm install angular-marked` + +or + +`jspm install angular-marked=npm:angular-marked` + +Depending on your setup you may need include script tags in your html: + +```html + + +``` + +## Usage + +```js +var app = angular.module('example-app', ['hc.marked']); +``` + +### Set default options (optional) + +```js +app.config(['markedProvider', function (markedProvider) { + markedProvider.setOptions({gfm: true}); +}]); +``` + +Example using [highlight.js Javascript syntax highlighter](http://highlightjs.org/) (must include highlight.js script). + +```js +app.config(['markedProvider', function (markedProvider) { + markedProvider.setOptions({ + gfm: true, + tables: true, + highlight: function (code, lang) { + if (lang) { + return hljs.highlight(lang, code, true).value; + } else { + return hljs.highlightAuto(code).value; + } + } + }); +}]); + +``` + +### Override Rendered Markdown Links + +Example overriding the way custom markdown links are displayed to open in new windows: + +```js +app.config(['markedProvider', function (markedProvider) { + markedProvider.setRenderer({ + link: function(href, title, text) { + return "" + text + ""; + } + }); +}]); +``` + +### Use as a directive + +```html + + # Markdown directive + *It works!* + +``` + +Bind the markdown input to a scope variable: + +```html +

+
+ +``` + +Include a markdown file: + +```html +
+
+ +``` + +Or a template (great for md that includes code blocks): + +```html + + +
+ +``` + +### As a service + +```js +app.controller('myCtrl', ['marked', function (marked) { + $scope.html = marked('#TEST'); +}]); +``` + +## Testing + +Install npm and bower dependencies: + +```sh +npm install +bower install +npm test +``` + +## Why? + +I wanted to use `marked` instead of `showdown` as used in `angular-markdown-directive` as well as expose the option to globally set defaults. Yes, it is probably best to avoid creating a bunch of angular wrapper modules... but I use this enough across multiple projects to make it worth while for me. Use it if you like. Pull requests are welcome. + +## Acknowledgments +Based on [angular-markdown-directive](https://github.com/btford/angular-markdown-directive) by [briantford](http://briantford.com/) which, in turn, is based on [this excellent tutorial](http://blog.angularjs.org/2012/05/custom-components-part-1.html) by [@johnlinquist](https://twitter.com/johnlindquist). + +## License +Copyright (c) 2013-2015 Jayson Harshbarger + +[MIT License] + +[npm]: https://npmjs.org/package/angular-marked +[bower]: https://npmjs.org/package/angular-marked +[semistandard]: https://github.com/Flet/semistandard +[Codacy]: https://www.codacy.com/app/hypercubed/angular-marked +[MIT License]: http://en.wikipedia.org/wiki/MIT_License +[travis-url]: https://travis-ci.org/Hypercubed/angular-marked + +[travis-image]: https://img.shields.io/travis/Hypercubed/angular-marked.svg +[npm-badge]: https://img.shields.io/npm/v/angular-marked.svg +[bower-badge]: https://img.shields.io/bower/v/angular-marked.svg +[standard-badge]: https://img.shields.io/badge/code%20style-semistandard-brightgreen.svg +[download-badge]: http://img.shields.io/npm/dm/angular-marked.svg +[codacy-badge]: https://api.codacy.com/project/badge/grade/eef9605446ce4555bebb1c55f7b93e2b +[license-badge]: https://img.shields.io/badge/license-MIT-blue.svg diff --git a/cdap-distributions/src/COPYRIGHT/angular-mocks@1.4.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-mocks@1.4.3/COPYRIGHT index 18967694ac24..440cce9b78ac 100644 --- a/cdap-distributions/src/COPYRIGHT/angular-mocks@1.4.3/COPYRIGHT +++ b/cdap-distributions/src/COPYRIGHT/angular-mocks@1.4.3/COPYRIGHT @@ -1,9 +1,63 @@ -The MIT License (MIT) +# packaged angular-mocks -Copyright (c) 2016 Angular +This repo is for distribution on `npm` and `bower`. The source for this module is in the +[main AngularJS repo](https://github.com/angular/angular.js/tree/master/src/ngMock). +Please file issues and pull requests against that repo. -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +## Install -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +You can install this package either with `npm` or with `bower`. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +### npm + +```shell +npm install angular-mocks +``` + +You can `require` ngMock modules: + +```js +var angular = require('angular'); +angular.module('myMod', [ + require('angular-animate'), + require('angular-mocks/ngMock') + require('angular-mocks/ngAnimateMock') +]); +``` + +### bower + +```shell +bower install angular-mocks +``` + +The mocks are then available at `bower_components/angular-mocks/angular-mocks.js`. + +## Documentation + +Documentation is available on the +[AngularJS docs site](https://docs.angularjs.org/guide/unit-testing). + +## License + +The MIT License + +Copyright (c) 2010-2015 Google, Inc. http://angularjs.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/angular-moment@0.10.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-moment@0.10.3/COPYRIGHT index 020faa2f2221..c27243df0248 100644 --- a/cdap-distributions/src/COPYRIGHT/angular-moment@0.10.3/COPYRIGHT +++ b/cdap-distributions/src/COPYRIGHT/angular-moment@0.10.3/COPYRIGHT @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2013-2016 Uri Shaked and contributors +Copyright (c) 2013-2015 Uri Shaked and contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/angular-motion@0.4.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-motion@0.4.3/COPYRIGHT index c6a38b7b062a..6615f14a9f90 100644 --- a/cdap-distributions/src/COPYRIGHT/angular-motion@0.4.3/COPYRIGHT +++ b/cdap-distributions/src/COPYRIGHT/angular-motion@0.4.3/COPYRIGHT @@ -2,8 +2,20 @@ The MIT License Copyright (c) 2014 Olivier Louvignes http://olouv.com -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/angular-resource@1.4.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-resource@1.4.3/COPYRIGHT index 18967694ac24..f3bd119ce22b 100644 --- a/cdap-distributions/src/COPYRIGHT/angular-resource@1.4.3/COPYRIGHT +++ b/cdap-distributions/src/COPYRIGHT/angular-resource@1.4.3/COPYRIGHT @@ -1,9 +1,68 @@ -The MIT License (MIT) +# packaged angular-resource -Copyright (c) 2016 Angular +This repo is for distribution on `npm` and `bower`. The source for this module is in the +[main AngularJS repo](https://github.com/angular/angular.js/tree/master/src/ngResource). +Please file issues and pull requests against that repo. -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +## Install -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +You can install this package either with `npm` or with `bower`. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +### npm + +```shell +npm install angular-resource +``` + +Then add `ngResource` as a dependency for your app: + +```javascript +angular.module('myApp', [require('angular-resource')]); +``` + +### bower + +```shell +bower install angular-resource +``` + +Add a ` +``` + +Then add `ngResource` as a dependency for your app: + +```javascript +angular.module('myApp', ['ngResource']); +``` + +## Documentation + +Documentation is available on the +[AngularJS docs site](http://docs.angularjs.org/api/ngResource). + +## License + +The MIT License + +Copyright (c) 2010-2015 Google, Inc. http://angularjs.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/angular-sanitize@1.4.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-sanitize@1.4.3/COPYRIGHT index 18967694ac24..b84aaf6dbf1c 100644 --- a/cdap-distributions/src/COPYRIGHT/angular-sanitize@1.4.3/COPYRIGHT +++ b/cdap-distributions/src/COPYRIGHT/angular-sanitize@1.4.3/COPYRIGHT @@ -1,9 +1,68 @@ -The MIT License (MIT) +# packaged angular-sanitize -Copyright (c) 2016 Angular +This repo is for distribution on `npm` and `bower`. The source for this module is in the +[main AngularJS repo](https://github.com/angular/angular.js/tree/master/src/ngSanitize). +Please file issues and pull requests against that repo. -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +## Install -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +You can install this package either with `npm` or with `bower`. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +### npm + +```shell +npm install angular-sanitize +``` + +Then add `ngSanitize` as a dependency for your app: + +```javascript +angular.module('myApp', [require('angular-sanitize')]); +``` + +### bower + +```shell +bower install angular-sanitize +``` + +Add a ` +``` + +Then add `ngSanitize` as a dependency for your app: + +```javascript +angular.module('myApp', ['ngSanitize']); +``` + +## Documentation + +Documentation is available on the +[AngularJS docs site](http://docs.angularjs.org/api/ngSanitize). + +## License + +The MIT License + +Copyright (c) 2010-2015 Google, Inc. http://angularjs.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/angular-strap@2.3.10/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-strap@2.3.10/COPYRIGHT index 6e0b72a3878e..c26ff0307d01 100644 --- a/cdap-distributions/src/COPYRIGHT/angular-strap@2.3.10/COPYRIGHT +++ b/cdap-distributions/src/COPYRIGHT/angular-strap@2.3.10/COPYRIGHT @@ -2,8 +2,20 @@ The MIT License Copyright (c) 2012-2014 Olivier Louvignes http://olouv.com -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/angular-ui-ace/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-ui-ace/COPYRIGHT deleted file mode 100644 index 945a3f5d3712..000000000000 --- a/cdap-distributions/src/COPYRIGHT/angular-ui-ace/COPYRIGHT +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License - -Copyright (c) 2012 the AngularUI Team, http://angular-ui.github.com - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file diff --git a/cdap-distributions/src/COPYRIGHT/angular-ui-bootstrap@1.1.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-ui-bootstrap@1.1.2/COPYRIGHT new file mode 100644 index 000000000000..9607c65f685a --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/angular-ui-bootstrap@1.1.2/COPYRIGHT @@ -0,0 +1,120 @@ +### UI Bootstrap - [AngularJS](http://angularjs.org/) directives specific to [Bootstrap](http://getbootstrap.com) + +[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/angular-ui/bootstrap?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Build Status](https://secure.travis-ci.org/angular-ui/bootstrap.svg)](http://travis-ci.org/angular-ui/bootstrap) +[![devDependency Status](https://david-dm.org/angular-ui/bootstrap/dev-status.svg?branch=master)](https://david-dm.org/angular-ui/bootstrap#info=devDependencies) + +### Quick links +- [Demo](#demo) +- [Installation](#installation) + - [NPM](#install-with-npm) + - [Bower](#install-with-bower) + - [NuGet](#install-with-nuget) + - [Custom](#custom-build) + - [Manual](#manual-download) +- [Support](#support) + - [FAQ](#faq) + - [Supported browsers](#supported-browsers) + - [Need help?](#need-help) + - [Found a bug?](#found-a-bug) +- [Contributing to the project](#contributing-to-the-project) +- [Development, meeting minutes, roadmap and more.](#development-meeting-minutes-roadmap-and-more) + + +# Demo + +Do you want to see directives in action? Visit http://angular-ui.github.io/bootstrap/! + +# Installation + +Installation is easy as UI Bootstrap has minimal dependencies - only the AngularJS and Twitter Bootstrap's CSS are required. +Note: Since version 0.13.0, UI Bootstrap depends on [ngAnimate](https://docs.angularjs.org/api/ngAnimate) for transitions and animations, such as the accordion, carousel, etc. Include `ngAnimate` in the module dependencies for your app in order to enable animation. + +#### Install with NPM + +```sh +$ npm install angular-ui-bootstrap +``` + +This will install AngularJS and Bootstrap NPM packages. + +#### Install with Bower +```sh +$ bower install angular-bootstrap +``` + +Note: do not install 'angular-ui-bootstrap'. A separate repository - [bootstrap-bower](https://github.com/angular-ui/bootstrap-bower) - hosts the compiled javascript file and bower.json. + +#### Install with NuGet +To install AngularJS UI Bootstrap, run the following command in the Package Manager Console + +```sh +PM> Install-Package Angular.UI.Bootstrap +``` + +#### Custom build + +Head over to http://angular-ui.github.io/bootstrap/ and hit the *Custom build* button to create your own custom UI Bootstrap build, just the way you like it. + +#### Manual download + +After downloading dependencies (or better yet, referencing them from your favorite CDN) you need to download build version of this project. All the files and their purposes are described here: +https://github.com/angular-ui/bootstrap/tree/gh-pages#build-files +Don't worry, if you are not sure which file to take, opt for `ui-bootstrap-tpls-[version].min.js`. + +### Adding dependency to your project + +When you are done downloading all the dependencies and project files the only remaining part is to add dependencies on the `ui.bootstrap` AngularJS module: + +```js +angular.module('myModule', ['ui.bootstrap']); +``` + +If you're a Browserify or Webpack user, you can do: + +```js +var uibs = require('angular-ui-bootstrap'); + +angular.module('myModule', [uibs]); +``` + +# Support + +## FAQ + +https://github.com/angular-ui/bootstrap/wiki/FAQ + +## Supported browsers + +Directives from this repository are automatically tested with the following browsers: +* Chrome (stable and canary channel) +* Firefox +* IE 9 and 10 +* Opera +* Safari + +Modern mobile browsers should work without problems. + + +## Need help? +Need help using UI Bootstrap? + +* Live help in the IRC (`#angularjs` channel at the `freenode` network). Use this [webchat](https://webchat.freenode.net/) or your own IRC client. +* Ask a question in [StackOverflow](http://stackoverflow.com/) under the [angular-ui-bootstrap](http://stackoverflow.com/questions/tagged/angular-ui-bootstrap) tag. + +**Please do not create new issues in this repository to ask questions about using UI Bootstrap** + +## Found a bug? +Please take a look at [CONTRIBUTING.md](CONTRIBUTING.md#you-think-youve-found-a-bug) and submit your issue [here](https://github.com/angular-ui/bootstrap/issues/new). + + +---- + + +# Contributing to the project + +We are always looking for the quality contributions! Please check the [CONTRIBUTING.md](CONTRIBUTING.md) for the contribution guidelines. + +# Development, meeting minutes, roadmap and more. + +Head over to the [Wiki](https://github.com/angular-ui/bootstrap/wiki) for notes on development for UI Bootstrap, meeting minutes from the UI Bootstrap team, roadmap plans, project philosophy and more. diff --git a/cdap-distributions/src/COPYRIGHT/angular-ui-router@0.3.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular-ui-router@0.3.0/COPYRIGHT index 2b44125a421f..6413b092d70f 100644 --- a/cdap-distributions/src/COPYRIGHT/angular-ui-router@0.3.0/COPYRIGHT +++ b/cdap-distributions/src/COPYRIGHT/angular-ui-router@0.3.0/COPYRIGHT @@ -1,6 +1,6 @@ The MIT License -Copyright (c) 2014 The AngularUI Team, Karsten Sperling +Copyright (c) 2013-2015 The AngularUI Team, Karsten Sperling Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/angular@1.4.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular@1.4.3/COPYRIGHT index 18967694ac24..d1bc0eddf448 100644 --- a/cdap-distributions/src/COPYRIGHT/angular@1.4.3/COPYRIGHT +++ b/cdap-distributions/src/COPYRIGHT/angular@1.4.3/COPYRIGHT @@ -1,9 +1,64 @@ -The MIT License (MIT) +# packaged angular -Copyright (c) 2016 Angular +This repo is for distribution on `npm` and `bower`. The source for this module is in the +[main AngularJS repo](https://github.com/angular/angular.js). +Please file issues and pull requests against that repo. -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +## Install -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +You can install this package either with `npm` or with `bower`. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +### npm + +```shell +npm install angular +``` + +Then add a ` +``` + +Or `require('angular')` from your code. + +### bower + +```shell +bower install angular +``` + +Then add a ` +``` + +## Documentation + +Documentation is available on the +[AngularJS docs site](http://docs.angularjs.org/). + +## License + +The MIT License + +Copyright (c) 2010-2015 Google, Inc. http://angularjs.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/angular@1.8.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angular@1.8.3/COPYRIGHT new file mode 100644 index 000000000000..2c395eef1bae --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/angular@1.8.3/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Angular + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/angularjs-dropdown-multiselect@1.5.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/angularjs-dropdown-multiselect@1.5.2/COPYRIGHT deleted file mode 100644 index 96862685ec2c..000000000000 --- a/cdap-distributions/src/COPYRIGHT/angularjs-dropdown-multiselect@1.5.2/COPYRIGHT +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2016 Dotan Simha - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/cdap-distributions/src/COPYRIGHT/ansi-align@1.1.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ansi-align@1.1.0/COPYRIGHT new file mode 100644 index 000000000000..ab601b657ea8 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ansi-align@1.1.0/COPYRIGHT @@ -0,0 +1,13 @@ +Copyright (c) 2016, Contributors + +Permission to use, copy, modify, and/or distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright notice +and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/ansi-colors@3.2.4/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ansi-colors@3.2.4/COPYRIGHT new file mode 100644 index 000000000000..8749cc754e5d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ansi-colors@3.2.4/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015-present, Brian Woodward. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/ansi-html@0.0.7/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ansi-html@0.0.7/COPYRIGHT new file mode 100644 index 000000000000..e06d2081865a --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ansi-html@0.0.7/COPYRIGHT @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/cdap-distributions/src/COPYRIGHT/is-finite@1.0.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ansi-regex@2.1.1/COPYRIGHT similarity index 100% rename from cdap-distributions/src/COPYRIGHT/is-finite@1.0.2/COPYRIGHT rename to cdap-distributions/src/COPYRIGHT/ansi-regex@2.1.1/COPYRIGHT diff --git a/cdap-distributions/src/COPYRIGHT/builtin-modules@1.1.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ansi-regex@3.0.0/COPYRIGHT similarity index 100% rename from cdap-distributions/src/COPYRIGHT/builtin-modules@1.1.1/COPYRIGHT rename to cdap-distributions/src/COPYRIGHT/ansi-regex@3.0.0/COPYRIGHT diff --git a/cdap-distributions/src/COPYRIGHT/is-builtin-module@1.0.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ansi-regex@4.1.0/COPYRIGHT similarity index 100% rename from cdap-distributions/src/COPYRIGHT/is-builtin-module@1.0.0/COPYRIGHT rename to cdap-distributions/src/COPYRIGHT/ansi-regex@4.1.0/COPYRIGHT diff --git a/cdap-distributions/src/COPYRIGHT/lowercase-keys@1.0.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ansi-styles@3.2.1/COPYRIGHT similarity index 100% rename from cdap-distributions/src/COPYRIGHT/lowercase-keys@1.0.0/COPYRIGHT rename to cdap-distributions/src/COPYRIGHT/ansi-styles@3.2.1/COPYRIGHT diff --git a/cdap-distributions/src/COPYRIGHT/ant:ant:1.6.5/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ant:ant:1.6.5/COPYRIGHT new file mode 100644 index 000000000000..6b0b1270ff0c --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ant:ant:1.6.5/COPYRIGHT @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/cdap-distributions/src/COPYRIGHT/antlr4-runtime@4.5.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/antlr4-runtime@4.5.3/COPYRIGHT deleted file mode 100644 index 2042d1bda6c9..000000000000 --- a/cdap-distributions/src/COPYRIGHT/antlr4-runtime@4.5.3/COPYRIGHT +++ /dev/null @@ -1,52 +0,0 @@ -[The "BSD 3-clause license"] -Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -===== - -MIT License for codepointat.js from https://git.io/codepointat -MIT License for fromcodepoint.js from https://git.io/vDW1m - -Copyright Mathias Bynens - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/antlr4-runtime@4.7/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/antlr4-runtime@4.7/COPYRIGHT deleted file mode 100644 index 2042d1bda6c9..000000000000 --- a/cdap-distributions/src/COPYRIGHT/antlr4-runtime@4.7/COPYRIGHT +++ /dev/null @@ -1,52 +0,0 @@ -[The "BSD 3-clause license"] -Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -===== - -MIT License for codepointat.js from https://git.io/codepointat -MIT License for fromcodepoint.js from https://git.io/vDW1m - -Copyright Mathias Bynens - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/antlr4@4.7/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/antlr4@4.7/COPYRIGHT deleted file mode 100644 index 2042d1bda6c9..000000000000 --- a/cdap-distributions/src/COPYRIGHT/antlr4@4.7/COPYRIGHT +++ /dev/null @@ -1,52 +0,0 @@ -[The "BSD 3-clause license"] -Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -===== - -MIT License for codepointat.js from https://git.io/codepointat -MIT License for fromcodepoint.js from https://git.io/vDW1m - -Copyright Mathias Bynens - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/antlr:antlr:2.7.7/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/antlr:antlr:2.7.7/COPYRIGHT new file mode 100644 index 000000000000..5d276941558a --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/antlr:antlr:2.7.7/COPYRIGHT @@ -0,0 +1,28 @@ +Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +3. Neither name of copyright holders nor the names of its contributors +may be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/cdap-distributions/src/COPYRIGHT/antlr@3.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/antlr@3.2/COPYRIGHT deleted file mode 100644 index 2042d1bda6c9..000000000000 --- a/cdap-distributions/src/COPYRIGHT/antlr@3.2/COPYRIGHT +++ /dev/null @@ -1,52 +0,0 @@ -[The "BSD 3-clause license"] -Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -===== - -MIT License for codepointat.js from https://git.io/codepointat -MIT License for fromcodepoint.js from https://git.io/vDW1m - -Copyright Mathias Bynens - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/anymatch@2.0.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/anymatch@2.0.0/COPYRIGHT new file mode 100644 index 000000000000..bc424705fb9c --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/anymatch@2.0.0/COPYRIGHT @@ -0,0 +1,15 @@ +The ISC License + +Copyright (c) 2014 Elan Shanker + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR +IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/anymatch@3.1.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/anymatch@3.1.1/COPYRIGHT new file mode 100644 index 000000000000..491766ca79a0 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/anymatch@3.1.1/COPYRIGHT @@ -0,0 +1,15 @@ +The ISC License + +Copyright (c) 2019 Elan Shanker, Paul Miller (https://paulmillr.com) + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR +IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/aopalliance:aopalliance:1.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/aopalliance:aopalliance:1.0/COPYRIGHT new file mode 100644 index 000000000000..effe5d2bf9cb --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/aopalliance:aopalliance:1.0/COPYRIGHT @@ -0,0 +1,12 @@ + +package org.aopalliance.aop; + +/** + * Tag interface for Advice. Implementations can be any type + * of advice, such as Interceptors. + * @author Rod Johnson + * @version $Id: Advice.java,v 1.1 2004/03/19 17:02:16 johnsonr Exp $ + */ +public interface Advice { + +} diff --git a/cdap-distributions/src/COPYRIGHT/api-asn1-api@1.0.0-M20/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/api-asn1-api@1.0.0-M20/COPYRIGHT deleted file mode 100644 index bab08118c9b6..000000000000 --- a/cdap-distributions/src/COPYRIGHT/api-asn1-api@1.0.0-M20/COPYRIGHT +++ /dev/null @@ -1,10 +0,0 @@ - -ApacheDS -Copyright 2003-2020 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This product includes software developped at -http://jdbm.sourceforge.net/ - diff --git a/cdap-distributions/src/COPYRIGHT/api-asn1-ber@1.0.0-M20/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/api-asn1-ber@1.0.0-M20/COPYRIGHT deleted file mode 100644 index bab08118c9b6..000000000000 --- a/cdap-distributions/src/COPYRIGHT/api-asn1-ber@1.0.0-M20/COPYRIGHT +++ /dev/null @@ -1,10 +0,0 @@ - -ApacheDS -Copyright 2003-2020 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This product includes software developped at -http://jdbm.sourceforge.net/ - diff --git a/cdap-distributions/src/COPYRIGHT/apollo-boost@0.4.7/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-boost@0.4.7/COPYRIGHT new file mode 100644 index 000000000000..278ee9d8c04b --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-boost@0.4.7/COPYRIGHT @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2018 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/cdap-distributions/src/COPYRIGHT/apollo-cache-control@0.11.6/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-cache-control@0.11.6/COPYRIGHT new file mode 100644 index 000000000000..bb81fb80c81d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-cache-control@0.11.6/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2020 Apollo Graph, Inc. (Formerly Meteor Development Group, Inc.) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-cache-inmemory@1.6.6/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-cache-inmemory@1.6.6/COPYRIGHT new file mode 100644 index 000000000000..278ee9d8c04b --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-cache-inmemory@1.6.6/COPYRIGHT @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2018 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/cdap-distributions/src/COPYRIGHT/apollo-cache@1.3.5/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-cache@1.3.5/COPYRIGHT new file mode 100644 index 000000000000..278ee9d8c04b --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-cache@1.3.5/COPYRIGHT @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2018 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/cdap-distributions/src/COPYRIGHT/apollo-client@2.6.10/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-client@2.6.10/COPYRIGHT new file mode 100644 index 000000000000..278ee9d8c04b --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-client@2.6.10/COPYRIGHT @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2018 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/cdap-distributions/src/COPYRIGHT/apollo-datasource@0.7.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-datasource@0.7.3/COPYRIGHT new file mode 100644 index 000000000000..bb81fb80c81d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-datasource@0.7.3/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2020 Apollo Graph, Inc. (Formerly Meteor Development Group, Inc.) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-engine-reporting-protobuf@0.4.4/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-engine-reporting-protobuf@0.4.4/COPYRIGHT new file mode 100644 index 000000000000..1558a68a8a8e --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-engine-reporting-protobuf@0.4.4/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-env@0.6.6/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-env@0.6.6/COPYRIGHT new file mode 100644 index 000000000000..1558a68a8a8e --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-env@0.6.6/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-graphql@0.6.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-graphql@0.6.1/COPYRIGHT new file mode 100644 index 000000000000..1558a68a8a8e --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-graphql@0.6.1/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-link-error@1.1.13/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-link-error@1.1.13/COPYRIGHT new file mode 100644 index 000000000000..07ea66258baf --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-link-error@1.1.13/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 - 2017 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-link-http-common@0.2.16/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-link-http-common@0.2.16/COPYRIGHT new file mode 100644 index 000000000000..07ea66258baf --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-link-http-common@0.2.16/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 - 2017 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-link-http@1.5.17/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-link-http@1.5.17/COPYRIGHT new file mode 100644 index 000000000000..07ea66258baf --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-link-http@1.5.17/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 - 2017 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-link@1.2.14/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-link@1.2.14/COPYRIGHT new file mode 100644 index 000000000000..07ea66258baf --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-link@1.2.14/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 - 2017 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-reporting-protobuf@0.6.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-reporting-protobuf@0.6.2/COPYRIGHT new file mode 100644 index 000000000000..bb81fb80c81d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-reporting-protobuf@0.6.2/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2020 Apollo Graph, Inc. (Formerly Meteor Development Group, Inc.) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-server-caching@0.5.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-server-caching@0.5.3/COPYRIGHT new file mode 100644 index 000000000000..bb81fb80c81d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-server-caching@0.5.3/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2020 Apollo Graph, Inc. (Formerly Meteor Development Group, Inc.) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-server-core@2.21.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-server-core@2.21.0/COPYRIGHT new file mode 100644 index 000000000000..bb81fb80c81d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-server-core@2.21.0/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2020 Apollo Graph, Inc. (Formerly Meteor Development Group, Inc.) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-server-env@2.4.5/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-server-env@2.4.5/COPYRIGHT new file mode 100644 index 000000000000..f7f2ca70818d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-server-env@2.4.5/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2020 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-server-env@3.0.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-server-env@3.0.0/COPYRIGHT new file mode 100644 index 000000000000..bb81fb80c81d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-server-env@3.0.0/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2020 Apollo Graph, Inc. (Formerly Meteor Development Group, Inc.) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-server-errors@2.4.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-server-errors@2.4.2/COPYRIGHT new file mode 100644 index 000000000000..f7f2ca70818d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-server-errors@2.4.2/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2020 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-server-express@2.21.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-server-express@2.21.0/COPYRIGHT new file mode 100644 index 000000000000..bb81fb80c81d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-server-express@2.21.0/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2020 Apollo Graph, Inc. (Formerly Meteor Development Group, Inc.) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-server-express@2.9.15/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-server-express@2.9.15/COPYRIGHT new file mode 100644 index 000000000000..f7f2ca70818d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-server-express@2.9.15/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2020 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-server-plugin-base@0.10.4/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-server-plugin-base@0.10.4/COPYRIGHT new file mode 100644 index 000000000000..bb81fb80c81d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-server-plugin-base@0.10.4/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2020 Apollo Graph, Inc. (Formerly Meteor Development Group, Inc.) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-server-types@0.2.10/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-server-types@0.2.10/COPYRIGHT new file mode 100644 index 000000000000..f7f2ca70818d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-server-types@0.2.10/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2020 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-server-types@0.6.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-server-types@0.6.3/COPYRIGHT new file mode 100644 index 000000000000..bb81fb80c81d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-server-types@0.6.3/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2020 Apollo Graph, Inc. (Formerly Meteor Development Group, Inc.) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-server@2.9.15/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-server@2.9.15/COPYRIGHT new file mode 100644 index 000000000000..f7f2ca70818d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-server@2.9.15/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2020 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-tracing@0.12.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-tracing@0.12.2/COPYRIGHT new file mode 100644 index 000000000000..bb81fb80c81d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-tracing@0.12.2/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2020 Apollo Graph, Inc. (Formerly Meteor Development Group, Inc.) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/apollo-utilities@1.3.4/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/apollo-utilities@1.3.4/COPYRIGHT new file mode 100644 index 000000000000..278ee9d8c04b --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/apollo-utilities@1.3.4/COPYRIGHT @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2018 Meteor Development Group, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/cdap-distributions/src/COPYRIGHT/are-we-there-yet@1.1.4/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/are-we-there-yet@1.1.4/COPYRIGHT deleted file mode 100644 index af4588069db8..000000000000 --- a/cdap-distributions/src/COPYRIGHT/are-we-there-yet@1.1.4/COPYRIGHT +++ /dev/null @@ -1,5 +0,0 @@ -Copyright (c) 2015, Rebecca Turner - -Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/argparse4j@0.7.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/argparse4j@0.7.0/COPYRIGHT deleted file mode 100644 index 773b0df0e359..000000000000 --- a/cdap-distributions/src/COPYRIGHT/argparse4j@0.7.0/COPYRIGHT +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2011-2017 Tatsuhiro Tsujikawa - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, copy, - * modify, merge, publish, distribute, sublicense, and/or sell copies - * of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ diff --git a/cdap-distributions/src/COPYRIGHT/arr-diff@4.0.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/arr-diff@4.0.0/COPYRIGHT new file mode 100644 index 000000000000..d734237bdedc --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/arr-diff@4.0.0/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014-2017, Jon Schlinkert + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/arr-flatten@1.1.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/arr-flatten@1.1.0/COPYRIGHT new file mode 100644 index 000000000000..3f2eca18f1bc --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/arr-flatten@1.1.0/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014-2017, Jon Schlinkert. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/arr-union@3.1.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/arr-union@3.1.0/COPYRIGHT new file mode 100644 index 000000000000..39245ac1c606 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/arr-union@3.1.0/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014-2016, Jon Schlinkert. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/path-to-regexp@1.7.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/array-flatten@2.1.2/COPYRIGHT similarity index 100% rename from cdap-distributions/src/COPYRIGHT/path-to-regexp@1.7.0/COPYRIGHT rename to cdap-distributions/src/COPYRIGHT/array-flatten@2.1.2/COPYRIGHT diff --git a/cdap-distributions/src/COPYRIGHT/array-union@1.0.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/array-union@1.0.2/COPYRIGHT new file mode 100644 index 000000000000..654d0bfe9434 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/array-union@1.0.2/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) Sindre Sorhus (sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/array-uniq@1.0.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/array-uniq@1.0.3/COPYRIGHT new file mode 100644 index 000000000000..654d0bfe9434 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/array-uniq@1.0.3/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) Sindre Sorhus (sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/array-unique@0.3.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/array-unique@0.3.2/COPYRIGHT new file mode 100644 index 000000000000..842218cf09a2 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/array-unique@0.3.2/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014-2016, Jon Schlinkert + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/asm@3.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/asm:asm:3.1/COPYRIGHT similarity index 100% rename from cdap-distributions/src/COPYRIGHT/asm@3.1/COPYRIGHT rename to cdap-distributions/src/COPYRIGHT/asm:asm:3.1/COPYRIGHT diff --git a/cdap-distributions/src/COPYRIGHT/asn1.js@5.4.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/asn1.js@5.4.1/COPYRIGHT new file mode 100644 index 000000000000..caaf4f211079 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/asn1.js@5.4.1/COPYRIGHT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Fedor Indutny + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/assert@1.5.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/assert@1.5.0/COPYRIGHT new file mode 100644 index 000000000000..e3d4e695a4cf --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/assert@1.5.0/COPYRIGHT @@ -0,0 +1,18 @@ +Copyright Joyent, Inc. and other Node contributors. All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/assign-symbols@1.0.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/assign-symbols@1.0.0/COPYRIGHT new file mode 100644 index 000000000000..65f90aca8c2f --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/assign-symbols@1.0.0/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015, Jon Schlinkert. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/async-each@1.0.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/async-each@1.0.3/COPYRIGHT new file mode 100644 index 000000000000..6444d9542958 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/async-each@1.0.3/COPYRIGHT @@ -0,0 +1,52 @@ +# async-each + +No-bullshit, ultra-simple, 35-lines-of-code async parallel forEach function for JavaScript. + +We don't need junky 30K async libs. Really. + +For browsers and node.js. + +## Installation +* Just include async-each before your scripts. +* `npm install async-each` if you’re using node.js. + +## Usage + +* `each(array, iterator, callback);` — `Array`, `Function`, `(optional) Function` +* `iterator(item, next)` receives current item and a callback that will mark the item as done. `next` callback receives optional `error, transformedItem` arguments. +* `callback(error, transformedArray)` optionally receives first error and transformed result `Array`. + +```javascript +var each = require('async-each'); +each(['a.js', 'b.js', 'c.js'], fs.readFile, function(error, contents) { + if (error) console.error(error); + console.log('Contents for a, b and c:', contents); +}); + +// Alternatively in browser: +asyncEach(list, fn, callback); +``` + +## License + +The MIT License (MIT) + +Copyright (c) 2016 Paul Miller [(paulmillr.com)](http://paulmillr.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the “Software”), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/async-limiter@1.0.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/async-limiter@1.0.1/COPYRIGHT new file mode 100644 index 000000000000..9c91fb26803a --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/async-limiter@1.0.1/COPYRIGHT @@ -0,0 +1,8 @@ +The MIT License (MIT) +Copyright (c) 2017 Samuel Reed + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/async-retry@1.3.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/async-retry@1.3.1/COPYRIGHT new file mode 100644 index 000000000000..34abe98ea9f8 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/async-retry@1.3.1/COPYRIGHT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 ZEIT, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/async@2.6.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/async@2.6.3/COPYRIGHT new file mode 100644 index 000000000000..b18aed692195 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/async@2.6.3/COPYRIGHT @@ -0,0 +1,19 @@ +Copyright (c) 2010-2018 Caolan McMahon + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/atob@2.1.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/atob@2.1.2/COPYRIGHT new file mode 100644 index 000000000000..2d9338bd9b8d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/atob@2.1.2/COPYRIGHT @@ -0,0 +1,230 @@ +At your option you may choose either of the following licenses: + + * The MIT License (MIT) + * The Apache License 2.0 (Apache-2.0) + + +The MIT License (MIT) + +Copyright (c) 2015 AJ ONeal + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015 AJ ONeal + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/cdap-distributions/src/COPYRIGHT/attr-accept@1.1.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/attr-accept@2.2.2/COPYRIGHT similarity index 100% rename from cdap-distributions/src/COPYRIGHT/attr-accept@1.1.0/COPYRIGHT rename to cdap-distributions/src/COPYRIGHT/attr-accept@2.2.2/COPYRIGHT diff --git a/cdap-distributions/src/COPYRIGHT/auto-common@0.10/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/auto-common@0.10/COPYRIGHT deleted file mode 100644 index 8aa899e4c293..000000000000 --- a/cdap-distributions/src/COPYRIGHT/auto-common@0.10/COPYRIGHT +++ /dev/null @@ -1,409 +0,0 @@ -The Checker Framework -Copyright 2004-present by the Checker Framework developers - - -Most of the Checker Framework is licensed under the GNU General Public -License, version 2 (GPL2), with the classpath exception. The text of this -license appears below. This is the same license used for OpenJDK. - -A few parts of the Checker Framework have more permissive licenses. - - * The annotations are licensed under the MIT License. (The text of this - license also appears below.) More specifically, all the parts of the - Checker Framework that you might want to include with your own program - use the MIT License. This is the checker-qual*.jar and - checker-compat-qual*.jar files and all the files that appear in them: - every file in a qual/ directory, plus utility files such as - NullnessUtil.java, RegexUtil.java, SignednessUtil.java, etc. In - addition, the cleanroom implementations of third-party annotations, - which the Checker Framework recognizes as aliases for its own - annotations, are licensed under the MIT License. - -Some external libraries that are included with the Checker Framework have -different licenses. - - * javaparser is dual licensed under the LGPL or the Apache license -- you - may use it under whichever one you want. (The javaparser source code - contains a file with the text of the GPL, but it is not clear why, since - javaparser does not use the GPL.) See file stubparser/LICENSE - and the source code of all its files. - - * Libraries in plume-lib (https://github.com/plume-lib/) are licensed - under the MIT License. - -The Checker Framework includes annotations for some libraries. Each annotated -library uses the same license as the unannotated version of the library. - -=========================================================================== - -The GNU General Public License (GPL) - -Version 2, June 1991 - -Copyright (C) 1989, 1991 Free Software Foundation, Inc. -59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -Everyone is permitted to copy and distribute verbatim copies of this license -document, but changing it is not allowed. - -Preamble - -The licenses for most software are designed to take away your freedom to share -and change it. By contrast, the GNU General Public License is intended to -guarantee your freedom to share and change free software--to make sure the -software is free for all its users. This General Public License applies to -most of the Free Software Foundation's software and to any other program whose -authors commit to using it. (Some other Free Software Foundation software is -covered by the GNU Library General Public License instead.) You can apply it to -your programs, too. - -When we speak of free software, we are referring to freedom, not price. Our -General Public Licenses are designed to make sure that you have the freedom to -distribute copies of free software (and charge for this service if you wish), -that you receive source code or can get it if you want it, that you can change -the software or use pieces of it in new free programs; and that you know you -can do these things. - -To protect your rights, we need to make restrictions that forbid anyone to deny -you these rights or to ask you to surrender the rights. These restrictions -translate to certain responsibilities for you if you distribute copies of the -software, or if you modify it. - -For example, if you distribute copies of such a program, whether gratis or for -a fee, you must give the recipients all the rights that you have. You must -make sure that they, too, receive or can get the source code. And you must -show them these terms so they know their rights. - -We protect your rights with two steps: (1) copyright the software, and (2) -offer you this license which gives you legal permission to copy, distribute -and/or modify the software. - -Also, for each author's protection and ours, we want to make certain that -everyone understands that there is no warranty for this free software. If the -software is modified by someone else and passed on, we want its recipients to -know that what they have is not the original, so that any problems introduced -by others will not reflect on the original authors' reputations. - -Finally, any free program is threatened constantly by software patents. We -wish to avoid the danger that redistributors of a free program will -individually obtain patent licenses, in effect making the program proprietary. -To prevent this, we have made it clear that any patent must be licensed for -everyone's free use or not licensed at all. - -The precise terms and conditions for copying, distribution and modification -follow. - -TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - -0. This License applies to any program or other work which contains a notice -placed by the copyright holder saying it may be distributed under the terms of -this General Public License. The "Program", below, refers to any such program -or work, and a "work based on the Program" means either the Program or any -derivative work under copyright law: that is to say, a work containing the -Program or a portion of it, either verbatim or with modifications and/or -translated into another language. (Hereinafter, translation is included -without limitation in the term "modification".) Each licensee is addressed as -"you". - -Activities other than copying, distribution and modification are not covered by -this License; they are outside its scope. The act of running the Program is -not restricted, and the output from the Program is covered only if its contents -constitute a work based on the Program (independent of having been made by -running the Program). Whether that is true depends on what the Program does. - -1. You may copy and distribute verbatim copies of the Program's source code as -you receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice and -disclaimer of warranty; keep intact all the notices that refer to this License -and to the absence of any warranty; and give any other recipients of the -Program a copy of this License along with the Program. - -You may charge a fee for the physical act of transferring a copy, and you may -at your option offer warranty protection in exchange for a fee. - -2. You may modify your copy or copies of the Program or any portion of it, thus -forming a work based on the Program, and copy and distribute such modifications -or work under the terms of Section 1 above, provided that you also meet all of -these conditions: - - a) You must cause the modified files to carry prominent notices stating - that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in whole or - in part contains or is derived from the Program or any part thereof, to be - licensed as a whole at no charge to all third parties under the terms of - this License. - - c) If the modified program normally reads commands interactively when run, - you must cause it, when started running for such interactive use in the - most ordinary way, to print or display an announcement including an - appropriate copyright notice and a notice that there is no warranty (or - else, saying that you provide a warranty) and that users may redistribute - the program under these conditions, and telling the user how to view a copy - of this License. (Exception: if the Program itself is interactive but does - not normally print such an announcement, your work based on the Program is - not required to print an announcement.) - -These requirements apply to the modified work as a whole. If identifiable -sections of that work are not derived from the Program, and can be reasonably -considered independent and separate works in themselves, then this License, and -its terms, do not apply to those sections when you distribute them as separate -works. But when you distribute the same sections as part of a whole which is a -work based on the Program, the distribution of the whole must be on the terms -of this License, whose permissions for other licensees extend to the entire -whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest your -rights to work written entirely by you; rather, the intent is to exercise the -right to control the distribution of derivative or collective works based on -the Program. - -In addition, mere aggregation of another work not based on the Program with the -Program (or with a work based on the Program) on a volume of a storage or -distribution medium does not bring the other work under the scope of this -License. - -3. You may copy and distribute the Program (or a work based on it, under -Section 2) in object code or executable form under the terms of Sections 1 and -2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable source - code, which must be distributed under the terms of Sections 1 and 2 above - on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three years, to - give any third party, for a charge no more than your cost of physically - performing source distribution, a complete machine-readable copy of the - corresponding source code, to be distributed under the terms of Sections 1 - and 2 above on a medium customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer to - distribute corresponding source code. (This alternative is allowed only - for noncommercial distribution and only if you received the program in - object code or executable form with such an offer, in accord with - Subsection b above.) - -The source code for a work means the preferred form of the work for making -modifications to it. For an executable work, complete source code means all -the source code for all modules it contains, plus any associated interface -definition files, plus the scripts used to control compilation and installation -of the executable. However, as a special exception, the source code -distributed need not include anything that is normally distributed (in either -source or binary form) with the major components (compiler, kernel, and so on) -of the operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering access to copy -from a designated place, then offering equivalent access to copy the source -code from the same place counts as distribution of the source code, even though -third parties are not compelled to copy the source along with the object code. - -4. You may not copy, modify, sublicense, or distribute the Program except as -expressly provided under this License. Any attempt otherwise to copy, modify, -sublicense or distribute the Program is void, and will automatically terminate -your rights under this License. However, parties who have received copies, or -rights, from you under this License will not have their licenses terminated so -long as such parties remain in full compliance. - -5. You are not required to accept this License, since you have not signed it. -However, nothing else grants you permission to modify or distribute the Program -or its derivative works. These actions are prohibited by law if you do not -accept this License. Therefore, by modifying or distributing the Program (or -any work based on the Program), you indicate your acceptance of this License to -do so, and all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - -6. Each time you redistribute the Program (or any work based on the Program), -the recipient automatically receives a license from the original licensor to -copy, distribute or modify the Program subject to these terms and conditions. -You may not impose any further restrictions on the recipients' exercise of the -rights granted herein. You are not responsible for enforcing compliance by -third parties to this License. - -7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), conditions -are imposed on you (whether by court order, agreement or otherwise) that -contradict the conditions of this License, they do not excuse you from the -conditions of this License. If you cannot distribute so as to satisfy -simultaneously your obligations under this License and any other pertinent -obligations, then as a consequence you may not distribute the Program at all. -For example, if a patent license would not permit royalty-free redistribution -of the Program by all those who receive copies directly or indirectly through -you, then the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under any -particular circumstance, the balance of the section is intended to apply and -the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any patents or -other property right claims or to contest validity of any such claims; this -section has the sole purpose of protecting the integrity of the free software -distribution system, which is implemented by public license practices. Many -people have made generous contributions to the wide range of software -distributed through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing to -distribute software through any other system and a licensee cannot impose that -choice. - -This section is intended to make thoroughly clear what is believed to be a -consequence of the rest of this License. - -8. If the distribution and/or use of the Program is restricted in certain -countries either by patents or by copyrighted interfaces, the original -copyright holder who places the Program under this License may add an explicit -geographical distribution limitation excluding those countries, so that -distribution is permitted only in or among countries not thus excluded. In -such case, this License incorporates the limitation as if written in the body -of this License. - -9. The Free Software Foundation may publish revised and/or new versions of the -General Public License from time to time. Such new versions will be similar in -spirit to the present version, but may differ in detail to address new problems -or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any later -version", you have the option of following the terms and conditions either of -that version or of any later version published by the Free Software Foundation. -If the Program does not specify a version number of this License, you may -choose any version ever published by the Free Software Foundation. - -10. If you wish to incorporate parts of the Program into other free programs -whose distribution conditions are different, write to the author to ask for -permission. For software which is copyrighted by the Free Software Foundation, -write to the Free Software Foundation; we sometimes make exceptions for this. -Our decision will be guided by the two goals of preserving the free status of -all derivatives of our free software and of promoting the sharing and reuse of -software generally. - -NO WARRANTY - -11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR -THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE -STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE -PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND -PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, -YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - -12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL -ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE -PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR -INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA -BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A -FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER -OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -END OF TERMS AND CONDITIONS - -How to Apply These Terms to Your New Programs - -If you develop a new program, and you want it to be of the greatest possible -use to the public, the best way to achieve this is to make it free software -which everyone can redistribute and change under these terms. - -To do so, attach the following notices to the program. It is safest to attach -them to the start of each source file to most effectively convey the exclusion -of warranty; and each file should have at least the "copyright" line and a -pointer to where the full notice is found. - - One line to give the program's name and a brief idea of what it does. - - Copyright (C) - - This program is free software; you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by the Free - Software Foundation; either version 2 of the License, or (at your option) - any later version. - - This program is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., 59 - Temple Place, Suite 330, Boston, MA 02111-1307 USA - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this when it -starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author Gnomovision comes - with ABSOLUTELY NO WARRANTY; for details type 'show w'. This is free - software, and you are welcome to redistribute it under certain conditions; - type 'show c' for details. - -The hypothetical commands 'show w' and 'show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may be -called something other than 'show w' and 'show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your school, -if any, to sign a "copyright disclaimer" for the program, if necessary. Here -is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - 'Gnomovision' (which makes passes at compilers) written by James Hacker. - - signature of Ty Coon, 1 April 1989 - - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General Public -License instead of this License. - - -"CLASSPATH" EXCEPTION TO THE GPL - -Certain source files distributed by Oracle America and/or its affiliates are -subject to the following clarification and special exception to the GPL, but -only where Oracle has expressly included in the particular source file's header -the words "Oracle designates this particular file as subject to the "Classpath" -exception as provided by Oracle in the LICENSE file that accompanied this code." - - Linking this library statically or dynamically with other modules is making - a combined work based on this library. Thus, the terms and conditions of - the GNU General Public License cover the whole combination. - - As a special exception, the copyright holders of this library give you - permission to link this library with independent modules to produce an - executable, regardless of the license terms of these independent modules, - and to copy and distribute the resulting executable under terms of your - choice, provided that you also meet, for each linked independent module, - the terms and conditions of the license of that module. An independent - module is a module which is not derived from or based on this library. If - you modify this library, you may extend this exception to your version of - the library, but you are not obligated to do so. If you do not wish to do - so, delete this exception statement from your version. - -=========================================================================== - -MIT License: - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -=========================================================================== diff --git a/cdap-distributions/src/COPYRIGHT/auto-service-annotations@1.0-rc5/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/auto-service-annotations@1.0-rc5/COPYRIGHT deleted file mode 100644 index 8aa899e4c293..000000000000 --- a/cdap-distributions/src/COPYRIGHT/auto-service-annotations@1.0-rc5/COPYRIGHT +++ /dev/null @@ -1,409 +0,0 @@ -The Checker Framework -Copyright 2004-present by the Checker Framework developers - - -Most of the Checker Framework is licensed under the GNU General Public -License, version 2 (GPL2), with the classpath exception. The text of this -license appears below. This is the same license used for OpenJDK. - -A few parts of the Checker Framework have more permissive licenses. - - * The annotations are licensed under the MIT License. (The text of this - license also appears below.) More specifically, all the parts of the - Checker Framework that you might want to include with your own program - use the MIT License. This is the checker-qual*.jar and - checker-compat-qual*.jar files and all the files that appear in them: - every file in a qual/ directory, plus utility files such as - NullnessUtil.java, RegexUtil.java, SignednessUtil.java, etc. In - addition, the cleanroom implementations of third-party annotations, - which the Checker Framework recognizes as aliases for its own - annotations, are licensed under the MIT License. - -Some external libraries that are included with the Checker Framework have -different licenses. - - * javaparser is dual licensed under the LGPL or the Apache license -- you - may use it under whichever one you want. (The javaparser source code - contains a file with the text of the GPL, but it is not clear why, since - javaparser does not use the GPL.) See file stubparser/LICENSE - and the source code of all its files. - - * Libraries in plume-lib (https://github.com/plume-lib/) are licensed - under the MIT License. - -The Checker Framework includes annotations for some libraries. Each annotated -library uses the same license as the unannotated version of the library. - -=========================================================================== - -The GNU General Public License (GPL) - -Version 2, June 1991 - -Copyright (C) 1989, 1991 Free Software Foundation, Inc. -59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -Everyone is permitted to copy and distribute verbatim copies of this license -document, but changing it is not allowed. - -Preamble - -The licenses for most software are designed to take away your freedom to share -and change it. By contrast, the GNU General Public License is intended to -guarantee your freedom to share and change free software--to make sure the -software is free for all its users. This General Public License applies to -most of the Free Software Foundation's software and to any other program whose -authors commit to using it. (Some other Free Software Foundation software is -covered by the GNU Library General Public License instead.) You can apply it to -your programs, too. - -When we speak of free software, we are referring to freedom, not price. Our -General Public Licenses are designed to make sure that you have the freedom to -distribute copies of free software (and charge for this service if you wish), -that you receive source code or can get it if you want it, that you can change -the software or use pieces of it in new free programs; and that you know you -can do these things. - -To protect your rights, we need to make restrictions that forbid anyone to deny -you these rights or to ask you to surrender the rights. These restrictions -translate to certain responsibilities for you if you distribute copies of the -software, or if you modify it. - -For example, if you distribute copies of such a program, whether gratis or for -a fee, you must give the recipients all the rights that you have. You must -make sure that they, too, receive or can get the source code. And you must -show them these terms so they know their rights. - -We protect your rights with two steps: (1) copyright the software, and (2) -offer you this license which gives you legal permission to copy, distribute -and/or modify the software. - -Also, for each author's protection and ours, we want to make certain that -everyone understands that there is no warranty for this free software. If the -software is modified by someone else and passed on, we want its recipients to -know that what they have is not the original, so that any problems introduced -by others will not reflect on the original authors' reputations. - -Finally, any free program is threatened constantly by software patents. We -wish to avoid the danger that redistributors of a free program will -individually obtain patent licenses, in effect making the program proprietary. -To prevent this, we have made it clear that any patent must be licensed for -everyone's free use or not licensed at all. - -The precise terms and conditions for copying, distribution and modification -follow. - -TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - -0. This License applies to any program or other work which contains a notice -placed by the copyright holder saying it may be distributed under the terms of -this General Public License. The "Program", below, refers to any such program -or work, and a "work based on the Program" means either the Program or any -derivative work under copyright law: that is to say, a work containing the -Program or a portion of it, either verbatim or with modifications and/or -translated into another language. (Hereinafter, translation is included -without limitation in the term "modification".) Each licensee is addressed as -"you". - -Activities other than copying, distribution and modification are not covered by -this License; they are outside its scope. The act of running the Program is -not restricted, and the output from the Program is covered only if its contents -constitute a work based on the Program (independent of having been made by -running the Program). Whether that is true depends on what the Program does. - -1. You may copy and distribute verbatim copies of the Program's source code as -you receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice and -disclaimer of warranty; keep intact all the notices that refer to this License -and to the absence of any warranty; and give any other recipients of the -Program a copy of this License along with the Program. - -You may charge a fee for the physical act of transferring a copy, and you may -at your option offer warranty protection in exchange for a fee. - -2. You may modify your copy or copies of the Program or any portion of it, thus -forming a work based on the Program, and copy and distribute such modifications -or work under the terms of Section 1 above, provided that you also meet all of -these conditions: - - a) You must cause the modified files to carry prominent notices stating - that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in whole or - in part contains or is derived from the Program or any part thereof, to be - licensed as a whole at no charge to all third parties under the terms of - this License. - - c) If the modified program normally reads commands interactively when run, - you must cause it, when started running for such interactive use in the - most ordinary way, to print or display an announcement including an - appropriate copyright notice and a notice that there is no warranty (or - else, saying that you provide a warranty) and that users may redistribute - the program under these conditions, and telling the user how to view a copy - of this License. (Exception: if the Program itself is interactive but does - not normally print such an announcement, your work based on the Program is - not required to print an announcement.) - -These requirements apply to the modified work as a whole. If identifiable -sections of that work are not derived from the Program, and can be reasonably -considered independent and separate works in themselves, then this License, and -its terms, do not apply to those sections when you distribute them as separate -works. But when you distribute the same sections as part of a whole which is a -work based on the Program, the distribution of the whole must be on the terms -of this License, whose permissions for other licensees extend to the entire -whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest your -rights to work written entirely by you; rather, the intent is to exercise the -right to control the distribution of derivative or collective works based on -the Program. - -In addition, mere aggregation of another work not based on the Program with the -Program (or with a work based on the Program) on a volume of a storage or -distribution medium does not bring the other work under the scope of this -License. - -3. You may copy and distribute the Program (or a work based on it, under -Section 2) in object code or executable form under the terms of Sections 1 and -2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable source - code, which must be distributed under the terms of Sections 1 and 2 above - on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three years, to - give any third party, for a charge no more than your cost of physically - performing source distribution, a complete machine-readable copy of the - corresponding source code, to be distributed under the terms of Sections 1 - and 2 above on a medium customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer to - distribute corresponding source code. (This alternative is allowed only - for noncommercial distribution and only if you received the program in - object code or executable form with such an offer, in accord with - Subsection b above.) - -The source code for a work means the preferred form of the work for making -modifications to it. For an executable work, complete source code means all -the source code for all modules it contains, plus any associated interface -definition files, plus the scripts used to control compilation and installation -of the executable. However, as a special exception, the source code -distributed need not include anything that is normally distributed (in either -source or binary form) with the major components (compiler, kernel, and so on) -of the operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering access to copy -from a designated place, then offering equivalent access to copy the source -code from the same place counts as distribution of the source code, even though -third parties are not compelled to copy the source along with the object code. - -4. You may not copy, modify, sublicense, or distribute the Program except as -expressly provided under this License. Any attempt otherwise to copy, modify, -sublicense or distribute the Program is void, and will automatically terminate -your rights under this License. However, parties who have received copies, or -rights, from you under this License will not have their licenses terminated so -long as such parties remain in full compliance. - -5. You are not required to accept this License, since you have not signed it. -However, nothing else grants you permission to modify or distribute the Program -or its derivative works. These actions are prohibited by law if you do not -accept this License. Therefore, by modifying or distributing the Program (or -any work based on the Program), you indicate your acceptance of this License to -do so, and all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - -6. Each time you redistribute the Program (or any work based on the Program), -the recipient automatically receives a license from the original licensor to -copy, distribute or modify the Program subject to these terms and conditions. -You may not impose any further restrictions on the recipients' exercise of the -rights granted herein. You are not responsible for enforcing compliance by -third parties to this License. - -7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), conditions -are imposed on you (whether by court order, agreement or otherwise) that -contradict the conditions of this License, they do not excuse you from the -conditions of this License. If you cannot distribute so as to satisfy -simultaneously your obligations under this License and any other pertinent -obligations, then as a consequence you may not distribute the Program at all. -For example, if a patent license would not permit royalty-free redistribution -of the Program by all those who receive copies directly or indirectly through -you, then the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under any -particular circumstance, the balance of the section is intended to apply and -the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any patents or -other property right claims or to contest validity of any such claims; this -section has the sole purpose of protecting the integrity of the free software -distribution system, which is implemented by public license practices. Many -people have made generous contributions to the wide range of software -distributed through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing to -distribute software through any other system and a licensee cannot impose that -choice. - -This section is intended to make thoroughly clear what is believed to be a -consequence of the rest of this License. - -8. If the distribution and/or use of the Program is restricted in certain -countries either by patents or by copyrighted interfaces, the original -copyright holder who places the Program under this License may add an explicit -geographical distribution limitation excluding those countries, so that -distribution is permitted only in or among countries not thus excluded. In -such case, this License incorporates the limitation as if written in the body -of this License. - -9. The Free Software Foundation may publish revised and/or new versions of the -General Public License from time to time. Such new versions will be similar in -spirit to the present version, but may differ in detail to address new problems -or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any later -version", you have the option of following the terms and conditions either of -that version or of any later version published by the Free Software Foundation. -If the Program does not specify a version number of this License, you may -choose any version ever published by the Free Software Foundation. - -10. If you wish to incorporate parts of the Program into other free programs -whose distribution conditions are different, write to the author to ask for -permission. For software which is copyrighted by the Free Software Foundation, -write to the Free Software Foundation; we sometimes make exceptions for this. -Our decision will be guided by the two goals of preserving the free status of -all derivatives of our free software and of promoting the sharing and reuse of -software generally. - -NO WARRANTY - -11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR -THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE -STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE -PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND -PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, -YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - -12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL -ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE -PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR -INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA -BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A -FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER -OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -END OF TERMS AND CONDITIONS - -How to Apply These Terms to Your New Programs - -If you develop a new program, and you want it to be of the greatest possible -use to the public, the best way to achieve this is to make it free software -which everyone can redistribute and change under these terms. - -To do so, attach the following notices to the program. It is safest to attach -them to the start of each source file to most effectively convey the exclusion -of warranty; and each file should have at least the "copyright" line and a -pointer to where the full notice is found. - - One line to give the program's name and a brief idea of what it does. - - Copyright (C) - - This program is free software; you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by the Free - Software Foundation; either version 2 of the License, or (at your option) - any later version. - - This program is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., 59 - Temple Place, Suite 330, Boston, MA 02111-1307 USA - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this when it -starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author Gnomovision comes - with ABSOLUTELY NO WARRANTY; for details type 'show w'. This is free - software, and you are welcome to redistribute it under certain conditions; - type 'show c' for details. - -The hypothetical commands 'show w' and 'show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may be -called something other than 'show w' and 'show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your school, -if any, to sign a "copyright disclaimer" for the program, if necessary. Here -is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - 'Gnomovision' (which makes passes at compilers) written by James Hacker. - - signature of Ty Coon, 1 April 1989 - - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General Public -License instead of this License. - - -"CLASSPATH" EXCEPTION TO THE GPL - -Certain source files distributed by Oracle America and/or its affiliates are -subject to the following clarification and special exception to the GPL, but -only where Oracle has expressly included in the particular source file's header -the words "Oracle designates this particular file as subject to the "Classpath" -exception as provided by Oracle in the LICENSE file that accompanied this code." - - Linking this library statically or dynamically with other modules is making - a combined work based on this library. Thus, the terms and conditions of - the GNU General Public License cover the whole combination. - - As a special exception, the copyright holders of this library give you - permission to link this library with independent modules to produce an - executable, regardless of the license terms of these independent modules, - and to copy and distribute the resulting executable under terms of your - choice, provided that you also meet, for each linked independent module, - the terms and conditions of the license of that module. An independent - module is a module which is not derived from or based on this library. If - you modify this library, you may extend this exception to your version of - the library, but you are not obligated to do so. If you do not wish to do - so, delete this exception statement from your version. - -=========================================================================== - -MIT License: - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -=========================================================================== diff --git a/cdap-distributions/src/COPYRIGHT/auto-service@1.0-rc5/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/auto-service@1.0-rc5/COPYRIGHT deleted file mode 100644 index 8aa899e4c293..000000000000 --- a/cdap-distributions/src/COPYRIGHT/auto-service@1.0-rc5/COPYRIGHT +++ /dev/null @@ -1,409 +0,0 @@ -The Checker Framework -Copyright 2004-present by the Checker Framework developers - - -Most of the Checker Framework is licensed under the GNU General Public -License, version 2 (GPL2), with the classpath exception. The text of this -license appears below. This is the same license used for OpenJDK. - -A few parts of the Checker Framework have more permissive licenses. - - * The annotations are licensed under the MIT License. (The text of this - license also appears below.) More specifically, all the parts of the - Checker Framework that you might want to include with your own program - use the MIT License. This is the checker-qual*.jar and - checker-compat-qual*.jar files and all the files that appear in them: - every file in a qual/ directory, plus utility files such as - NullnessUtil.java, RegexUtil.java, SignednessUtil.java, etc. In - addition, the cleanroom implementations of third-party annotations, - which the Checker Framework recognizes as aliases for its own - annotations, are licensed under the MIT License. - -Some external libraries that are included with the Checker Framework have -different licenses. - - * javaparser is dual licensed under the LGPL or the Apache license -- you - may use it under whichever one you want. (The javaparser source code - contains a file with the text of the GPL, but it is not clear why, since - javaparser does not use the GPL.) See file stubparser/LICENSE - and the source code of all its files. - - * Libraries in plume-lib (https://github.com/plume-lib/) are licensed - under the MIT License. - -The Checker Framework includes annotations for some libraries. Each annotated -library uses the same license as the unannotated version of the library. - -=========================================================================== - -The GNU General Public License (GPL) - -Version 2, June 1991 - -Copyright (C) 1989, 1991 Free Software Foundation, Inc. -59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -Everyone is permitted to copy and distribute verbatim copies of this license -document, but changing it is not allowed. - -Preamble - -The licenses for most software are designed to take away your freedom to share -and change it. By contrast, the GNU General Public License is intended to -guarantee your freedom to share and change free software--to make sure the -software is free for all its users. This General Public License applies to -most of the Free Software Foundation's software and to any other program whose -authors commit to using it. (Some other Free Software Foundation software is -covered by the GNU Library General Public License instead.) You can apply it to -your programs, too. - -When we speak of free software, we are referring to freedom, not price. Our -General Public Licenses are designed to make sure that you have the freedom to -distribute copies of free software (and charge for this service if you wish), -that you receive source code or can get it if you want it, that you can change -the software or use pieces of it in new free programs; and that you know you -can do these things. - -To protect your rights, we need to make restrictions that forbid anyone to deny -you these rights or to ask you to surrender the rights. These restrictions -translate to certain responsibilities for you if you distribute copies of the -software, or if you modify it. - -For example, if you distribute copies of such a program, whether gratis or for -a fee, you must give the recipients all the rights that you have. You must -make sure that they, too, receive or can get the source code. And you must -show them these terms so they know their rights. - -We protect your rights with two steps: (1) copyright the software, and (2) -offer you this license which gives you legal permission to copy, distribute -and/or modify the software. - -Also, for each author's protection and ours, we want to make certain that -everyone understands that there is no warranty for this free software. If the -software is modified by someone else and passed on, we want its recipients to -know that what they have is not the original, so that any problems introduced -by others will not reflect on the original authors' reputations. - -Finally, any free program is threatened constantly by software patents. We -wish to avoid the danger that redistributors of a free program will -individually obtain patent licenses, in effect making the program proprietary. -To prevent this, we have made it clear that any patent must be licensed for -everyone's free use or not licensed at all. - -The precise terms and conditions for copying, distribution and modification -follow. - -TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - -0. This License applies to any program or other work which contains a notice -placed by the copyright holder saying it may be distributed under the terms of -this General Public License. The "Program", below, refers to any such program -or work, and a "work based on the Program" means either the Program or any -derivative work under copyright law: that is to say, a work containing the -Program or a portion of it, either verbatim or with modifications and/or -translated into another language. (Hereinafter, translation is included -without limitation in the term "modification".) Each licensee is addressed as -"you". - -Activities other than copying, distribution and modification are not covered by -this License; they are outside its scope. The act of running the Program is -not restricted, and the output from the Program is covered only if its contents -constitute a work based on the Program (independent of having been made by -running the Program). Whether that is true depends on what the Program does. - -1. You may copy and distribute verbatim copies of the Program's source code as -you receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice and -disclaimer of warranty; keep intact all the notices that refer to this License -and to the absence of any warranty; and give any other recipients of the -Program a copy of this License along with the Program. - -You may charge a fee for the physical act of transferring a copy, and you may -at your option offer warranty protection in exchange for a fee. - -2. You may modify your copy or copies of the Program or any portion of it, thus -forming a work based on the Program, and copy and distribute such modifications -or work under the terms of Section 1 above, provided that you also meet all of -these conditions: - - a) You must cause the modified files to carry prominent notices stating - that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in whole or - in part contains or is derived from the Program or any part thereof, to be - licensed as a whole at no charge to all third parties under the terms of - this License. - - c) If the modified program normally reads commands interactively when run, - you must cause it, when started running for such interactive use in the - most ordinary way, to print or display an announcement including an - appropriate copyright notice and a notice that there is no warranty (or - else, saying that you provide a warranty) and that users may redistribute - the program under these conditions, and telling the user how to view a copy - of this License. (Exception: if the Program itself is interactive but does - not normally print such an announcement, your work based on the Program is - not required to print an announcement.) - -These requirements apply to the modified work as a whole. If identifiable -sections of that work are not derived from the Program, and can be reasonably -considered independent and separate works in themselves, then this License, and -its terms, do not apply to those sections when you distribute them as separate -works. But when you distribute the same sections as part of a whole which is a -work based on the Program, the distribution of the whole must be on the terms -of this License, whose permissions for other licensees extend to the entire -whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest your -rights to work written entirely by you; rather, the intent is to exercise the -right to control the distribution of derivative or collective works based on -the Program. - -In addition, mere aggregation of another work not based on the Program with the -Program (or with a work based on the Program) on a volume of a storage or -distribution medium does not bring the other work under the scope of this -License. - -3. You may copy and distribute the Program (or a work based on it, under -Section 2) in object code or executable form under the terms of Sections 1 and -2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable source - code, which must be distributed under the terms of Sections 1 and 2 above - on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three years, to - give any third party, for a charge no more than your cost of physically - performing source distribution, a complete machine-readable copy of the - corresponding source code, to be distributed under the terms of Sections 1 - and 2 above on a medium customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer to - distribute corresponding source code. (This alternative is allowed only - for noncommercial distribution and only if you received the program in - object code or executable form with such an offer, in accord with - Subsection b above.) - -The source code for a work means the preferred form of the work for making -modifications to it. For an executable work, complete source code means all -the source code for all modules it contains, plus any associated interface -definition files, plus the scripts used to control compilation and installation -of the executable. However, as a special exception, the source code -distributed need not include anything that is normally distributed (in either -source or binary form) with the major components (compiler, kernel, and so on) -of the operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering access to copy -from a designated place, then offering equivalent access to copy the source -code from the same place counts as distribution of the source code, even though -third parties are not compelled to copy the source along with the object code. - -4. You may not copy, modify, sublicense, or distribute the Program except as -expressly provided under this License. Any attempt otherwise to copy, modify, -sublicense or distribute the Program is void, and will automatically terminate -your rights under this License. However, parties who have received copies, or -rights, from you under this License will not have their licenses terminated so -long as such parties remain in full compliance. - -5. You are not required to accept this License, since you have not signed it. -However, nothing else grants you permission to modify or distribute the Program -or its derivative works. These actions are prohibited by law if you do not -accept this License. Therefore, by modifying or distributing the Program (or -any work based on the Program), you indicate your acceptance of this License to -do so, and all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - -6. Each time you redistribute the Program (or any work based on the Program), -the recipient automatically receives a license from the original licensor to -copy, distribute or modify the Program subject to these terms and conditions. -You may not impose any further restrictions on the recipients' exercise of the -rights granted herein. You are not responsible for enforcing compliance by -third parties to this License. - -7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), conditions -are imposed on you (whether by court order, agreement or otherwise) that -contradict the conditions of this License, they do not excuse you from the -conditions of this License. If you cannot distribute so as to satisfy -simultaneously your obligations under this License and any other pertinent -obligations, then as a consequence you may not distribute the Program at all. -For example, if a patent license would not permit royalty-free redistribution -of the Program by all those who receive copies directly or indirectly through -you, then the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under any -particular circumstance, the balance of the section is intended to apply and -the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any patents or -other property right claims or to contest validity of any such claims; this -section has the sole purpose of protecting the integrity of the free software -distribution system, which is implemented by public license practices. Many -people have made generous contributions to the wide range of software -distributed through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing to -distribute software through any other system and a licensee cannot impose that -choice. - -This section is intended to make thoroughly clear what is believed to be a -consequence of the rest of this License. - -8. If the distribution and/or use of the Program is restricted in certain -countries either by patents or by copyrighted interfaces, the original -copyright holder who places the Program under this License may add an explicit -geographical distribution limitation excluding those countries, so that -distribution is permitted only in or among countries not thus excluded. In -such case, this License incorporates the limitation as if written in the body -of this License. - -9. The Free Software Foundation may publish revised and/or new versions of the -General Public License from time to time. Such new versions will be similar in -spirit to the present version, but may differ in detail to address new problems -or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any later -version", you have the option of following the terms and conditions either of -that version or of any later version published by the Free Software Foundation. -If the Program does not specify a version number of this License, you may -choose any version ever published by the Free Software Foundation. - -10. If you wish to incorporate parts of the Program into other free programs -whose distribution conditions are different, write to the author to ask for -permission. For software which is copyrighted by the Free Software Foundation, -write to the Free Software Foundation; we sometimes make exceptions for this. -Our decision will be guided by the two goals of preserving the free status of -all derivatives of our free software and of promoting the sharing and reuse of -software generally. - -NO WARRANTY - -11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR -THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE -STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE -PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND -PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, -YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - -12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL -ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE -PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR -INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA -BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A -FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER -OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -END OF TERMS AND CONDITIONS - -How to Apply These Terms to Your New Programs - -If you develop a new program, and you want it to be of the greatest possible -use to the public, the best way to achieve this is to make it free software -which everyone can redistribute and change under these terms. - -To do so, attach the following notices to the program. It is safest to attach -them to the start of each source file to most effectively convey the exclusion -of warranty; and each file should have at least the "copyright" line and a -pointer to where the full notice is found. - - One line to give the program's name and a brief idea of what it does. - - Copyright (C) - - This program is free software; you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by the Free - Software Foundation; either version 2 of the License, or (at your option) - any later version. - - This program is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., 59 - Temple Place, Suite 330, Boston, MA 02111-1307 USA - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this when it -starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author Gnomovision comes - with ABSOLUTELY NO WARRANTY; for details type 'show w'. This is free - software, and you are welcome to redistribute it under certain conditions; - type 'show c' for details. - -The hypothetical commands 'show w' and 'show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may be -called something other than 'show w' and 'show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your school, -if any, to sign a "copyright disclaimer" for the program, if necessary. Here -is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - 'Gnomovision' (which makes passes at compilers) written by James Hacker. - - signature of Ty Coon, 1 April 1989 - - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General Public -License instead of this License. - - -"CLASSPATH" EXCEPTION TO THE GPL - -Certain source files distributed by Oracle America and/or its affiliates are -subject to the following clarification and special exception to the GPL, but -only where Oracle has expressly included in the particular source file's header -the words "Oracle designates this particular file as subject to the "Classpath" -exception as provided by Oracle in the LICENSE file that accompanied this code." - - Linking this library statically or dynamically with other modules is making - a combined work based on this library. Thus, the terms and conditions of - the GNU General Public License cover the whole combination. - - As a special exception, the copyright holders of this library give you - permission to link this library with independent modules to produce an - executable, regardless of the license terms of these independent modules, - and to copy and distribute the resulting executable under terms of your - choice, provided that you also meet, for each linked independent module, - the terms and conditions of the license of that module. An independent - module is a module which is not derived from or based on this library. If - you modify this library, you may extend this exception to your version of - the library, but you are not obligated to do so. If you do not wish to do - so, delete this exception statement from your version. - -=========================================================================== - -MIT License: - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -=========================================================================== diff --git a/cdap-distributions/src/COPYRIGHT/autobind-decorator@2.1.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/autobind-decorator@2.4.0/COPYRIGHT similarity index 100% rename from cdap-distributions/src/COPYRIGHT/autobind-decorator@2.1.0/COPYRIGHT rename to cdap-distributions/src/COPYRIGHT/autobind-decorator@2.4.0/COPYRIGHT diff --git a/cdap-distributions/src/COPYRIGHT/aws4@1.8.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/aws4@1.11.0/COPYRIGHT similarity index 100% rename from cdap-distributions/src/COPYRIGHT/aws4@1.8.0/COPYRIGHT rename to cdap-distributions/src/COPYRIGHT/aws4@1.11.0/COPYRIGHT diff --git a/cdap-distributions/src/COPYRIGHT/babel-plugin-styled-components@1.13.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/babel-plugin-styled-components@1.13.2/COPYRIGHT new file mode 100644 index 000000000000..fe8eebee93d0 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/babel-plugin-styled-components@1.13.2/COPYRIGHT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016-present Vladimir Danchenkov and Maximilian Stoiber + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/babel-plugin-syntax-jsx@6.18.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/babel-plugin-syntax-jsx@6.18.0/COPYRIGHT new file mode 100644 index 000000000000..e8c7e6b5fc9b --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/babel-plugin-syntax-jsx@6.18.0/COPYRIGHT @@ -0,0 +1,35 @@ +# babel-plugin-syntax-jsx + + + +## Installation + +```sh +$ npm install babel-plugin-syntax-jsx +``` + +## Usage + +### Via `.babelrc` (Recommended) + +**.babelrc** + +```json +{ + "plugins": ["syntax-jsx"] +} +``` + +### Via CLI + +```sh +$ babel --plugins syntax-jsx script.js +``` + +### Via Node API + +```javascript +require("babel-core").transform("code", { + plugins: ["syntax-jsx"] +}); +``` diff --git a/cdap-distributions/src/COPYRIGHT/backo2@1.0.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/backo2@1.0.2/COPYRIGHT new file mode 100644 index 000000000000..0df2a399b195 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/backo2@1.0.2/COPYRIGHT @@ -0,0 +1,34 @@ +# backo + + Simple exponential backoff because the others seem to have weird abstractions. + +## Installation + +``` +$ npm install backo +``` + +## Options + + - `min` initial timeout in milliseconds [100] + - `max` max timeout [10000] + - `jitter` [0] + - `factor` [2] + +## Example + +```js +var Backoff = require('backo'); +var backoff = new Backoff({ min: 100, max: 20000 }); + +setTimeout(function(){ + something.reconnect(); +}, backoff.duration()); + +// later when something works +backoff.reset() +``` + +# License + + MIT diff --git a/cdap-distributions/src/COPYRIGHT/backport-util-concurrent:backport-util-concurrent:3.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/backport-util-concurrent:backport-util-concurrent:3.1/COPYRIGHT new file mode 100644 index 000000000000..e9b095683b2c --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/backport-util-concurrent:backport-util-concurrent:3.1/COPYRIGHT @@ -0,0 +1,41 @@ + +Creative Commons Public Domain + + + +

Copyright-Only Dedication (based on United States law) or Public Domain Certification

+ +

The +person or persons who have associated work with this document (the +"Dedicator" or "Certifier") hereby either (a) certifies that, to the +best of his knowledge, the work of authorship identified is in the +public domain of the country from which the work is published, or (b) +hereby dedicates whatever copyright the dedicators holds in the work of +authorship identified below (the "Work") to the public domain. A +certifier, morever, dedicates any copyright interest he may have in the +associated work, and for these purposes, is described as a "dedicator" +below.

+ +

A certifier has taken reasonable steps to verify +the copyright status of this work. Certifier recognizes that his good +faith efforts may not shield him from liability if in fact the work +certified is not in the public domain.

+ +

Dedicator makes +this dedication for the benefit of the public at large and to the +detriment of the Dedicator's heirs and successors. Dedicator intends +this dedication to be an overt act of relinquishment in perpetuity of +all present and future rights under copyright law, whether vested or +contingent, in the Work. Dedicator understands that such relinquishment +of all rights includes the relinquishment of all rights to enforce (by +lawsuit or otherwise) those copyrights in the Work.

+ +

Dedicator +recognizes that, once placed in the public domain, the Work may be +freely reproduced, distributed, transmitted, used, modified, built +upon, or otherwise exploited by anyone for any purpose, commercial or +non-commercial, and in any way, including by methods that have not yet +been invented or conceived.

+ + + \ No newline at end of file diff --git a/cdap-distributions/src/COPYRIGHT/base64-js@1.5.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/base64-js@1.5.1/COPYRIGHT new file mode 100644 index 000000000000..6d52b8acfbe7 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/base64-js@1.5.1/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Jameson Little + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/base@0.11.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/base@0.11.2/COPYRIGHT new file mode 100644 index 000000000000..e33d14b754e8 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/base@0.11.2/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015-2017, Jon Schlinkert. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/batch@0.6.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/batch@0.6.1/COPYRIGHT new file mode 100644 index 000000000000..b7409302c44e --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/batch@0.6.1/COPYRIGHT @@ -0,0 +1,22 @@ +(The MIT License) + +Copyright (c) 2013 TJ Holowaychuk + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/bcel:bcel:5.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/bcel:bcel:5.1/COPYRIGHT new file mode 100644 index 000000000000..6b0b1270ff0c --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/bcel:bcel:5.1/COPYRIGHT @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/cdap-distributions/src/COPYRIGHT/big.js@3.2.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/big.js@3.2.0/COPYRIGHT deleted file mode 100644 index c30d30b252dd..000000000000 --- a/cdap-distributions/src/COPYRIGHT/big.js@3.2.0/COPYRIGHT +++ /dev/null @@ -1,23 +0,0 @@ -The MIT Licence (Expat). - -Copyright (c) 2018 Michael Mclaughlin - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -'Software'), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/cdap-distributions/src/COPYRIGHT/big.js@5.2.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/big.js@5.2.2/COPYRIGHT new file mode 100644 index 000000000000..f7e12c838527 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/big.js@5.2.2/COPYRIGHT @@ -0,0 +1,23 @@ +The MIT Licence (Expat). + +Copyright (c) 2018 Michael Mclaughlin + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/cdap-distributions/src/COPYRIGHT/mimic-fn@1.1.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/binary-extensions@1.13.1/COPYRIGHT similarity index 100% rename from cdap-distributions/src/COPYRIGHT/mimic-fn@1.1.0/COPYRIGHT rename to cdap-distributions/src/COPYRIGHT/binary-extensions@1.13.1/COPYRIGHT diff --git a/cdap-distributions/src/COPYRIGHT/binary-extensions@2.2.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/binary-extensions@2.2.0/COPYRIGHT new file mode 100644 index 000000000000..401b1c731bcd --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/binary-extensions@2.2.0/COPYRIGHT @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2019 Sindre Sorhus (https://sindresorhus.com), Paul Miller (https://paulmillr.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/blob-polyfill@1.0.20150320/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/blob-polyfill@1.0.20150320/COPYRIGHT new file mode 100644 index 000000000000..11520c90c0ab --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/blob-polyfill@1.0.20150320/COPYRIGHT @@ -0,0 +1,25 @@ +Copyright © 2014 [Eli Grey][1]. + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + + + [1]: http://eligrey.com diff --git a/cdap-distributions/src/COPYRIGHT/blob-tmp@1.0.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/blob-tmp@1.0.0/COPYRIGHT new file mode 100644 index 000000000000..11520c90c0ab --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/blob-tmp@1.0.0/COPYRIGHT @@ -0,0 +1,25 @@ +Copyright © 2014 [Eli Grey][1]. + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + + + [1]: http://eligrey.com diff --git a/cdap-distributions/src/COPYRIGHT/bluebird@3.7.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/bluebird@3.7.2/COPYRIGHT new file mode 100644 index 000000000000..b24e6350cafa --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/bluebird@3.7.2/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2013-2018 Petka Antonov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/bn.js@4.12.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/bn.js@4.12.0/COPYRIGHT new file mode 100644 index 000000000000..c328f0401e39 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/bn.js@4.12.0/COPYRIGHT @@ -0,0 +1,19 @@ +Copyright Fedor Indutny, 2015. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/bn.js@5.2.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/bn.js@5.2.0/COPYRIGHT new file mode 100644 index 000000000000..c328f0401e39 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/bn.js@5.2.0/COPYRIGHT @@ -0,0 +1,19 @@ +Copyright Fedor Indutny, 2015. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/body-parser@1.18.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/body-parser@1.19.0/COPYRIGHT similarity index 100% rename from cdap-distributions/src/COPYRIGHT/body-parser@1.18.2/COPYRIGHT rename to cdap-distributions/src/COPYRIGHT/body-parser@1.19.0/COPYRIGHT diff --git a/cdap-distributions/src/COPYRIGHT/bonjour@3.5.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/bonjour@3.5.0/COPYRIGHT new file mode 100644 index 000000000000..ee5acbb947c1 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/bonjour@3.5.0/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015-2016 Thomas Watson Steen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/boom@4.3.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/boom@4.3.1/COPYRIGHT deleted file mode 100644 index aa2404c43f8e..000000000000 --- a/cdap-distributions/src/COPYRIGHT/boom@4.3.1/COPYRIGHT +++ /dev/null @@ -1,10 +0,0 @@ -Copyright (c) 2012-2019, Sideway Inc, and project contributors -Copyright (c) 2012-2014, Walmart. -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -* The names of any contributors may not be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS OFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/cdap-distributions/src/COPYRIGHT/boom@5.2.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/boom@5.2.0/COPYRIGHT deleted file mode 100644 index aa2404c43f8e..000000000000 --- a/cdap-distributions/src/COPYRIGHT/boom@5.2.0/COPYRIGHT +++ /dev/null @@ -1,10 +0,0 @@ -Copyright (c) 2012-2019, Sideway Inc, and project contributors -Copyright (c) 2012-2014, Walmart. -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -* The names of any contributors may not be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS OFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/cdap-distributions/src/COPYRIGHT/bootstrap@3.3.6/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/bootstrap@3.3.6/COPYRIGHT index 26fa3f4acbe8..f4c52d6a4b3e 100644 --- a/cdap-distributions/src/COPYRIGHT/bootstrap@3.3.6/COPYRIGHT +++ b/cdap-distributions/src/COPYRIGHT/bootstrap@3.3.6/COPYRIGHT @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2011-2019 Twitter, Inc. +Copyright (c) 2011-2015 Twitter, Inc Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/bootstrap@4.1.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/bootstrap@4.1.3/COPYRIGHT deleted file mode 100644 index 7d1e2311f44f..000000000000 --- a/cdap-distributions/src/COPYRIGHT/bootstrap@4.1.3/COPYRIGHT +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2011-2019 Twitter, Inc. -Copyright (c) 2011-2019 The Bootstrap Authors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/bootstrap@4.6.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/bootstrap@4.6.1/COPYRIGHT new file mode 100644 index 000000000000..72dda234edaa --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/bootstrap@4.6.1/COPYRIGHT @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2011-2021 Twitter, Inc. +Copyright (c) 2011-2021 The Bootstrap Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/braces@2.3.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/braces@2.3.2/COPYRIGHT new file mode 100644 index 000000000000..d32ab4426a5f --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/braces@2.3.2/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014-2018, Jon Schlinkert. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/braces@3.0.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/braces@3.0.2/COPYRIGHT new file mode 100644 index 000000000000..d32ab4426a5f --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/braces@3.0.2/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014-2018, Jon Schlinkert. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/brcast@3.0.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/brcast@3.0.1/COPYRIGHT deleted file mode 100644 index ada2af59063c..000000000000 --- a/cdap-distributions/src/COPYRIGHT/brcast@3.0.1/COPYRIGHT +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2017-present Alessandro Arnodo - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/breeze-macros_2.10@0.11.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/breeze-macros_2.10@0.11.2/COPYRIGHT deleted file mode 100644 index e40679a31a99..000000000000 --- a/cdap-distributions/src/COPYRIGHT/breeze-macros_2.10@0.11.2/COPYRIGHT +++ /dev/null @@ -1,17 +0,0 @@ -Breeze is distributed under an Apache License V2.0 (See LICENSE) - -=============================================================================== - -Proximal algorithms outlined in Proximal.scala (package breeze.optimize.proximal) -are based on https://github.com/cvxgrp/proximal (see LICENSE for details) and distributed with -Copyright (c) 2014 by Debasish Das (Verizon), all rights reserved. - -=============================================================================== - -QuadraticMinimizer class in package breeze.optimize.proximal is distributed with Copyright (c) -2014, Debasish Das (Verizon), all rights reserved. - -=============================================================================== - -NonlinearMinimizer class in package breeze.optimize.proximal is distributed with Copyright (c) -2015, Debasish Das (Verizon), all rights reserved. diff --git a/cdap-distributions/src/COPYRIGHT/breeze_2.10@0.11.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/breeze_2.10@0.11.2/COPYRIGHT deleted file mode 100644 index e40679a31a99..000000000000 --- a/cdap-distributions/src/COPYRIGHT/breeze_2.10@0.11.2/COPYRIGHT +++ /dev/null @@ -1,17 +0,0 @@ -Breeze is distributed under an Apache License V2.0 (See LICENSE) - -=============================================================================== - -Proximal algorithms outlined in Proximal.scala (package breeze.optimize.proximal) -are based on https://github.com/cvxgrp/proximal (see LICENSE for details) and distributed with -Copyright (c) 2014 by Debasish Das (Verizon), all rights reserved. - -=============================================================================== - -QuadraticMinimizer class in package breeze.optimize.proximal is distributed with Copyright (c) -2014, Debasish Das (Verizon), all rights reserved. - -=============================================================================== - -NonlinearMinimizer class in package breeze.optimize.proximal is distributed with Copyright (c) -2015, Debasish Das (Verizon), all rights reserved. diff --git a/cdap-distributions/src/COPYRIGHT/brorand@1.1.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/brorand@1.1.0/COPYRIGHT new file mode 100644 index 000000000000..f80437d141e5 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/brorand@1.1.0/COPYRIGHT @@ -0,0 +1,26 @@ +# Brorand + +#### LICENSE + +This software is licensed under the MIT License. + +Copyright Fedor Indutny, 2014. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/browserify-aes@1.2.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/browserify-aes@1.2.0/COPYRIGHT new file mode 100644 index 000000000000..c6e36b5fa73d --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/browserify-aes@1.2.0/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014-2017 browserify-aes contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/browserify-cipher@1.0.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/browserify-cipher@1.0.1/COPYRIGHT new file mode 100644 index 000000000000..a49f062688d1 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/browserify-cipher@1.0.1/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014-2017 Calvin Metcalf & contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/browserify-des@1.0.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/browserify-des@1.0.2/COPYRIGHT new file mode 100644 index 000000000000..798de7dd4fc3 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/browserify-des@1.0.2/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014-2017 Calvin Metcalf, Fedor Indutny & contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/browserify-rsa@4.1.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/browserify-rsa@4.1.0/COPYRIGHT new file mode 100644 index 000000000000..c1dd3c5c0da1 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/browserify-rsa@4.1.0/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014-2016 Calvin Metcalf & contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/browserify-sign@4.2.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/browserify-sign@4.2.1/COPYRIGHT new file mode 100644 index 000000000000..870bcf1f7bb8 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/browserify-sign@4.2.1/COPYRIGHT @@ -0,0 +1,13 @@ +Copyright (c) 2014-2015 Calvin Metcalf and browserify-sign contributors + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/browserify-zlib@0.2.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/browserify-zlib@0.2.0/COPYRIGHT new file mode 100644 index 000000000000..81c6c0c6f276 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/browserify-zlib@0.2.0/COPYRIGHT @@ -0,0 +1,70 @@ +The MIT License (MIT) + +Copyright (c) 2014-2015 Devon Govett + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +This project contains parts of Node.js. +Node.js is licensed for use as follows: + +""" +Copyright Node.js contributors. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. +""" + +This license applies to parts of Node.js originating from the +https://github.com/joyent/node repository: + +""" +Copyright Joyent, Inc. and other Node contributors. All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. +""" diff --git a/cdap-distributions/src/COPYRIGHT/buffer-from@1.1.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/buffer-from@1.1.1/COPYRIGHT new file mode 100644 index 000000000000..e4bf1d69b1bb --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/buffer-from@1.1.1/COPYRIGHT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016, 2018 Linus Unnebäck + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/buffer-indexof@1.1.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/buffer-indexof@1.1.1/COPYRIGHT new file mode 100644 index 000000000000..f6079dfdb675 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/buffer-indexof@1.1.1/COPYRIGHT @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013 Ryan Day + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/buffer-xor@1.0.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/buffer-xor@1.0.3/COPYRIGHT new file mode 100644 index 000000000000..bba52181d52e --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/buffer-xor@1.0.3/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Daniel Cousens + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/buffer@4.9.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/buffer@4.9.2/COPYRIGHT new file mode 100644 index 000000000000..d6bf75dcf1f6 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/buffer@4.9.2/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) Feross Aboukhadijeh, and other contributors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/builtin-status-codes@3.0.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/builtin-status-codes@3.0.0/COPYRIGHT new file mode 100644 index 000000000000..25c624701389 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/builtin-status-codes@3.0.0/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) Ben Drucker (bendrucker.me) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/busboy@0.3.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/busboy@0.3.1/COPYRIGHT new file mode 100644 index 000000000000..290762e94f4e --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/busboy@0.3.1/COPYRIGHT @@ -0,0 +1,19 @@ +Copyright Brian White. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. \ No newline at end of file diff --git a/cdap-distributions/src/COPYRIGHT/bytes@3.1.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/bytes@3.1.0/COPYRIGHT new file mode 100644 index 000000000000..63e95a96338a --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/bytes@3.1.0/COPYRIGHT @@ -0,0 +1,23 @@ +(The MIT License) + +Copyright (c) 2012-2014 TJ Holowaychuk +Copyright (c) 2015 Jed Watson + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/c3@0.4.10/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/c3@0.4.10/COPYRIGHT deleted file mode 100644 index 1c71957bb31e..000000000000 --- a/cdap-distributions/src/COPYRIGHT/c3@0.4.10/COPYRIGHT +++ /dev/null @@ -1,20 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2013 Masayuki Tanaka - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-base:2.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-base:2.2/COPYRIGHT new file mode 100644 index 000000000000..35f38a015ed6 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-base:2.2/COPYRIGHT @@ -0,0 +1,13 @@ +Copyright 2015, University Health Network +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this work except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v21:2.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v21:2.2/COPYRIGHT new file mode 100644 index 000000000000..35f38a015ed6 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v21:2.2/COPYRIGHT @@ -0,0 +1,13 @@ +Copyright 2015, University Health Network +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this work except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v22:2.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v22:2.2/COPYRIGHT new file mode 100644 index 000000000000..35f38a015ed6 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v22:2.2/COPYRIGHT @@ -0,0 +1,13 @@ +Copyright 2015, University Health Network +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this work except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v231:2.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v231:2.2/COPYRIGHT new file mode 100644 index 000000000000..35f38a015ed6 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v231:2.2/COPYRIGHT @@ -0,0 +1,13 @@ +Copyright 2015, University Health Network +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this work except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v23:2.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v23:2.2/COPYRIGHT new file mode 100644 index 000000000000..35f38a015ed6 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v23:2.2/COPYRIGHT @@ -0,0 +1,13 @@ +Copyright 2015, University Health Network +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this work except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v24:2.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v24:2.2/COPYRIGHT new file mode 100644 index 000000000000..35f38a015ed6 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v24:2.2/COPYRIGHT @@ -0,0 +1,13 @@ +Copyright 2015, University Health Network +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this work except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v251:2.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v251:2.2/COPYRIGHT new file mode 100644 index 000000000000..35f38a015ed6 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v251:2.2/COPYRIGHT @@ -0,0 +1,13 @@ +Copyright 2015, University Health Network +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this work except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v25:2.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v25:2.2/COPYRIGHT new file mode 100644 index 000000000000..35f38a015ed6 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v25:2.2/COPYRIGHT @@ -0,0 +1,13 @@ +Copyright 2015, University Health Network +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this work except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v26:2.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v26:2.2/COPYRIGHT new file mode 100644 index 000000000000..35f38a015ed6 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ca.uhn.hapi:hapi-structures-v26:2.2/COPYRIGHT @@ -0,0 +1,13 @@ +Copyright 2015, University Health Network +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this work except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/cdap-distributions/src/COPYRIGHT/cacache@12.0.4/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/cacache@12.0.4/COPYRIGHT new file mode 100644 index 000000000000..8d28acf866d9 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/cacache@12.0.4/COPYRIGHT @@ -0,0 +1,16 @@ +ISC License + +Copyright (c) npm, Inc. + +Permission to use, copy, modify, and/or distribute this software for +any purpose with or without fee is hereby granted, provided that the +above copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE COPYRIGHT HOLDER DISCLAIMS +ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE +USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/cache-base@1.0.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/cache-base@1.0.1/COPYRIGHT new file mode 100644 index 000000000000..943e71d05511 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/cache-base@1.0.1/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014-2017, Jon Schlinkert. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/cdap-distributions/src/COPYRIGHT/call-bind@1.0.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/call-bind@1.0.2/COPYRIGHT new file mode 100644 index 000000000000..48f05d01d0ac --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/call-bind@1.0.2/COPYRIGHT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Jordan Harband + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/camelcase@2.1.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/camelcase@2.1.1/COPYRIGHT new file mode 100644 index 000000000000..654d0bfe9434 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/camelcase@2.1.1/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) Sindre Sorhus (sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/camelcase@3.0.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/camelcase@3.0.0/COPYRIGHT new file mode 100644 index 000000000000..654d0bfe9434 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/camelcase@3.0.0/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) Sindre Sorhus (sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/p-limit@1.1.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/camelcase@5.3.1/COPYRIGHT similarity index 100% rename from cdap-distributions/src/COPYRIGHT/p-limit@1.1.0/COPYRIGHT rename to cdap-distributions/src/COPYRIGHT/camelcase@5.3.1/COPYRIGHT diff --git a/cdap-distributions/src/COPYRIGHT/canvas-prebuilt@1.6.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/canvas-prebuilt@1.6.0/COPYRIGHT deleted file mode 100644 index d13cc4b26a53..000000000000 --- a/cdap-distributions/src/COPYRIGHT/canvas-prebuilt@1.6.0/COPYRIGHT +++ /dev/null @@ -1,19 +0,0 @@ -The MIT License (MIT) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/capture-stack-trace@1.0.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/capture-stack-trace@1.0.1/COPYRIGHT similarity index 100% rename from cdap-distributions/src/COPYRIGHT/capture-stack-trace@1.0.0/COPYRIGHT rename to cdap-distributions/src/COPYRIGHT/capture-stack-trace@1.0.1/COPYRIGHT diff --git a/cdap-distributions/src/COPYRIGHT/cdap-ui@6.2.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/cdap-ui@6.2.0/COPYRIGHT new file mode 100644 index 000000000000..c20c4a503205 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/cdap-ui@6.2.0/COPYRIGHT @@ -0,0 +1,232 @@ +Copyright © 2014-2021 Cask Data, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APACHE HADOOP SUBCOMPONENTS + +The Apache Hadoop project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + +For the org.apache.hadoop.util.bloom.* classes: + +/** + * + * Copyright (c) 2005, European Commission project OneLab under contract + * 034819 (http://www.one-lab.org) + * All rights reserved. + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the distribution. + * - Neither the name of the University Catholique de Louvain - UCL + * nor the names of its contributors may be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ diff --git a/cdap-distributions/src/COPYRIGHT/ch.ethz.ganymed:ganymed-ssh2:262/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ch.ethz.ganymed:ganymed-ssh2:262/COPYRIGHT new file mode 100644 index 000000000000..892eaed8f1ce --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ch.ethz.ganymed:ganymed-ssh2:262/COPYRIGHT @@ -0,0 +1,203 @@ +Google Gson + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2008-2011 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-classic:1.0.9/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-classic:1.0.9/COPYRIGHT new file mode 100644 index 000000000000..126ef23a2a88 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-classic:1.0.9/COPYRIGHT @@ -0,0 +1,15 @@ +Logback LICENSE +--------------- + +Logback: the reliable, generic, fast and flexible logging framework. +Copyright (C) 1999-2024, QOS.ch. All rights reserved. + +This program and the accompanying materials are dual-licensed under +either the terms of the Eclipse Public License v1.0 as published by +the Eclipse Foundation + + or (per the licensee's choosing) + +under the terms of the GNU Lesser General Public License version 2.1 +as published by the Free Software Foundation. + diff --git a/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-classic:1.2.11/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-classic:1.2.11/COPYRIGHT new file mode 100644 index 000000000000..126ef23a2a88 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-classic:1.2.11/COPYRIGHT @@ -0,0 +1,15 @@ +Logback LICENSE +--------------- + +Logback: the reliable, generic, fast and flexible logging framework. +Copyright (C) 1999-2024, QOS.ch. All rights reserved. + +This program and the accompanying materials are dual-licensed under +either the terms of the Eclipse Public License v1.0 as published by +the Eclipse Foundation + + or (per the licensee's choosing) + +under the terms of the GNU Lesser General Public License version 2.1 +as published by the Free Software Foundation. + diff --git a/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-classic:1.2.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-classic:1.2.3/COPYRIGHT new file mode 100644 index 000000000000..126ef23a2a88 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-classic:1.2.3/COPYRIGHT @@ -0,0 +1,15 @@ +Logback LICENSE +--------------- + +Logback: the reliable, generic, fast and flexible logging framework. +Copyright (C) 1999-2024, QOS.ch. All rights reserved. + +This program and the accompanying materials are dual-licensed under +either the terms of the Eclipse Public License v1.0 as published by +the Eclipse Foundation + + or (per the licensee's choosing) + +under the terms of the GNU Lesser General Public License version 2.1 +as published by the Free Software Foundation. + diff --git a/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-core:1.0.9/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-core:1.0.9/COPYRIGHT new file mode 100644 index 000000000000..126ef23a2a88 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-core:1.0.9/COPYRIGHT @@ -0,0 +1,15 @@ +Logback LICENSE +--------------- + +Logback: the reliable, generic, fast and flexible logging framework. +Copyright (C) 1999-2024, QOS.ch. All rights reserved. + +This program and the accompanying materials are dual-licensed under +either the terms of the Eclipse Public License v1.0 as published by +the Eclipse Foundation + + or (per the licensee's choosing) + +under the terms of the GNU Lesser General Public License version 2.1 +as published by the Free Software Foundation. + diff --git a/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-core:1.2.11/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-core:1.2.11/COPYRIGHT new file mode 100644 index 000000000000..126ef23a2a88 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-core:1.2.11/COPYRIGHT @@ -0,0 +1,15 @@ +Logback LICENSE +--------------- + +Logback: the reliable, generic, fast and flexible logging framework. +Copyright (C) 1999-2024, QOS.ch. All rights reserved. + +This program and the accompanying materials are dual-licensed under +either the terms of the Eclipse Public License v1.0 as published by +the Eclipse Foundation + + or (per the licensee's choosing) + +under the terms of the GNU Lesser General Public License version 2.1 +as published by the Free Software Foundation. + diff --git a/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-core:1.2.3/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-core:1.2.3/COPYRIGHT new file mode 100644 index 000000000000..126ef23a2a88 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/ch.qos.logback:logback-core:1.2.3/COPYRIGHT @@ -0,0 +1,15 @@ +Logback LICENSE +--------------- + +Logback: the reliable, generic, fast and flexible logging framework. +Copyright (C) 1999-2024, QOS.ch. All rights reserved. + +This program and the accompanying materials are dual-licensed under +either the terms of the Eclipse Public License v1.0 as published by +the Eclipse Foundation + + or (per the licensee's choosing) + +under the terms of the GNU Lesser General Public License version 2.1 +as published by the Free Software Foundation. + diff --git a/cdap-distributions/src/COPYRIGHT/google-api-client-jackson2@1.30.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/ch.qos.reload4j:reload4j:1.2.18.3/COPYRIGHT similarity index 100% rename from cdap-distributions/src/COPYRIGHT/google-api-client-jackson2@1.30.1/COPYRIGHT rename to cdap-distributions/src/COPYRIGHT/ch.qos.reload4j:reload4j:1.2.18.3/COPYRIGHT diff --git a/cdap-distributions/src/COPYRIGHT/query-string@4.3.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/chalk@2.4.2/COPYRIGHT similarity index 100% rename from cdap-distributions/src/COPYRIGHT/query-string@4.3.2/COPYRIGHT rename to cdap-distributions/src/COPYRIGHT/chalk@2.4.2/COPYRIGHT diff --git a/cdap-distributions/src/COPYRIGHT/checker-compat-qual@2.5.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/checker-compat-qual@2.5.2/COPYRIGHT deleted file mode 100644 index 8aa899e4c293..000000000000 --- a/cdap-distributions/src/COPYRIGHT/checker-compat-qual@2.5.2/COPYRIGHT +++ /dev/null @@ -1,409 +0,0 @@ -The Checker Framework -Copyright 2004-present by the Checker Framework developers - - -Most of the Checker Framework is licensed under the GNU General Public -License, version 2 (GPL2), with the classpath exception. The text of this -license appears below. This is the same license used for OpenJDK. - -A few parts of the Checker Framework have more permissive licenses. - - * The annotations are licensed under the MIT License. (The text of this - license also appears below.) More specifically, all the parts of the - Checker Framework that you might want to include with your own program - use the MIT License. This is the checker-qual*.jar and - checker-compat-qual*.jar files and all the files that appear in them: - every file in a qual/ directory, plus utility files such as - NullnessUtil.java, RegexUtil.java, SignednessUtil.java, etc. In - addition, the cleanroom implementations of third-party annotations, - which the Checker Framework recognizes as aliases for its own - annotations, are licensed under the MIT License. - -Some external libraries that are included with the Checker Framework have -different licenses. - - * javaparser is dual licensed under the LGPL or the Apache license -- you - may use it under whichever one you want. (The javaparser source code - contains a file with the text of the GPL, but it is not clear why, since - javaparser does not use the GPL.) See file stubparser/LICENSE - and the source code of all its files. - - * Libraries in plume-lib (https://github.com/plume-lib/) are licensed - under the MIT License. - -The Checker Framework includes annotations for some libraries. Each annotated -library uses the same license as the unannotated version of the library. - -=========================================================================== - -The GNU General Public License (GPL) - -Version 2, June 1991 - -Copyright (C) 1989, 1991 Free Software Foundation, Inc. -59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -Everyone is permitted to copy and distribute verbatim copies of this license -document, but changing it is not allowed. - -Preamble - -The licenses for most software are designed to take away your freedom to share -and change it. By contrast, the GNU General Public License is intended to -guarantee your freedom to share and change free software--to make sure the -software is free for all its users. This General Public License applies to -most of the Free Software Foundation's software and to any other program whose -authors commit to using it. (Some other Free Software Foundation software is -covered by the GNU Library General Public License instead.) You can apply it to -your programs, too. - -When we speak of free software, we are referring to freedom, not price. Our -General Public Licenses are designed to make sure that you have the freedom to -distribute copies of free software (and charge for this service if you wish), -that you receive source code or can get it if you want it, that you can change -the software or use pieces of it in new free programs; and that you know you -can do these things. - -To protect your rights, we need to make restrictions that forbid anyone to deny -you these rights or to ask you to surrender the rights. These restrictions -translate to certain responsibilities for you if you distribute copies of the -software, or if you modify it. - -For example, if you distribute copies of such a program, whether gratis or for -a fee, you must give the recipients all the rights that you have. You must -make sure that they, too, receive or can get the source code. And you must -show them these terms so they know their rights. - -We protect your rights with two steps: (1) copyright the software, and (2) -offer you this license which gives you legal permission to copy, distribute -and/or modify the software. - -Also, for each author's protection and ours, we want to make certain that -everyone understands that there is no warranty for this free software. If the -software is modified by someone else and passed on, we want its recipients to -know that what they have is not the original, so that any problems introduced -by others will not reflect on the original authors' reputations. - -Finally, any free program is threatened constantly by software patents. We -wish to avoid the danger that redistributors of a free program will -individually obtain patent licenses, in effect making the program proprietary. -To prevent this, we have made it clear that any patent must be licensed for -everyone's free use or not licensed at all. - -The precise terms and conditions for copying, distribution and modification -follow. - -TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - -0. This License applies to any program or other work which contains a notice -placed by the copyright holder saying it may be distributed under the terms of -this General Public License. The "Program", below, refers to any such program -or work, and a "work based on the Program" means either the Program or any -derivative work under copyright law: that is to say, a work containing the -Program or a portion of it, either verbatim or with modifications and/or -translated into another language. (Hereinafter, translation is included -without limitation in the term "modification".) Each licensee is addressed as -"you". - -Activities other than copying, distribution and modification are not covered by -this License; they are outside its scope. The act of running the Program is -not restricted, and the output from the Program is covered only if its contents -constitute a work based on the Program (independent of having been made by -running the Program). Whether that is true depends on what the Program does. - -1. You may copy and distribute verbatim copies of the Program's source code as -you receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice and -disclaimer of warranty; keep intact all the notices that refer to this License -and to the absence of any warranty; and give any other recipients of the -Program a copy of this License along with the Program. - -You may charge a fee for the physical act of transferring a copy, and you may -at your option offer warranty protection in exchange for a fee. - -2. You may modify your copy or copies of the Program or any portion of it, thus -forming a work based on the Program, and copy and distribute such modifications -or work under the terms of Section 1 above, provided that you also meet all of -these conditions: - - a) You must cause the modified files to carry prominent notices stating - that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in whole or - in part contains or is derived from the Program or any part thereof, to be - licensed as a whole at no charge to all third parties under the terms of - this License. - - c) If the modified program normally reads commands interactively when run, - you must cause it, when started running for such interactive use in the - most ordinary way, to print or display an announcement including an - appropriate copyright notice and a notice that there is no warranty (or - else, saying that you provide a warranty) and that users may redistribute - the program under these conditions, and telling the user how to view a copy - of this License. (Exception: if the Program itself is interactive but does - not normally print such an announcement, your work based on the Program is - not required to print an announcement.) - -These requirements apply to the modified work as a whole. If identifiable -sections of that work are not derived from the Program, and can be reasonably -considered independent and separate works in themselves, then this License, and -its terms, do not apply to those sections when you distribute them as separate -works. But when you distribute the same sections as part of a whole which is a -work based on the Program, the distribution of the whole must be on the terms -of this License, whose permissions for other licensees extend to the entire -whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest your -rights to work written entirely by you; rather, the intent is to exercise the -right to control the distribution of derivative or collective works based on -the Program. - -In addition, mere aggregation of another work not based on the Program with the -Program (or with a work based on the Program) on a volume of a storage or -distribution medium does not bring the other work under the scope of this -License. - -3. You may copy and distribute the Program (or a work based on it, under -Section 2) in object code or executable form under the terms of Sections 1 and -2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable source - code, which must be distributed under the terms of Sections 1 and 2 above - on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three years, to - give any third party, for a charge no more than your cost of physically - performing source distribution, a complete machine-readable copy of the - corresponding source code, to be distributed under the terms of Sections 1 - and 2 above on a medium customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer to - distribute corresponding source code. (This alternative is allowed only - for noncommercial distribution and only if you received the program in - object code or executable form with such an offer, in accord with - Subsection b above.) - -The source code for a work means the preferred form of the work for making -modifications to it. For an executable work, complete source code means all -the source code for all modules it contains, plus any associated interface -definition files, plus the scripts used to control compilation and installation -of the executable. However, as a special exception, the source code -distributed need not include anything that is normally distributed (in either -source or binary form) with the major components (compiler, kernel, and so on) -of the operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering access to copy -from a designated place, then offering equivalent access to copy the source -code from the same place counts as distribution of the source code, even though -third parties are not compelled to copy the source along with the object code. - -4. You may not copy, modify, sublicense, or distribute the Program except as -expressly provided under this License. Any attempt otherwise to copy, modify, -sublicense or distribute the Program is void, and will automatically terminate -your rights under this License. However, parties who have received copies, or -rights, from you under this License will not have their licenses terminated so -long as such parties remain in full compliance. - -5. You are not required to accept this License, since you have not signed it. -However, nothing else grants you permission to modify or distribute the Program -or its derivative works. These actions are prohibited by law if you do not -accept this License. Therefore, by modifying or distributing the Program (or -any work based on the Program), you indicate your acceptance of this License to -do so, and all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - -6. Each time you redistribute the Program (or any work based on the Program), -the recipient automatically receives a license from the original licensor to -copy, distribute or modify the Program subject to these terms and conditions. -You may not impose any further restrictions on the recipients' exercise of the -rights granted herein. You are not responsible for enforcing compliance by -third parties to this License. - -7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), conditions -are imposed on you (whether by court order, agreement or otherwise) that -contradict the conditions of this License, they do not excuse you from the -conditions of this License. If you cannot distribute so as to satisfy -simultaneously your obligations under this License and any other pertinent -obligations, then as a consequence you may not distribute the Program at all. -For example, if a patent license would not permit royalty-free redistribution -of the Program by all those who receive copies directly or indirectly through -you, then the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under any -particular circumstance, the balance of the section is intended to apply and -the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any patents or -other property right claims or to contest validity of any such claims; this -section has the sole purpose of protecting the integrity of the free software -distribution system, which is implemented by public license practices. Many -people have made generous contributions to the wide range of software -distributed through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing to -distribute software through any other system and a licensee cannot impose that -choice. - -This section is intended to make thoroughly clear what is believed to be a -consequence of the rest of this License. - -8. If the distribution and/or use of the Program is restricted in certain -countries either by patents or by copyrighted interfaces, the original -copyright holder who places the Program under this License may add an explicit -geographical distribution limitation excluding those countries, so that -distribution is permitted only in or among countries not thus excluded. In -such case, this License incorporates the limitation as if written in the body -of this License. - -9. The Free Software Foundation may publish revised and/or new versions of the -General Public License from time to time. Such new versions will be similar in -spirit to the present version, but may differ in detail to address new problems -or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any later -version", you have the option of following the terms and conditions either of -that version or of any later version published by the Free Software Foundation. -If the Program does not specify a version number of this License, you may -choose any version ever published by the Free Software Foundation. - -10. If you wish to incorporate parts of the Program into other free programs -whose distribution conditions are different, write to the author to ask for -permission. For software which is copyrighted by the Free Software Foundation, -write to the Free Software Foundation; we sometimes make exceptions for this. -Our decision will be guided by the two goals of preserving the free status of -all derivatives of our free software and of promoting the sharing and reuse of -software generally. - -NO WARRANTY - -11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR -THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE -STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE -PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND -PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, -YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - -12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL -ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE -PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR -INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA -BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A -FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER -OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -END OF TERMS AND CONDITIONS - -How to Apply These Terms to Your New Programs - -If you develop a new program, and you want it to be of the greatest possible -use to the public, the best way to achieve this is to make it free software -which everyone can redistribute and change under these terms. - -To do so, attach the following notices to the program. It is safest to attach -them to the start of each source file to most effectively convey the exclusion -of warranty; and each file should have at least the "copyright" line and a -pointer to where the full notice is found. - - One line to give the program's name and a brief idea of what it does. - - Copyright (C) - - This program is free software; you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by the Free - Software Foundation; either version 2 of the License, or (at your option) - any later version. - - This program is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., 59 - Temple Place, Suite 330, Boston, MA 02111-1307 USA - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this when it -starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author Gnomovision comes - with ABSOLUTELY NO WARRANTY; for details type 'show w'. This is free - software, and you are welcome to redistribute it under certain conditions; - type 'show c' for details. - -The hypothetical commands 'show w' and 'show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may be -called something other than 'show w' and 'show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your school, -if any, to sign a "copyright disclaimer" for the program, if necessary. Here -is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - 'Gnomovision' (which makes passes at compilers) written by James Hacker. - - signature of Ty Coon, 1 April 1989 - - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General Public -License instead of this License. - - -"CLASSPATH" EXCEPTION TO THE GPL - -Certain source files distributed by Oracle America and/or its affiliates are -subject to the following clarification and special exception to the GPL, but -only where Oracle has expressly included in the particular source file's header -the words "Oracle designates this particular file as subject to the "Classpath" -exception as provided by Oracle in the LICENSE file that accompanied this code." - - Linking this library statically or dynamically with other modules is making - a combined work based on this library. Thus, the terms and conditions of - the GNU General Public License cover the whole combination. - - As a special exception, the copyright holders of this library give you - permission to link this library with independent modules to produce an - executable, regardless of the license terms of these independent modules, - and to copy and distribute the resulting executable under terms of your - choice, provided that you also meet, for each linked independent module, - the terms and conditions of the license of that module. An independent - module is a module which is not derived from or based on this library. If - you modify this library, you may extend this exception to your version of - the library, but you are not obligated to do so. If you do not wish to do - so, delete this exception statement from your version. - -=========================================================================== - -MIT License: - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -=========================================================================== diff --git a/cdap-distributions/src/COPYRIGHT/chill_2.10@0.5.0/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/chill_2.10@0.5.0/COPYRIGHT deleted file mode 100644 index 62854a4e7b76..000000000000 --- a/cdap-distributions/src/COPYRIGHT/chill_2.10@0.5.0/COPYRIGHT +++ /dev/null @@ -1,12 +0,0 @@ -Chill is a set of Scala extensions for Kryo. -Copyright 2012 Twitter, Inc. - -Third Party Dependencies: - -Kryo 2.17 -BSD 3-Clause License -http://code.google.com/p/kryo - -Commons-Codec 1.7 -Apache Public License 2.0 -http://hadoop.apache.org diff --git a/cdap-distributions/src/COPYRIGHT/chokidar@2.1.8/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/chokidar@2.1.8/COPYRIGHT new file mode 100644 index 000000000000..ab66c179bb92 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/chokidar@2.1.8/COPYRIGHT @@ -0,0 +1,294 @@ +# Chokidar [![Weekly downloads](https://img.shields.io/npm/dw/chokidar.svg)](https://github.com/paulmillr/chokidar) [![Yearly downloads](https://img.shields.io/npm/dy/chokidar.svg)](https://github.com/paulmillr/chokidar) [![Mac/Linux Build Status](https://img.shields.io/travis/paulmillr/chokidar/master.svg?label=Mac%20OSX%20%26%20Linux)](https://travis-ci.org/paulmillr/chokidar) [![Windows Build status](https://img.shields.io/appveyor/ci/paulmillr/chokidar/master.svg?label=Windows)](https://ci.appveyor.com/project/paulmillr/chokidar/branch/master) [![Coverage Status](https://coveralls.io/repos/paulmillr/chokidar/badge.svg)](https://coveralls.io/r/paulmillr/chokidar) + +> A neat wrapper around node.js fs.watch / fs.watchFile / FSEvents. + +[![NPM](https://nodei.co/npm/chokidar.png)](https://www.npmjs.com/package/chokidar) + +## Why? +Node.js `fs.watch`: + +* Doesn't report filenames on MacOS. +* Doesn't report events at all when using editors like Sublime on MacOS. +* Often reports events twice. +* Emits most changes as `rename`. +* Has [a lot of other issues](https://github.com/nodejs/node/search?q=fs.watch&type=Issues) +* Does not provide an easy way to recursively watch file trees. + +Node.js `fs.watchFile`: + +* Almost as bad at event handling. +* Also does not provide any recursive watching. +* Results in high CPU utilization. + +Chokidar resolves these problems. + +Initially made for **[Brunch](http://brunch.io)** (an ultra-swift web app build tool), it is now used in +[gulp](https://github.com/gulpjs/gulp/), +[karma](http://karma-runner.github.io), +[PM2](https://github.com/Unitech/PM2), +[browserify](http://browserify.org/), +[webpack](http://webpack.github.io/), +[BrowserSync](http://www.browsersync.io/), +[Microsoft's Visual Studio Code](https://github.com/microsoft/vscode), +and [many others](https://www.npmjs.org/browse/depended/chokidar/). +It has proven itself in production environments. + +## How? +Chokidar does still rely on the Node.js core `fs` module, but when using +`fs.watch` and `fs.watchFile` for watching, it normalizes the events it +receives, often checking for truth by getting file stats and/or dir contents. + +On MacOS, chokidar by default uses a native extension exposing the Darwin +`FSEvents` API. This provides very efficient recursive watching compared with +implementations like `kqueue` available on most \*nix platforms. Chokidar still +does have to do some work to normalize the events received that way as well. + +On other platforms, the `fs.watch`-based implementation is the default, which +avoids polling and keeps CPU usage down. Be advised that chokidar will initiate +watchers recursively for everything within scope of the paths that have been +specified, so be judicious about not wasting system resources by watching much +more than needed. + +## Getting started +Install with npm: + +```sh +npm install chokidar +``` + +Then `require` and use it in your code: + +```javascript +var chokidar = require('chokidar'); + +// One-liner for current directory, ignores .dotfiles +chokidar.watch('.', {ignored: /(^|[\/\\])\../}).on('all', (event, path) => { + console.log(event, path); +}); +``` + +```javascript +// Example of a more typical implementation structure: + +// Initialize watcher. +var watcher = chokidar.watch('file, dir, glob, or array', { + ignored: /(^|[\/\\])\../, + persistent: true +}); + +// Something to use when events are received. +var log = console.log.bind(console); +// Add event listeners. +watcher + .on('add', path => log(`File ${path} has been added`)) + .on('change', path => log(`File ${path} has been changed`)) + .on('unlink', path => log(`File ${path} has been removed`)); + +// More possible events. +watcher + .on('addDir', path => log(`Directory ${path} has been added`)) + .on('unlinkDir', path => log(`Directory ${path} has been removed`)) + .on('error', error => log(`Watcher error: ${error}`)) + .on('ready', () => log('Initial scan complete. Ready for changes')) + .on('raw', (event, path, details) => { + log('Raw event info:', event, path, details); + }); + +// 'add', 'addDir' and 'change' events also receive stat() results as second +// argument when available: http://nodejs.org/api/fs.html#fs_class_fs_stats +watcher.on('change', (path, stats) => { + if (stats) console.log(`File ${path} changed size to ${stats.size}`); +}); + +// Watch new files. +watcher.add('new-file'); +watcher.add(['new-file-2', 'new-file-3', '**/other-file*']); + +// Get list of actual paths being watched on the filesystem +var watchedPaths = watcher.getWatched(); + +// Un-watch some files. +watcher.unwatch('new-file*'); + +// Stop watching. +watcher.close(); + +// Full list of options. See below for descriptions. (do not use this example) +chokidar.watch('file', { + persistent: true, + + ignored: '*.txt', + ignoreInitial: false, + followSymlinks: true, + cwd: '.', + disableGlobbing: false, + + usePolling: true, + interval: 100, + binaryInterval: 300, + alwaysStat: false, + depth: 99, + awaitWriteFinish: { + stabilityThreshold: 2000, + pollInterval: 100 + }, + + ignorePermissionErrors: false, + atomic: true // or a custom 'atomicity delay', in milliseconds (default 100) +}); + +``` + +## API + +`chokidar.watch(paths, [options])` + +* `paths` (string or array of strings). Paths to files, dirs to be watched +recursively, or glob patterns. +* `options` (object) Options object as defined below: + +#### Persistence + +* `persistent` (default: `true`). Indicates whether the process +should continue to run as long as files are being watched. If set to +`false` when using `fsevents` to watch, no more events will be emitted +after `ready`, even if the process continues to run. + +#### Path filtering + +* `ignored` ([anymatch](https://github.com/es128/anymatch)-compatible definition) +Defines files/paths to be ignored. The whole relative or absolute path is +tested, not just filename. If a function with two arguments is provided, it +gets called twice per path - once with a single argument (the path), second +time with two arguments (the path and the +[`fs.Stats`](http://nodejs.org/api/fs.html#fs_class_fs_stats) +object of that path). +* `ignoreInitial` (default: `false`). If set to `false` then `add`/`addDir` events are also emitted for matching paths while +instantiating the watching as chokidar discovers these file paths (before the `ready` event). +* `followSymlinks` (default: `true`). When `false`, only the +symlinks themselves will be watched for changes instead of following +the link references and bubbling events through the link's path. +* `cwd` (no default). The base directory from which watch `paths` are to be +derived. Paths emitted with events will be relative to this. +* `disableGlobbing` (default: `false`). If set to `true` then the strings passed to `.watch()` and `.add()` are treated as +literal path names, even if they look like globs. + +#### Performance + +* `usePolling` (default: `false`). +Whether to use fs.watchFile (backed by polling), or fs.watch. If polling +leads to high CPU utilization, consider setting this to `false`. It is +typically necessary to **set this to `true` to successfully watch files over +a network**, and it may be necessary to successfully watch files in other +non-standard situations. Setting to `true` explicitly on MacOS overrides the +`useFsEvents` default. You may also set the CHOKIDAR_USEPOLLING env variable +to true (1) or false (0) in order to override this option. +* _Polling-specific settings_ (effective when `usePolling: true`) + * `interval` (default: `100`). Interval of file system polling. You may also + set the CHOKIDAR_INTERVAL env variable to override this option. + * `binaryInterval` (default: `300`). Interval of file system + polling for binary files. + ([see list of binary extensions](https://github.com/sindresorhus/binary-extensions/blob/master/binary-extensions.json)) +* `useFsEvents` (default: `true` on MacOS). Whether to use the +`fsevents` watching interface if available. When set to `true` explicitly +and `fsevents` is available this supercedes the `usePolling` setting. When +set to `false` on MacOS, `usePolling: true` becomes the default. +* `alwaysStat` (default: `false`). If relying upon the +[`fs.Stats`](http://nodejs.org/api/fs.html#fs_class_fs_stats) +object that may get passed with `add`, `addDir`, and `change` events, set +this to `true` to ensure it is provided even in cases where it wasn't +already available from the underlying watch events. +* `depth` (default: `undefined`). If set, limits how many levels of +subdirectories will be traversed. +* `awaitWriteFinish` (default: `false`). +By default, the `add` event will fire when a file first appears on disk, before +the entire file has been written. Furthermore, in some cases some `change` +events will be emitted while the file is being written. In some cases, +especially when watching for large files there will be a need to wait for the +write operation to finish before responding to a file creation or modification. +Setting `awaitWriteFinish` to `true` (or a truthy value) will poll file size, +holding its `add` and `change` events until the size does not change for a +configurable amount of time. The appropriate duration setting is heavily +dependent on the OS and hardware. For accurate detection this parameter should +be relatively high, making file watching much less responsive. +Use with caution. + * *`options.awaitWriteFinish` can be set to an object in order to adjust + timing params:* + * `awaitWriteFinish.stabilityThreshold` (default: 2000). Amount of time in + milliseconds for a file size to remain constant before emitting its event. + * `awaitWriteFinish.pollInterval` (default: 100). File size polling interval. + +#### Errors +* `ignorePermissionErrors` (default: `false`). Indicates whether to watch files +that don't have read permissions if possible. If watching fails due to `EPERM` +or `EACCES` with this set to `true`, the errors will be suppressed silently. +* `atomic` (default: `true` if `useFsEvents` and `usePolling` are `false`). +Automatically filters out artifacts that occur when using editors that use +"atomic writes" instead of writing directly to the source file. If a file is +re-added within 100 ms of being deleted, Chokidar emits a `change` event +rather than `unlink` then `add`. If the default of 100 ms does not work well +for you, you can override it by setting `atomic` to a custom value, in +milliseconds. + +### Methods & Events + +`chokidar.watch()` produces an instance of `FSWatcher`. Methods of `FSWatcher`: + +* `.add(path / paths)`: Add files, directories, or glob patterns for tracking. +Takes an array of strings or just one string. +* `.on(event, callback)`: Listen for an FS event. +Available events: `add`, `addDir`, `change`, `unlink`, `unlinkDir`, `ready`, +`raw`, `error`. +Additionally `all` is available which gets emitted with the underlying event +name and path for every event other than `ready`, `raw`, and `error`. +* `.unwatch(path / paths)`: Stop watching files, directories, or glob patterns. +Takes an array of strings or just one string. +* `.close()`: Removes all listeners from watched files. +* `.getWatched()`: Returns an object representing all the paths on the file +system being watched by this `FSWatcher` instance. The object's keys are all the +directories (using absolute paths unless the `cwd` option was used), and the +values are arrays of the names of the items contained in each directory. + +## CLI + +If you need a CLI interface for your file watching, check out +[chokidar-cli](https://github.com/kimmobrunfeldt/chokidar-cli), allowing you to +execute a command on each change, or get a stdio stream of change events. + +## Install Troubleshooting + +* `npm WARN optional dep failed, continuing fsevents@n.n.n` + * This message is normal part of how `npm` handles optional dependencies and is + not indicative of a problem. Even if accompanied by other related error messages, + Chokidar should function properly. + +* `ERR! stack Error: Python executable "python" is v3.4.1, which is not supported by gyp.` + * You should be able to resolve this by installing python 2.7 and running: + `npm config set python python2.7` + +* `gyp ERR! stack Error: not found: make` + * On Mac, install the XCode command-line tools + +## License + +The MIT License (MIT) + +Copyright (c) 2012-2019 Paul Miller (https://paulmillr.com) & Elan Shanker + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the “Software”), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/chokidar@3.5.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/chokidar@3.5.1/COPYRIGHT new file mode 100644 index 000000000000..fa9162b51acb --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/chokidar@3.5.1/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2012-2019 Paul Miller (https://paulmillr.com), Elan Shanker + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the “Software”), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/block-stream@0.0.9/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/chownr@1.1.4/COPYRIGHT similarity index 100% rename from cdap-distributions/src/COPYRIGHT/block-stream@0.0.9/COPYRIGHT rename to cdap-distributions/src/COPYRIGHT/chownr@1.1.4/COPYRIGHT diff --git a/cdap-distributions/src/COPYRIGHT/chrome-trace-event@1.0.2/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/chrome-trace-event@1.0.2/COPYRIGHT new file mode 100644 index 000000000000..427a1364ca2b --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/chrome-trace-event@1.0.2/COPYRIGHT @@ -0,0 +1,23 @@ +# This is the MIT license + +Copyright (c) 2015 Joyent Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/cdap-distributions/src/COPYRIGHT/cipher-base@1.0.4/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/cipher-base@1.0.4/COPYRIGHT new file mode 100644 index 000000000000..f06007ae3233 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/cipher-base@1.0.4/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 crypto-browserify contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/class-utils@0.3.6/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/class-utils@0.3.6/COPYRIGHT new file mode 100644 index 000000000000..27c85370f182 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/class-utils@0.3.6/COPYRIGHT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015, 2017-2018, Jon Schlinkert. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/classnames@2.2.5/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/classnames@2.2.5/COPYRIGHT deleted file mode 100644 index 4117bfae0309..000000000000 --- a/cdap-distributions/src/COPYRIGHT/classnames@2.2.5/COPYRIGHT +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 Jed Watson - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/cdap-distributions/src/COPYRIGHT/classworlds:classworlds:1.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/classworlds:classworlds:1.1/COPYRIGHT new file mode 100644 index 000000000000..a8e9a986f620 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/classworlds:classworlds:1.1/COPYRIGHT @@ -0,0 +1,47 @@ + + +/* + $Id$ + + Copyright 2002 (C) The Codehaus. All Rights Reserved. + + Redistribution and use of this software and associated documentation + ("Software"), with or without modification, are permitted provided + that the following conditions are met: + + 1. Redistributions of source code must retain copyright + statements and notices. Redistributions must also contain a + copy of this document. + + 2. Redistributions in binary form must reproduce the + above copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + + 3. The name "classworlds" must not be used to endorse or promote + products derived from this Software without prior written + permission of The Codehaus. For written permission, please + contact bob@codehaus.org. + + 4. Products derived from this Software may not be called "classworlds" + nor may "classworlds" appear in their names without prior written + permission of The Codehaus. "classworlds" is a registered + trademark of The Codehaus. + + 5. Due credit should be given to The Codehaus. + (http://classworlds.codehaus.org/). + + THIS SOFTWARE IS PROVIDED BY THE CODEHAUS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT + NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + THE CODEHAUS OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + OF THE POSSIBILITY OF SUCH DAMAGE. + + */ diff --git a/cdap-distributions/src/COPYRIGHT/clipboard@1.7.1/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/clipboard@1.7.1/COPYRIGHT deleted file mode 100644 index 810f7338da4e..000000000000 --- a/cdap-distributions/src/COPYRIGHT/clipboard@1.7.1/COPYRIGHT +++ /dev/null @@ -1,73 +0,0 @@ - - - -MIT License - - - - - - - - - - - -
- -

The MIT License (MIT)

- -

Copyright © 2019 Zeno Rocha <[email protected]>

- -

Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the “Software”), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions:

- -

The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software.

- -

THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE.

-
- - - - - diff --git a/cdap-distributions/src/COPYRIGHT/clipboard@2.0.4/COPYRIGHT b/cdap-distributions/src/COPYRIGHT/clipboard@2.0.4/COPYRIGHT new file mode 100644 index 000000000000..76692ff9efb4 --- /dev/null +++ b/cdap-distributions/src/COPYRIGHT/clipboard@2.0.4/COPYRIGHT @@ -0,0 +1,189 @@ +# clipboard.js + +[![Build Status](http://img.shields.io/travis/zenorocha/clipboard.js/master.svg?style=flat)](https://travis-ci.org/zenorocha/clipboard.js) +![Killing Flash](https://img.shields.io/badge/killing-flash-brightgreen.svg?style=flat) + +> Modern copy to clipboard. No Flash. Just 3kb gzipped. + +Demo + +## Why + +Copying text to the clipboard shouldn't be hard. It shouldn't require dozens of steps to configure or hundreds of KBs to load. But most of all, it shouldn't depend on Flash or any bloated framework. + +That's why clipboard.js exists. + +## Install + +You can get it on npm. + +``` +npm install clipboard --save +``` + +Or if you're not into package management, just [download a ZIP](https://github.com/zenorocha/clipboard.js/archive/master.zip) file. + +## Setup + +First, include the script located on the `dist` folder or load it from [a third-party CDN provider](https://github.com/zenorocha/clipboard.js/wiki/CDN-Providers). + +```html + +``` + +Now, you need to instantiate it by [passing a DOM selector](https://github.com/zenorocha/clipboard.js/blob/master/demo/constructor-selector.html#L18), [HTML element](https://github.com/zenorocha/clipboard.js/blob/master/demo/constructor-node.html#L16-L17), or [list of HTML elements](https://github.com/zenorocha/clipboard.js/blob/master/demo/constructor-nodelist.html#L18-L19). + +```js +new ClipboardJS('.btn'); +``` + +Internally, we need to fetch all elements that matches with your selector and attach event listeners for each one. But guess what? If you have hundreds of matches, this operation can consume a lot of memory. + +For this reason we use [event delegation](http://stackoverflow.com/questions/1687296/what-is-dom-event-delegation) which replaces multiple event listeners with just a single listener. After all, [#perfmatters](https://twitter.com/hashtag/perfmatters). + +# Usage + +We're living a _declarative renaissance_, that's why we decided to take advantage of [HTML5 data attributes](https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/Using_data_attributes) for better usability. + +### Copy text from another element + +A pretty common use case is to copy content from another element. You can do that by adding a `data-clipboard-target` attribute in your trigger element. + +The value you include on this attribute needs to match another's element selector. + +example-2 + +```html + + + + + +``` + +### Cut text from another element + +Additionally, you can define a `data-clipboard-action` attribute to specify if you want to either `copy` or `cut` content. + +If you omit this attribute, `copy` will be used by default. + +example-3 + +```html + + + + + +``` + +As you may expect, the `cut` action only works on `` or `