fix(registration): apply mitigations to Cryostat registration flooding

README.md

@@ -246,7 +246,13 @@ and how it advertises itself to a Cryostat server instance. Properties that requ
 - [ ] `cryostat.agent.webserver.credentials.pass.length` [`int`]: the length of the generated password used for `Basic` authorization on the embedded webserver. Default `24`.
 - [ ] `cryostat.agent.webserver.credentials.pass.hash-function` [`String`]: the name of the hash function to use when generating passwords. Default `SHA-256`.
 - [ ] `cryostat.agent.app.jmx.port` [`int`]: the JMX RMI port that the application is listening on. The default is to attempt to determine this from the `com.sun.management.jmxremote.port` system property.
-- [ ] `cryostat.agent.registration.retry-ms` [`long`]: the duration in milliseconds between attempts to register with the Cryostat server. Default `5000`.
+- [ ] `cryostat.agent.registration.retry-ms` [`long`]: the duration in milliseconds between attempts to register with the Cryostat server. This is the base value used for exponential backoff calculations. Default `15000`.
+- [ ] `cryostat.agent.registration.max-backoff-ms` [`long`]: the maximum duration in milliseconds for exponential backoff between registration retry attempts. Default `300000` (5 minutes).
+- [ ] `cryostat.agent.registration.backoff-multiplier` [`double`]: the multiplier used for exponential backoff calculations. Each consecutive failure multiplies the retry delay by this factor. Default `2.0`.
+- [ ] `cryostat.agent.registration.circuit-breaker-threshold` [`int`]: the number of consecutive registration failures before the circuit breaker opens. Default `10`.
+- [ ] `cryostat.agent.registration.circuit-breaker-duration` [`Duration`]: the duration the circuit breaker remains open before attempting to close. Default `PT5M` (5 minutes).
+- [ ] `cryostat.agent.registration.min-cooldown-ms` [`long`]: the minimum cooldown period in milliseconds after registration failures. This ensures the cooldown always exceeds Cryostat's plugin ping interval, allowing the server to detect and clean up stale registrations. Default `300000` (5 minutes).
+- [ ] `cryostat.agent.registration.check-ms` [`long`]: the duration in milliseconds between checks to verify the Agent's registration status with the Cryostat server. Default `300000` (5 minutes).
 - [ ] `cryostat.agent.registration.jmx.ignore` [`boolean`]: if the Agent detects that the host JVM has its JMX server enabled, then setting this property to `true` will cause the Agent to ignore the JMX server and not publish a JMX Service URL after registering with the Cryostat server. Default `false`.
 - [ ] `cryostat.agent.registration.jmx.use-callback-host` [`boolean`]: if the Agent should use the host part of the callback URL when constructing the JMX Service URL for registration. If set to `false` then the URL will contain the automatically detected hostname instead. Default `true`.
 - [ ] `cryostat.agent.exit.signals` [`[String]`]: a comma-separated list of signals that the agent should handle. When any of these signals is caught the agent initiates an orderly shutdown, deregistering from the Cryostat server and potentially uploading the latest harvested JFR data. Default `INT,TERM`.

src/main/java/io/cryostat/agent/Agent.java

@@ -23,13 +23,16 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
+import java.util.stream.Collectors;
 
 import javax.inject.Named;
 import javax.inject.Singleton;
@@ -226,6 +229,9 @@ public void accept(AgentArgs args) {
             Registration registration = client.registration();
             Harvester harvester = client.harvester();
             WebServer webServer = client.webServer();
+            CredentialCleanupJob credentialCleanupJob = client.credentialCleanupJob();
+            CredentialTracker credentialTracker = client.credentialTracker();
+            CryostatClient cryostatClient = client.cryostatClient();
             ExecutorService executor = client.executor();
             List<String> exitSignals = client.exitSignals();
             long exitDeregistrationTimeout = client.exitDeregistrationTimeout();
@@ -237,6 +243,9 @@ public void accept(AgentArgs args) {
                             registration,
                             harvester,
                             webServer,
+                            credentialCleanupJob,
+                            credentialTracker,
+                            cryostatClient,
                             executor,
                             exitDeregistrationTimeout);
             final AgentExitHandler fHandler = agentExitHandler;
@@ -272,6 +281,7 @@ public void accept(AgentArgs args) {
                     });
             webServer.start();
             registration.start();
+            credentialCleanupJob.start();
             client.triggerEvaluator().start(args.getSmartTriggers());
             log.trace("Startup complete");
         } catch (Exception e) {
@@ -288,11 +298,21 @@ private static AgentExitHandler installSignalHandlers(
             Registration registration,
             Harvester harvester,
             WebServer webServer,
+            CredentialCleanupJob credentialCleanupJob,
+            CredentialTracker credentialTracker,
+            CryostatClient cryostatClient,
             ExecutorService executor,
             long exitDeregistrationTimeout) {
         AgentExitHandler agentExitHandler =
                 new AgentExitHandler(
-                        registration, harvester, webServer, executor, exitDeregistrationTimeout);
+                        registration,
+                        harvester,
+                        webServer,
+                        credentialCleanupJob,
+                        credentialTracker,
+                        cryostatClient,
+                        executor,
+                        exitDeregistrationTimeout);
         for (String s : exitSignals) {
             Signal signal = new Signal(s);
             try {
@@ -344,6 +364,12 @@ interface Client {
 
         TriggerEvaluator triggerEvaluator();
 
+        CredentialCleanupJob credentialCleanupJob();
+
+        CredentialTracker credentialTracker();
+
+        CryostatClient cryostatClient();
+
         ScheduledExecutorService executor();
 
         @Named(ConfigModule.CRYOSTAT_AGENT_EXIT_SIGNALS)
@@ -366,18 +392,27 @@ private static class AgentExitHandler implements SignalHandler {
         private final Registration registration;
         private final Harvester harvester;
         private final WebServer webServer;
+        private final CredentialCleanupJob credentialCleanupJob;
+        private final CredentialTracker credentialTracker;
+        private final CryostatClient cryostatClient;
         private final ExecutorService executor;
         private final long exitDeregistrationTimeout;
 
         private AgentExitHandler(
                 Registration registration,
                 Harvester harvester,
                 WebServer webServer,
+                CredentialCleanupJob credentialCleanupJob,
+                CredentialTracker credentialTracker,
+                CryostatClient cryostatClient,
                 ExecutorService executor,
                 long exitDeregistrationTimeout) {
             this.registration = Objects.requireNonNull(registration);
             this.harvester = Objects.requireNonNull(harvester);
             this.webServer = Objects.requireNonNull(webServer);
+            this.credentialCleanupJob = Objects.requireNonNull(credentialCleanupJob);
+            this.credentialTracker = Objects.requireNonNull(credentialTracker);
+            this.cryostatClient = Objects.requireNonNull(cryostatClient);
             this.executor = Objects.requireNonNull(executor);
             this.exitDeregistrationTimeout = exitDeregistrationTimeout;
         }
@@ -413,10 +448,33 @@ void performCleanup(Signal sig) {
                                     if (t != null) {
                                         log.warn("Exception during deregistration", t);
                                     }
+
+                                    try {
+                                        Set<Integer> orphaned =
+                                                credentialTracker.getOrphanedCredentials();
+                                        if (!orphaned.isEmpty()) {
+                                            log.debug(
+                                                    "Cleaning up {} credentials on shutdown",
+                                                    orphaned.size());
+                                            List<CompletableFuture<Void>> deletions =
+                                                    orphaned.stream()
+                                                            .map(cryostatClient::deleteCredentials)
+                                                            .collect(Collectors.toList());
+                                            CompletableFuture.allOf(
+                                                            deletions.toArray(
+                                                                    new CompletableFuture[0]))
+                                                    .get(30, TimeUnit.SECONDS);
+                                            log.debug("Cleaned up all credentials");
+                                        }
+                                    } catch (Exception e) {
+                                        log.error("Error during credential cleanup on shutdown", e);
+                                    }
+
                                     try {
                                         log.debug("Shutting down...");
                                         safeCall(webServer::stop);
                                         safeCall(registration::stop);
+                                        safeCall(credentialCleanupJob::stop);
                                         safeCall(executor::shutdown);
                                     } finally {
                                         log.debug("Shutdown complete");

src/main/java/io/cryostat/agent/ConfigModule.java

@@ -31,6 +31,7 @@
 import java.security.NoSuchAlgorithmException;
 import java.security.PrivilegedExceptionAction;
 import java.security.SecureRandom;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -205,6 +206,16 @@ public abstract class ConfigModule {
             "cryostat.agent.registration.jmx.ignore";
     public static final String CRYOSTAT_AGENT_REGISTRATION_JMX_USE_CALLBACK_HOST =
             "cryostat.agent.registration.jmx.use-callback-host";
+    public static final String CRYOSTAT_AGENT_REGISTRATION_MAX_BACKOFF_MS =
+            "cryostat.agent.registration.max-backoff-ms";
+    public static final String CRYOSTAT_AGENT_REGISTRATION_BACKOFF_MULTIPLIER =
+            "cryostat.agent.registration.backoff-multiplier";
+    public static final String CRYOSTAT_AGENT_REGISTRATION_CIRCUIT_BREAKER_THRESHOLD =
+            "cryostat.agent.registration.circuit-breaker-threshold";
+    public static final String CRYOSTAT_AGENT_REGISTRATION_CIRCUIT_BREAKER_DURATION =
+            "cryostat.agent.registration.circuit-breaker-duration";
+    public static final String CRYOSTAT_AGENT_REGISTRATION_MIN_COOLDOWN_MS =
+            "cryostat.agent.registration.min-cooldown-ms";
     public static final String CRYOSTAT_AGENT_EXIT_SIGNALS = "cryostat.agent.exit.signals";
     public static final String CRYOSTAT_AGENT_EXIT_DEREGISTRATION_TIMEOUT_MS =
             "cryostat.agent.exit.deregistration.timeout-ms";
@@ -255,6 +266,11 @@ public abstract class ConfigModule {
     public static final String CRYOSTAT_AGENT_FLEET_SAMPLING_RATIO =
             "cryostat.agent.fleet-sampling-ratio";
 
+    public static final String CRYOSTAT_AGENT_CREDENTIAL_CLEANUP_INTERVAL =
+            "cryostat.agent.credential.cleanup.interval";
+    public static final String CRYOSTAT_AGENT_CREDENTIAL_CLEANUP_MAX_RETRIES =
+            "cryostat.agent.credential.cleanup.max-retries";
+
     @Provides
     @Singleton
     public static SmallRyeConfig provideConfig() {
@@ -951,6 +967,46 @@ public static boolean provideCryostatAgentRegistrationJmxUseCallbackHost(
         return config.getValue(CRYOSTAT_AGENT_REGISTRATION_JMX_USE_CALLBACK_HOST, boolean.class);
     }
 
+    @Provides
+    @Singleton
+    @Named(CRYOSTAT_AGENT_REGISTRATION_MAX_BACKOFF_MS)
+    public static int provideCryostatAgentRegistrationMaxBackoffMs(SmallRyeConfig config) {
+        return config.getValue(CRYOSTAT_AGENT_REGISTRATION_MAX_BACKOFF_MS, int.class);
+    }
+
+    @Provides
+    @Singleton
+    @Named(CRYOSTAT_AGENT_REGISTRATION_BACKOFF_MULTIPLIER)
+    public static double provideCryostatAgentRegistrationBackoffMultiplier(SmallRyeConfig config) {
+        return config.getValue(CRYOSTAT_AGENT_REGISTRATION_BACKOFF_MULTIPLIER, double.class);
+    }
+
+    @Provides
+    @Singleton
+    @Named(CRYOSTAT_AGENT_REGISTRATION_CIRCUIT_BREAKER_THRESHOLD)
+    public static int provideCryostatAgentRegistrationCircuitBreakerThreshold(
+            SmallRyeConfig config) {
+        return config.getValue(CRYOSTAT_AGENT_REGISTRATION_CIRCUIT_BREAKER_THRESHOLD, int.class);
+    }
+
+    @Provides
+    @Singleton
+    @Named(CRYOSTAT_AGENT_REGISTRATION_CIRCUIT_BREAKER_DURATION)
+    public static Duration provideCryostatAgentRegistrationCircuitBreakerDuration(
+            SmallRyeConfig config) {
+        String durationStr =
+                config.getValue(CRYOSTAT_AGENT_REGISTRATION_CIRCUIT_BREAKER_DURATION, String.class);
+        return Duration.parse(durationStr);
+    }
+
+    @Provides
+    @Singleton
+    @Named(CRYOSTAT_AGENT_REGISTRATION_MIN_COOLDOWN_MS)
+    public static Duration provideCryostatAgentRegistrationMinCooldownMs(SmallRyeConfig config) {
+        int cooldownMs = config.getValue(CRYOSTAT_AGENT_REGISTRATION_MIN_COOLDOWN_MS, int.class);
+        return Duration.ofMillis(cooldownMs);
+    }
+
     @Provides
     @Singleton
     @Named(CRYOSTAT_AGENT_HARVESTER_PERIOD_MS)
@@ -1097,6 +1153,20 @@ public static double provideCryostatAgentFleetSamplingRatio(SmallRyeConfig confi
         return config.getValue(CRYOSTAT_AGENT_FLEET_SAMPLING_RATIO, double.class);
     }
 
+    @Provides
+    @Singleton
+    @Named(CRYOSTAT_AGENT_CREDENTIAL_CLEANUP_INTERVAL)
+    public static Duration provideCryostatAgentCredentialCleanupInterval(SmallRyeConfig config) {
+        return config.getValue(CRYOSTAT_AGENT_CREDENTIAL_CLEANUP_INTERVAL, Duration.class);
+    }
+
+    @Provides
+    @Singleton
+    @Named(CRYOSTAT_AGENT_CREDENTIAL_CLEANUP_MAX_RETRIES)
+    public static int provideCryostatAgentCredentialCleanupMaxRetries(SmallRyeConfig config) {
+        return config.getValue(CRYOSTAT_AGENT_CREDENTIAL_CLEANUP_MAX_RETRIES, int.class);
+    }
+
     public enum URIRange {
         LOOPBACK(u -> check(u, u2 -> true, InetAddress::isLoopbackAddress)),
         LINK_LOCAL(