diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java new file mode 100644 index 000000000000..ba08dcf3aaed --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.local; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.Locale; +import java.util.Objects; + +/** + * Configuration for a local single-node Ozone runtime. + */ +public final class LocalOzoneClusterConfig { + + static final Path DEFAULT_DATA_DIR = + Paths.get(System.getProperty("user.home"), ".ozone", "local") + .toAbsolutePath() + .normalize(); + static final FormatMode DEFAULT_FORMAT_MODE = FormatMode.IF_NEEDED; + static final int DEFAULT_DATANODES = 1; + static final String DEFAULT_HOST = "127.0.0.1"; + static final String DEFAULT_BIND_HOST = "0.0.0.0"; + static final int DEFAULT_PORT = 0; + static final boolean DEFAULT_S3G_ENABLED = true; + static final boolean DEFAULT_EPHEMERAL = false; + static final Duration DEFAULT_STARTUP_TIMEOUT = Duration.ofMinutes(2); + static final String DEFAULT_S3_ACCESS_KEY = "admin"; + static final String DEFAULT_S3_SECRET_KEY = "admin123"; + static final String DEFAULT_S3_REGION = "us-east-1"; + + private final Path dataDir; + private final FormatMode formatMode; + private final int datanodes; + private final String host; + private final String bindHost; + private final int scmPort; + private final int omPort; + private final int s3gPort; + private final boolean s3gEnabled; + private final boolean ephemeral; + private final Duration startupTimeout; + private final String s3AccessKey; + private final String s3SecretKey; + private final String s3Region; + + private LocalOzoneClusterConfig(Builder builder) { + dataDir = Objects.requireNonNull(builder.dataDir, "dataDir") + .toAbsolutePath() + .normalize(); + formatMode = Objects.requireNonNull(builder.formatMode, "formatMode"); + datanodes = builder.datanodes; + host = Objects.requireNonNull(builder.host, "host"); + bindHost = Objects.requireNonNull(builder.bindHost, "bindHost"); + scmPort = builder.scmPort; + omPort = builder.omPort; + s3gPort = builder.s3gPort; + s3gEnabled = builder.s3gEnabled; + ephemeral = builder.ephemeral; + startupTimeout = Objects.requireNonNull(builder.startupTimeout, + "startupTimeout"); + s3AccessKey = Objects.requireNonNull(builder.s3AccessKey, "s3AccessKey"); + s3SecretKey = Objects.requireNonNull(builder.s3SecretKey, "s3SecretKey"); + s3Region = Objects.requireNonNull(builder.s3Region, "s3Region"); + } + + public Path getDataDir() { + return dataDir; + } + + public FormatMode getFormatMode() { + return formatMode; + } + + public int getDatanodes() { + return datanodes; + } + + public String getHost() { + return host; + } + + public String getBindHost() { + return bindHost; + } + + public int getScmPort() { + return scmPort; + } + + public int getOmPort() { + return omPort; + } + + public int getS3gPort() { + return s3gPort; + } + + public boolean isS3gEnabled() { + return s3gEnabled; + } + + public boolean isEphemeral() { + return ephemeral; + } + + public Duration getStartupTimeout() { + return startupTimeout; + } + + public String getS3AccessKey() { + return s3AccessKey; + } + + public String getS3SecretKey() { + return s3SecretKey; + } + + public String getS3Region() { + return s3Region; + } + + public static Builder builder() { + return new Builder(DEFAULT_DATA_DIR); + } + + public static Builder builder(Path dataDir) { + return new Builder(dataDir); + } + + /** + * Storage initialization mode for the local runtime. + */ + public enum FormatMode { + IF_NEEDED, + ALWAYS, + NEVER; + + public static FormatMode fromString(String value) { + String normalized = value.trim().toUpperCase(Locale.ROOT) + .replace('-', '_'); + return valueOf(normalized); + } + } + + /** + * Builder for {@link LocalOzoneClusterConfig}. + */ + public static final class Builder { + + private final Path dataDir; + private FormatMode formatMode = DEFAULT_FORMAT_MODE; + private int datanodes = DEFAULT_DATANODES; + private String host = DEFAULT_HOST; + private String bindHost = DEFAULT_BIND_HOST; + private int scmPort = DEFAULT_PORT; + private int omPort = DEFAULT_PORT; + private int s3gPort = DEFAULT_PORT; + private boolean s3gEnabled = DEFAULT_S3G_ENABLED; + private boolean ephemeral = DEFAULT_EPHEMERAL; + private Duration startupTimeout = DEFAULT_STARTUP_TIMEOUT; + private String s3AccessKey = DEFAULT_S3_ACCESS_KEY; + private String s3SecretKey = DEFAULT_S3_SECRET_KEY; + private String s3Region = DEFAULT_S3_REGION; + + private Builder(Path dataDir) { + this.dataDir = dataDir; + } + + public Builder setFormatMode(FormatMode value) { + formatMode = value; + return this; + } + + public Builder setDatanodes(int value) { + datanodes = value; + return this; + } + + public Builder setHost(String value) { + host = value; + return this; + } + + public Builder setBindHost(String value) { + bindHost = value; + return this; + } + + public Builder setScmPort(int value) { + scmPort = value; + return this; + } + + public Builder setOmPort(int value) { + omPort = value; + return this; + } + + public Builder setS3gPort(int value) { + s3gPort = value; + return this; + } + + public Builder setS3gEnabled(boolean value) { + s3gEnabled = value; + return this; + } + + public Builder setEphemeral(boolean value) { + ephemeral = value; + return this; + } + + public Builder setStartupTimeout(Duration value) { + startupTimeout = value; + return this; + } + + public Builder setS3AccessKey(String value) { + s3AccessKey = value; + return this; + } + + public Builder setS3SecretKey(String value) { + s3SecretKey = value; + return this; + } + + public Builder setS3Region(String value) { + s3Region = value; + return this; + } + + public LocalOzoneClusterConfig build() { + return new LocalOzoneClusterConfig(this); + } + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java new file mode 100644 index 000000000000..54a08e476ff6 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.local; + +/** + * Runtime contract for local single-node Ozone commands. + */ +public interface LocalOzoneRuntime extends AutoCloseable { + + void start() throws Exception; + + String getDisplayHost(); + + int getScmPort(); + + int getOmPort(); + + int getS3gPort(); + + String getS3Endpoint(); + + @Override + void close() throws Exception; +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java new file mode 100644 index 000000000000..6d75331c8bfb --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java @@ -0,0 +1,385 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.local; + +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.time.format.DateTimeParseException; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hdds.cli.GenericCli; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +/** + * Internal CLI entry point for local single-node Ozone commands. + */ +@Command(name = "ozone local", + hidden = true, + description = "Internal commands for local single-node Ozone", + versionProvider = HddsVersionProvider.class, + mixinStandardHelpOptions = true, + subcommands = { + OzoneLocal.RunCommand.class + }) +public class OzoneLocal extends GenericCli { + + static final String ENV_DATA_DIR = "OZONE_LOCAL_DATA_DIR"; + static final String ENV_FORMAT = "OZONE_LOCAL_FORMAT"; + static final String ENV_DATANODES = "OZONE_LOCAL_DATANODES"; + static final String ENV_HOST = "OZONE_LOCAL_HOST"; + static final String ENV_BIND_HOST = "OZONE_LOCAL_BIND_HOST"; + static final String ENV_SCM_PORT = "OZONE_LOCAL_SCM_PORT"; + static final String ENV_OM_PORT = "OZONE_LOCAL_OM_PORT"; + static final String ENV_S3G_ENABLED = "OZONE_LOCAL_S3G_ENABLED"; + static final String ENV_S3G_PORT = "OZONE_LOCAL_S3G_PORT"; + static final String ENV_EPHEMERAL = "OZONE_LOCAL_EPHEMERAL"; + static final String ENV_STARTUP_TIMEOUT = "OZONE_LOCAL_STARTUP_TIMEOUT"; + static final String ENV_S3_ACCESS_KEY = "OZONE_LOCAL_S3_ACCESS_KEY"; + static final String ENV_S3_SECRET_KEY = "OZONE_LOCAL_S3_SECRET_KEY"; + static final String ENV_S3_REGION = "OZONE_LOCAL_S3_REGION"; + + public OzoneLocal() { + super(); + } + + OzoneLocal(CommandLine.IFactory factory) { + super(factory); + } + + public static void main(String[] args) { + new OzoneLocal().run(args); + } + + @Command(name = "run", + hidden = true, + description = "Internal placeholder for a local Ozone runtime") + static class RunCommand implements Callable { + + private final Map environment; + + @Option(names = "--data-dir", + description = "Persistent data directory for the local cluster") + private String dataDir; + + @Option(names = "--format", + description = "Storage init mode: if-needed, always, never") + private String format; + + @Option(names = "--datanodes", + description = "Number of datanodes to start") + private Integer datanodes; + + @Option(names = "--host", + description = "Advertised host to write into local service addresses") + private String host; + + @Option(names = "--bind-host", + description = "Bind host for HTTP and RPC listeners") + private String bindHost; + + @Option(names = "--scm-port", + description = "SCM client RPC port (0 means auto-allocate)") + private Integer scmPort; + + @Option(names = "--om-port", + description = "OM RPC port (0 means auto-allocate)") + private Integer omPort; + + @Option(names = "--s3g-port", + description = "S3 Gateway HTTP port (0 means auto-allocate)") + private Integer s3gPort; + + @Option(names = "--without-s3g", + description = "Disable S3 Gateway") + private boolean withoutS3g; + + @Option(names = "--ephemeral", + description = "Delete the data directory on shutdown") + private boolean ephemeral; + + @Option(names = "--startup-timeout", + description = "How long to wait for the local cluster to become ready") + private String startupTimeout; + + @Option(names = "--s3-access-key", + description = "Suggested local AWS access key to print on startup") + private String s3AccessKey; + + @Option(names = "--s3-secret-key", + description = "Suggested local AWS secret key to print on startup") + private String s3SecretKey; + + @Option(names = "--s3-region", + description = "Suggested local AWS region to print on startup") + private String s3Region; + + RunCommand() { + this(System.getenv()); + } + + RunCommand(Map environment) { + this.environment = Objects.requireNonNull(environment, "environment"); + } + + @Override + public Void call() { + resolveConfig(new OzoneConfiguration()); + return null; + } + + LocalOzoneClusterConfig resolveConfig( + OzoneConfiguration baseConfiguration) { + Path resolvedDataDir = resolvePath(dataDir, environment.get(ENV_DATA_DIR), + "--data-dir", ENV_DATA_DIR, LocalOzoneClusterConfig.DEFAULT_DATA_DIR); + LocalOzoneClusterConfig.FormatMode resolvedFormat = format != null + ? parseFormat(format, "--format") + : parseFormat(environment.get(ENV_FORMAT), ENV_FORMAT, + LocalOzoneClusterConfig.DEFAULT_FORMAT_MODE); + int resolvedDatanodes = datanodes != null + ? datanodes + : parseInt(environment.get(ENV_DATANODES), ENV_DATANODES, + LocalOzoneClusterConfig.DEFAULT_DATANODES); + if (resolvedDatanodes < 1) { + throw new IllegalArgumentException( + "Datanode count for " + + (datanodes != null ? "--datanodes" : ENV_DATANODES) + + " must be at least 1."); + } + + String resolvedHost = firstNonBlank(host, environment.get(ENV_HOST), + LocalOzoneClusterConfig.DEFAULT_HOST); + String resolvedBindHost = firstNonBlank(bindHost, + environment.get(ENV_BIND_HOST), + LocalOzoneClusterConfig.DEFAULT_BIND_HOST); + + int resolvedScmPort = scmPort != null + ? validatePort(scmPort, "--scm-port") + : parsePort(environment.get(ENV_SCM_PORT), ENV_SCM_PORT, + LocalOzoneClusterConfig.DEFAULT_PORT); + int resolvedOmPort = omPort != null + ? validatePort(omPort, "--om-port") + : parsePort(environment.get(ENV_OM_PORT), ENV_OM_PORT, + LocalOzoneClusterConfig.DEFAULT_PORT); + int resolvedS3gPort = s3gPort != null + ? validatePort(s3gPort, "--s3g-port") + : parsePort(environment.get(ENV_S3G_PORT), ENV_S3G_PORT, + LocalOzoneClusterConfig.DEFAULT_PORT); + + boolean resolvedS3gEnabled = withoutS3g ? false + : parseBoolean(environment.get(ENV_S3G_ENABLED), ENV_S3G_ENABLED, + LocalOzoneClusterConfig.DEFAULT_S3G_ENABLED); + boolean resolvedEphemeral = ephemeral || parseBoolean( + environment.get(ENV_EPHEMERAL), ENV_EPHEMERAL, + LocalOzoneClusterConfig.DEFAULT_EPHEMERAL); + + Duration resolvedStartupTimeout = startupTimeout != null + ? parseDuration(startupTimeout, "--startup-timeout", + baseConfiguration) + : parseDuration(environment.get(ENV_STARTUP_TIMEOUT), + ENV_STARTUP_TIMEOUT, + LocalOzoneClusterConfig.DEFAULT_STARTUP_TIMEOUT, + baseConfiguration); + if (resolvedStartupTimeout.isZero() + || resolvedStartupTimeout.isNegative()) { + throw new IllegalArgumentException("Startup timeout for " + + (startupTimeout != null ? "--startup-timeout" + : ENV_STARTUP_TIMEOUT) + + " must be greater than zero."); + } + + String resolvedAccessKey = firstNonBlank(s3AccessKey, + environment.get(ENV_S3_ACCESS_KEY), + LocalOzoneClusterConfig.DEFAULT_S3_ACCESS_KEY); + String resolvedSecretKey = firstNonBlank(s3SecretKey, + environment.get(ENV_S3_SECRET_KEY), + LocalOzoneClusterConfig.DEFAULT_S3_SECRET_KEY); + String resolvedRegion = firstNonBlank(s3Region, + environment.get(ENV_S3_REGION), + LocalOzoneClusterConfig.DEFAULT_S3_REGION); + + return LocalOzoneClusterConfig.builder(resolvedDataDir) + .setFormatMode(resolvedFormat) + .setDatanodes(resolvedDatanodes) + .setHost(resolvedHost) + .setBindHost(resolvedBindHost) + .setScmPort(resolvedScmPort) + .setOmPort(resolvedOmPort) + .setS3gPort(resolvedS3gPort) + .setS3gEnabled(resolvedS3gEnabled) + .setEphemeral(resolvedEphemeral) + .setStartupTimeout(resolvedStartupTimeout) + .setS3AccessKey(resolvedAccessKey) + .setS3SecretKey(resolvedSecretKey) + .setS3Region(resolvedRegion) + .build(); + } + + private static String firstNonBlank(String... values) { + for (String value : values) { + if (!isBlank(value)) { + return trimWhitespace(value); + } + } + return null; + } + + private static Path resolvePath(String cliValue, String envValue, + String cliSource, String envSource, Path fallback) { + if (!isBlank(cliValue)) { + return parsePath(trimWhitespace(cliValue), cliSource); + } + if (!isBlank(envValue)) { + return parsePath(trimWhitespace(envValue), envSource); + } + return fallback.toAbsolutePath().normalize(); + } + + private static Path parsePath(String rawValue, String source) { + try { + return Paths.get(rawValue).toAbsolutePath().normalize(); + } catch (InvalidPathException ex) { + throw new IllegalArgumentException("Unable to parse " + source + + " as a filesystem path: " + rawValue, ex); + } + } + + private static LocalOzoneClusterConfig.FormatMode parseFormat( + String rawValue, String source) { + try { + return LocalOzoneClusterConfig.FormatMode.fromString(rawValue); + } catch (IllegalArgumentException ex) { + throw new IllegalArgumentException("Unable to parse " + source + + ". Expected one of: if-needed, always, never.", ex); + } + } + + private static LocalOzoneClusterConfig.FormatMode parseFormat( + String rawValue, String source, + LocalOzoneClusterConfig.FormatMode fallback) { + if (isBlank(rawValue)) { + return fallback; + } + return parseFormat(rawValue, source); + } + + private static int parseInt(String rawValue, String source, int fallback) { + if (isBlank(rawValue)) { + return fallback; + } + try { + return Integer.parseInt(rawValue.trim()); + } catch (NumberFormatException ex) { + throw new IllegalArgumentException("Unable to parse " + source + + " as an integer: " + rawValue, ex); + } + } + + private static boolean parseBoolean(String rawValue, String source, + boolean fallback) { + if (isBlank(rawValue)) { + return fallback; + } + String value = rawValue.trim(); + if ("true".equalsIgnoreCase(value) + || "yes".equalsIgnoreCase(value) + || "on".equalsIgnoreCase(value)) { + return true; + } + if ("false".equalsIgnoreCase(value) + || "no".equalsIgnoreCase(value) + || "off".equalsIgnoreCase(value)) { + return false; + } + throw new IllegalArgumentException("Unable to parse " + source + + " as a boolean. Use true/false, yes/no, or on/off."); + } + + private static int parsePort(String rawValue, String source, int fallback) { + return isBlank(rawValue) ? fallback + : validatePort(parseInt(rawValue, source, fallback), source); + } + + private static int validatePort(int value, String source) { + if (value < 0 || value > 65_535) { + throw new IllegalArgumentException("Port value for " + source + + " must be between 0 and 65535."); + } + return value; + } + + private static Duration parseDuration(String rawValue, String source, + Duration fallback, OzoneConfiguration baseConfiguration) { + if (isBlank(rawValue)) { + return fallback; + } + return parseDuration(rawValue, source, baseConfiguration); + } + + private static Duration parseDuration(String rawValue, String source, + OzoneConfiguration baseConfiguration) { + try { + return Duration.parse(rawValue.trim()); + } catch (DateTimeParseException ignored) { + OzoneConfiguration conf = new OzoneConfiguration(baseConfiguration); + String configKey = "ozone.local.duration.parse"; + conf.set(configKey, rawValue.trim()); + long millis; + try { + millis = conf.getTimeDuration(configKey, -1L, + TimeUnit.MILLISECONDS); + } catch (RuntimeException ex) { + throw new IllegalArgumentException(durationMessage(source), ex); + } + if (millis < 0) { + throw new IllegalArgumentException(durationMessage(source)); + } + return Duration.ofMillis(millis); + } + } + + private static String durationMessage(String source) { + return "Unable to parse " + source + + " as a duration. Use ISO-8601 like PT2M or " + + "Hadoop-style values like 120s."; + } + + private static boolean isBlank(String value) { + return value == null || trimWhitespace(value).isEmpty(); + } + + private static String trimWhitespace(String value) { + int start = 0; + int end = value.length(); + while (start < end && Character.isWhitespace(value.charAt(start))) { + start++; + } + while (end > start && Character.isWhitespace(value.charAt(end - 1))) { + end--; + } + return value.substring(start, end); + } + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/package-info.java new file mode 100644 index 000000000000..ca9e55fd6cee --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Internal local single-node Ozone runtime support. + */ +package org.apache.hadoop.ozone.local; diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java new file mode 100644 index 000000000000..6407db14973c --- /dev/null +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.local; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Paths; +import java.time.Duration; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link LocalOzoneClusterConfig}. + */ +class TestLocalOzoneClusterConfig { + + @Test + void builderProvidesLocalSingleNodeDefaults() { + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder().build(); + + assertEquals(LocalOzoneClusterConfig.DEFAULT_DATA_DIR, + config.getDataDir()); + assertEquals(LocalOzoneClusterConfig.FormatMode.IF_NEEDED, + config.getFormatMode()); + assertEquals(1, config.getDatanodes()); + assertEquals("127.0.0.1", config.getHost()); + assertEquals("0.0.0.0", config.getBindHost()); + assertEquals(0, config.getScmPort()); + assertEquals(0, config.getOmPort()); + assertEquals(0, config.getS3gPort()); + assertTrue(config.isS3gEnabled()); + assertFalse(config.isEphemeral()); + assertEquals(Duration.ofMinutes(2), config.getStartupTimeout()); + assertEquals("admin", config.getS3AccessKey()); + assertEquals("admin123", config.getS3SecretKey()); + assertEquals("us-east-1", config.getS3Region()); + } + + @Test + void builderAcceptsExplicitOverrides() { + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder( + Paths.get("target", "custom-local-ozone")) + .setFormatMode(LocalOzoneClusterConfig.FormatMode.ALWAYS) + .setDatanodes(3) + .setHost("localhost") + .setBindHost("127.0.0.1") + .setScmPort(9860) + .setOmPort(9862) + .setS3gPort(9878) + .setS3gEnabled(false) + .setEphemeral(true) + .setStartupTimeout(Duration.ofSeconds(45)) + .setS3AccessKey("dev") + .setS3SecretKey("secret") + .setS3Region("ap-south-1") + .build(); + + assertEquals(Paths.get("target", "custom-local-ozone") + .toAbsolutePath().normalize(), config.getDataDir()); + assertEquals(LocalOzoneClusterConfig.FormatMode.ALWAYS, + config.getFormatMode()); + assertEquals(3, config.getDatanodes()); + assertEquals("localhost", config.getHost()); + assertEquals("127.0.0.1", config.getBindHost()); + assertEquals(9860, config.getScmPort()); + assertEquals(9862, config.getOmPort()); + assertEquals(9878, config.getS3gPort()); + assertFalse(config.isS3gEnabled()); + assertTrue(config.isEphemeral()); + assertEquals(Duration.ofSeconds(45), config.getStartupTimeout()); + assertEquals("dev", config.getS3AccessKey()); + assertEquals("secret", config.getS3SecretKey()); + assertEquals("ap-south-1", config.getS3Region()); + } + + @Test + void formatModeParsesUserFacingValues() { + assertEquals(LocalOzoneClusterConfig.FormatMode.IF_NEEDED, + LocalOzoneClusterConfig.FormatMode.fromString("if-needed")); + assertEquals(LocalOzoneClusterConfig.FormatMode.ALWAYS, + LocalOzoneClusterConfig.FormatMode.fromString(" always ")); + assertEquals(LocalOzoneClusterConfig.FormatMode.NEVER, + LocalOzoneClusterConfig.FormatMode.fromString("NEVER")); + } + + @Test + void formatModeRejectsUnknownValues() { + assertThrows(IllegalArgumentException.class, + () -> LocalOzoneClusterConfig.FormatMode.fromString("sometimes")); + } +} diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java new file mode 100644 index 000000000000..1ec310dbb816 --- /dev/null +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java @@ -0,0 +1,352 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.local; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.junit.jupiter.api.Test; +import picocli.CommandLine; +import picocli.CommandLine.Command; + +/** + * Tests for {@link OzoneLocal}. + */ +class TestOzoneLocal { + + @Test + void localCommandMetadataIsPresentAndHidden() { + Command command = OzoneLocal.class.getAnnotation(Command.class); + + assertNotNull(command); + assertEquals("ozone local", command.name()); + assertTrue(command.hidden()); + } + + @Test + void runCommandMetadataIsPresentAndHidden() { + Command command = OzoneLocal.RunCommand.class.getAnnotation(Command.class); + + assertNotNull(command); + assertEquals("run", command.name()); + assertTrue(command.hidden()); + } + + @Test + void genericCliRegistersRunPlaceholder() { + OzoneLocal local = new OzoneLocal(); + + assertTrue(local.getCmd().getSubcommands().containsKey("run")); + } + + @Test + void rootHelpHidesRunPlaceholder() throws Exception { + OzoneLocal local = new OzoneLocal(); + CommandLine commandLine = local.getCmd(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayOutputStream err = new ByteArrayOutputStream(); + commandLine.setOut(new PrintWriter(new OutputStreamWriter(out, UTF_8), + true)); + commandLine.setErr(new PrintWriter(new OutputStreamWriter(err, UTF_8), + true)); + + int exitCode = local.execute(new String[] {"--help"}); + + String help = out.toString(UTF_8.name()); + assertEquals(0, exitCode); + assertTrue(help.contains("Usage: ozone local")); + assertFalse(help.contains("run")); + assertEquals("", err.toString(UTF_8.name())); + } + + @Test + void runCommandIsQuietNoOpPlaceholder() throws Exception { + OzoneLocal local = new OzoneLocal(); + CommandLine commandLine = local.getCmd(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayOutputStream err = new ByteArrayOutputStream(); + commandLine.setOut(new PrintWriter(new OutputStreamWriter(out, UTF_8), + true)); + commandLine.setErr(new PrintWriter(new OutputStreamWriter(err, UTF_8), + true)); + + int exitCode = local.execute(new String[] {"run"}); + + assertEquals(0, exitCode); + assertEquals("", out.toString(UTF_8.name())); + assertEquals("", err.toString(UTF_8.name())); + } + + @Test + void resolveConfigUsesDefaults() { + LocalOzoneClusterConfig config = resolve(Collections.emptyMap()); + + assertEquals(LocalOzoneClusterConfig.DEFAULT_DATA_DIR, + config.getDataDir()); + assertEquals(LocalOzoneClusterConfig.FormatMode.IF_NEEDED, + config.getFormatMode()); + assertEquals(1, config.getDatanodes()); + assertEquals("127.0.0.1", config.getHost()); + assertEquals("0.0.0.0", config.getBindHost()); + assertEquals(0, config.getScmPort()); + assertEquals(0, config.getOmPort()); + assertEquals(0, config.getS3gPort()); + assertTrue(config.isS3gEnabled()); + assertFalse(config.isEphemeral()); + assertEquals(Duration.ofMinutes(2), config.getStartupTimeout()); + assertEquals("admin", config.getS3AccessKey()); + assertEquals("admin123", config.getS3SecretKey()); + assertEquals("us-east-1", config.getS3Region()); + } + + @Test + void resolveConfigUsesEnvironmentOverrides() { + Map env = new HashMap<>(); + env.put(OzoneLocal.ENV_DATA_DIR, "target/ozone-local-test"); + env.put(OzoneLocal.ENV_FORMAT, "always"); + env.put(OzoneLocal.ENV_DATANODES, "2"); + env.put(OzoneLocal.ENV_HOST, "localhost"); + env.put(OzoneLocal.ENV_BIND_HOST, "127.0.0.1"); + env.put(OzoneLocal.ENV_SCM_PORT, "9860"); + env.put(OzoneLocal.ENV_OM_PORT, "9862"); + env.put(OzoneLocal.ENV_S3G_ENABLED, "false"); + env.put(OzoneLocal.ENV_S3G_PORT, "9878"); + env.put(OzoneLocal.ENV_EPHEMERAL, "true"); + env.put(OzoneLocal.ENV_STARTUP_TIMEOUT, "120s"); + env.put(OzoneLocal.ENV_S3_ACCESS_KEY, "dev"); + env.put(OzoneLocal.ENV_S3_SECRET_KEY, "devsecret"); + env.put(OzoneLocal.ENV_S3_REGION, "eu-central-1"); + + LocalOzoneClusterConfig config = resolve(env); + + assertEquals(Paths.get("target/ozone-local-test") + .toAbsolutePath().normalize(), config.getDataDir()); + assertEquals(LocalOzoneClusterConfig.FormatMode.ALWAYS, + config.getFormatMode()); + assertEquals(2, config.getDatanodes()); + assertEquals("localhost", config.getHost()); + assertEquals("127.0.0.1", config.getBindHost()); + assertEquals(9860, config.getScmPort()); + assertEquals(9862, config.getOmPort()); + assertEquals(9878, config.getS3gPort()); + assertFalse(config.isS3gEnabled()); + assertTrue(config.isEphemeral()); + assertEquals(Duration.ofSeconds(120), config.getStartupTimeout()); + assertEquals("dev", config.getS3AccessKey()); + assertEquals("devsecret", config.getS3SecretKey()); + assertEquals("eu-central-1", config.getS3Region()); + } + + @Test + void resolveConfigAllowsCliFlagsToOverrideEnvironment() { + Map env = new HashMap<>(); + env.put(OzoneLocal.ENV_DATA_DIR, "target/env-local"); + env.put(OzoneLocal.ENV_FORMAT, "never"); + env.put(OzoneLocal.ENV_DATANODES, "2"); + env.put(OzoneLocal.ENV_HOST, "env-host"); + env.put(OzoneLocal.ENV_BIND_HOST, "0.0.0.0"); + env.put(OzoneLocal.ENV_SCM_PORT, "100"); + env.put(OzoneLocal.ENV_OM_PORT, "101"); + env.put(OzoneLocal.ENV_S3G_ENABLED, "true"); + env.put(OzoneLocal.ENV_S3G_PORT, "102"); + env.put(OzoneLocal.ENV_EPHEMERAL, "false"); + env.put(OzoneLocal.ENV_STARTUP_TIMEOUT, "30s"); + env.put(OzoneLocal.ENV_S3_ACCESS_KEY, "env-access"); + env.put(OzoneLocal.ENV_S3_SECRET_KEY, "env-secret"); + env.put(OzoneLocal.ENV_S3_REGION, "env-region"); + + LocalOzoneClusterConfig config = resolve(env, + "--data-dir", "target/cli-local", + "--format", "always", + "--datanodes", "3", + "--host", "cli-host", + "--bind-host", "127.0.0.1", + "--scm-port", "200", + "--om-port", "201", + "--s3g-port", "202", + "--without-s3g", + "--ephemeral", + "--startup-timeout", "PT45S", + "--s3-access-key", "cli-access", + "--s3-secret-key", "cli-secret", + "--s3-region", "cli-region"); + + assertEquals(Paths.get("target/cli-local").toAbsolutePath().normalize(), + config.getDataDir()); + assertEquals(LocalOzoneClusterConfig.FormatMode.ALWAYS, + config.getFormatMode()); + assertEquals(3, config.getDatanodes()); + assertEquals("cli-host", config.getHost()); + assertEquals("127.0.0.1", config.getBindHost()); + assertEquals(200, config.getScmPort()); + assertEquals(201, config.getOmPort()); + assertEquals(202, config.getS3gPort()); + assertFalse(config.isS3gEnabled()); + assertTrue(config.isEphemeral()); + assertEquals(Duration.ofSeconds(45), config.getStartupTimeout()); + assertEquals("cli-access", config.getS3AccessKey()); + assertEquals("cli-secret", config.getS3SecretKey()); + assertEquals("cli-region", config.getS3Region()); + } + + @Test + void resolveConfigTreatsBlankEnvironmentValuesAsUnset() { + Map env = new HashMap<>(); + env.put(OzoneLocal.ENV_DATA_DIR, " "); + env.put(OzoneLocal.ENV_DATANODES, "\t"); + env.put(OzoneLocal.ENV_S3_ACCESS_KEY, ""); + env.put(OzoneLocal.ENV_S3_REGION, " "); + + LocalOzoneClusterConfig config = resolve(env); + + assertEquals(LocalOzoneClusterConfig.DEFAULT_DATA_DIR, + config.getDataDir()); + assertEquals(1, config.getDatanodes()); + assertEquals("admin", config.getS3AccessKey()); + assertEquals("us-east-1", config.getS3Region()); + } + + @Test + void resolveConfigRejectsInvalidBooleanEnvironmentValue() { + assertConfigError(OzoneLocal.ENV_S3G_ENABLED, "sometimes", + OzoneLocal.ENV_S3G_ENABLED); + } + + @Test + void resolveConfigRejectsInvalidFormatEnvironmentValue() { + assertConfigError(OzoneLocal.ENV_FORMAT, "sometimes", + OzoneLocal.ENV_FORMAT); + } + + @Test + void resolveConfigRejectsInvalidIntegerEnvironmentValue() { + assertConfigError(OzoneLocal.ENV_DATANODES, "two", + OzoneLocal.ENV_DATANODES); + } + + @Test + void resolveConfigRejectsInvalidPortEnvironmentValue() { + assertConfigError(OzoneLocal.ENV_SCM_PORT, "65536", + OzoneLocal.ENV_SCM_PORT); + } + + @Test + void resolveConfigRejectsDatanodeCountBelowOne() { + assertConfigError(OzoneLocal.ENV_DATANODES, "0", + OzoneLocal.ENV_DATANODES); + } + + @Test + void resolveConfigRejectsInvalidDurationEnvironmentValue() { + assertConfigError(OzoneLocal.ENV_STARTUP_TIMEOUT, "forever", + OzoneLocal.ENV_STARTUP_TIMEOUT); + } + + @Test + void resolveConfigRejectsNonPositiveDurationEnvironmentValue() { + assertConfigError(OzoneLocal.ENV_STARTUP_TIMEOUT, "0s", + OzoneLocal.ENV_STARTUP_TIMEOUT); + } + + @Test + void resolveConfigRejectsInvalidPathEnvironmentValue() { + assertConfigError(OzoneLocal.ENV_DATA_DIR, "\0", + OzoneLocal.ENV_DATA_DIR); + } + + @Test + void resolveConfigRejectsInvalidPathCliValue() { + assertCliConfigError("--data-dir", "\0", "--data-dir"); + } + + @Test + void genericCliErrorOutputIncludesOffendingConfigSource() + throws Exception { + Map env = new HashMap<>(); + env.put(OzoneLocal.ENV_S3G_ENABLED, "sometimes"); + OzoneLocal local = localWithEnvironment(env); + ByteArrayOutputStream err = new ByteArrayOutputStream(); + local.getCmd().setErr(new PrintWriter(new OutputStreamWriter(err, UTF_8), + true)); + + int exitCode = local.execute(new String[] {"run"}); + + assertEquals(-1, exitCode); + assertTrue(err.toString(UTF_8.name()) + .contains(OzoneLocal.ENV_S3G_ENABLED)); + } + + private static LocalOzoneClusterConfig resolve(Map env, + String... args) { + OzoneLocal.RunCommand command = new OzoneLocal.RunCommand(env); + new CommandLine(command).parseArgs(args); + return command.resolveConfig(new OzoneConfiguration()); + } + + private static void assertConfigError(String key, String value, + String expectedMessage) { + Map env = new HashMap<>(); + env.put(key, value); + OzoneLocal.RunCommand command = new OzoneLocal.RunCommand(env); + + IllegalArgumentException error = assertThrows(IllegalArgumentException.class, + () -> command.resolveConfig(new OzoneConfiguration())); + + assertTrue(error.getMessage().contains(expectedMessage), + error.getMessage()); + } + + private static void assertCliConfigError(String option, String value, + String expectedMessage) { + OzoneLocal.RunCommand command = new OzoneLocal.RunCommand( + Collections.emptyMap()); + new CommandLine(command).parseArgs(option, value); + + IllegalArgumentException error = assertThrows(IllegalArgumentException.class, + () -> command.resolveConfig(new OzoneConfiguration())); + + assertTrue(error.getMessage().contains(expectedMessage), + error.getMessage()); + } + + private static OzoneLocal localWithEnvironment(Map env) { + CommandLine.IFactory defaultFactory = CommandLine.defaultFactory(); + return new OzoneLocal(new CommandLine.IFactory() { + @Override + public K create(Class clazz) throws Exception { + if (clazz == OzoneLocal.RunCommand.class) { + return clazz.cast(new OzoneLocal.RunCommand(env)); + } + return defaultFactory.create(clazz); + } + }); + } +}