ACOMMONS-217 Introduce shared IP address extraction with IpAddressExtractor

commons/README.md

@@ -11,6 +11,7 @@ Logic useful for an average language plugin
 * [`ExternalReportProvider`](./src/main/java/org/sonarsource/analyzer/commons/ExternalReportProvider.java) - to get the list of io.File with external reports
 * [`ProgressReport`](./src/main/java/org/sonarsource/analyzer/commons/ProgressReport.java) - to produce logs with number of analyzed files
 * [`FileProvider`](./src/main/java/org/sonarsource/analyzer/commons/FileProvider.java) - to get files matching given pattern in the given directory
+* [`IpAddressCandidateExtractor`](./src/main/java/org/sonarsource/analyzer/commons/appsec/IpAddressCandidateExtractor.java) - to extract IP address candidates from bare addresses, endpoints, and URLs
 * [`IpAddressClassifier`](./src/main/java/org/sonarsource/analyzer/commons/appsec/IpAddressClassifier.java) - to parse numeric IP addresses and CIDRs and query their classification
 
 ### License

commons/src/main/java/org/sonarsource/analyzer/commons/appsec/IpAddressCandidateExtractor.java

@@ -0,0 +1,141 @@
+/*
+ * SonarSource Analyzers Commons
+ * Copyright (C) SonarSource Sàrl
+ * mailto:info AT sonarsource DOT com
+ *
+ * You can redistribute and/or modify this program under the terms of
+ * the Sonar Source-Available License Version 1, as published by SonarSource Sàrl.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the Sonar Source-Available License for more details.
+ *
+ * You should have received a copy of the Sonar Source-Available License
+ * along with this program; if not, see https://sonarsource.com/license/ssal/
+ */
+package org.sonarsource.analyzer.commons.appsec;
+
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/** Heuristically extracts an IP address candidate string from a string literal's content. */
+public final class IpAddressCandidateExtractor {
+
+  private static final Pattern URL_PREFIX = Pattern.compile(
+    "^(?:[a-zA-Z][a-zA-Z0-9+.-]*:)?//(?:[^/?#,@\\s]*@)?", Pattern.UNICODE_CHARACTER_CLASS);
+  private static final Pattern ADDRESS_DELIMITER = Pattern.compile("[?#,\\s]", Pattern.UNICODE_CHARACTER_CLASS);
+  private static final Pattern ASCII_DIGITS = Pattern.compile("\\d+");
+
+  private IpAddressCandidateExtractor() {
+  }
+
+  /**
+   * Heuristically extracts an IP address candidate from the given string.
+   * Supports bare IP addresses, socket endpoints, URLs, and delimited lists of these.
+   * The result is not guaranteed to be a valid IP address.
+   *
+   * <p><strong>Note:</strong> Extraction starts at the beginning of the string.
+   * For delimited lists, the candidate is extracted from the first entry only.
+   * If extraction fails, no attempt is made to find another candidate later in the string.
+   *
+   * More specifically:
+   *
+   * <ul>
+   *   <li>Skips URL user information before the host, such as {@code user@} or {@code user:pass@}.</li>
+   *   <li>Removes a single enclosing pair of brackets from IPv6 addresses, and ASCII decimal ports from 0 through 65535.
+   *     Brackets delimit IPv6 hosts in URLs and endpoints so address colons are distinct from the port separator,
+   *     for example {@code [2003::1]:8080}.</li>
+   *   <li>Preserves IPv6 zones and recognized CIDR suffixes on bare addresses.
+   *     Slash suffixes in URLs or after ports are always paths.
+   *     Other slash suffixes on bare addresses are also treated as paths.</li>
+   *   <li>Stops at a query, fragment, comma, or whitespace; leading whitespace is not accepted.</li>
+   * </ul>
+   *
+   * <p>Returns empty for missing candidates or malformed endpoints.
+   * The returned text is not validated as an IP address or CIDR and may be a host name or malformed IP address.
+   * Use {@link IpAddressClassifier#parse(String)} to validate and parse the candidate.
+   */
+  public static Optional<String> extract(String content) {
+    // Isolate the URL host: http://127.0.0.1/7 has a path, not a CIDR prefix.
+    Matcher protocol = URL_PREFIX.matcher(content);
+    String host = content;
+    if (protocol.find()) {
+      host = content.substring(protocol.end());
+      int path = host.indexOf('/');
+      if (path >= 0) {
+        host = host.substring(0, path);
+      }
+    }
+
+    // Stop only at recognized delimiters; retain other characters so parsing rejects malformed candidates like 1.2.3.4-foo.
+    Matcher delimiter = ADDRESS_DELIMITER.matcher(host);
+    String candidate = delimiter.find() ? host.substring(0, delimiter.start()) : host;
+
+    // Preserve bare CIDRs (1.2.3.4/24); strip paths (1.2.3.4/path).
+    int slash = candidate.indexOf('/');
+    String suffix = "";
+    if (slash >= 0) {
+      suffix = candidate.substring(slash + 1);
+      candidate = candidate.substring(0, slash);
+    }
+    // Three digits cover both IPv4 and IPv6 prefixes.
+    int maxPrefixDigits = 3;
+    String cidr = (suffix.length() <= maxPrefixDigits && ASCII_DIGITS.matcher(suffix).matches()) ? ("/" + suffix) : "";
+
+    return stripBracketsAndPort(candidate)
+      .filter(stripped -> !stripped.candidate().isEmpty())
+      .map(stripped -> stripped.candidate() +
+        // An IP address with a port identifies an endpoint, where a CIDR prefix doesn't make sense.
+        // Treat the suffix as a URL path.
+        (stripped.portRemoved() ? "" : cidr));
+  }
+
+  private static Optional<StrippedCandidate> stripBracketsAndPort(String candidate) {
+    // Brackets distinguish IPv6 address colons from the port separator.
+    String address = candidate;
+    int portSeparator = candidate.length();
+    if (candidate.startsWith("[")) {
+      int closing = candidate.indexOf(']');
+      if (closing < 0) {
+        return Optional.empty();
+      }
+      address = candidate.substring(1, closing);
+      if (address.indexOf(':') < 0) {
+        // No `:` found => it is not an IPv6 address.
+        // IPv4 addresses must not be enclosed in brackets.
+        return Optional.empty();
+      }
+      portSeparator = closing + 1;
+    } else {
+      int colon = candidate.indexOf(':');
+      if (colon > 0 && candidate.substring(0, colon).indexOf('.') >= 0) {
+        address = candidate.substring(0, colon);
+        portSeparator = colon;
+      }
+    }
+
+    boolean hasPort = portSeparator < candidate.length();
+    if (hasPort && (candidate.charAt(portSeparator) != ':' || !isPort(candidate.substring(portSeparator + 1)))) {
+      return Optional.empty();
+    }
+    // After unwrapping, any remaining bracket makes the address invalid.
+    if (address.indexOf('[') >= 0 || address.indexOf(']') >= 0) {
+      return Optional.empty();
+    }
+
+    return Optional.of(new StrippedCandidate(address, hasPort));
+  }
+
+  private static boolean isPort(String port) {
+    if (port.length() > 5 || !ASCII_DIGITS.matcher(port).matches()) {
+      return false;
+    }
+    return Integer.parseInt(port) <= 65535;
+  }
+
+  private record StrippedCandidate(String candidate, boolean portRemoved) {
+  }
+
+}

commons/src/test/java/org/sonarsource/analyzer/commons/appsec/IpAddressCandidateExtractorTest.java

@@ -0,0 +1,173 @@
+/*
+ * SonarSource Analyzers Commons
+ * Copyright (C) SonarSource Sàrl
+ * mailto:info AT sonarsource DOT com
+ *
+ * You can redistribute and/or modify this program under the terms of
+ * the Sonar Source-Available License Version 1, as published by SonarSource Sàrl.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the Sonar Source-Available License for more details.
+ *
+ * You should have received a copy of the Sonar Source-Available License
+ * along with this program; if not, see https://sonarsource.com/license/ssal/
+ */
+package org.sonarsource.analyzer.commons.appsec;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class IpAddressCandidateExtractorTest {
+
+  @ParameterizedTest
+  @CsvSource({
+    // Bare addresses and endpoints
+    "192.0.2.1, 192.0.2.1",
+    "192.0.2.1:0, 192.0.2.1",
+    "192.0.2.1:65535, 192.0.2.1",
+    "192.0.2.1:00080, 192.0.2.1",
+    "2001:db8::1, 2001:db8::1",
+    "::00000:2, ::00000:2",
+    "::ffff:192.0.2.1, ::ffff:192.0.2.1",
+    "[::ffff:192.0.2.1], ::ffff:192.0.2.1",
+    "[2001:db8::1]:65535, 2001:db8::1",
+    // URL hosts, including scheme-relative URLs and mixed notation
+    "http://192.0.2.1:8080/path, 192.0.2.1",
+    "http://192.0.2.1/path with space, 192.0.2.1",
+    "git+ssh://192.0.2.1/path, 192.0.2.1",
+    "custom-scheme.v1://192.0.2.1/path, 192.0.2.1",
+    "//[2001:db8::1]:8080/path, 2001:db8::1",
+    "https://[2001:db8::192.0.2.1]:8080/path, 2001:db8::192.0.2.1",
+    "http://[::FFFF:7f00:1]:8080/7, ::FFFF:7f00:1",
+    // URL user information is separate from the host
+    "http://user@192.0.2.1/path, 192.0.2.1",
+    "http://user:pass@[2001:db8::1]:8080/path, 2001:db8::1",
+    "//user@192.0.2.1/path, 192.0.2.1",
+    // An @ outside the URL authority does not introduce user information
+    "http://192.0.2.1/path@192.0.2.2, 192.0.2.1",
+    "http://192.0.2.1?email=user@example.org, 192.0.2.1",
+    "http://192.0.2.1#user@example.org, 192.0.2.1",
+    "'http://192.0.2.1,user@example.org', 192.0.2.1",
+    "'http://192.0.2.1 user@example.org', 192.0.2.1",
+    // URL lists yield only the first host's address
+    "'http://192.0.2.1,https://192.0.2.2', 192.0.2.1",
+    "http://192.0.2.1 https://192.0.2.2, 192.0.2.1",
+    "'http://192.0.2.1:8080/path,https://192.0.2.2/other', 192.0.2.1",
+    "https://[2001:db8::1]:8080/path http://192.0.2.2/other, 2001:db8::1",
+    // The comma delimiter heuristic can extract an IP from user information rather than the host
+    "'http://192.0.2.1,user:pass@192.0.2.2/path', 192.0.2.1",
+    // Delimiters after the host
+    "'192.0.2.1,192.0.2.2', 192.0.2.1",
+    "'192.0.2.1 ', 192.0.2.1",
+    "'192.0.2.1\u00a0rest', 192.0.2.1", // Nonbreaking space is a whitespace delimiter
+    "http://192.0.2.1?query, 192.0.2.1",
+    "192.0.2.1#fragment, 192.0.2.1",
+    // Query strings do not affect host extraction
+    "http://192.0.2.1?foo=bar&baz=qux, 192.0.2.1",
+    "https://[2001:db8::1]:8443?foo=bar;baz=qux, 2001:db8::1",
+    "http://192.0.2.1:8080?q=hello+world%20%26%3B&next=%2Fdocs, 192.0.2.1",
+    "'https://user:pass@[2001:db8::1]/search?tags=a,b&filter=one;two#results', 2001:db8::1",
+    "http://192.0.2.1?redirect=https://user@example.org/path?a=1&b=2, 192.0.2.1",
+    // Bare CIDRs retain their prefix; URL paths do not become CIDRs
+    "192.0.2.1/0, 192.0.2.1/0",
+    "192.0.2.1/32, 192.0.2.1/32",
+    "192.0.2.1/032, 192.0.2.1/032",
+    "2001:db8::1/0, 2001:db8::1/0",
+    "2001:db8::1/64, 2001:db8::1/64",
+    "2001:db8::1/064, 2001:db8::1/064",
+    "2001:db8::1/128, 2001:db8::1/128",
+    "127.0.0.1/7, 127.0.0.1/7",
+    "http://127.0.0.1/7, 127.0.0.1",
+    "[2001:db8::1]/64, 2001:db8::1/64",
+    "//[2001:db8::1]/64, 2001:db8::1",
+    "http://192.0.2.1/123456, 192.0.2.1",
+    "//[2001:db8::1]/123456, 2001:db8::1",
+    // Overlong CIDR suffixes are treated as paths
+    "192.0.2.1/1000, 192.0.2.1",
+    "192.0.2.1/0032, 192.0.2.1",
+    "2001:db8::1/1000, 2001:db8::1",
+    "2001:db8::1/0128, 2001:db8::1",
+    // A slash after a port starts a path, even when it contains only digits
+    "192.0.2.1:8080/0, 192.0.2.1",
+    "[2001:db8::1]:8080/0, 2001:db8::1",
+    "[2001:db8::1]:8080/64, 2001:db8::1",
+    "192.0.2.1:8080/33, 192.0.2.1",
+    "[2001:db8::1]:8080/129, 2001:db8::1",
+    "192.0.2.1:8080/123456, 192.0.2.1",
+    "[2001:db8::1]:8080/123456, 2001:db8::1",
+    "192.0.2.1/path, 192.0.2.1",
+    "192.0.2.1/３３, 192.0.2.1", // Non-ASCII digits form a path, not an ASCII CIDR suffix
+    "192.0.2.1/, 192.0.2.1",
+    // Zone suffixes remain opaque and preserve bare CIDRs
+    "2001:db8::1%, 2001:db8::1%",
+    "[2001:db8::1%接口]/64, 2001:db8::1%接口/64",
+    "http://[2001:db8::1%eth0]:8080/path, 2001:db8::1%eth0"
+  })
+  void shouldExtractCandidate(String content, String expected) {
+    assertThat(IpAddressCandidateExtractor.extract(content)).hasValue(expected);
+  }
+
+  @ParameterizedTest
+  @CsvSource({
+    "https://example.com/path, example.com",
+    "999.2.3.4/33, 999.2.3.4/33",
+    "[192.0.2.1:8080], 192.0.2.1:8080", // Port inside the brackets
+    // Text appended with punctuation remains part of the candidate
+    "192.0.2.1-foo, 192.0.2.1-foo",
+    "192.0.2.1~foo, 192.0.2.1~foo",
+    "192.0.2.1!x, 192.0.2.1!x",
+    "http://192.0.2.1;x/path, 192.0.2.1;x", // Punctuation in the URL host
+    "192.0.2.1., 192.0.2.1.", // Trailing dot
+    "192.0.2.1.5, 192.0.2.1.5", // Too many octets
+    "prefix192.0.2.1, prefix192.0.2.1", // Text prepended to an address
+    "192.0.2.1/33, 192.0.2.1/33", // Invalid IPv4 CIDR prefix
+    "192.0.2.1/100, 192.0.2.1/100", // Invalid IPv4 CIDR prefix (at most 2 digits possible)
+    "2001:db8::1/129, 2001:db8::1/129", // Invalid IPv6 CIDR prefix
+    "prefix/192.0.2.1, prefix", // Address does not start the content
+    "http://host192.0.2.1/path, host192.0.2.1", // Host name instead of an IP address
+    "http://192.0.2.1@example.org/path, example.org" // IP address is user information, not the host
+  })
+  void couldProduceNonIpAddressResult(String content, String expected) {
+    assertThat(IpAddressCandidateExtractor.extract(content)).hasValue(expected);
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {
+    "", // No address candidate
+    " 192.0.2.1", // Leading whitespace
+    "http:///path", // No URL host
+    "/64", // CIDR suffix without an address
+    "[2001:db8::1", // Missing closing bracket
+    "2001:db8::1]", // Missing opening bracket
+    "[[2001:db8::1]]", // Nested bracket pairs
+    "[2001:db8::1]extra", // Unexpected text after the closing bracket
+    "[2001:db8::1]8080", // Missing port separator
+    "[192.0.2.1]", // Bracketed IPv4
+    "[192.0.2.1]:8080", // Bracketed IPv4 endpoint
+    "http://[192.0.2.1]/path", // Bracketed IPv4 URL host
+    "2001:db8::1%eth[0", // Stray opening bracket in an opaque zone
+    "2001:db8::1%eth0]", // Stray closing bracket in an opaque zone
+    "192.0.2.1:", // Empty port
+    "192.0.2.1:bad", // Nonnumeric port
+    "192.0.2.1:-1", // Negative port
+    "192.0.2.1:+80", // Signed port
+    "192.0.2.1:８０", // Non-ASCII port digits
+    "192.0.2.1:65536", // Port exceeds 65535
+    "192.0.2.1:000080", // Port exceeds five digits despite its value fitting the range
+    "192.0.2.1:99999999999999999999", // Port exceeds five digits
+    "[2001:db8::1]:", // Empty port
+    "[2001:db8::1]:-1", // Negative port
+    "[2001:db8::1]:65536", // Port exceeds 65535
+    "[2001:db8::1]:000000", // Port exceeds five digits
+    "http://[2001:db8::1]:path/instead/of/port", // Nonnumeric port
+    "http://192.0.2.1:8080;https://192.0.2.2/other" // Limitation: semicolon is not a recognized delimiter
+  })
+  void shouldRejectMissingCandidateOrMalformedEndpoint(String content) {
+    assertThat(IpAddressCandidateExtractor.extract(content)).isEmpty();
+  }
+}