ACOMMONS-231 Find IP addresses throughout string literals

commons/README.md

@@ -11,7 +11,7 @@ Logic useful for an average language plugin
 * [`ExternalReportProvider`](./src/main/java/org/sonarsource/analyzer/commons/ExternalReportProvider.java) - to get the list of io.File with external reports
 * [`ProgressReport`](./src/main/java/org/sonarsource/analyzer/commons/ProgressReport.java) - to produce logs with number of analyzed files
 * [`FileProvider`](./src/main/java/org/sonarsource/analyzer/commons/FileProvider.java) - to get files matching given pattern in the given directory
-* [`IpAddressCandidateExtractor`](./src/main/java/org/sonarsource/analyzer/commons/appsec/IpAddressCandidateExtractor.java) - to extract IP address candidates from bare addresses, endpoints, and URLs
+* [`IpAddressCandidateExtractor`](./src/main/java/org/sonarsource/analyzer/commons/appsec/IpAddressCandidateExtractor.java) - to extract IP address candidates from bare addresses, endpoints, and URLs; `extractAll` finds validated addresses throughout text and lists
 * [`IpAddressClassifier`](./src/main/java/org/sonarsource/analyzer/commons/appsec/IpAddressClassifier.java) - to parse numeric IP addresses and CIDRs and query their classification
 
 ### License

commons/src/main/java/org/sonarsource/analyzer/commons/appsec/IpAddressCandidateExtractor.java

@@ -16,18 +16,27 @@
  */
 package org.sonarsource.analyzer.commons.appsec;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Optional;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-/** Heuristically extracts an IP address candidate string from a string literal's content. */
+/** Extracts IP address candidates from a string literal's content. */
 public final class IpAddressCandidateExtractor {
 
   private static final Pattern URL_PREFIX = Pattern.compile(
     "^(?:[a-zA-Z][a-zA-Z0-9+.-]*:)?//(?:[^/?#,@\\s]*@)?", Pattern.UNICODE_CHARACTER_CLASS);
   private static final Pattern ADDRESS_DELIMITER = Pattern.compile("[?#,\\s]", Pattern.UNICODE_CHARACTER_CLASS);
   private static final Pattern ASCII_DIGITS = Pattern.compile("\\d+");
 
+  private static final Pattern ENTRY = Pattern.compile("[^\\s\"'()]+", Pattern.UNICODE_CHARACTER_CLASS);
+  private static final Pattern LIST_ENTRY = Pattern.compile("[^,;]+");
+  private static final Pattern URL_START = Pattern.compile("(?<![a-zA-Z0-9+.-])(?:[a-zA-Z][a-zA-Z0-9+.-]*:)?//");
+  // Keep entire identifiers and malformed endpoints together, rather than recovering shorter valid addresses.
+  // Zones are opaque to address validation, but stop at path and value separators.
+  private static final Pattern ADDRESS_TOKEN = Pattern.compile("[\\p{L}\\p{N}_.:\\[\\]+-]+(?:%[^,;/?#=&]*)?");
+
   private IpAddressCandidateExtractor() {
   }
 
@@ -92,6 +101,110 @@ public static Optional<String> extract(String content) {
         (stripped.portRemoved() ? "" : cidr));
   }
 
+  /**
+   * Finds all syntactically valid IP address literals throughout the given string, in source order.
+   * Repeated occurrences are retained. Returns an empty list when no address is found.
+   * Unlike {@link #extract(String)}, every result is validated with {@link IpAddressClassifier#parse(String)};
+   * address classification and security-rule exceptions are left to the caller.
+   *
+   * <p>Supports text, paths, values after an equals sign, and lists separated by whitespace, commas, or semicolons.
+   * Quotes and parentheses may enclose entries. Addresses touching letters, digits, dots, hyphens, or underscores
+   * are rejected, including sentence-ending dots. A complete IPv6 address, including a dotted tail or zone,
+   * is returned as one result with its spelling preserved.
+   *
+   * <p>URL schemes and authority user information are skipped; hosts, paths, queries, and fragments are searched.
+   * IPv6 endpoint brackets and valid ports on IPv4 or bracketed IPv6 endpoints are removed.
+   * Invalid endpoints are rejected. Bracketed IPv4 endpoints and bracketed lists are not supported.
+   * The last group of an unbracketed IPv6 address is never interpreted as a port.
+   *
+   * <p>A standalone address with a decimal CIDR suffix retains that suffix if its prefix length is valid.
+   * Invalid prefix lengths invalidate the entry. Slashes in URLs, after ports, or in paths are separators.
+   * A single slash followed by a digit or sign on a bare address is treated as a CIDR attempt;
+   * named path segments, leading slashes, and multiple slashes identify paths.
+   *
+   * @throws NullPointerException if {@code content} is null
+   */
+  public static List<String> extractAll(String content) {
+    List<String> addresses = new ArrayList<>();
+    Matcher entries = ENTRY.matcher(content);
+    while (entries.find()) {
+      extractEntry(entries.group(), addresses);
+    }
+    return addresses;
+  }
+
+  private static void extractEntry(String entry, List<String> addresses) {
+    Matcher urls = URL_START.matcher(entry);
+    int start = 0;
+    boolean inUrl = false;
+    while (urls.find(start)) {
+      extractText(entry.substring(start, urls.start()), inUrl, addresses);
+      int authorityEnd = urls.end();
+      while (authorityEnd < entry.length() && "/?#".indexOf(entry.charAt(authorityEnd)) < 0) {
+        authorityEnd++;
+      }
+      String authority = entry.substring(urls.end(), authorityEnd);
+      int hostStart = authority.lastIndexOf('@') + 1;
+      int hostEnd = hostStart;
+      while (hostEnd < authority.length() && ",;".indexOf(authority.charAt(hostEnd)) < 0) {
+        hostEnd++;
+      }
+      String host = authority.substring(hostStart, hostEnd);
+      stripBracketsAndPort(host).map(StrippedCandidate::candidate).ifPresent(candidate -> addValid(candidate, addresses));
+      start = urls.end() + hostEnd;
+      inUrl = true;
+    }
+    extractText(entry.substring(start), inUrl, addresses);
+  }
+
+  private static void extractText(String text, boolean inUrl, List<String> addresses) {
+    Matcher entries = LIST_ENTRY.matcher(text);
+    boolean inUrlValues = false;
+    while (entries.find()) {
+      String entry = entries.group();
+      inUrlValues |= entry.indexOf('?') >= 0 || entry.indexOf('#') >= 0;
+      extractListEntry(entry, inUrl && (entries.start() == 0 || inUrlValues || entry.indexOf('=') >= 0), addresses);
+    }
+  }
+
+  private static void extractListEntry(String text, boolean inUrl, List<String> addresses) {
+    Matcher tokens = ADDRESS_TOKEN.matcher(text);
+    while (tokens.find()) {
+      Optional<StrippedCandidate> stripped = stripBracketsAndPort(tokens.group());
+      if (stripped.isEmpty()) {
+        continue;
+      }
+      StrippedCandidate endpoint = stripped.get();
+      String candidate = endpoint.candidate();
+      if (!inUrl && !endpoint.portRemoved() && isStandalonePrefix(text, tokens.end())) {
+        candidate += text.substring(tokens.end());
+        // Consume even an invalid CIDR suffix so it cannot yield a shorter address.
+        addValid(candidate, addresses);
+        return;
+      }
+      addValid(candidate, addresses);
+    }
+  }
+
+  private static boolean isStandalonePrefix(String text, int end) {
+    if (end + 1 >= text.length() || text.charAt(end) != '/' || text.indexOf('/') != end || text.lastIndexOf('/') != end) {
+      return false;
+    }
+    String suffix = text.substring(end + 1);
+    // An address in the next segment identifies a path, not a prefix length.
+    if (stripBracketsAndPort(suffix).filter(endpoint -> IpAddressClassifier.parse(endpoint.candidate()).isPresent()).isPresent()) {
+      return false;
+    }
+    char first = suffix.charAt(0);
+    return Character.isDigit(first) || first == '+' || first == '-';
+  }
+
+  private static void addValid(String candidate, List<String> addresses) {
+    if (IpAddressClassifier.parse(candidate).isPresent()) {
+      addresses.add(candidate);
+    }
+  }
+
   private static Optional<StrippedCandidate> stripBracketsAndPort(String candidate) {
     // Brackets distinguish IPv6 address colons from the port separator.
     String address = candidate;

commons/src/test/java/org/sonarsource/analyzer/commons/appsec/IpAddressCandidateExtractorAllTest.java

@@ -0,0 +1,173 @@
+/*
+ * SonarSource Analyzers Commons
+ * Copyright (C) SonarSource Sàrl
+ * mailto:info AT sonarsource DOT com
+ *
+ * You can redistribute and/or modify this program under the terms of
+ * the Sonar Source-Available License Version 1, as published by SonarSource Sàrl.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the Sonar Source-Available License for more details.
+ *
+ * You should have received a copy of the Sonar Source-Available License
+ * along with this program; if not, see https://sonarsource.com/license/ssal/
+ */
+package org.sonarsource.analyzer.commons.appsec;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class IpAddressCandidateExtractorAllTest {
+
+  @ParameterizedTest
+  @CsvSource({
+    // Text, paths, and value separators
+    "/registry/ipaddresses/192.168.2.3, 192.168.2.3",
+    "connect to 10.0.0.1, 10.0.0.1",
+    "'  10.0.0.1', 10.0.0.1",
+    "'\u00a010.0.0.1\u2003', 10.0.0.1",
+    "address=10.0.0.1, 10.0.0.1",
+    "(10.0.0.1), 10.0.0.1",
+    "\"10.0.0.1\", 10.0.0.1",
+    "'''10.0.0.1''', 10.0.0.1",
+    "10.0.0.1!, 10.0.0.1",
+    "path/10.0.0.1/24, 10.0.0.1",
+    "/10.0.0.1/33, 10.0.0.1",
+    "10.0.0.1/path, 10.0.0.1",
+    "10.0.0.1/path/33, 10.0.0.1",
+    // Entire IPv6 literals, including mixed notation and opaque zones
+    "2001:DB8:0000::1, 2001:DB8:0000::1",
+    "::00000:2, ::00000:2",
+    "::ffff:192.0.2.1, ::ffff:192.0.2.1",
+    "2001:db8::192.0.2.1, 2001:db8::192.0.2.1",
+    "fe80::1%, fe80::1%",
+    "fe80::1%eth0, fe80::1%eth0",
+    "fe80::1%接口, fe80::1%接口",
+    "fe80::1%eth-0_1, fe80::1%eth-0_1",
+    "fe80::1%eth~0, fe80::1%eth~0",
+    "[fe80::1%25eth0]:8080, fe80::1%25eth0",
+    "[fe80::1%接口]/64, fe80::1%接口/64",
+    // Port handling
+    "10.0.0.1:0, 10.0.0.1",
+    "10.0.0.1:65535, 10.0.0.1",
+    "10.0.0.1:00080, 10.0.0.1",
+    "[2001:db8::1], 2001:db8::1",
+    "[2001:db8::1]:65535, 2001:db8::1",
+    "[::ffff:192.0.2.1]:0, ::ffff:192.0.2.1",
+    "2001:db8::1:8080, 2001:db8::1:8080",
+    "10.0.0.1:8080/33, 10.0.0.1",
+    "[2001:db8::1]:8080/129, 2001:db8::1",
+    // CIDR spelling and address-family bounds
+    "10.0.0.1/0, 10.0.0.1/0",
+    "10.0.0.1/32, 10.0.0.1/32",
+    "10.0.0.1/032, 10.0.0.1/032",
+    "ip=10.0.0.1/24, 10.0.0.1/24",
+    "2001:db8::1/0, 2001:db8::1/0",
+    "2001:db8::1/128, 2001:db8::1/128",
+    "2001:db8::1/064, 2001:db8::1/064",
+    "::ffff:192.0.2.1/120, ::ffff:192.0.2.1/120",
+    "[2001:db8::1]/64, 2001:db8::1/64",
+    // URL schemes, credentials, and numeric paths
+    "http://10.0.0.1:8080/33, 10.0.0.1",
+    "//[2001:db8::1]:8080/129, 2001:db8::1",
+    "custom-scheme.v1://user:pass@10.0.0.1/path, 10.0.0.1",
+    "http://10.0.0.2:pass@10.0.0.1/path, 10.0.0.1",
+    "http://[2001:db8::2]@10.0.0.1/path, 10.0.0.1",
+    "'http://10.0.0.2,user@10.0.0.1/path', 10.0.0.1",
+    "http://10.0.0.2;user@10.0.0.1/path, 10.0.0.1",
+    "http://example.org/10.0.0.1/33, 10.0.0.1",
+    "http://example.org?ip=10.0.0.1, 10.0.0.1",
+    "http://example.org#10.0.0.1, 10.0.0.1",
+    "url=https://user@example.org/10.0.0.1, 10.0.0.1",
+    "http://example.org?ip=10.0.0.1/33, 10.0.0.1"
+  })
+  void findsSingleAddress(String content, String expected) {
+    assertThat(IpAddressCandidateExtractor.extractAll(content)).containsExactly(expected);
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {
+    "", "no addresses here", "example.org", "http://example.org/path",
+    "999.2.3.4", "192.168.01.1", "1.2.3", "1.2.3.4.5", "v1.2.3.4", "1.2.3.4.",
+    "prefix10.0.0.1", "10.0.0.1suffix", "10.0.0.1-foo", "10.0.0.1_foo", "-10.0.0.1", "_10.0.0.1",
+    "接口10.0.0.1", "10.0.0.1接口", "10.0.0.1９", "９10.0.0.1",
+    "2001:db8:::1", "2001:db8::1::2", "gggg::1", "::ffff:999.2.3.4", "::ffff:192.0.02.1",
+    "2001:db8::1.", "2001:db8::1-foo", "[2001:db8::1", "2001:db8::1]", "[[2001:db8::1]]",
+    "[192.0.2.1]", "[192.0.2.1]:8080", "[192.0.2.1:8080]", "[2001:db8::1]extra", "[2001:db8::1]8080",
+    "[2001:db8::1].", "[2001:db8::1]suffix", "fe80::1%eth[0", "fe80::1%eth0]",
+    "10.0.0.1:", "10.0.0.1:bad", "10.0.0.1:-1", "10.0.0.1:+80", "10.0.0.1:８０",
+    "10.0.0.1:65536", "10.0.0.1:000080", "10.0.0.1:99999999999999999999", "10.0.0.1:80:90",
+    "[2001:db8::1]:", "[2001:db8::1]:bad", "[2001:db8::1]:-1", "[2001:db8::1]:+80",
+    "[2001:db8::1]:８０", "[2001:db8::1]:65536", "[2001:db8::1]:000080",
+    "10.0.0.1/33", "10.0.0.1/9999999999999999999999", "10.0.0.1/-1", "10.0.0.1/+24",
+    "10.0.0.1/２４", "10.0.0.1/24bad", "10.0.0.1/24.", "10.0.0.1/33.5", "2001:db8::1/129", "[2001:db8::1]/129",
+    "http://10.0.0.1@host.example/path", "http://10.0.0.1:65536/path", "http://[2001:db8::1]:bad/path",
+    "http://host10.0.0.1/path", "http://host:2001:db8::1/path", "http+10.0.0.1://example.org"
+  })
+  void rejectsMalformedAddressesAndEndpoints(String content) {
+    assertThat(IpAddressCandidateExtractor.extractAll(content)).isEmpty();
+  }
+
+  @Test
+  void retainsOrderAndRepeatedOccurrencesAcrossLists() {
+    assertThat(IpAddressCandidateExtractor.extractAll("connect to 10.0.0.1, 10.0.0.2;[2001:DB8::1]:443 10.0.0.1"))
+      .containsExactly("10.0.0.1", "10.0.0.2", "2001:DB8::1", "10.0.0.1");
+    assertThat(IpAddressCandidateExtractor.extractAll("10.0.0.1/24,2001:db8::1/64;10.0.0.1/24"))
+      .containsExactly("10.0.0.1/24", "2001:db8::1/64", "10.0.0.1/24");
+  }
+
+  @Test
+  void searchesEveryPartOfUrlsAndContinuesAfterNamedHosts() {
+    assertThat(IpAddressCandidateExtractor.extractAll(
+      "http://10.0.0.9:secret@10.0.0.1:80/10.0.0.2?peer=10.0.0.3#10.0.0.4 " +
+      "https://example.org/10.0.0.5?other=[2001:DB8::1]:443"))
+      .containsExactly("10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.0.4", "10.0.0.5", "2001:DB8::1");
+    assertThat(IpAddressCandidateExtractor.extractAll("http://10.0.0.1,https://user@10.0.0.2;10.0.0.3/24"))
+      .containsExactly("10.0.0.1", "10.0.0.2", "10.0.0.3/24");
+    assertThat(IpAddressCandidateExtractor.extractAll("http://example.org?next=https://10.0.0.9@10.0.0.1/10.0.0.2"))
+      .containsExactly("10.0.0.1", "10.0.0.2");
+    assertThat(IpAddressCandidateExtractor.extractAll("http://example.org?one=10.0.0.1/33;two=10.0.0.2/33"))
+      .containsExactly("10.0.0.1", "10.0.0.2");
+    assertThat(IpAddressCandidateExtractor.extractAll("http://example.org?peers=10.0.0.1,10.0.0.2/24"))
+      .containsExactly("10.0.0.1", "10.0.0.2");
+    assertThat(IpAddressCandidateExtractor.extractAll("http://example.org#10.0.0.1;10.0.0.2/33"))
+      .containsExactly("10.0.0.1", "10.0.0.2");
+    assertThat(IpAddressCandidateExtractor.extractAll("http://example.org/path;peer=10.0.0.1/33"))
+      .containsExactly("10.0.0.1");
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {
+    "[2001:db8::1", "2001:db8::1]", "[[2001:db8::1]]", "10.0.0.1:65536", "10.0.0.1/33",
+    "http://[2001:db8::1]:bad/path", "http://[2001:db8::1/path", "[2001:db8::1]:", "[192.0.2.1]"
+  })
+  void malformedEntriesDoNotHideLaterAddresses(String malformed) {
+    for (String separator : new String[] {" ", ",", ";", ") (", "\" \""}) {
+      assertThat(IpAddressCandidateExtractor.extractAll(malformed + separator + "10.0.0.2"))
+        .as("after %s separated by %s", malformed, separator)
+        .containsExactly("10.0.0.2");
+    }
+  }
+
+  @Test
+  void treatsSlashesBetweenAddressesAsPathSeparators() {
+    assertThat(IpAddressCandidateExtractor.extractAll("/10.0.0.1/2001:db8::1/10.0.0.2"))
+      .containsExactly("10.0.0.1", "2001:db8::1", "10.0.0.2");
+    assertThat(IpAddressCandidateExtractor.extractAll("10.0.0.1/10.0.0.2"))
+      .containsExactly("10.0.0.1", "10.0.0.2");
+    assertThat(IpAddressCandidateExtractor.extractAll("10.0.0.1/10.0.0.2:80"))
+      .containsExactly("10.0.0.1", "10.0.0.2");
+  }
+
+  @Test
+  void leavesSecurityRuleClassificationToTheCaller() {
+    assertThat(IpAddressCandidateExtractor.extractAll("0.0.0.0 127.0.0.1 224.0.0.1 255.255.255.255 8.8.8.8 :: ff02::1"))
+      .containsExactly("0.0.0.0", "127.0.0.1", "224.0.0.1", "255.255.255.255", "8.8.8.8", "::", "ff02::1");
+  }
+}