ACOMMONS-231 Find IP addresses throughout string literals

commons/README.md

@@ -11,7 +11,7 @@ Logic useful for an average language plugin
 * [`ExternalReportProvider`](./src/main/java/org/sonarsource/analyzer/commons/ExternalReportProvider.java) - to get the list of io.File with external reports
 * [`ProgressReport`](./src/main/java/org/sonarsource/analyzer/commons/ProgressReport.java) - to produce logs with number of analyzed files
 * [`FileProvider`](./src/main/java/org/sonarsource/analyzer/commons/FileProvider.java) - to get files matching given pattern in the given directory
-* [`IpAddressCandidateExtractor`](./src/main/java/org/sonarsource/analyzer/commons/appsec/IpAddressCandidateExtractor.java) - to extract IP address candidates from bare addresses, endpoints, and URLs
+* [`IpAddressCandidateExtractor`](./src/main/java/org/sonarsource/analyzer/commons/appsec/IpAddressCandidateExtractor.java) - to extract IP address candidates from bare addresses, endpoints, and URLs; `extractAll` finds validated addresses throughout text and lists
 * [`IpAddressClassifier`](./src/main/java/org/sonarsource/analyzer/commons/appsec/IpAddressClassifier.java) - to parse numeric IP addresses and CIDRs and query their classification
 
 ### License

commons/src/main/java/org/sonarsource/analyzer/commons/appsec/IpAddressCandidateExtractor.java

@@ -16,18 +16,28 @@
  */
 package org.sonarsource.analyzer.commons.appsec;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Optional;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-/** Heuristically extracts an IP address candidate string from a string literal's content. */
+/** Extracts IP address candidates from a string literal's content. */
 public final class IpAddressCandidateExtractor {
 
   private static final Pattern URL_PREFIX = Pattern.compile(
     "^(?:[a-zA-Z][a-zA-Z0-9+.-]*:)?//(?:[^/?#,@\\s]*@)?", Pattern.UNICODE_CHARACTER_CLASS);
   private static final Pattern ADDRESS_DELIMITER = Pattern.compile("[?#,\\s]", Pattern.UNICODE_CHARACTER_CLASS);
   private static final Pattern ASCII_DIGITS = Pattern.compile("\\d+");
 
+  private static final Pattern ENTRY = Pattern.compile("[^\\s\"'()]+", Pattern.UNICODE_CHARACTER_CLASS);
+  private static final Pattern LIST_ENTRY = Pattern.compile("[^,;]+");
+  private static final Pattern URL_START = Pattern.compile("(?<![a-zA-Z0-9+.-])(?:[a-zA-Z][a-zA-Z0-9+.-]*:)?//");
+  // Keep entire identifiers and malformed endpoints together, rather than recovering shorter valid addresses.
+  // Zones are opaque to address validation, but stop at path and value separators.
+  private static final Pattern ADDRESS_TOKEN = Pattern.compile("[\\p{L}\\p{N}_.:\\[\\]+-]+(?:%[^,;/?#=&]*)?");
+  private static final Pattern CIDR_SUFFIX = Pattern.compile("[\\p{L}\\p{N}_.:%\\[\\]+-]+");
+
   private IpAddressCandidateExtractor() {
   }
 
@@ -92,6 +102,164 @@ public static Optional<String> extract(String content) {
         (stripped.portRemoved() ? "" : cidr));
   }
 
+  /**
+   * Finds all syntactically valid IP address literals throughout the given string, in source order.
+   * Repeated occurrences are retained. Returns an empty list when no address is found.
+   * Unlike {@link #extract(String)}, every result is validated with {@link IpAddressClassifier#parse(String)};
+   * address classification and security-rule exceptions are left to the caller.
+   *
+   * <p>Supports text, paths, values after an equals sign, and lists separated by whitespace, commas, or semicolons.
+   * Quotes and parentheses may enclose entries. Addresses touching letters, digits, dots, hyphens, or underscores
+   * are rejected, including sentence-ending dots. A complete IPv6 address, including a dotted tail or zone,
+   * is returned as one result with its spelling preserved.
+   *
+   * <p>URL schemes and authority user information are skipped; hosts, paths, queries, and fragments are searched.
+   * IPv6 endpoint brackets and valid ports on IPv4 or bracketed IPv6 endpoints are removed.
+   * Invalid endpoints are rejected. Bracketed IPv4 endpoints and bracketed lists are not supported.
+   * The last group of an unbracketed IPv6 address is never interpreted as a port.
+   *
+   * <p>A standalone address with a decimal CIDR suffix retains that suffix if its prefix length is valid.
+   * Invalid prefix lengths invalidate the entry. Slashes in URLs, after ports, or in paths are separators.
+   * A single slash followed by a digit or sign on a bare address is treated as a CIDR attempt;
+   * named path segments, leading slashes, and multiple slashes identify paths.
+   *
+   * @throws NullPointerException if {@code content} is null
+   */
+  public static List<String> extractAll(String content) {
+    List<String> addresses = new ArrayList<>();
+    Matcher entries = ENTRY.matcher(content);
+    while (entries.find()) {
+      extractEntry(entries.group(), addresses);
+    }
+    return addresses;
+  }
+
+  private static void extractEntry(String entry, List<String> addresses) {
+    Matcher urls = URL_START.matcher(entry);
+    int start = 0;
+    boolean inUrl = false;
+    while (urls.find(start)) {
+      int urlStart = urls.start();
+      int authorityStart = urls.end();
+      extractText(entry.substring(start, urlStart), inUrl, addresses);
+      boolean wrapped = urlStart > 0 && entry.charAt(urlStart - 1) == '[';
+      int wrapperEnd = wrapped ? urlWrapperEnd(entry, urlStart) : -1;
+      if (wrapperEnd >= 0) {
+        // Remove the URL wrapper, keeping the IPv6 endpoint's own brackets and any attached suffix.
+        entry = entry.substring(0, wrapperEnd) + entry.substring(wrapperEnd + 1);
+        urls = URL_START.matcher(entry);
+      }
+      int authorityEnd = authorityStart;
+      while (authorityEnd < entry.length() && "/?#".indexOf(entry.charAt(authorityEnd)) < 0) {
+        authorityEnd++;
+      }
+      String authority = entry.substring(authorityStart, authorityEnd);
+      int hostStart = authority.lastIndexOf('@') + 1;
+      int hostEnd = urlHostEnd(authority, hostStart);
+      if (!wrapped || wrapperEnd >= 0) {
+        stripBracketsAndPort(authority.substring(hostStart, hostEnd))
+          .map(StrippedCandidate::candidate).ifPresent(candidate -> addValid(candidate, addresses));
+      }
+      start = authorityStart + hostEnd;
+      inUrl = true;
+    }
+    extractText(entry.substring(start), inUrl, addresses);
+  }
+
+  private static int urlWrapperEnd(String entry, int urlStart) {
+    int depth = 1;
+    for (int i = urlStart; i < entry.length(); i++) {
+      char current = entry.charAt(i);
+      if (current == '[') {
+        depth++;
+      } else if (current == ']') {
+        depth--;
+        if (depth == 0) {
+          return i;
+        }
+      }
+    }
+    return -1;
+  }
+
+  private static int urlHostEnd(String authority, int hostStart) {
+    int end = hostStart;
+    while (end < authority.length() && ",;".indexOf(authority.charAt(end)) < 0) {
+      end++;
+    }
+    int tokenStart = hostStart;
+    if (hostStart < end && authority.charAt(hostStart) == '[') {
+      int closing = authority.indexOf(']', hostStart);
+      if (closing >= 0 && closing < end) {
+        // A closing endpoint bracket bounds even an opaque zone. Keep the complete port or malformed suffix attached.
+        tokenStart = closing + 1;
+      }
+    }
+    Matcher token = ADDRESS_TOKEN.matcher(authority).region(tokenStart, end);
+    if (token.lookingAt()) {
+      return token.end();
+    }
+    return tokenStart > hostStart ? tokenStart : end;
+  }
+
+  private static void extractText(String text, boolean inUrl, List<String> addresses) {
+    Matcher entries = LIST_ENTRY.matcher(text);
+    boolean inUrlValues = false;
+    while (entries.find()) {
+      String entry = entries.group();
+      inUrlValues |= entry.indexOf('?') >= 0 || entry.indexOf('#') >= 0;
+      extractListEntry(entry, inUrl && (entries.start() == 0 || inUrlValues || entry.indexOf('=') >= 0), addresses);
+    }
+  }
+
+  private static void extractListEntry(String text, boolean inUrl, List<String> addresses) {
+    Matcher tokens = ADDRESS_TOKEN.matcher(text);
+    int previousEnd = 0;
+    boolean inPath = false;
+    while (tokens.find()) {
+      if (tokens.start() != previousEnd) {
+        inPath = text.charAt(tokens.start() - 1) == '/';
+      }
+      previousEnd = tokens.end();
+      Optional<StrippedCandidate> stripped = stripBracketsAndPort(tokens.group());
+      if (stripped.isEmpty()) {
+        continue;
+      }
+      StrippedCandidate endpoint = stripped.get();
+      String candidate = endpoint.candidate();
+      if (!inUrl && !inPath && !endpoint.portRemoved()) {
+        int prefixEnd = standalonePrefixEnd(text, tokens.end());
+        candidate += text.substring(tokens.end(), prefixEnd);
+        // Consume malformed prefixes too, while leaving later separate values available to the scanner.
+        tokens.region(prefixEnd, text.length());
+        previousEnd = prefixEnd;
+      }
+      addValid(candidate, addresses);
+    }
+  }
+
+  private static int standalonePrefixEnd(String text, int end) {
+    if (end + 1 >= text.length() || text.charAt(end) != '/') {
+      return end;
+    }
+    Matcher suffix = CIDR_SUFFIX.matcher(text).region(end + 1, text.length());
+    if (!suffix.lookingAt() || (suffix.end() < text.length() && text.charAt(suffix.end()) == '/')) {
+      return end;
+    }
+    // An address in the next segment identifies a path, not a prefix length.
+    if (stripBracketsAndPort(suffix.group()).filter(endpoint -> IpAddressClassifier.parse(endpoint.candidate()).isPresent()).isPresent()) {
+      return end;
+    }
+    char first = text.charAt(end + 1);
+    return (Character.isDigit(first) || first == '+' || first == '-') ? suffix.end() : end;
+  }
+
+  private static void addValid(String candidate, List<String> addresses) {
+    if (IpAddressClassifier.parse(candidate).isPresent()) {
+      addresses.add(candidate);
+    }
+  }
+
   private static Optional<StrippedCandidate> stripBracketsAndPort(String candidate) {
     // Brackets distinguish IPv6 address colons from the port separator.
     String address = candidate;