diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d5a6f33 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,34 @@ +name: CI + +on: + push: + branches: ["**"] + pull_request: + +jobs: + tests: + name: Tests (py${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + + - name: Install deps + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -e . + pip install pytest + + - name: Run tests + run: | + python -m pytest -q diff --git a/.github/workflows/python_publish.yml b/.github/workflows/python_publish.yml index 2dccf6f..bc7193b 100644 --- a/.github/workflows/python_publish.yml +++ b/.github/workflows/python_publish.yml @@ -1,26 +1,33 @@ -name: Upload Python Package +name: Publish to PyPI on: release: - types: [created] + types: [published] + +permissions: + contents: read jobs: - deploy: + build-and-publish: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* \ No newline at end of file + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Build dists + run: | + python -m pip install --upgrade pip + pip install build twine + python -m build + twine check dist/* + + - name: Publish to PyPI + # Uses an API token stored as a GitHub Actions secret: PYPI_TOKEN + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_TOKEN }} diff --git a/.gitignore b/.gitignore index 9af229b..cbbc1f0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,26 @@ -dist/ -venv3/ +# Python cache/artifacts +__pycache__/ +*.py[cod] +*.so + +# Virtual environments +venv/ +venv*/ +.venv/ + +# Test/cache +.pytest_cache/ +.coverage +htmlcov/ + +# Packaging/build build/ -knockpy.egg-info/ -*.pyc -knockpy_report/ -knock_subdomains.egg-info/ -.env \ No newline at end of file +dist/ +*.egg-info/ + +# Local environment/config +.env +.Rhistory + +# Generated reports +*_report_*.html diff --git a/README.md b/README.md index f7f1c9e..dcb4668 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,25 @@ -# ๐Ÿ” Knock Subdomain Scan v8 +# ๐Ÿ” Knockpy Subdomain Scan v9.0 โœ… Fast & Async โ€ข ๐Ÿ” Recon + Brute โ€ข ๐Ÿ”ง Easy to Extend **KnockPy** is a modular Python 3 tool to enumerate subdomains via passive reconnaissance and bruteforce, now with **async/await support**, enhanced performance, and modern HTTP/TLS handling. -![knockpy8](https://github.com/guelfoweb/knock/blob/master/assets/knockpy8.png) - --- -## ๐Ÿš€ Features (v8) +## ๐Ÿš€ Features (v9) - โœ… **Async scanning** with `httpx` and DNS resolution - โœ… Modular: plug new passive sources easily - ๐Ÿ” Supports **passive recon**, **bruteforce**, or both +- ๐ŸŽจ Formatted terminal output with **Rich** (tables, progress, panels) - ๐Ÿ“œ Validates **HTTP/HTTPS status**, **TLS cert**, and **IP** +- โš ๏ธ Detects legacy TLS support (**TLS 1.0/1.1**) as warning in CLI/verbose/HTML report +- ๐Ÿงพ Checks **AXFR (zone transfer)** on root domain during domain-mode scans +- ๐Ÿ”Ž `--verbose` single-domain diagnostics (DNS/TCP/TLS/redirect chains/request errors + security checks) - ๐Ÿ’ก Supports **wildcard DNS** detection -- ๐Ÿงช Output as **JSON**, optional save & reload +- ๐Ÿงช SQLite reports with interactive catalog (`show/delete/export/search`) - ๐Ÿ” Supports **VirusTotal** and **Shodan** API +- ๐Ÿš€ Optimized bruteforce runtime with TLS-probe endpoint caching (no timeout changes required) --- @@ -25,13 +28,42 @@ ### From GitHub source (recommended) ```bash -git clone https://github.com/guelfoweb/knock.git -cd knock +git clone https://github.com/guelfoweb/knockpy.git +cd knockpy pip install . ``` โš ๏ธ Recommended Python version: 3.9+ +## ๐Ÿงฑ Project Structure + +The codebase is organized by responsibility, with stable facades for backward compatibility: + +```text +knockpy/ + cli.py # CLI entrypoint (facade/orchestration) + cli_parts/ + status.py # runtime/status panel rendering + setup.py # interactive setup and persisted runtime defaults + report.py # interactive report mode + scan_flow.py # exclude rules, recon-test, wildcard helpers + core.py # public core facade (compatibility) + engine/ + runtime.py # scanning engine implementation + storage.py # public storage facade (compatibility) + storage_parts/ + db.py # SQLite persistence/settings + export.py # report export orchestration + html_report.py # HTML report rendering + output.py # terminal output rendering + server_versions.py # web-server versions catalog + knockpy.py # compatibility module exports +``` + +Compatibility note: +- Preferred external imports: `import knockpy` or `from knockpy import KNOCKPY`. +- Internal modules are split into `engine/`, `cli_parts/`, and `storage_parts/`. + ### Using pip @@ -51,25 +83,57 @@ knockpy -d domain.com [options] | Flag | Description | | ----------------- | ---------------------------------- | -| `-d`, `--domain` | Target domain | +| `-d`, `--domain` | Target domain (or stdin if used without value) | | `-f`, `--file` | File with list of domains | | `--recon` | Enable passive reconnaissance | -| `--bruteforce`,`brute` | Enable bruteforce using wordlist | -| `--wordlist` | Custom wordlist (default included) | -| `--dns` | Custom DNS resolver | -| `--useragent` | Custom HTTP user-agent | -| `--timeout` | Request timeout in seconds | -| `--threads` | Number of concurrent workers | +| `--bruteforce`, `--brute` | Enable bruteforce using wordlist | +| `--exclude TYPE VALUE` | Exclude matches (`status`, `length/lenght`, `body`) | +| `--verbose` | Deep diagnostics for single-domain scans only | | `--wildcard` | Test wildcard DNS and exit | -| `--json` | Output results in JSON | -| `--save FOLDER` | Save report to folder | -| `--report FILE` | Load and show a saved report | +| `--test` | With `--recon`, test each recon source (failed/empty/data) | +| `--setup` | Interactive setup (runtime defaults + API keys in DB) | +| `--update-versions` | Update local latest web-server versions catalog | +| `--report [ID|latest|list]` | Report mode (interactive show/delete/export/search/reset db, export HTML) | +| `--check-update` | Check online if a newer Knockpy release is available on PyPI | +| `--wordlist` | Runtime override for wordlist | +| `--dns` | Runtime override for DNS resolver | +| `--useragent` | Runtime override for HTTP user-agent | +| `--timeout` | Runtime override for timeout (seconds) | +| `--threads` | Runtime override for concurrent workers | | `--silent` | Hide progress bar | -| `--logfile` | Write debug log to file | -| `--show-settings` | Print scan settings and continue | -| `--version` | Show KnockPy version | +| `--json` | JSON-only output (forces `--silent`) | +| `--status` | Print runtime status and continue | | `-h`, `--help` | Show help message | +### Performance Tuning: `--threads` and `--timeout` + +These two options have the biggest impact on runtime for large scans. + +- `--threads` controls concurrency (how many targets are processed in parallel) +- `--timeout` controls how long each network step waits before giving up + +Trade-off: + +- higher `threads` = faster scans, but more load on CPU/network/DNS and higher risk of remote rate-limits +- lower `timeout` = faster scans, but higher risk of missing slow yet valid targets (false negatives) + +Recommended profiles: + +- small/accurate scan (few domains): `--threads 50 --timeout 5` +- balanced scan: `--threads 150 --timeout 4` +- large scan (10k+ domains): start with `--threads 250 --timeout 3` + +If you need both speed and completeness on very large lists, use 2-pass strategy: + +1. fast pass: `--threads 250 --timeout 3` +2. retry pass only on missing/uncertain targets: `--threads 80 --timeout 5` (or higher) + +Notes: + +- CLI values always override saved setup values +- saved setup values (`--setup`) override built-in defaults +- current built-in defaults are `threads=250`, `timeout=3` + ## ๐Ÿ“Œ Examples @@ -79,6 +143,61 @@ knockpy -d domain.com [options] knockpy -d example.com --recon --bruteforce ``` +### ๐Ÿงช Recon services test + +```bash +knockpy -d example.com --recon --test +``` + +### ๐Ÿ”„ Update web-server latest versions catalog + +```bash +knockpy --update-versions +``` + +### ๐Ÿ†• Check for Knockpy updates + +```bash +knockpy --check-update +``` + +### โš™๏ธ Recon sources config (editable) + +At first run, KnockPy creates: + +```bash +~/.knockpy/recon_services.json +``` + +You can add/disable sources by editing the `services` array. +You can also point to a custom file path without changing code: + +```bash +export KNOCK_RECON_SERVICES=/path/to/recon_services.json +``` + +Each service supports: + +- `name` +- `enabled` (`true`/`false`) +- `parser` +- `url_template` (supports `{domain}`, `{virustotal_key}`, `{shodan_key}`) +- `requires_api` (`virustotal` or `shodan`, optional) + +Supported parsers: + +- `csv_first_column` +- `rapiddns_html_td` +- `json_list` +- `virustotal_subdomains` +- `shodan_subdomains` + +### ๐Ÿ“ฅ Domain from stdin + +```bash +echo "example.com" | knockpy -d +``` + ### ๐Ÿง  API Keys (optional) ```bash @@ -93,11 +212,32 @@ API_KEY_VIRUSTOTAL=your-virustotal-api-key API_KEY_SHODAN=your-shodan-api-key ``` -### ๐Ÿ’พ Save and reload report +### ๐Ÿ’พ Reports (SQLite + Interactive HTML export) ```bash -knockpy -d example.com --recon --bruteforce --save report/ -knockpy --report report/example.com_2025_10_25_14_00_00.json +knockpy -d example.com --recon --bruteforce +knockpy --report list +knockpy --report latest +knockpy --report +``` + +Interactive report menu: + +- `1 show` +- `2 delete` +- `3 export` +- `4 search` +- `0 reset db` (asks explicit confirmation) + +Exit report mode: + +- press `Enter` on empty action prompt +- or press `CTRL+C` + +### ๐Ÿ” Single-domain diagnostics + +```bash +knockpy -d forum.example.com --verbose ``` ### ๐Ÿงช Wildcard test only @@ -111,7 +251,16 @@ knockpy -d example.com --wildcard KnockPy can be used as a Python module: ```python -from knock import KNOCKPY +import knockpy + +result = knockpy.KNOCKPY("example.com", timeout=5.0, threads=20) +print(result["domain"], result["ip"]) +``` + +or: + +```python +from knockpy import KNOCKPY domain = 'example.com' @@ -119,7 +268,7 @@ results = KNOCKPY( domain, dns="8.8.8.8", useragent="Mozilla/5.0", - timeout=2, + timeout=5, threads=10, recon=True, bruteforce=True, @@ -133,17 +282,20 @@ for entry in results: ## ๐Ÿ“‚ Wordlist -A default wordlist is included in `knock/wordlist/wordlist.txt`. +A default wordlist is included in `knockpy/wordlist/wordlist.txt`. You can supply your own with `--wordlist`. ## Test ```bash -python tests/poc.py +python3 -m pytest + +# (optional) smoke-run example script +python3 examples/poc.py ``` ## ๐Ÿ“– License Licensed under the GPLv3 license. -Gianni Amato (@guelfoweb) \ No newline at end of file +Gianni Amato (@guelfoweb) diff --git a/assets/knockpy8.png b/assets/knockpy8.png deleted file mode 100644 index 4c69152..0000000 Binary files a/assets/knockpy8.png and /dev/null differ diff --git a/tests/poc.py b/examples/poc.py similarity index 98% rename from tests/poc.py rename to examples/poc.py index b93aa6f..9d2a962 100644 --- a/tests/poc.py +++ b/examples/poc.py @@ -9,9 +9,8 @@ # Root to sys.path (only for test) sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) -from knock import KNOCKPY +from knockpy import KNOCKPY from datetime import datetime -import json # ---- Configurable parameters ---- domain = "github.com" # target domain diff --git a/knock/__init__.py b/knock/__init__.py deleted file mode 100644 index ce8a2e4..0000000 --- a/knock/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -__version__ = "7.0.0" - -from .knockpy import KNOCKPY diff --git a/knock/knockpy.py b/knock/knockpy.py deleted file mode 100644 index ff1462b..0000000 --- a/knock/knockpy.py +++ /dev/null @@ -1,719 +0,0 @@ -#!/usr/bin/env python3 -""" -KnockPy - subdomain scan. -https://github.com/guelfoweb/knock -Gianni 'guelfoweb' Amato -""" - -from __future__ import annotations -import os, sys, re, json, socket, ssl, argparse, logging -from datetime import datetime, timedelta -from collections import OrderedDict -from typing import List, Optional, Union, Iterable, Tuple, Dict, Any -import asyncio -import random -import string -import functools -import httpx -import bs4 -import dns.resolver # fallback resolver (sync) -import OpenSSL -from dotenv import load_dotenv -load_dotenv() - -__version__ = "8.0.0" - -ROOT = os.path.abspath(os.path.dirname(__file__)) - -USER_AGENTS = [ - "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:33.0) Gecko/20120101 Firefox/33.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A", - "Mozilla/5.0 (MSIE 10.0; Windows NT 6.1; Trident/5.0)", -] - -# --------------------------- -# Logging -# --------------------------- - -# Simple logger used by the tool. -# Level INFO by default. Use --logfile to write debug logs to a file. -logger = logging.getLogger("knockpy") -logger.setLevel(logging.INFO) -handler = logging.StreamHandler(sys.stderr) -handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s", "%H:%M:%S")) -logger.addHandler(handler) - - -# --------------------------- -# Elapsed time -# --------------------------- - -# fmt_td: -# - Format time elapsed in hours, minutes, seconds. -def fmt_td(td: Optional[timedelta]) -> str: - """Format a timedelta as HH:MM:SS, return '-' if None.""" - if td is None: - return "-" - total_seconds = int(td.total_seconds()) - hours, rem = divmod(total_seconds, 3600) - minutes, seconds = divmod(rem, 60) - return f"{hours:02d}:{minutes:02d}:{seconds:02d}" - -# --------------------------- -# Bruteforce -# --------------------------- - -# Bruteforce class: -# - Load words from a file. -# - Make subdomains by joining each word with the root domain. -# - It also can create a random wildcard test subdomain. -# -# Example: -# b = Bruteforce("example.com") -# list = b.start() # ["www.example.com", "mail.example.com", ...] -class Bruteforce: - """Simple wordlist-based bruteforcer.""" - def __init__(self, domain: str, wordlist: Optional[str] = None): - self.domain = domain - self.wordlist = wordlist or os.path.join(ROOT, "wordlist", "wordlist.txt") - - # load wordlist lines; return list of words - # Example: ["www", "mail", "api"] - def load(self) -> List[str]: - try: - with open(self.wordlist, "r", encoding="utf-8", errors="ignore") as fh: - return [ln.strip() for ln in fh if ln.strip()] - except FileNotFoundError: - logger.error("Wordlist not found: %s", self.wordlist) - return [] - - # create a random subdomain for wildcard test - # Example: "xkqzabc.example.com" - def wildcard(self) -> str: - rnd = "".join(random.choice(string.ascii_lowercase) for _ in range(random.randint(10, 15))) - return f"{rnd}.{self.domain}" - - # Build full FQDN list from words and deduplicate preserving order - # Example: ["www.example.com", "mail.example.com"] - def start(self) -> List[str]: - words = self.load() - return list(OrderedDict.fromkeys(f"{w}.{self.domain}" for w in words if w)) - -# --------------------------- -# Recon (async) -# --------------------------- - -# Recon class: -# - Define a set of external services (crt.sh, certspotter, AlienVault, etc.). -# - Query them in parallel and parse returned subdomains. -# - It returns a sorted unique list of discovered subdomains. -# -# Example: -# r = Recon("example.com") -# subs = await r.start() -# # subs -> ["sub.example.com", "x.example.com"] -class Recon: - """Query a set of web services asynchronously to collect subdomains.""" - def __init__(self, domain: str, timeout: float = 3.0, silent: bool = False, max_concurrency: int = 10): - self.domain = domain - self.timeout = timeout - self.silent = silent - self.max_concurrency = max_concurrency - self.headers = {"User-Agent": random.choice(USER_AGENTS)} - # read optional env API keys - self.vt_key = os.getenv("API_KEY_VIRUSTOTAL") - self.shodan_key = os.getenv("API_KEY_SHODAN") - if not silent: - print(f"- VirusTotal: {'โœ”๏ธ' if self.vt_key else 'โŒ'}") - print(f"- Shodan: {'โœ”๏ธ' if self.shodan_key else 'โŒ'}") - - # Build the service list. If env keys are present, add VT and Shodan. - # Example return: [("crtsh", "https://crt.sh/?q=example.com&output=json"), ...] - def services(self) -> List[Tuple[str, str]]: - s = [ - ("alienvault", f"https://otx.alienvault.com/api/v1/indicators/domain/{self.domain}/passive_dns"), - ("certspotter", f"https://api.certspotter.com/v1/issuances?domain={self.domain}&include_subdomains=true&expand=dns_names"), - ("crtsh", f"https://crt.sh/?q={self.domain}&output=json"), - ("hackertarget", f"https://api.hackertarget.com/hostsearch/?q={self.domain}"), - ("rapiddns", f"https://rapiddns.io/subdomain/{self.domain}"), - ("webarchive", f"https://web.archive.org/cdx/search/cdx?url=*.{self.domain}/*&output=txt"), - ] - - if self.vt_key: - s.append(("virustotal", f"https://www.virustotal.com/vtapi/v2/domain/report?apikey={self.vt_key}&domain={self.domain}")) - if self.shodan_key: - s.append(("shodan", f"https://api.shodan.io/dns/domain/{self.domain}?key={self.shodan_key}")) - - return s - - # _fetch: perform a single HTTP GET to a service endpoint. - # It returns (name, text). If failure, return empty text. - # Example: await _fetch(client, "crtsh", url) -> ("crtsh", "[{...}]") - async def _fetch(self, client: httpx.AsyncClient, name: str, url: str) -> Tuple[str, str]: - try: - r = await client.get(url, timeout=self.timeout) - r.raise_for_status() - return name, r.text - except Exception as e: - if not self.silent: - logger.debug("Recon fetch error %s: %s", name, e) - return name, "" - - # _parse: parse the response text from a named service. - # It extracts candidate subdomains and returns them as a list. - # Example: _parse("crtsh", "[{...}]") -> ["sub.example.com"] - def _parse(self, name: str, text: str) -> List[str]: - # best-effort parsers similar to original - out: List[str] = [] - if not text: - return out - try: - if name == "alienvault": - data = json.loads(text) - out = [it["hostname"] for it in data.get("passive_dns", []) if it.get("hostname", "").endswith(self.domain)] - elif name == "virustotal": - data = json.loads(text) - out = [s for s in data.get("subdomains", []) if s.endswith(self.domain)] - elif name == "shodan": - data = json.loads(text) - out = [f"{s}.{self.domain}" for s in data.get("subdomains", [])] - elif name == "certspotter": - data = json.loads(text) - for it in data: - for dn in it.get("dns_names", []): - if dn.endswith(self.domain): - out.append(dn) - elif name == "crtsh": - data = json.loads(text) - out = [it.get("common_name") for it in data if it.get("common_name", "").endswith(self.domain)] - elif name == "hackertarget": - lines = [l for l in text.splitlines() if l.strip()] - out = [l.split(",")[0].strip() for l in lines if l.split(",")[0].strip()] - elif name == "rapiddns": - soup = bs4.BeautifulSoup(text, "html.parser") - out = [td.text.strip() for td in soup.find_all("td") if td.text.strip().endswith(self.domain)] - elif name == "webarchive": - pat = re.compile(r"https?://([^/]+)") - hosts = [m.group(1) for m in (pat.search(l) for l in text.splitlines()) if m] - out = [h for h in hosts if h.endswith(self.domain) and re.match(r"^[A-Za-z0-9\-\._]+$", h)] - except Exception: - # silent best-effort - pass - return out - - # start: run all services in parallel and collect subdomains. - # Returns a sorted unique list of subdomains. - # Example: subs = await Recon("example.com").start() - async def start(self) -> List[str]: - services = self.services() - results: List[str] = [] - limits = asyncio.Semaphore(self.max_concurrency) - async with httpx.AsyncClient(headers=self.headers, http2=True) as client: - tasks = [] - for name, url in services: - async def worker(n=name, u=url): - async with limits: - return await self._fetch(client, n, u) - tasks.append(asyncio.create_task(worker())) - for coro in asyncio.as_completed(tasks): - name, text = await coro - results += self._parse(name, text) - unique = [s for s in OrderedDict.fromkeys(results) if "*" not in s and s] - return sorted(unique) - -# --------------------------- -# Async DNS + HTTP + Cert checks -# --------------------------- - -# AsyncScanner class: -# - Resolve a domain to IP(s). -# - Check http and https responses and headers. -# - Fetch certificate details and check TLS versions. -# - It caches DNS results to avoid repeated lookups. -# -# Example: -# s = AsyncScanner("sub.example.com") -# info = await s.scan() -# # info -> {"domain":"sub.example.com", "ip": [...], "http": [...], "https": [...], "cert": [...]} -class AsyncScanner: - DNS_CACHE: Dict[str, Optional[List[str]]] = {} - CERT_SEMAPHORE = asyncio.Semaphore(50) - - def __init__(self, domain: str, dns_server: Optional[str] = None, useragent: Optional[str] = None, - timeout: float = 2.0, concurrency: int = 200, ssl_verify: bool = False, - root_domain: Optional[str] = None, silent: bool = False): - # Initialize basic scanner state. - self.domain = domain - self.dns_server = dns_server or "8.8.8.8" - self.headers = {"User-Agent": useragent or random.choice(USER_AGENTS)} - self.timeout = timeout - self.concurrency = concurrency - self.ssl_verify = ssl_verify - self.client = None - # store the original root domain to check certificate SAN against it - # if not provided, fallback to the instance domain - self.root_domain = root_domain or domain - - # _resolve_async: - # - Resolve A records for a domain using the synchronous dns.resolver in a threadpool. - # - Cache the results in DNS_CACHE. - # Example: ips = await _resolve_async("sub.example.com") -> ["1.2.3.4"] - async def _resolve_async(self, domain: str) -> Optional[List[str]]: - if not hasattr(self, "DNS_CACHE"): - self.DNS_CACHE = {} - - """Resolve A records asynchronously or via threadpool; with caching.""" - if domain in self.DNS_CACHE: - return self.DNS_CACHE[domain] - - ips: Optional[List[str]] = None # ensure variable is always defined - - loop = asyncio.get_running_loop() - def sync_resolve(): - r = dns.resolver.Resolver() - r.nameservers = [self.dns_server] - r.timeout = self.timeout - r.lifetime = self.timeout - try: - answers = r.resolve(domain, "A") - return [str(a) for a in answers] - except Exception: - return None - ips = await loop.run_in_executor(None, sync_resolve) - - # cache result (even None) to avoid repeated lookups - self.DNS_CACHE[domain] = ips - self.DNS_CACHE[f"{domain}_timestamp"] = asyncio.get_event_loop().time() - return ips - - - # _http_check: - # - Perform an HTTP GET to the given URL using the AsyncClient stored in self.client. - # - Return (status_code, redirect_location, server_header, content_length). - # Example: await _http_check("http://sub.example.com") -> (200, None, "nginx", 1250) - async def _http_check(self, url: str) -> Tuple[Optional[int], Optional[str], Optional[str], Optional[int]]: - try: - r = await self.client.get(url, timeout=self.timeout, follow_redirects=False) - return r.status_code, r.headers.get("Location"), r.headers.get("Server"), len(r.content) - except Exception: - return None, None, None, None - - # _cert_fetch: - # - Blocking part (run in threadpool) gets certificate, expiry, common name and checks SANs. - # - It uses a semaphore self.CERT_SEMAPHORE to limit concurrent cert checks. - # - Then it calls _check_tls_versions() to get a list of TLS versions supported by the host. - # - The function returns (is_valid, expiry_iso, common_name, tls_versions_list). - # - # Example: - # cert_info = await _cert_fetch() - # # cert_info -> (True, "2026-09-15", "example.com", ["TLS 1.3", "TLS 1.2"]) - async def _cert_fetch(self) -> Tuple[Optional[bool], Optional[str], Optional[str], Optional[List[str]]]: - """ - Fetch and analyze the SSL certificate for self.domain, then check supported TLS versions. - """ - loop = asyncio.get_running_loop() - - # sync part: runs in threadpool - def sync_cert(): - try: - pem = ssl.get_server_certificate((self.domain, 443)) - x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, pem) - not_after = x509.get_notAfter().decode("utf-8") - expiry = datetime.strptime(not_after, "%Y%m%d%H%M%S%z").date().isoformat() - is_valid = expiry >= datetime.now().date().isoformat() - cn = x509.get_subject().commonName - # Check SANs: ensure the original root_domain is present - root = self.root_domain - for i in range(x509.get_extension_count()): - ext = x509.get_extension(i) - if b"subjectAltName" in ext.get_short_name(): - san = str(ext) - if root not in san and f"*.{root}" not in san: - is_valid = False - return is_valid, expiry, cn - except Exception: - return None, None, None - - # run sync_cert() in a threadpool - async with self.CERT_SEMAPHORE: - is_valid, expiry, cn = await loop.run_in_executor(None, sync_cert) - - # run TLS check asynchronously - tls_versions = await self._check_tls_versions() - weak_versions = [v for v in tls_versions if v.startswith("SSLv") or v in ("TLS 1.0", "TLS 1.1")] - if weak_versions: - is_valid = False - - return is_valid, expiry, cn, tls_versions - - - # _check_tls_versions: - # - Test which TLS protocol versions the host supports. - # - Each test uses a specific SSL context or a generic client context for TLS 1.3. - # - This function runs the blocking checks in a threadpool and returns a list of version names. - # - # Example: - # versions = await _check_tls_versions() - # # versions -> ["TLS 1.3", "TLS 1.2"] - async def _check_tls_versions(self) -> List[str]: - """ - Detect which TLS protocol versions are supported by the target host. - Runs the blocking socket tests in a threadpool. - Returns a list like ["TLS 1.3", "TLS 1.2"]. - """ - loop = asyncio.get_running_loop() - - def sync_tls_check() -> List[str]: - """Return supported TLS versions for this host, safely across Python versions.""" - # Build dictionary only with versions actually supported by current ssl module - tls_versions = {} - if hasattr(ssl, "PROTOCOL_TLSv1"): - tls_versions["TLS 1.0"] = ssl.PROTOCOL_TLSv1 - if hasattr(ssl, "PROTOCOL_TLSv1_1"): - tls_versions["TLS 1.1"] = ssl.PROTOCOL_TLSv1_1 - if hasattr(ssl, "PROTOCOL_TLSv1_2"): - tls_versions["TLS 1.2"] = ssl.PROTOCOL_TLSv1_2 - # TLS 1.3 constant may not exist; try generic client context instead - if hasattr(ssl, "PROTOCOL_TLSv1_3"): - tls_versions["TLS 1.3"] = ssl.PROTOCOL_TLSv1_3 - else: - # fallback: test with PROTOCOL_TLS_CLIENT which will negotiate TLS 1.3 if supported - tls_versions["TLS 1.3"] = ssl.PROTOCOL_TLS_CLIENT # via generic context - - supported = [] - for name, proto in tls_versions.items(): - try: - ctx = ssl.SSLContext(proto) - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE - with socket.create_connection((self.domain, 443), timeout=self.timeout) as sock: - with ctx.wrap_socket(sock, server_hostname=self.domain): - supported.append(name) - except ssl.SSLError: - continue - except Exception: - continue - return supported - - return await loop.run_in_executor(None, sync_tls_check) - - # scan: - # - Main function to check the domain. - # - It resolves DNS, checks http and https, handles http redirects, and runs certificate check if https is present. - # - It returns a dict with domain, ip, http, https and cert info. - # - # Example: - # result = await AsyncScanner("sub.example.com").scan() - # # result keys: domain, ip, http, https, cert - async def scan(self) -> Optional[Dict[str, Any]]: - # create client per-scan to respect headers / http2 - async with httpx.AsyncClient(headers=self.headers, http2=True, verify=self.ssl_verify) as client: - self.client = client - ips = await self._resolve_async(self.domain) - if not ips: - return None - http = await self._http_check(f"http://{self.domain}") - https = await self._http_check(f"https://{self.domain}") - # handle redirect to other domain for https - if http[0] and http[1] and not https[0]: - loc = http[1] - if not loc.startswith(("http://", "https://")): - loc = "http://" + loc - redirected = loc.split("://", 1)[1].split("/")[0] - https = await self._http_check(f"https://{redirected}") - cert = (None, None, None, None) - if https[0]: - #cert = await self._cert_fetch(self.domain) - cert = await self._cert_fetch() - return {"domain": self.domain, "ip": ips, "http": list(http), "https": list(https), "cert": list(cert)} - -# --------------------------- -# Orchestration: top-level async runner -# --------------------------- - -# _run_async: -# - Expand domains when recon or bruteforce is requested. -# - If single domain, scan directly. -# - If many domains, schedule many AsyncScanner workers with concurrency limits. -# - Uses a CERT_SEMAPHORE to limit concurrent certificate checks (heavy operations). -# -# Example: -# results = asyncio.run(_run_async(["example.com"], ...)) -async def _run_async(domains: List[str], dns: Optional[str], useragent: Optional[str], - timeout: Optional[float], threads: Optional[int], - recon: bool, bruteforce: bool, wordlist: Optional[str], - silent: bool) -> Union[List[dict], dict, None]: - # expansion - if recon or bruteforce: - expanded: List[str] = [] - base = domains[0] - if recon: - if not silent: - print("Reconnaissance...") - r = Recon(base, timeout=(timeout or 2.0), silent=silent, max_concurrency=40) - expanded += await r.start() - if bruteforce: - if not silent: - print("Bruteforcing...") - b = Bruteforce(base, wordlist) - expanded += b.start() - domains = list(OrderedDict.fromkeys(domains + expanded)) - # single domain shortcut - if len(domains) == 1: - scanner = AsyncScanner(domains[0], dns_server=dns, useragent=useragent, timeout=(timeout or 1.5), root_domain=domains[0]) - return await scanner.scan() - # many domains concurrently with bounded concurrency - max_workers = threads or min(300, max(20, len(domains))) - sem = asyncio.Semaphore(max_workers) - CERT_SEMAPHORE = asyncio.Semaphore(50) - - async def worker(d): - async with sem: - s = AsyncScanner(d, dns_server=dns, useragent=useragent, timeout=(timeout or 1.5), root_domain=domains[0], silent=silent) - # give AsyncScanner access to semaphores (optionally) - s.CERT_SEMAPHORE = CERT_SEMAPHORE - return await s.scan() - - tasks = [asyncio.create_task(worker(d)) for d in domains] - - results = [] - if not silent: - total = len(tasks) - done = 0 - for fut in asyncio.as_completed(tasks): - res = await fut - done += 1 - if res: - results.append(res) - print(f"\rScanned {done}/{total} domains...", end="", flush=True) - print() - else: - for fut in asyncio.as_completed(tasks): - res = await fut - if res: - results.append(res) - - return results - -# KNOCKPY: -# - Synchronous wrapper for _run_async so the CLI can call it. -# - Accepts domain string or iterable of domains. -# - Example: res = KNOCKPY("example.com") -def KNOCKPY(domain: Union[str, Iterable[str]], - dns: Optional[str]=None, useragent: Optional[str]=None, timeout: Optional[float]=None, - threads: Optional[int]=None, recon: bool=False, bruteforce: bool=False, - wordlist: Optional[str]=None, silent: bool=False) -> Union[List[dict], dict, None]: - """Synchronous entrypoint that wraps the async runner.""" - if isinstance(domain, str): - domains = [domain] - else: - domains = list(domain) - return asyncio.run(_run_async(domains, dns, useragent, timeout, threads, recon, bruteforce, wordlist, silent)) - -# --------------------------- -# Output & utilities -# --------------------------- - -# output: -# - Nice console formatter for results. -# - If json_output True, dump JSON to stdout. -# - Example: output(result) -def output(results: Union[dict, List[dict], None], json_output: bool=False, elapsed: Optional[timedelta]=None) -> None: - if not results: - return - if json_output: - json.dump(results, sys.stdout, indent=2) - sys.exit(0) - if isinstance(results, dict): - results = [results] - RED, MAG, YEL, CYA, END = "\033[1;31m", "\033[1;35m", "\033[1;33m", "\033[1;36m", "\033[0m" - print() - for item in results: - ok_http = item["http"][0] != None - ok_https = item["https"][0] != None - ok_cert = bool(item["cert"][0]) - if all(v is None for v in item["http"]) and all(v is None for v in item["https"]): - print(f"{MAG}{item['domain']}{END}", item["ip"]) - elif ok_http and not ok_cert: - print(f"{RED}{item['domain']}{END}", item["ip"]) - else: - print(f"{CYA}{item['domain']}{END}", item["ip"]) - http_label = "http "# if ok_http else f"{YEL}http {END}" - https_label = f"{YEL}https {END}" if not ok_https and ok_http else "https " - cert_label = f"{YEL}cert {END}" if not ok_cert and ok_https else "cert " - print(http_label, item["http"]) - print(https_label, item["https"]) - print(cert_label, item["cert"]) - #print() - print("-" * 60) - print(len(results), "domains in", fmt_td(elapsed)) - -# save: -# - Save scan results as JSON file with timestamp. -# - Example: save("example.com", results, "reports") -def save(domain: str, results: List[dict], folder: Optional[str]) -> None: - dt = datetime.now().strftime("%Y_%m_%d_%H_%M_%S") - if folder: - os.makedirs(folder, exist_ok=True) - path = os.path.join(folder, f"{domain}_{dt}.json") - else: - path = f"{domain}_{dt}.json" - with open(path, "w", encoding="utf-8") as fh: - json.dump(results, fh, indent=2) - -# show_report: -# - Load a saved JSON report and print it. -# - If the file is missing, print helpful hints instead of crashing. -# - Example: show_report("reports/example_2025_01_01_12_00_00.json", False) -def show_report(report_name: str, json_output: bool) -> None: - """ - Load a saved report and display it. - If the file does not exist, print a helpful message instead of raising. - """ - if not report_name: - print("No report file provided.") - return - - if not os.path.exists(report_name): - print(f"Report file not found: {report_name}") - # suggest possible saved files - saved = [f for f in os.listdir('.') if f.endswith('.json')] - if saved: - print("Saved JSON files in current dir:") - for s in saved: - print(" -", s) - else: - print("No saved JSON reports found in current directory.") - return - - try: - with open(report_name, "r", encoding="utf-8") as f: - report = json.load(f) - except Exception as e: - print(f"Error reading report '{report_name}': {e}") - return - - output(report, json_output) - -def print_scan_settings(timeout, threads, dns, useragent, recon, brute, save): - print("\nScan settings:") - print(f"- Timeout: {timeout}") - print(f"- Threads: {threads or 'auto'}") - print(f"- DNS: {dns}") - print(f"- User-agent: {useragent}") - print(f"- Recon: {recon}") - print(f"- Bruteforce: {brute}") - print(f"- Save: {save}") - - -# --------------------------- -# CLI -# --------------------------- - -# main: -# - Parse arguments and run the tool. -# - Accepts domain or file, recon, bruteforce, wildcard, save, report, json, etc. -# - Example: python knockpy.py -d example.com --recon --bruteforce --threads 100 -def main(): - p = argparse.ArgumentParser(prog="KNOCKPY", description=f"knockpy v.{__version__} - Async Subdomain Scan\nhttps://github.com/guelfoweb/knock") - p.add_argument("-d","--domain", help="Domain to analyze.") - p.add_argument("-f","--file", help="File with domains, one per line.") - p.add_argument("--dns", help="Custom DNS server.", dest="dns", required=False) - p.add_argument("--useragent", help="Custom User-Agent.", dest="useragent", required=False) - p.add_argument("--timeout", help="Timeout (seconds).", dest="timeout", type=float, required=False) - p.add_argument("--threads", help="Number of concurrent workers.", dest="threads", type=int, required=False) - p.add_argument("--recon", help="Enable reconnaissance.", action="store_true") - p.add_argument("--bruteforce", "--brute", help="Enable bruteforce.", action="store_true") - p.add_argument("--wordlist", help="Path to wordlist file.", dest="wordlist", required=False) - p.add_argument("--wildcard", help="Test wildcard and exit.", action="store_true") - p.add_argument("--json", help="Output JSON.", action="store_true") - p.add_argument("--save", help="Directory to save results.", dest="folder", required=False) - p.add_argument("--report", help="Show saved report.", dest="report", required=False) - p.add_argument("--silent", help="Silent mode (fewer prints).", action="store_true") - p.add_argument("--logfile", help="Write logs to file.", dest="logfile", required=False) - p.add_argument("--show-settings", help="Print effective scan settings and continue.", action="store_true") - args = p.parse_args() - - if args.logfile: - fh = logging.FileHandler(args.logfile) - fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s")) - logger.addHandler(fh) - logger.setLevel(logging.DEBUG) - - # stdin handling as original - if not args.domain and not args.file and not sys.stdin.isatty(): - lines = [l.strip() for l in sys.stdin.read().splitlines() if l.strip()] - if len(lines) == 1: - if os.path.isfile(lines[0]): - args.file = lines[0] - else: - args.domain = lines[0] - elif len(lines) > 1: - args.domain = lines - - if args.report: - show_report(args.report, args.json) - return - - if not args.domain and not args.file: - p.print_help(sys.stderr) - return - - if args.domain: - domain_input = args.domain - - # determine actual values - timeout = args.timeout if args.timeout is not None else 2.0 - threads = args.threads if args.threads is not None else None - dns = args.dns or "8.8.8.8" - useragent = args.useragent or "random" - - if not args.silent and args.show_settings: - print_scan_settings(timeout, threads, dns, useragent, args.recon, args.bruteforce, bool(args.folder)) - - if args.wildcard and isinstance(domain_input, str): - w = Bruteforce(domain_input).wildcard() - start_time = datetime.now() - res = KNOCKPY(w, dns=dns, useragent=useragent, timeout=timeout, threads=threads, recon=False, bruteforce=False, wordlist=None, silent=args.silent) - elapsed = datetime.now() - start_time - output(res, args.json, elapsed) - return - - # KNOCKPY handles recon/bruteforce expansions - start_time = datetime.now() - res = KNOCKPY(domain_input, dns=dns, useragent=useragent, timeout=timeout, threads=threads, recon=args.recon, bruteforce=args.bruteforce, wordlist=args.wordlist, silent=args.silent) - elapsed = datetime.now() - start_time - - if (args.recon or args.bruteforce) and isinstance(res, list): - save(args.domain if isinstance(args.domain, str) else args.domain[0], res, args.folder) - - output(res, args.json, elapsed) - return - - if args.file: - # reuse values - timeout = args.timeout if args.timeout is not None else 2.0 - threads = args.threads if args.threads is not None else None - dns = args.dns or "8.8.8.8" - useragent = args.useragent or "random" - - if not args.silent and args.show_settings: - print_scan_settings(timeout, threads, dns, useragent, False, False, bool(args.folder)) - - with open(args.file, "r", encoding="utf-8") as fh: - domains = [l.strip() for l in fh if l.strip()] - start_time = datetime.now() - res = KNOCKPY(domains, dns=args.dns, useragent=args.useragent, timeout=args.timeout, threads=args.threads, recon=False, bruteforce=False, wordlist=None, silent=args.silent) - elapsed = datetime.now() - start_time - output(res, args.json, elapsed) - -if __name__ == "__main__": - try: - main() - except KeyboardInterrupt: - print("\nInterrupted", file=sys.stderr) - try: - sys.exit(0) - except SystemExit: - os._exit(0) diff --git a/knockpy.py b/knockpy.py index 356c0b5..7c31c78 100644 --- a/knockpy.py +++ b/knockpy.py @@ -1,10 +1,48 @@ #!/usr/bin/python3 # -*- coding: utf-8 -*- +"""Top-level executable and import-compatible shim. + +Purpose: +- `python knockpy.py ...` command execution +- backward-compatible imports for users that import from repository root +""" import os import sys -from knock.knockpy import main +from knockpy.knockpy import ( + AsyncScanner, + Bruteforce, + KNOCKPY, + Recon, + __version__, + export_report, + fmt_td, + get_report, + list_reports, + main, + output, + print_scan_status, + save_scan, + show_reports_catalog, +) + +__all__ = [ + "__version__", + "AsyncScanner", + "Bruteforce", + "KNOCKPY", + "Recon", + "fmt_td", + "main", + "output", + "print_scan_status", + "show_reports_catalog", + "save_scan", + "list_reports", + "get_report", + "export_report", +] if __name__ == "__main__": try: @@ -14,4 +52,4 @@ try: sys.exit(0) except SystemExit: - os._exit(0) \ No newline at end of file + os._exit(0) diff --git a/knockpy/__init__.py b/knockpy/__init__.py new file mode 100644 index 0000000..f85d910 --- /dev/null +++ b/knockpy/__init__.py @@ -0,0 +1,10 @@ +"""Public package surface for knockpy. + +Importing `knockpy` exposes the high-level API function (`KNOCKPY`) and package +version, keeping internals hidden by default. +""" + +from .core import KNOCKPY +from .version import __version__ + +__all__ = ["KNOCKPY", "__version__"] diff --git a/knockpy/cli.py b/knockpy/cli.py new file mode 100644 index 0000000..58f2e3b --- /dev/null +++ b/knockpy/cli.py @@ -0,0 +1,613 @@ +from __future__ import annotations + +"""Command-line interface for knockpy. + +This module translates CLI flags into runtime settings, executes scans through +`knockpy.core`, and handles setup/report workflows backed by local storage. +""" + +import argparse +import os +import sys +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Optional, Union +from urllib.parse import urlparse + +from rich.progress import ( + BarColumn, + Progress, + SpinnerColumn, + TextColumn, + TimeElapsedColumn, +) + +from .core import AsyncScanner, Bruteforce, KNOCKPY, _run_async, _run_coro_sync, pick_user_agent +from .cli_parts.report import report_mode as _report_mode +from .cli_parts.scan_flow import ( + apply_exclude_rules as _apply_exclude_rules, + parse_exclude_rules as _parse_exclude_rules, + print_json_output as _print_json_output, + print_server_versions_catalog as _print_server_versions_catalog, + render_recon_test_table as _render_recon_test_table, + run_recon_test as _run_recon_test, + wildcard_consistent_length as _wildcard_consistent_length, + wildcard_exclude_suggestions as _wildcard_exclude_suggestions, +) +from .cli_parts.setup import load_saved_runtime_settings as _load_saved_runtime_settings, setup_mode as _setup_mode +from .cli_parts.status import render_runtime_status_panel as _render_runtime_status_panel, wordlist_start_message as _wordlist_start_message +from .output import console, err_console, output, output_verbose, print_scan_status +from .storage import save_scan +from .server_versions import update_server_versions_catalog +from .version import __version__, check_latest_version + + +def _load_domains_from_file(file_path: str) -> List[str]: + with Path(file_path).open("r", encoding="utf-8") as fh: + return [line.strip() for line in fh if line.strip()] + + +def _normalize_domain_input(value: str) -> Optional[str]: + raw = (value or "").strip() + if "://" in raw: + parsed = urlparse(raw) + host = (parsed.hostname or "").strip().lower() + if host.startswith("www."): + host = host[4:] + return AsyncScanner._normalize_domain(host) + return AsyncScanner._normalize_domain(raw) + + +def _normalize_domain_list(values: List[str]) -> List[str]: + normalized: List[str] = [] + seen: set[str] = set() + for raw in values: + domain = _normalize_domain_input(raw) + if not domain or domain in seen: + continue + seen.add(domain) + normalized.append(domain) + return normalized + + +def _run_with_rich_progress( + domains: List[str], + dns: Optional[str], + useragent: Optional[str], + timeout: Optional[float], + threads: Optional[int], + recon: bool, + bruteforce: bool, + wordlist: Optional[str], + verbose: bool = False, + enable_axfr: bool = True, + api_key_virustotal: Optional[str] = None, + api_key_shodan: Optional[str] = None, +): + """Execute scanning with a Rich progress bar bound to async callbacks.""" + with Progress( + SpinnerColumn(), + TextColumn("[bold cyan]{task.description}"), + BarColumn(), + TextColumn("{task.completed}/{task.total}"), + TimeElapsedColumn(), + console=console, + ) as progress: + task_id = progress.add_task("Scanning domains", total=max(len(domains), 1)) + + def cb(done: int, total: int) -> None: + progress.update(task_id, total=max(total, 1), completed=done) + + return _run_coro_sync( + _run_async( + domains, + dns, + useragent, + timeout, + threads, + recon, + bruteforce, + wordlist, + verbose=verbose, + enable_axfr=enable_axfr, + api_key_virustotal=api_key_virustotal, + api_key_shodan=api_key_shodan, + progress_callback=cb, + ) + ) + + +def _should_save_scan(recon: bool, bruteforce: bool) -> bool: + return recon or bruteforce + + +def main() -> None: + """CLI entrypoint. + + This function is responsible for argument parsing, config layering + (CLI > saved setup > built-in defaults), mode dispatch and result handling. + """ + stdin_domain_sentinel = "__stdin_domain__" + parser = argparse.ArgumentParser( + prog="KNOCKPY", + description=( + f"knockpy v.{__version__} - Subdomain discovery and security checks\n" + "CLI options > saved setup (--setup) > built-in defaults.\n" + "https://github.com/guelfoweb/knockpy" + ), + ) + target_group = parser.add_argument_group("Target") + target_group.add_argument( + "-d", + "--domain", + nargs="?", + const=stdin_domain_sentinel, + help="Domain to analyze. If used without value, reads from stdin.", + ) + target_group.add_argument("-f", "--file", help="File with domains, one per line.") + + mode_group = parser.add_argument_group("Scan Modes") + mode_group.add_argument("--recon", help="Enable reconnaissance.", action="store_true") + mode_group.add_argument("--bruteforce", "--brute", help="Enable bruteforce.", action="store_true") + mode_group.add_argument("--wildcard", help="Test wildcard and exit.", action="store_true") + mode_group.add_argument( + "--exclude", + nargs=2, + action="append", + metavar=("TYPE", "VALUE"), + help="Exclude matches. TYPE=status, length/lenght (e.g. 275), or body. Repeatable.", + ) + mode_group.add_argument( + "--verbose", + help="Verbose checks for single-domain scan only (without --recon/--bruteforce).", + action="store_true", + ) + mode_group.add_argument( + "--test", + help="With --recon, test each recon service and show which fails or returns data.", + action="store_true", + ) + + setup_group = parser.add_argument_group("Setup and Reports") + setup_group.add_argument( + "--setup", + help="Interactive setup: save runtime defaults and API keys in the local DB.", + action="store_true", + ) + setup_group.add_argument( + "--report", + nargs="?", + const="choose", + help="Report mode. In interactive terminals it opens the menu (show/delete/export/search/findings).", + ) + setup_group.add_argument( + "--update-versions", + help="Update local web server versions catalog and show latest references.", + action="store_true", + ) + setup_group.add_argument( + "--check-update", + help="Check if a newer Knockpy version is available online (PyPI).", + action="store_true", + ) + + runtime_group = parser.add_argument_group("Runtime Overrides (Advanced)") + runtime_group.add_argument("--dns", help="DNS server (overrides saved setup).", dest="dns", required=False) + runtime_group.add_argument( + "--useragent", + help="User-Agent string or 'random' (overrides saved setup).", + dest="useragent", + required=False, + ) + runtime_group.add_argument( + "--timeout", + help="Timeout in seconds (overrides saved setup).", + dest="timeout", + type=float, + required=False, + ) + runtime_group.add_argument( + "--threads", + help="Concurrent workers (overrides saved setup).", + dest="threads", + type=int, + required=False, + ) + runtime_group.add_argument("--wordlist", help="Wordlist path (overrides saved setup).", dest="wordlist", required=False) + + output_group = parser.add_argument_group("Output") + output_group.add_argument("--silent", help="Silent mode (hide progress).", action="store_true") + output_group.add_argument("--json", help="JSON-only output (forces --silent).", action="store_true") + output_group.add_argument("--status", help="Print effective runtime status (including saved setup) and continue.", action="store_true") + args = parser.parse_args() + if args.json: + args.silent = True + + if args.domain == stdin_domain_sentinel: + args.domain = None + + if args.setup: + _setup_mode() + return + if args.update_versions: + catalog = update_server_versions_catalog() + _print_server_versions_catalog(catalog) + return + if args.check_update: + info = check_latest_version(current=__version__) + if info.get("ok"): + latest = str(info.get("latest") or __version__) + if info.get("update_available"): + console.print( + f"[yellow]Update available:[/yellow] current={__version__} latest={latest} " + "[cyan](pip install -U knock-subdomains)[/cyan]" + ) + else: + console.print(f"[green]You are up-to-date:[/green] v{__version__}") + else: + reason = str(info.get("error") or "unknown error") + err_console.print(f"[yellow]Update check failed:[/yellow] {reason}") + return + + if not args.domain and not args.file and not sys.stdin.isatty() and args.report is None: + lines = [line.strip() for line in sys.stdin.read().splitlines() if line.strip()] + if len(lines) == 1: + if Path(lines[0]).is_file(): + args.file = lines[0] + else: + args.domain = lines[0] + elif len(lines) > 1: + args.domain = lines + + if isinstance(args.domain, str): + normalized_domain = _normalize_domain_input(args.domain) + if not normalized_domain: + err_console.print(f"[red]Invalid domain input:[/red] {args.domain}") + return + args.domain = normalized_domain + elif isinstance(args.domain, list): + normalized_domains = _normalize_domain_list([str(d) for d in args.domain]) + if not normalized_domains: + err_console.print("[red]No valid domains found in input.[/red]") + return + args.domain = normalized_domains + + saved = _load_saved_runtime_settings() + try: + exclude_rules = _parse_exclude_rules(args.exclude) + except ValueError as exc: + err_console.print(f"[red]{exc}[/red]") + return + timeout = args.timeout if args.timeout is not None else float(saved["timeout"]) + threads = args.threads if args.threads is not None else saved["threads"] + dns = args.dns or saved["dns"] or "8.8.8.8" + useragent = args.useragent or saved["useragent"] or "random" + effective_useragent = pick_user_agent(useragent) + wordlist_path = args.wordlist if args.wordlist is not None else saved["wordlist"] + api_key_virustotal = saved["api_key_virustotal"] + api_key_shodan = saved["api_key_shodan"] + + if args.report is not None: + _report_mode(args.report) + return + + status_target_count: Optional[int] = None + if args.domain: + status_target_count = 1 if isinstance(args.domain, str) else len(args.domain) + elif args.file and Path(args.file).is_file(): + try: + status_target_count = len(_load_domains_from_file(args.file)) + except Exception: + status_target_count = None + + if args.status and not args.json: + print_scan_status( + timeout, + threads, + dns, + effective_useragent, + args.recon, + args.bruteforce, + wordlist_path, + status_target_count, + ) + + if not args.domain and not args.file: + if args.status: + return + parser.print_help(sys.stderr) + return + + if args.test and not args.recon: + if not args.json: + err_console.print("[yellow]--test works with --recon; ignoring.[/yellow]") + args.test = False + + if args.domain: + domain_input: Union[str, List[str]] = args.domain + verbose_enabled = ( + args.verbose + and isinstance(domain_input, str) + and not args.recon + and not args.bruteforce + and not args.wildcard + ) + if args.verbose and not verbose_enabled and not args.json: + err_console.print( + "[yellow]--verbose works only with single-domain scan without --recon/--bruteforce/--wildcard.[/yellow]" + ) + settings = { + "dns": dns, + "useragent": effective_useragent, + "timeout": timeout, + "threads": threads, + "recon": args.recon, + "bruteforce": args.bruteforce, + "wordlist": wordlist_path, + "wildcard": args.wildcard, + "api_key_virustotal": bool(api_key_virustotal), + "api_key_shodan": bool(api_key_shodan), + } + + mode = "domain" + scan_target = domain_input if isinstance(domain_input, str) else "multi-domain" + recon_test_rows: Optional[List[Dict[str, Union[str, int, bool, None]]]] = None + recon_test_domain: Optional[str] = domain_input if isinstance(domain_input, str) else (domain_input[0] if domain_input else None) + if args.test and args.recon and recon_test_domain: + recon_test_rows = _run_recon_test( + recon_test_domain, + timeout=timeout, + useragent=effective_useragent, + api_key_virustotal=api_key_virustotal, + api_key_shodan=api_key_shodan, + ) + if not args.json: + _render_recon_test_table(recon_test_domain, recon_test_rows) + + if args.wildcard and isinstance(domain_input, str): + wildcard_domain = Bruteforce(domain_input).wildcard() + mode = "wildcard" + scan_target = wildcard_domain + if not args.silent: + _render_runtime_status_panel( + target=str(scan_target), + mode=mode, + source="domain", + timeout=timeout, + threads=threads, + target_count=1, + dns=dns, + useragent=effective_useragent, + recon=False, + bruteforce=False, + wildcard=True, + wordlist=wordlist_path, + silent=args.silent, + api_key_virustotal=api_key_virustotal, + api_key_shodan=api_key_shodan, + ) + start_time = datetime.now() + results = KNOCKPY( + wildcard_domain, + dns=dns, + useragent=effective_useragent, + timeout=timeout, + threads=threads, + recon=False, + bruteforce=False, + wordlist=None, + silent=True, + api_key_virustotal=api_key_virustotal, + api_key_shodan=api_key_shodan, + ) + elapsed = datetime.now() - start_time + if results: + if not args.json: + err_console.print("[yellow]Wildcard seems active for this domain.[/yellow]") + for suggestion in _wildcard_exclude_suggestions(results if isinstance(results, dict) else None): + console.print(f"[cyan]{suggestion}[/cyan]") + consistent_len = _wildcard_consistent_length( + domain_input, + results, + dns=dns, + useragent=effective_useragent, + timeout=timeout, + threads=threads, + api_key_virustotal=api_key_virustotal, + api_key_shodan=api_key_shodan, + ) + if consistent_len is not None: + console.print(f"[cyan]Try excluding wildcard body length: --exclude lenght {consistent_len}[/cyan]") + else: + console.print( + "[yellow]Wildcard body length is not consistent across random probes; skipping length suggestion.[/yellow]" + ) + if args.json: + _print_json_output(results) + else: + output(results, elapsed) + return + + start_time = datetime.now() + domains = [domain_input] if isinstance(domain_input, str) else list(domain_input) + if not args.silent: + _render_runtime_status_panel( + target=", ".join(domains[:3]) + (" ..." if len(domains) > 3 else ""), + mode=( + "recon+bruteforce" + if args.recon and args.bruteforce + else "recon" + if args.recon + else "bruteforce" + if args.bruteforce + else "domain" + ), + source="domain", + timeout=timeout, + threads=threads, + target_count=len(domains), + dns=dns, + useragent=effective_useragent, + recon=args.recon, + bruteforce=args.bruteforce, + wildcard=args.wildcard, + wordlist=wordlist_path, + silent=args.silent, + api_key_virustotal=api_key_virustotal, + api_key_shodan=api_key_shodan, + ) + if args.bruteforce and not args.json: + console.print(f"[cyan]{_wordlist_start_message(wordlist_path)}[/cyan]") + + if args.silent: + results = _run_coro_sync( + _run_async( + domains, + dns, + effective_useragent, + timeout, + threads, + args.recon, + args.bruteforce, + wordlist_path, + verbose=verbose_enabled, + enable_axfr=True, + api_key_virustotal=api_key_virustotal, + api_key_shodan=api_key_shodan, + ) + ) + else: + results = _run_with_rich_progress( + domains, + dns, + effective_useragent, + timeout, + threads, + args.recon, + args.bruteforce, + wordlist_path, + verbose=verbose_enabled, + enable_axfr=True, + api_key_virustotal=api_key_virustotal, + api_key_shodan=api_key_shodan, + ) + + elapsed = datetime.now() - start_time + results, excluded_count = _apply_exclude_rules(results, exclude_rules) + if excluded_count > 0 and not args.silent and not args.json: + console.print(f"[yellow]Excluded results:[/yellow] {excluded_count}") + if args.recon and args.bruteforce: + mode = "recon+bruteforce" + elif args.recon: + mode = "recon" + elif args.bruteforce: + mode = "bruteforce" + + if _should_save_scan(args.recon, args.bruteforce): + report_id = save_scan(str(scan_target), mode, settings, results, elapsed) + if not args.silent and not args.json: + console.print(f"[green]Saved report #[/green]{report_id}") + if args.json: + if recon_test_rows is not None: + _print_json_output({"recon_test": recon_test_rows, "results": results}) + else: + _print_json_output(results) + return + output(results, elapsed) + if verbose_enabled and not args.json: + output_verbose(results) + return + + if args.file: + if not Path(args.file).is_file(): + err_console.print(f"[red]File not found:[/red] {args.file}") + return + + try: + domains = _load_domains_from_file(args.file) + except Exception as exc: + err_console.print(f"[red]Cannot read file:[/red] {args.file} ({exc})") + return + + domains = _normalize_domain_list(domains) + if not domains: + err_console.print(f"[yellow]No domains found in file:[/yellow] {args.file}") + return + + settings = { + "dns": dns, + "useragent": effective_useragent, + "timeout": timeout, + "threads": threads, + "recon": False, + "bruteforce": False, + "source_file": args.file, + } + + start_time = datetime.now() + if not args.silent: + _render_runtime_status_panel( + target=f"{len(domains)} domains", + mode="file", + source=args.file, + timeout=timeout, + threads=threads, + target_count=len(domains), + dns=dns, + useragent=effective_useragent, + recon=False, + bruteforce=False, + wildcard=False, + wordlist=None, + silent=args.silent, + api_key_virustotal=api_key_virustotal, + api_key_shodan=api_key_shodan, + ) + if args.silent: + results = _run_coro_sync( + _run_async( + domains, + dns, + effective_useragent, + timeout, + threads, + recon=False, + bruteforce=False, + wordlist=None, + verbose=False, + enable_axfr=False, + ) + ) + else: + results = _run_with_rich_progress( + domains, + dns, + effective_useragent, + timeout, + threads, + recon=False, + bruteforce=False, + wordlist=None, + verbose=False, + enable_axfr=False, + ) + + elapsed = datetime.now() - start_time + results, excluded_count = _apply_exclude_rules(results, exclude_rules) + if excluded_count > 0 and not args.silent and not args.json: + console.print(f"[yellow]Excluded results:[/yellow] {excluded_count}") + if args.json: + _print_json_output(results) + else: + output(results, elapsed) + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + err_console.print("\n[yellow]Interrupted[/yellow]") + try: + sys.exit(0) + except SystemExit: + os._exit(0) diff --git a/knockpy/cli_parts/__init__.py b/knockpy/cli_parts/__init__.py new file mode 100644 index 0000000..15cbde5 --- /dev/null +++ b/knockpy/cli_parts/__init__.py @@ -0,0 +1 @@ +"""CLI support modules grouped by concern.""" diff --git a/knockpy/cli_parts/report.py b/knockpy/cli_parts/report.py new file mode 100644 index 0000000..9f1a779 --- /dev/null +++ b/knockpy/cli_parts/report.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import sys +from datetime import timedelta +from typing import Any, Dict, List, Optional, Tuple + +from rich.panel import Panel +from rich.table import Table + +from ..output import console, err_console, output, show_reports_catalog +from ..storage import delete_report, export_report, get_report, list_reports, reset_reports +from ..storage_parts.export import _rows_for_export +from ..storage_parts.html_report import _detail_issues + + +def _parse_report_ids(raw_ids: str) -> List[int]: + ids: List[int] = [] + seen: set[int] = set() + for chunk in raw_ids.split(","): + value = chunk.strip() + if not value: + continue + if not value.isdigit(): + raise ValueError(f"Invalid report id: {value}") + report_id = int(value) + if report_id not in seen: + seen.add(report_id) + ids.append(report_id) + if not ids: + raise ValueError("No report IDs provided") + return ids + + +def _show_single_report(report: dict) -> None: + elapsed = None + if report.get("elapsed_seconds") is not None: + elapsed = timedelta(seconds=float(report["elapsed_seconds"])) + + console.print( + Panel.fit( + f"[bold]Report[/bold] #{report['id']} [bold]Target:[/bold] {report['target']} " + f"[bold]Created:[/bold] {report['created_at']} [bold]Mode:[/bold] {report['mode']}", + border_style="blue", + ) + ) + output(report.get("results"), elapsed) + + +def _describe_report_findings(report: dict) -> None: + results = report.get("results") or [] + if not results: + console.print("[yellow]No results in this report.[/yellow]") + return + + rows = _rows_for_export(results) + results_by_domain = {str(item.get("domain") or ""): item for item in results} + findings: Dict[str, List[Tuple[str, str]]] = {} + + def _build_evidence(detail_obj: Dict[str, Any], row: Dict[str, Any]) -> str: + takeover = detail_obj.get("takeover") or {} + takeover_status = str(takeover.get("status") or "").strip().lower() + takeover_provider = str(takeover.get("provider") or "-").strip() + takeover_cname = str(takeover.get("cname") or "-").strip() + http = detail_obj.get("http") or [] + https = detail_obj.get("https") or [] + cert = detail_obj.get("cert") or [] + http_status = http[0] if len(http) > 0 else row.get("http_status") + https_status = https[0] if len(https) > 0 else row.get("https_status") + cert_expiry = cert[1] if len(cert) > 1 else row.get("cert_expiry") + tls_versions = cert[3] if len(cert) > 3 else row.get("tls_versions") + parts: List[str] = [] + + if http_status is not None or https_status is not None: + parts.append(f"HTTP={http_status if http_status is not None else '-'} HTTPS={https_status if https_status is not None else '-'}") + if takeover_status in {"likely", "possible"}: + parts.append(f"takeover={takeover_status} provider={takeover_provider} cname={takeover_cname}") + if cert_expiry: + parts.append(f"cert_expiry={cert_expiry}") + if isinstance(tls_versions, list) and tls_versions: + parts.append(f"tls={', '.join(str(v) for v in tls_versions)}") + + return " | ".join(parts) if parts else "-" + + for row in rows: + domain = str(row.get("domain") or "").strip() + detail_obj = results_by_domain.get(domain, {}) + issues = _detail_issues(detail_obj, row) + for level, text in issues: + if str(level).strip().lower() == "ok": + continue + evidence = _build_evidence(detail_obj, row) + findings.setdefault(str(text), []).append((domain, evidence)) + + console.print( + Panel.fit( + f"[bold]Findings[/bold] report #{report['id']} " + f"[bold]Target:[/bold] {report['target']} " + f"[bold]Created:[/bold] {report['created_at']}", + border_style="magenta", + ) + ) + if not findings: + console.print("- No critical confirmed issue detected.") + return + + ordered = sorted(findings.items(), key=lambda item: len(item[1]), reverse=True) + for issue_text, entries in ordered: + uniq_entries: List[Tuple[str, str]] = [] + seen_domains: set[str] = set() + for domain, evidence in entries: + if not domain or domain in seen_domains: + continue + seen_domains.add(domain) + uniq_entries.append((domain, evidence)) + + console.print( + Panel.fit( + f"[bold red]Problem:[/bold red] {issue_text}\n" + f"[bold]Affected domains:[/bold] {len(uniq_entries)}", + border_style="red", + ) + ) + table = Table(box=None, show_header=True, header_style="bold cyan") + table.add_column("Domain", overflow="fold") + table.add_column("Evidence", overflow="fold") + for domain, evidence in uniq_entries: + table.add_row(domain, evidence) + console.print(table) + + +def report_mode(report_selector: Optional[str]) -> None: + """Interactive report manager used by `--report`. + + It supports listing, searching, showing, deleting, exporting and findings, + including multi-selection for delete/export operations. + """ + # Keep non-interactive behavior available for direct selectors. + if not sys.stdin.isatty(): + if report_selector and report_selector not in {"choose", "list"}: + report = get_report(report_selector) + if not report: + err_console.print(f"[red]Report not found:[/red] {report_selector}") + return + _show_single_report(report) + return + show_reports_catalog(list_reports(limit=100)) + err_console.print("[yellow]Interactive report menu requires a TTY.[/yellow]") + return + + search_term = "" + try: + while True: + reports = list_reports(limit=100) + if search_term: + filtered_reports = [r for r in reports if search_term in str(r.get("target", "")).lower()] + else: + filtered_reports = reports + + show_reports_catalog(filtered_reports) + + console.print("1 show") + console.print("2 delete") + console.print("3 export") + console.print("4 search") + console.print("5 findings") + console.print("99 reset db") + action_choice = input("Select action [1-5,99] (Enter to exit): ").strip() + if action_choice == "": + return + action_map = {"99": "reset", "1": "show", "2": "delete", "3": "export", "4": "search", "5": "findings"} + action = action_map.get(action_choice) + if action is None: + err_console.print("[red]Invalid action.[/red] Use: 1, 2, 3, 4, 5, 99.") + continue + + if action == "reset": + confirm = input("Type RESET to confirm DB reset (all reports will be deleted): ").strip() + if confirm != "RESET": + console.print("[yellow]Reset cancelled.[/yellow]") + continue + deleted = reset_reports() + console.print(f"[green]DB reset completed.[/green] Removed reports: {deleted}") + continue + + if action == "search": + term = input("Search domain (partial, empty = reset): ").strip().lower() + search_term = term + continue + + if action == "show": + selector = input("Report ID or target [latest]: ").strip() or "latest" + report = get_report(selector) + if not report: + err_console.print(f"[red]Report not found:[/red] {selector}") + continue + _show_single_report(report) + continue + + if action == "findings": + selector = input("Report ID or target [latest]: ").strip() or "latest" + report = get_report(selector) + if not report: + err_console.print(f"[red]Report not found:[/red] {selector}") + continue + _describe_report_findings(report) + continue + + if action == "delete": + raw_ids = input("Report IDs (comma-separated): ").strip() + try: + report_ids = _parse_report_ids(raw_ids) + except ValueError as exc: + err_console.print(f"[red]{exc}[/red]") + continue + + deleted_count = 0 + for report_id in report_ids: + if delete_report(report_id): + deleted_count += 1 + console.print(f"[green]Report deleted:[/green] #{report_id}") + else: + err_console.print(f"[red]Report not found:[/red] {report_id}") + console.print(f"[cyan]Delete summary:[/cyan] {deleted_count}/{len(report_ids)} deleted") + continue + + if action == "export": + fmt = "html" + console.print("[cyan]Export format:[/cyan] html") + raw_ids = input("Report IDs (comma-separated): ").strip() + try: + report_ids = _parse_report_ids(raw_ids) + except ValueError as exc: + err_console.print(f"[red]{exc}[/red]") + continue + + exported_count = 0 + for report_id in report_ids: + report = get_report(str(report_id)) + if not report: + err_console.print(f"[red]Report not found:[/red] {report_id}") + continue + try: + export_file = export_report(report, fmt) + console.print(f"[green]Export completed:[/green] {export_file}") + exported_count += 1 + except Exception as exc: + err_console.print(f"[red]Export failed for #{report_id}:[/red] {exc}") + console.print(f"[cyan]Export summary:[/cyan] {exported_count}/{len(report_ids)} exported") + except KeyboardInterrupt: + console.print("\n[yellow]Report mode interrupted.[/yellow]") + return diff --git a/knockpy/cli_parts/scan_flow.py b/knockpy/cli_parts/scan_flow.py new file mode 100644 index 0000000..b772418 --- /dev/null +++ b/knockpy/cli_parts/scan_flow.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Union + +from rich import box +from rich.table import Table + +from ..core import Bruteforce, KNOCKPY, Recon, _run_coro_sync +from ..output import console +from ..server_versions import CATALOG_PATH + + +def _compact_home(path: Path) -> str: + home = Path.home().resolve() + resolved = path.expanduser().resolve() + try: + rel = resolved.relative_to(home) + return f"~/{rel.as_posix()}" if str(rel) != "." else "~" + except ValueError: + return str(resolved) + + +def parse_exclude_rules(excludes: Optional[List[List[str]]]) -> Dict[str, Union[Set[int], List[str]]]: + """Validate and normalize `--exclude` rules for fast runtime filtering.""" + rules: Dict[str, Union[Set[int], List[str]]] = {"status": set(), "body": [], "length": set()} + if not excludes: + return rules + + for raw_type, raw_value in excludes: + rule_type = str(raw_type or "").strip().lower() + value = str(raw_value or "").strip() + if rule_type == "status": + for part in value.split(","): + part = part.strip() + if not part: + continue + if not part.isdigit(): + raise ValueError(f"Invalid status in --exclude: {part}") + status_code = int(part) + if status_code < 100 or status_code > 599: + raise ValueError(f"Out-of-range status in --exclude: {status_code}") + cast_set: Set[int] = rules["status"] # type: ignore[assignment] + cast_set.add(status_code) + elif rule_type in {"length", "lenght"}: + for part in value.split(","): + part = part.strip().lower() + if not part: + continue + match = re.search(r"(\d+)", part) + if not match: + raise ValueError(f"Invalid length in --exclude: {part}") + length_value = int(match.group(1)) + if length_value < 0: + raise ValueError(f"Invalid length in --exclude: {part}") + cast_len: Set[int] = rules["length"] # type: ignore[assignment] + cast_len.add(length_value) + elif rule_type == "body": + if not value: + raise ValueError("--exclude body requires non-empty text") + cast_list: List[str] = rules["body"] # type: ignore[assignment] + cast_list.append(value.lower()) + else: + raise ValueError(f"Unsupported --exclude type: {rule_type} (use 'status', 'length', or 'body')") + return rules + + +def _result_matches_exclude(item: dict, rules: Dict[str, Union[Set[int], List[str]]]) -> bool: + status_rules: Set[int] = rules["status"] # type: ignore[assignment] + body_rules: List[str] = rules["body"] # type: ignore[assignment] + length_rules: Set[int] = rules["length"] # type: ignore[assignment] + + http = item.get("http") or [] + https = item.get("https") or [] + http_status = http[0] if len(http) > 0 and isinstance(http[0], int) else None + https_status = https[0] if len(https) > 0 and isinstance(https[0], int) else None + if status_rules and (http_status in status_rules or https_status in status_rules): + return True + + http_len = http[3] if len(http) > 3 and isinstance(http[3], int) else None + https_len = https[3] if len(https) > 3 and isinstance(https[3], int) else None + if length_rules and (http_len in length_rules or https_len in length_rules): + return True + + if body_rules: + http_body = str(http[5]).lower() if len(http) > 5 and http[5] is not None else "" + https_body = str(https[5]).lower() if len(https) > 5 and https[5] is not None else "" + for needle in body_rules: + if needle in http_body or needle in https_body: + return True + return False + + +def apply_exclude_rules( + results: Union[dict, List[dict], None], + rules: Dict[str, Union[Set[int], List[str]]], +) -> tuple[Union[dict, List[dict], None], int]: + if not results: + return results, 0 + + if isinstance(results, dict): + return (None, 1) if _result_matches_exclude(results, rules) else (results, 0) + + filtered: List[dict] = [] + excluded = 0 + for item in results: + if _result_matches_exclude(item, rules): + excluded += 1 + else: + filtered.append(item) + return filtered, excluded + + +def print_json_output(results: Union[dict, List[dict], None]) -> None: + try: + sys.stdout.write(json.dumps(results, ensure_ascii=False, indent=2)) + sys.stdout.write("\n") + except BrokenPipeError: + # Preserve CLI behavior on piped output (e.g. `| head`) without traceback noise. + return + + +def run_recon_test( + domain: str, + timeout: float, + useragent: str, + api_key_virustotal: Optional[str], + api_key_shodan: Optional[str], +) -> List[Dict[str, Union[str, int, bool, None]]]: + """Probe all enabled recon providers and return per-provider diagnostics.""" + engine = Recon( + domain, + timeout=timeout, + max_concurrency=40, + useragent=useragent, + virustotal_key=api_key_virustotal, + shodan_key=api_key_shodan, + ) + return _run_coro_sync(engine.test_services()) + + +def render_recon_test_table(domain: str, rows: List[Dict[str, Union[str, int, bool, None]]]) -> None: + table = Table(title=f"Recon Services Test: {domain}", box=box.SIMPLE_HEAVY) + table.add_column("Service", style="cyan") + table.add_column("HTTP", justify="right") + table.add_column("Parsed", justify="right") + table.add_column("Status") + table.add_column("Error", overflow="fold") + + ok_count = 0 + data_count = 0 + for row in rows: + status_code = row.get("status_code") + parsed_count = int(row.get("parsed_count") or 0) + ok = bool(row.get("ok")) + error = str(row.get("error") or "-") + if ok: + ok_count += 1 + if parsed_count > 0: + data_count += 1 + + if parsed_count > 0: + status_text = "[green]data[/green]" + elif ok: + status_text = "[yellow]empty[/yellow]" + else: + status_text = "[red]failed[/red]" + http_text = str(status_code if status_code is not None else "-") + table.add_row(str(row.get("service") or "-"), http_text, str(parsed_count), status_text, error) + + console.print(table) + console.print(f"[cyan]Recon test summary:[/cyan] reachable={ok_count}/{len(rows)} | with_data={data_count}/{len(rows)}") + + +def print_server_versions_catalog(catalog: Dict[str, Any]) -> None: + products = catalog.get("products") or {} + table = Table(title="Web Server Versions Catalog", box=box.SIMPLE_HEAVY) + table.add_column("Product", style="cyan") + table.add_column("Latest") + table.add_column("Mode") + table.add_column("Source", overflow="fold") + for key, item in products.items(): + if not isinstance(item, dict): + continue + label = str(item.get("label") or key) + latest = str(item.get("latest") or "-") + mode = str(item.get("check_mode") or "strict") + source = str(item.get("source") or "-") + table.add_row(label, latest, mode, source) + console.print(table) + updated_at = catalog.get("updated_at") or "-" + console.print(f"[cyan]Catalog path:[/cyan] {_compact_home(CATALOG_PATH)}") + console.print(f"[cyan]Updated at:[/cyan] {updated_at}") + + +def wildcard_exclude_suggestions(result: Optional[dict]) -> List[str]: + if not result: + return [] + suggestions: List[str] = [] + http = result.get("http") or [] + https = result.get("https") or [] + status_codes: List[int] = [] + for candidate in (http[0] if len(http) > 0 else None, https[0] if len(https) > 0 else None): + if isinstance(candidate, int) and candidate not in status_codes: + status_codes.append(candidate) + if status_codes: + status_csv = ",".join(str(s) for s in status_codes) + suggestions.append(f"Try excluding by status: --exclude status {status_csv}") + + return suggestions + + +def _preferred_body_length(result: Optional[Union[dict, List[dict]]]) -> Optional[int]: + if not isinstance(result, dict): + return None + http = result.get("http") or [] + https = result.get("https") or [] + https_len = https[3] if len(https) > 3 and isinstance(https[3], int) else None + http_len = http[3] if len(http) > 3 and isinstance(http[3], int) else None + return https_len if https_len is not None else http_len + + +def wildcard_consistent_length( + base_domain: str, + first_result: Optional[Union[dict, List[dict]]], + dns: str, + useragent: str, + timeout: float, + threads: Optional[int], + api_key_virustotal: Optional[str], + api_key_shodan: Optional[str], + extra_samples: int = 3, +) -> Optional[int]: + first_len = _preferred_body_length(first_result) + if first_len is None: + return None + + lengths: List[int] = [first_len] + for _ in range(extra_samples): + probe_domain = Bruteforce(base_domain).wildcard() + probe_result = KNOCKPY( + probe_domain, + dns=dns, + useragent=useragent, + timeout=timeout, + threads=threads, + recon=False, + bruteforce=False, + wordlist=None, + silent=True, + api_key_virustotal=api_key_virustotal, + api_key_shodan=api_key_shodan, + ) + probe_len = _preferred_body_length(probe_result) + if probe_len is None: + return None + lengths.append(probe_len) + + return first_len if all(value == first_len for value in lengths) else None diff --git a/knockpy/cli_parts/setup.py b/knockpy/cli_parts/setup.py new file mode 100644 index 0000000..e7d0eb6 --- /dev/null +++ b/knockpy/cli_parts/setup.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Optional + +from rich import box +from rich.panel import Panel +from rich.table import Table + +from ..core import ROOT +from ..output import console, err_console +from ..storage import get_settings, set_setting + + +def _compact_home(path: Path) -> str: + home = Path.home().resolve() + resolved = path.expanduser().resolve() + try: + rel = resolved.relative_to(home) + return f"~/{rel.as_posix()}" if str(rel) != "." else "~" + except ValueError: + return str(resolved) + + +def _normalize_optional(value: Optional[str]) -> Optional[str]: + if value is None: + return None + text = value.strip() + return text if text else None + + +def load_saved_runtime_settings() -> dict: + saved = get_settings() + + def _parse_float(value: Optional[str], default: float) -> float: + if value is None: + return default + try: + return float(value) + except ValueError: + return default + + def _parse_int(value: Optional[str]) -> Optional[int]: + if value is None or value == "": + return None + try: + return int(value) + except ValueError: + return None + + return { + "dns": saved.get("runtime.dns") or "8.8.8.8", + "useragent": saved.get("runtime.useragent") or "random", + "timeout": _parse_float(saved.get("runtime.timeout"), 3.0), + "threads": _parse_int(saved.get("runtime.threads")) or 250, + "wordlist": _normalize_optional(saved.get("runtime.wordlist")), + "api_key_virustotal": _normalize_optional(saved.get("apikey.virustotal")), + "api_key_shodan": _normalize_optional(saved.get("apikey.shodan")), + } + + +def setup_mode() -> None: + """Interactive setup editor for persisted runtime defaults and API keys.""" + if not sys.stdin.isatty(): + err_console.print("[red]--setup requires interactive terminal.[/red]") + return + + current = load_saved_runtime_settings() + config = { + "dns": current["dns"], + "useragent": current["useragent"], + "timeout": float(current["timeout"]), + "threads": current["threads"], + "wordlist": current["wordlist"], + "api_key_virustotal": current["api_key_virustotal"], + "api_key_shodan": current["api_key_shodan"], + } + default_wordlist_path = _compact_home(ROOT / "wordlist" / "wordlist.txt") + console.print(Panel.fit("Knockpy Setup", border_style="blue")) + console.print("Select ID 1-7 to edit a single field. Use 0 to save and exit.") + console.print("Use '-' to clear optional values (wordlist/API keys, or threads=auto).") + + def _render_table(title: str) -> None: + table = Table(title=title, box=box.SIMPLE_HEAVY) + table.add_column("ID", style="cyan", justify="right") + table.add_column("Key", style="cyan") + table.add_column("Value", overflow="fold") + table.add_row("1", "DNS", str(config["dns"])) + table.add_row( + "2", + "User-Agent", + "random (browser)" if str(config["useragent"]).strip().lower() == "random" else str(config["useragent"]), + ) + table.add_row("3", "Timeout", str(config["timeout"])) + table.add_row("4", "Threads", str(config["threads"]) if config["threads"] is not None else "auto") + table.add_row( + "5", + "Wordlist", + str(config["wordlist"]) if config["wordlist"] else f"default ({default_wordlist_path})", + ) + table.add_row("6", "VirusTotal API", str(config["api_key_virustotal"] or "")) + table.add_row("7", "Shodan API", str(config["api_key_shodan"] or "")) + console.print(table) + + def _ask_text(label: str, current_value: Optional[str], optional: bool = False) -> Optional[str]: + prompt = f"{label} [{current_value if current_value is not None else ''}]: " + raw = input(prompt).strip() + if raw == "": + return current_value + if optional and raw == "-": + return None + return raw + + while True: + _render_table("Current Setup") + console.print("0 save and exit") + choice = input("Select field [1-7] or 0 to save: ").strip() + if choice == "0": + break + if choice == "1": + config["dns"] = _ask_text("DNS server", str(config["dns"])) or "8.8.8.8" + continue + if choice == "2": + config["useragent"] = _ask_text("User-Agent (or 'random')", str(config["useragent"])) or "random" + continue + if choice == "3": + raw_timeout = input(f"Timeout seconds [{config['timeout']}]: ").strip() + if raw_timeout: + try: + config["timeout"] = float(raw_timeout) + except ValueError: + err_console.print("[yellow]Invalid timeout, value unchanged.[/yellow]") + continue + if choice == "4": + raw_threads = input(f"Threads [{config['threads'] if config['threads'] is not None else 'auto'}]: ").strip() + if raw_threads: + if raw_threads == "-": + config["threads"] = None + else: + try: + config["threads"] = int(raw_threads) + except ValueError: + err_console.print("[yellow]Invalid threads, value unchanged.[/yellow]") + continue + if choice == "5": + config["wordlist"] = _ask_text("Wordlist path", config["wordlist"], optional=True) + continue + if choice == "6": + config["api_key_virustotal"] = _ask_text("API key VirusTotal", config["api_key_virustotal"], optional=True) + continue + if choice == "7": + config["api_key_shodan"] = _ask_text("API key Shodan", config["api_key_shodan"], optional=True) + continue + err_console.print("[red]Invalid selection.[/red] Use 0-7.") + + set_setting("runtime.dns", str(config["dns"])) + set_setting("runtime.useragent", str(config["useragent"])) + set_setting("runtime.timeout", str(config["timeout"])) + set_setting("runtime.threads", None if config["threads"] is None else str(config["threads"])) + set_setting("runtime.wordlist", config["wordlist"]) + set_setting("apikey.virustotal", config["api_key_virustotal"]) + set_setting("apikey.shodan", config["api_key_shodan"]) + + _render_table("Saved Setup") diff --git a/knockpy/cli_parts/status.py b/knockpy/cli_parts/status.py new file mode 100644 index 0000000..913525c --- /dev/null +++ b/knockpy/cli_parts/status.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import os +from pathlib import Path +from typing import Optional + +from rich import box +from rich.columns import Columns +from rich.console import Group +from rich.panel import Panel +from rich.table import Table + +from ..core import ROOT +from ..output import console +from ..storage import get_db_path +from ..version import __version__ + + +def _compact_home(path: Path) -> str: + home = Path.home().resolve() + resolved = path.expanduser().resolve() + try: + rel = resolved.relative_to(home) + return f"~/{rel.as_posix()}" if str(rel) != "." else "~" + except ValueError: + return str(resolved) + + +def render_runtime_status_panel( + target: str, + mode: str, + source: str, + timeout: float, + threads: Optional[int], + target_count: Optional[int], + dns: str, + useragent: str, + recon: bool, + bruteforce: bool, + wildcard: bool, + wordlist: Optional[str], + silent: bool, + api_key_virustotal: Optional[str] = None, + api_key_shodan: Optional[str] = None, +) -> None: + """Render the startup header with runtime, scan modes and API-key status.""" + term_width = console.size.width + runtime_width = 80 + half_width = 40 + card_height = 13 + + key_col_width = 11 + value_col_width = runtime_width - key_col_width - 6 + + def _fit_value(value: object) -> str: + text = str(value) + # Keep cells on one line so all cards remain visually aligned. + max_len = max(20, value_col_width) + if len(text) <= max_len: + return text + if max_len <= 3: + return "." * max_len + return f"{text[: max_len - 3]}..." + + default_wordlist = ROOT / "wordlist" / "wordlist.txt" + wordlist_display = wordlist or _compact_home(default_wordlist) + + if threads is None: + if target_count is None: + threads_value = "auto (dynamic)" + else: + threads_value = f"auto ({min(300, max(20, target_count))})" + else: + threads_value = str(threads) + + status = Table(box=box.MINIMAL, show_header=False, pad_edge=False, expand=False) + status.add_column("Key", width=key_col_width, no_wrap=True) + status.add_column("Value", width=value_col_width, no_wrap=True, overflow="crop") + status.add_row("Target", _fit_value(target)) + status.add_row("Mode", _fit_value(mode)) + status.add_row("Timeout", _fit_value(timeout)) + status.add_row("Threads", _fit_value(threads_value)) + status.add_row("DNS", _fit_value(dns)) + status.add_row("User-Agent", _fit_value(useragent)) + status.add_row("Wordlist", _fit_value(wordlist_display)) + status.add_row("Reports DB", _fit_value(_compact_home(get_db_path()))) + status.add_row("Silent", _fit_value(silent)) + + scan_modes = Table(title="Scans", box=box.SIMPLE_HEAVY) + scan_modes.add_column("Mode", style="cyan") + scan_modes.add_column("Status", style="white") + scan_modes.add_row("Recon", "โœ… enabled" if recon else "โŒ disabled") + scan_modes.add_row("Bruteforce", "โœ… enabled" if bruteforce else "โŒ disabled") + scan_modes.add_row("Wildcard", "โœ… enabled" if wildcard else "โŒ disabled") + + api_keys = Table(title="API Keys", box=box.SIMPLE_HEAVY) + api_keys.add_column("Provider", style="cyan") + api_keys.add_column("Status", style="white") + vt_enabled = bool(api_key_virustotal or os.getenv("API_KEY_VIRUSTOTAL")) + shodan_enabled = bool(api_key_shodan or os.getenv("API_KEY_SHODAN")) + api_keys.add_row("VirusTotal", "โœ… enabled" if vt_enabled else "โŒ disabled") + api_keys.add_row("Shodan", "โœ… enabled" if shodan_enabled else "โŒ disabled") + + # Fixed dimensions: Runtime 80, Scans 40 + API Keys 40. + status_panel = Panel(status, title="Runtime", border_style="cyan", width=runtime_width, height=card_height) + scans_panel = Panel(scan_modes, title="Scans", border_style="cyan", width=half_width, height=card_height) + keys_panel = Panel(api_keys, title="API Keys", border_style="cyan", width=half_width, height=card_height) + + if term_width >= 90: + # No horizontal gap so bottom row width is exactly 40 + 40 = 80. + content = Group(status_panel, Columns([scans_panel, keys_panel], equal=True, expand=False, padding=0)) + else: + content = Group(status_panel, scans_panel, keys_panel) + + outer_width = runtime_width + 4 + console.print(Panel(content, title=f"Knockpy v{__version__}", border_style="blue", width=outer_width, expand=False)) + + +def wordlist_start_message(wordlist: Optional[str]) -> str: + default_wordlist = ROOT / "wordlist" / "wordlist.txt" + if wordlist: + return f"Wordlist selected: custom ({wordlist})" + return f"Wordlist selected: default ({_compact_home(default_wordlist)})" diff --git a/knockpy/core.py b/knockpy/core.py new file mode 100644 index 0000000..48e2905 --- /dev/null +++ b/knockpy/core.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +"""Compatibility facade for knockpy core engine. + +Public imports remain stable while implementation lives in `knockpy.engine.runtime`. +""" + +from .engine.runtime import * # noqa: F401,F403 +from .engine.runtime import _run_async, _run_coro_sync + +__all__ = [ + "ROOT", + "Bruteforce", + "Recon", + "AsyncScanner", + "KNOCKPY", + "pick_user_agent", + "fmt_td", + "logger", + "_run_async", + "_run_coro_sync", +] diff --git a/knockpy/engine/__init__.py b/knockpy/engine/__init__.py new file mode 100644 index 0000000..68ff1c8 --- /dev/null +++ b/knockpy/engine/__init__.py @@ -0,0 +1 @@ +"""Scanning engine modules.""" diff --git a/knockpy/engine/runtime.py b/knockpy/engine/runtime.py new file mode 100644 index 0000000..1355f4e --- /dev/null +++ b/knockpy/engine/runtime.py @@ -0,0 +1,2790 @@ +from __future__ import annotations + +"""Core scanning engine for knockpy. + +This module contains the runtime used by both CLI and Python API: +- domain expansion (`Recon`, `Bruteforce`) +- asynchronous DNS/HTTP/HTTPS checks (`AsyncScanner`) +- orchestration helpers (`_run_async`, `KNOCKPY`) + +Keep logic in this file side-effect free where possible, because it is imported +from both `knockpy/cli.py` and external user scripts. +""" + +import asyncio +import hashlib +import ipaddress +import json +import logging +import os +import random +import re +import socket +import ssl +import string +import sys +import threading +import time +from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse +from collections import OrderedDict +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union + +import dns.exception +import dns.query +import dns.resolver +import dns.zone +import httpx +import OpenSSL +from dotenv import load_dotenv + +from ..server_versions import assess_server_banner, load_server_versions_catalog + +load_dotenv() + +ROOT = Path(__file__).resolve().parent.parent +CONFIG_DIR = Path.home() / ".knockpy" +RECON_SERVICES_PATH = CONFIG_DIR / "recon_services.json" + +USER_AGENTS = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 14.4; rv:124.0) Gecko/20100101 Firefox/124.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", +] + +logger = logging.getLogger("knockpy") +logger.setLevel(logging.INFO) +if not logger.handlers: + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s", "%H:%M:%S")) + logger.addHandler(handler) + + +def pick_user_agent(useragent: Optional[str]) -> str: + if useragent and useragent.strip().lower() != "random": + return useragent.strip() + return random.choice(USER_AGENTS) + + +def _www_fallback_domain(domain: str) -> Optional[str]: + normalized = AsyncScanner._normalize_domain(domain) + if not normalized: + return None + if normalized.startswith("www."): + return None + return f"www.{normalized}" + + +def fmt_td(td: Optional[timedelta]) -> str: + if td is None: + return "-" + total_seconds = int(td.total_seconds()) + hours, rem = divmod(total_seconds, 3600) + minutes, seconds = divmod(rem, 60) + return f"{hours:02d}:{minutes:02d}:{seconds:02d}" + + +async def _axfr_check_root(domain: str, dns_server: Optional[str], timeout: float) -> Dict[str, Any]: + loop = asyncio.get_running_loop() + + def sync_axfr() -> Dict[str, Any]: + resolver = dns.resolver.Resolver() + if dns_server: + resolver.nameservers = [dns_server] + resolver.timeout = timeout + resolver.lifetime = max(timeout * 2, timeout + 1.0) + + report: Dict[str, Any] = {"domain": domain, "status": "not-tested", "checks": []} + try: + ns_answers = resolver.resolve(domain, "NS") + ns_list = sorted({str(rr).strip().rstrip(".") for rr in ns_answers if str(rr).strip()}) + except Exception as exc: + report["status"] = "no-ns" + report["error"] = f"{exc.__class__.__name__}: {exc}" + return report + + if not ns_list: + report["status"] = "no-ns" + report["error"] = "No NS records found" + return report + + checks: List[Dict[str, Any]] = [] + for ns in ns_list[:3]: + entry: Dict[str, Any] = {"ns": ns, "status": "failed", "record_count": None, "error": None} + try: + xfr = dns.query.xfr(where=ns, zone=domain, timeout=timeout, lifetime=max(timeout * 2, timeout + 1.0)) + zone = dns.zone.from_xfr(xfr, relativize=False) + if zone and zone.nodes: + entry["status"] = "allowed" + entry["record_count"] = len(zone.nodes) + checks.append(entry) + report["checks"] = checks + report["status"] = "allowed" + report["allowed_ns"] = ns + report["record_count"] = entry["record_count"] + return report + entry["status"] = "denied" + entry["error"] = "Empty AXFR response" + except dns.exception.Timeout: + entry["status"] = "timeout" + entry["error"] = "Timeout" + except Exception as exc: + msg = str(exc) + low = msg.lower() + if "refused" in low: + entry["status"] = "refused" + elif "transfer failed" in low or "not authoritative" in low: + entry["status"] = "denied" + else: + entry["status"] = "failed" + entry["error"] = f"{exc.__class__.__name__}: {exc}" + checks.append(entry) + + report["checks"] = checks + statuses = {str(c.get("status")) for c in checks} + if "refused" in statuses: + report["status"] = "refused" + elif "denied" in statuses: + report["status"] = "denied" + elif "timeout" in statuses and len(statuses) == 1: + report["status"] = "timeout" + elif "failed" in statuses and len(statuses) == 1: + report["status"] = "failed" + else: + report["status"] = "closed" + return report + + return await loop.run_in_executor(None, sync_axfr) + + +class Bruteforce: + _WORDLIST_CACHE: Dict[str, Tuple[float, List[str]]] = {} + + def __init__(self, domain: str, wordlist: Optional[str] = None): + self.domain = domain + self.wordlist = Path(wordlist) if wordlist else ROOT / "wordlist" / "wordlist.txt" + + def load(self) -> List[str]: + try: + wordlist_str = str(self.wordlist) + mtime = self.wordlist.stat().st_mtime + cached = self._WORDLIST_CACHE.get(wordlist_str) + if cached and cached[0] == mtime: + return cached[1] + + words: List[str] = [] + seen: set[str] = set() + with open(self.wordlist, "r", encoding="utf-8", errors="ignore") as fh: + for raw in fh: + word = raw.strip().lower() + if not word or word.startswith("#"): + continue + if word in seen: + continue + seen.add(word) + words.append(word) + + self._WORDLIST_CACHE[wordlist_str] = (mtime, words) + return words + except FileNotFoundError: + logger.error("Wordlist not found: %s", self.wordlist) + return [] + + def wildcard(self) -> str: + rnd = "".join(random.choice(string.ascii_lowercase) for _ in range(random.randint(10, 15))) + return f"{rnd}.{self.domain}" + + def start(self) -> List[str]: + words = self.load() + suffix = f".{self.domain}" + return [f"{w}{suffix}" for w in words if w] + + +class Recon: + """Collect subdomains from configurable external reconnaissance services. + + Used only when `--recon` is enabled (or `recon=True` via Python API). + Services are read from `~/.knockpy/recon_services.json` and validated to + avoid leaking API keys to untrusted endpoints. + """ + HOST_RE = re.compile(r"^[A-Za-z0-9._-]+$") + RAPIDDNS_RE = re.compile(r"([^<]+)", re.IGNORECASE) + REMOVED_SERVICES = {"alienvault", "certspotter", "crtsh", "webarchive", "bufferover", "anubis"} + TRUSTED_API_HOSTS = { + "virustotal": ("virustotal.com",), + "shodan": ("shodan.io",), + } + + def __init__( + self, + domain: str, + timeout: float = 3.0, + max_concurrency: int = 10, + useragent: Optional[str] = None, + virustotal_key: Optional[str] = None, + shodan_key: Optional[str] = None, + ): + self.domain = domain + self.timeout = timeout + self.max_concurrency = max_concurrency + self.headers = {"User-Agent": pick_user_agent(useragent)} + self.vt_key = virustotal_key if virustotal_key is not None else os.getenv("API_KEY_VIRUSTOTAL") + self.shodan_key = shodan_key if shodan_key is not None else os.getenv("API_KEY_SHODAN") + + @staticmethod + def _default_services() -> List[Dict[str, Any]]: + return [ + { + "name": "hackertarget", + "enabled": True, + "parser": "csv_first_column", + "url_template": "https://api.hackertarget.com/hostsearch/?q={domain}", + }, + { + "name": "rapiddns", + "enabled": True, + "parser": "rapiddns_html_td", + "url_template": "https://rapiddns.io/subdomain/{domain}", + }, + { + "name": "subdomaincenter", + "enabled": True, + "parser": "json_list", + "url_template": "https://api.subdomain.center/?domain={domain}", + }, + { + "name": "virustotal", + "enabled": True, + "requires_api": "virustotal", + "parser": "virustotal_subdomains", + "url_template": "https://www.virustotal.com/vtapi/v2/domain/report?apikey={virustotal_key}&domain={domain}", + }, + { + "name": "shodan", + "enabled": True, + "requires_api": "shodan", + "parser": "shodan_subdomains", + "url_template": "https://api.shodan.io/dns/domain/{domain}?key={shodan_key}", + }, + ] + + def _recon_config_path(self) -> Path: + custom = (os.getenv("KNOCK_RECON_SERVICES") or "").strip() + return Path(custom).expanduser() if custom else RECON_SERVICES_PATH + + def _ensure_recon_config(self) -> Path: + path = self._recon_config_path() + if path.is_file(): + try: + os.chmod(path, 0o600) + except Exception: + pass + return path + try: + path.parent.mkdir(parents=True, exist_ok=True) + payload = {"version": 1, "services": self._default_services()} + path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + try: + os.chmod(path, 0o600) + except Exception: + pass + except Exception as exc: + logger.debug("Cannot create recon services config %s: %s", path, exc) + return path + + def _load_service_specs(self) -> List[Dict[str, Any]]: + """Load, sanitize and auto-migrate recon service definitions. + + Why this exists: + - users can customize service list over time + - defaults evolve across releases + - removed/broken services must be filtered out safely + """ + defaults = self._default_services() + defaults_by_name = {str(s.get("name") or "").strip().lower(): s for s in defaults} + path = self._ensure_recon_config() + try: + data = json.loads(path.read_text(encoding="utf-8")) + services = data.get("services") if isinstance(data, dict) else None + if isinstance(services, list): + loaded = [s for s in services if isinstance(s, dict)] + filtered_loaded: List[Dict[str, Any]] = [] + dropped_removed = False + for spec in loaded: + service_name = str(spec.get("name") or "").strip().lower() + if service_name in self.REMOVED_SERVICES: + dropped_removed = True + continue + filtered_loaded.append(spec) + loaded = filtered_loaded + loaded_names = {str(s.get("name") or "").strip().lower() for s in loaded} + missing: List[Dict[str, Any]] = [] + for name, default_spec in defaults_by_name.items(): + if name and name not in loaded_names: + loaded.append(default_spec) + missing.append(default_spec) + if (missing or dropped_removed) and isinstance(data, dict): + try: + data["services"] = loaded + path.write_text(json.dumps(data, indent=2), encoding="utf-8") + try: + os.chmod(path, 0o600) + except Exception: + pass + except Exception as exc: + logger.debug("Cannot update recon services config %s: %s", path, exc) + return loaded + except Exception as exc: + logger.debug("Cannot read recon services config %s: %s", path, exc) + return defaults + + def _render_service_url(self, template: str) -> Optional[str]: + """Render and validate a service URL template. + + This function is a security boundary: it blocks malformed or unsafe URLs + before any network request is sent. + """ + mapping = { + "domain": self.domain, + "virustotal_key": self.vt_key or "", + "shodan_key": self.shodan_key or "", + } + try: + rendered = template.format(**mapping).strip() + except Exception: + return None + if not rendered.startswith(("http://", "https://")): + return None + try: + parsed = urlparse(rendered) + except Exception: + return None + if parsed.scheme.lower() != "https": + return None + if parsed.username or parsed.password: + return None + host = (parsed.hostname or "").strip().lower() + if not host: + return None + # Avoid leaking API keys to local/private endpoints from tampered configs. + if host in {"localhost", "localhost.localdomain"} or host.endswith(".local"): + return None + try: + ip_obj = ipaddress.ip_address(host) + if ip_obj.is_private or ip_obj.is_loopback or ip_obj.is_link_local or ip_obj.is_reserved: + return None + except ValueError: + pass + return rendered + + @staticmethod + def _host_matches_suffix(host: str, suffixes: Tuple[str, ...]) -> bool: + host_l = (host or "").strip().lower() + for suffix in suffixes: + s = suffix.strip().lower() + if not s: + continue + if host_l == s or host_l.endswith("." + s): + return True + return False + + def _redact_url_secrets(self, url: Optional[str]) -> Optional[str]: + if not url: + return url + try: + parts = urlparse(url) + query = parse_qsl(parts.query, keep_blank_values=True) + redacted: List[Tuple[str, str]] = [] + for k, v in query: + key = k.lower() + if key in {"apikey", "api_key", "key", "token", "access_token"}: + redacted.append((k, "***")) + else: + redacted.append((k, v)) + new_query = urlencode(redacted, doseq=True) + return urlunparse((parts.scheme, parts.netloc, parts.path, parts.params, new_query, parts.fragment)) + except Exception: + return url + + def _redact_text_secrets(self, text: Optional[str]) -> Optional[str]: + if not text: + return text + sanitized = text + for secret in (self.vt_key, self.shodan_key): + if secret: + sanitized = sanitized.replace(secret, "***") + return sanitized + + def services(self) -> List[Dict[str, str]]: + """Return executable recon services after policy checks. + + Where used: + - called by `start()` to build request task list + Why: + - centralizes `enabled` filtering, API-key requirements and host policy. + """ + services: List[Dict[str, str]] = [] + for spec in self._load_service_specs(): + name = str(spec.get("name") or "").strip().lower() + parser = str(spec.get("parser") or "").strip().lower() + url_template = str(spec.get("url_template") or "").strip() + enabled = bool(spec.get("enabled", True)) + requires_api = str(spec.get("requires_api") or "").strip().lower() + if not enabled or not name or not parser or not url_template or name in self.REMOVED_SERVICES: + continue + if requires_api == "virustotal" and not self.vt_key: + continue + if requires_api == "shodan" and not self.shodan_key: + continue + url = self._render_service_url(url_template) + if not url: + continue + if requires_api in self.TRUSTED_API_HOSTS: + parsed = urlparse(url) + host = parsed.hostname or "" + if not self._host_matches_suffix(host, self.TRUSTED_API_HOSTS[requires_api]): + logger.debug("Skipping %s: API key target host not trusted (%s)", name, host) + continue + services.append({"name": name, "parser": parser, "url": url}) + return services + + async def _fetch(self, client: httpx.AsyncClient, name: str, url: str) -> Tuple[str, str]: + fetch = await self._fetch_with_meta(client, url) + return name, str(fetch.get("text") or "") + + async def _fetch_with_meta(self, client: httpx.AsyncClient, url: str) -> Dict[str, Any]: + try: + response = await client.get(url, timeout=self.timeout, follow_redirects=True) + response.raise_for_status() + return { + "ok": True, + "status_code": response.status_code, + "text": response.text, + "bytes": len(response.content or b""), + "final_url": str(response.url), + "error": None, + } + except Exception as exc: # pragma: no cover - network errors are expected + return { + "ok": False, + "status_code": None, + "text": "", + "bytes": 0, + "final_url": None, + "error": f"{exc.__class__.__name__}: {exc}", + } + + def _normalize_candidate(self, host: str) -> Optional[str]: + candidate = (host or "").strip().lower().rstrip(".") + if not candidate: + return None + if candidate.startswith("*."): + candidate = candidate[2:] + if not candidate.endswith(self.domain): + return None + if not self.HOST_RE.match(candidate): + return None + return candidate + + def _parse(self, parser: str, text: str) -> List[str]: + out: List[str] = [] + if not text: + return out + try: + if parser == "virustotal_subdomains": + data = json.loads(text) + out = [s for s in data.get("subdomains", []) if s] + elif parser == "shodan_subdomains": + data = json.loads(text) + out = [f"{s}.{self.domain}" for s in data.get("subdomains", [])] + elif parser == "csv_first_column": + out = [line.split(",")[0].strip() for line in text.splitlines() if line.strip() and "," in line] + elif parser == "rapiddns_html_td": + out = [m.group(1).strip() for m in self.RAPIDDNS_RE.finditer(text)] + elif parser in ("json_list", "anubis_list"): + data = json.loads(text) + if isinstance(data, list): + out = [str(s).strip() for s in data if str(s).strip()] + except Exception as exc: # pragma: no cover - parser failures are input-dependent + logger.debug("Recon parse error %s: %s", parser, exc) + + normalized: List[str] = [] + seen: set[str] = set() + for candidate in out: + host = self._normalize_candidate(candidate) + if host and host not in seen: + seen.add(host) + normalized.append(host) + return normalized + + async def start(self) -> List[str]: + services = self.services() + limiter = asyncio.Semaphore(self.max_concurrency) + timeout_cfg = httpx.Timeout(self.timeout) + limits_cfg = httpx.Limits( + max_connections=max(20, self.max_concurrency * 2), + max_keepalive_connections=max(10, self.max_concurrency), + ) + results: List[str] = [] + seen: set[str] = set() + + async with httpx.AsyncClient(headers=self.headers, http2=True, timeout=timeout_cfg, limits=limits_cfg) as client: + tasks = [] + for service in services: + name = service.get("name", "-") + parser = service.get("parser", "") + url = service.get("url", "") + if not parser or not url: + continue + + async def worker(service_name: str = name, service_parser: str = parser, service_url: str = url) -> Tuple[str, str, str]: + async with limiter: + fetch_name, fetch_text = await self._fetch(client, service_name, service_url) + return fetch_name, service_parser, fetch_text + + tasks.append(asyncio.create_task(worker())) + + for coro in asyncio.as_completed(tasks): + _, parser, text = await coro + for host in self._parse(parser, text): + if host not in seen: + seen.add(host) + results.append(host) + + return sorted(results) + + async def test_services(self) -> List[Dict[str, Any]]: + services = self.services() + limiter = asyncio.Semaphore(self.max_concurrency) + timeout_cfg = httpx.Timeout(self.timeout) + limits_cfg = httpx.Limits( + max_connections=max(20, self.max_concurrency * 2), + max_keepalive_connections=max(10, self.max_concurrency), + ) + report: List[Dict[str, Any]] = [] + + async with httpx.AsyncClient(headers=self.headers, http2=True, timeout=timeout_cfg, limits=limits_cfg) as client: + tasks = [] + for service in services: + name = service.get("name", "-") + parser = service.get("parser", "") + url = service.get("url", "") + if not parser or not url: + continue + + async def worker(service_name: str = name, service_parser: str = parser, service_url: str = url) -> Dict[str, Any]: + async with limiter: + fetch = await self._fetch_with_meta(client, service_url) + parsed_count = 0 + if fetch.get("text"): + parsed_count = len(self._parse(service_parser, str(fetch.get("text")))) + return { + "service": service_name, + "parser": service_parser, + "url": self._redact_url_secrets(service_url), + "ok": bool(fetch.get("ok")), + "status_code": fetch.get("status_code"), + "response_bytes": int(fetch.get("bytes") or 0), + "parsed_count": parsed_count, + "error": self._redact_text_secrets(fetch.get("error")), + "final_url": self._redact_url_secrets(fetch.get("final_url")), + } + + tasks.append(asyncio.create_task(worker())) + + for item in await asyncio.gather(*tasks): + report.append(item) + + return report + + +class AsyncScanner: + """Run DNS, HTTP, HTTPS and certificate checks for one hostname. + + It is instantiated by `_run_async` for each target domain and returns a + single normalized result object consumed by output/storage layers. + """ + DNS_CACHE: Dict[str, Tuple[float, Optional[List[str]]]] = {} + DNS_POSITIVE_TTL = 300.0 + DNS_NEGATIVE_TTL = 30.0 + CNAME_CACHE: Dict[str, Tuple[float, Optional[str]]] = {} + CNAME_POSITIVE_TTL = 300.0 + CNAME_NEGATIVE_TTL = 60.0 + TLS_PROBE_CACHE: Dict[str, Tuple[float, List[str]]] = {} + TLS_PROBE_TTL = 600.0 + META_TAG_RE = re.compile(r"""]*http-equiv\s*=\s*["']?refresh["']?[^>]*>""", re.IGNORECASE) + META_CONTENT_RE = re.compile(r"""content\s*=\s*(["'])(.*?)\1""", re.IGNORECASE) + META_URL_RE = re.compile(r"""url\s*=\s*['"]?([^'";\s>]+)""", re.IGNORECASE) + EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b") + TAKEOVER_PROVIDERS: List[Dict[str, Any]] = [ + { + "name": "github-pages", + "cname_suffixes": ("github.io",), + "fingerprints": ( + "there isn't a github pages site here", + "for root urls (like http://example.com/) you must provide an index.html file", + ), + }, + { + "name": "heroku", + "cname_suffixes": ("herokudns.com", "herokuapp.com"), + "fingerprints": ( + "no such app", + "there is no app configured at that hostname", + ), + }, + { + "name": "fastly", + "cname_suffixes": ("fastly.net",), + "fingerprints": ( + "fastly error: unknown domain", + "please check that this domain has been added to a service", + ), + }, + { + "name": "azure", + "cname_suffixes": ("azurewebsites.net",), + "fingerprints": ( + "404 web site not found", + "the resource you are looking for has been removed", + ), + }, + { + "name": "s3-website", + "cname_suffixes": ("amazonaws.com",), + "fingerprints": ( + "nosuchbucket", + "the specified bucket does not exist", + ), + }, + ] + + def __init__( + self, + domain: str, + client: httpx.AsyncClient, + dns_server: Optional[str] = None, + useragent: Optional[str] = None, + timeout: float = 2.0, + cert_semaphore: Optional[asyncio.Semaphore] = None, + root_domain: Optional[str] = None, + io_executor: Optional[ThreadPoolExecutor] = None, + verbose: bool = False, + api_key_virustotal: Optional[str] = None, + api_key_shodan: Optional[str] = None, + ): + self.domain = domain + self.client = client + self.dns_server = dns_server or "8.8.8.8" + self.headers = {"User-Agent": pick_user_agent(useragent)} + self.timeout = timeout + self.cert_semaphore = cert_semaphore or asyncio.Semaphore(50) + self.root_domain = root_domain or domain + self.io_executor = io_executor + self.verbose = verbose + self.vt_key = api_key_virustotal if api_key_virustotal is not None else os.getenv("API_KEY_VIRUSTOTAL") + self.shodan_key = api_key_shodan if api_key_shodan is not None else os.getenv("API_KEY_SHODAN") + + async def _resolve_async(self, domain: str) -> Optional[List[str]]: + normalized = self._normalize_domain(domain) + if not normalized: + return None + + now = time.monotonic() + cached = self.DNS_CACHE.get(normalized) + if cached is not None: + expiry, cached_ips = cached + if expiry > now: + return cached_ips + + loop = asyncio.get_running_loop() + + def sync_resolve() -> Tuple[Optional[List[str]], float]: + resolver = dns.resolver.Resolver() + resolver.nameservers = [self.dns_server] + resolver.timeout = self.timeout + resolver.lifetime = max(self.timeout * 2, self.timeout + 1.0) + + ips: List[str] = [] + min_ttl = self.DNS_POSITIVE_TTL + + for qtype in ("A", "AAAA"): + try: + answers = resolver.resolve(normalized, qtype) + except dns.resolver.NXDOMAIN: + return None, self.DNS_NEGATIVE_TTL + except dns.resolver.NoAnswer: + continue + except dns.resolver.NoNameservers: + continue + except dns.exception.Timeout: + continue + except dns.resolver.YXDOMAIN: + return None, self.DNS_NEGATIVE_TTL + except Exception: + continue + + if answers.rrset is not None and answers.rrset.ttl is not None: + min_ttl = min(min_ttl, float(max(answers.rrset.ttl, 1))) + + for rr in answers: + ip_text = str(rr).strip() + try: + ipaddress.ip_address(ip_text) + except ValueError: + continue + if ip_text not in ips: + ips.append(ip_text) + + if not ips: + return None, self.DNS_NEGATIVE_TTL + return ips, min(min_ttl, self.DNS_POSITIVE_TTL) + + ips, ttl = await loop.run_in_executor(self.io_executor, sync_resolve) + self.DNS_CACHE[normalized] = (now + ttl, ips) + return ips + + async def _resolve_cname_async(self, domain: str) -> Optional[str]: + normalized = self._normalize_domain(domain) + if not normalized: + return None + + now = time.monotonic() + cached = self.CNAME_CACHE.get(normalized) + if cached is not None: + expiry, cached_cname = cached + if expiry > now: + return cached_cname + + loop = asyncio.get_running_loop() + + def sync_cname() -> Tuple[Optional[str], float]: + resolver = dns.resolver.Resolver() + resolver.nameservers = [self.dns_server] + resolver.timeout = self.timeout + resolver.lifetime = max(self.timeout * 2, self.timeout + 1.0) + try: + answers = resolver.resolve(normalized, "CNAME") + except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.resolver.YXDOMAIN, dns.resolver.NoNameservers, dns.exception.Timeout): + return None, self.CNAME_NEGATIVE_TTL + except Exception: + return None, self.CNAME_NEGATIVE_TTL + + cname_value: Optional[str] = None + ttl = self.CNAME_POSITIVE_TTL + if answers.rrset is not None and answers.rrset.ttl is not None: + ttl = min(ttl, float(max(answers.rrset.ttl, 1))) + for rr in answers: + text = str(rr).strip().rstrip(".").lower() + if text: + cname_value = text + break + if not cname_value: + return None, self.CNAME_NEGATIVE_TTL + return cname_value, ttl + + cname, ttl = await loop.run_in_executor(self.io_executor, sync_cname) + self.CNAME_CACHE[normalized] = (now + ttl, cname) + return cname + + @staticmethod + def _status_from_http_tuple(http_data: Tuple[Optional[int], Optional[str], Optional[str], Optional[int], Optional[str], Optional[str], Optional[str]]) -> Optional[int]: + code = http_data[0] if len(http_data) > 0 else None + return code if isinstance(code, int) else None + + async def _takeover_check( + self, + http: Tuple[Optional[int], Optional[str], Optional[str], Optional[int], Optional[str], Optional[str], Optional[str]], + https: Tuple[Optional[int], Optional[str], Optional[str], Optional[int], Optional[str], Optional[str], Optional[str]], + ) -> Optional[Dict[str, Any]]: + cname = await self._resolve_cname_async(self.domain) + if not cname: + return None + + provider: Optional[Dict[str, Any]] = None + for item in self.TAKEOVER_PROVIDERS: + suffixes = item.get("cname_suffixes") or () + if any(cname == s or cname.endswith("." + s) for s in suffixes): + provider = item + break + if not provider: + return None + + http_status = self._status_from_http_tuple(http) + https_status = self._status_from_http_tuple(https) + preview_text = " ".join( + str(part or "").lower() + for part in ( + http[5] if len(http) > 5 else "", + https[5] if len(https) > 5 else "", + http[6] if len(http) > 6 else "", + https[6] if len(https) > 6 else "", + ) + ) + + fingerprints = [str(f).strip().lower() for f in provider.get("fingerprints") or [] if str(f).strip()] + matched = [f for f in fingerprints if f in preview_text] + likely = len(matched) > 0 + possible = not likely and any(code in {404, 410} for code in (http_status, https_status) if code is not None) + + if likely: + status = "likely" + elif possible: + status = "possible" + else: + status = "none" + + return { + "status": status, + "provider": str(provider.get("name") or "unknown"), + "cname": cname, + "http_status": http_status, + "https_status": https_status, + "matched_fingerprints": matched[:2], + } + + @staticmethod + def _normalize_domain(domain: str) -> Optional[str]: + host = (domain or "").strip().lower() + if not host: + return None + + host = re.sub(r"^\w+://", "", host) + host = host.split("/", 1)[0].strip(".") + if not host or " " in host: + return None + + try: + host = host.encode("idna").decode("ascii") + except Exception: + return None + + if len(host) > 253: + return None + labels = host.split(".") + if any(not lbl or len(lbl) > 63 for lbl in labels): + return None + if any(not re.match(r"^[a-z0-9-]+$", lbl) or lbl.startswith("-") or lbl.endswith("-") for lbl in labels): + return None + return host + + def _extract_app_redirect(self, headers: httpx.Headers, content: bytes) -> Optional[str]: + refresh = headers.get("Refresh") + if refresh: + # Examples: "0; URL=https://example.com" / "5;url=/path" + lower = refresh.lower() + if "url=" in lower: + raw = refresh.split("=", 1)[1].strip().strip("'\"") + if raw: + return raw + if refresh.strip(): + return refresh.strip() + + content_type = (headers.get("Content-Type") or "").lower() + if "html" not in content_type: + return None + if not content: + return None + + probe = content[:16384].decode("utf-8", errors="ignore") + meta = self.META_TAG_RE.search(probe) + if meta: + tag_text = meta.group(0) + content_match = self.META_CONTENT_RE.search(tag_text) + if content_match: + content_value = content_match.group(2).strip() + url_match = self.META_URL_RE.search(content_value) + if url_match: + target = url_match.group(1).strip().strip("'\"") + return target or None + return None + + def _body_preview(self, content: bytes, limit: int = 300) -> Optional[str]: + if not content: + return None + text = content[:4096].decode("utf-8", errors="ignore") + text = re.sub(r"\s+", " ", text).strip() + if not text: + return None + return text[:limit] + + async def _http_check( + self, url: str + ) -> Tuple[Optional[int], Optional[str], Optional[str], Optional[int], Optional[str], Optional[str], Optional[str]]: + try: + response = await self.client.get( + url, + timeout=self.timeout, + follow_redirects=False, + headers=self.headers, + ) + app_redirect = None + if not response.headers.get("Location"): + app_redirect = self._extract_app_redirect(response.headers, response.content) + return ( + response.status_code, + response.headers.get("Location"), + response.headers.get("Server"), + len(response.content), + app_redirect, + self._body_preview(response.content), + None, + ) + except Exception as exc: + message = str(exc).strip() + err = f"{exc.__class__.__name__}: {message}" if message else exc.__class__.__name__ + return None, None, None, None, None, None, err + + def _extract_title(self, content: bytes) -> Optional[str]: + if not content: + return None + text = content[:65536].decode("utf-8", errors="ignore") + match = re.search(r"]*>(.*?)", text, flags=re.IGNORECASE | re.DOTALL) + if not match: + return None + title = re.sub(r"\s+", " ", match.group(1)).strip() + return title or None + + def _security_headers(self, headers: httpx.Headers) -> Dict[str, Optional[str]]: + keys = [ + "Strict-Transport-Security", + "Content-Security-Policy", + "X-Frame-Options", + "X-Content-Type-Options", + "Referrer-Policy", + "Permissions-Policy", + ] + return {k: headers.get(k) for k in keys} + + def _infra_headers(self, headers: httpx.Headers) -> Dict[str, Optional[str]]: + keys = ["Server", "Via", "X-Cache", "CF-Ray", "X-Served-By", "X-Powered-By"] + return {k: headers.get(k) for k in keys} + + async def _dns_detail(self) -> Dict[str, Any]: + loop = asyncio.get_running_loop() + + def sync_dns() -> Dict[str, Any]: + resolver = dns.resolver.Resolver() + resolver.nameservers = [self.dns_server] + resolver.timeout = self.timeout + resolver.lifetime = max(self.timeout * 2, self.timeout + 1.0) + out: Dict[str, Any] = {"a": [], "aaaa": [], "cname": [], "ttl": {}, "errors": {}} + for qtype in ("CNAME", "A", "AAAA"): + try: + answers = resolver.resolve(self.domain, qtype) + if answers.rrset is not None and answers.rrset.ttl is not None: + out["ttl"][qtype] = int(answers.rrset.ttl) + values = [str(rr).strip().rstrip(".") for rr in answers if str(rr).strip()] + out[qtype.lower()] = values + except Exception as exc: + out["errors"][qtype] = exc.__class__.__name__ + return out + + return await loop.run_in_executor(self.io_executor, sync_dns) + + async def _tcp_ports(self, ports: Iterable[int] = (80, 443)) -> Dict[str, bool]: + loop = asyncio.get_running_loop() + + def sync_ports() -> Dict[str, bool]: + out: Dict[str, bool] = {} + for port in ports: + ok = False + try: + with socket.create_connection((self.domain, int(port)), timeout=self.timeout): + ok = True + except Exception: + ok = False + out[str(port)] = ok + return out + + return await loop.run_in_executor(self.io_executor, sync_ports) + + async def _tls_handshake_detail(self, port: int = 443) -> Dict[str, Any]: + loop = asyncio.get_running_loop() + + def sync_tls() -> Dict[str, Any]: + detail: Dict[str, Any] = { + "strict_ok": False, + "strict_error": None, + "protocol": None, + "cipher": None, + "alpn": None, + "issuer": None, + "subject": None, + "san": [], + "san_count": 0, + "self_signed": None, + "key_type": None, + "key_bits": None, + "signature_algorithm": None, + "port": port, + } + try: + ctx = ssl.create_default_context() + ctx.check_hostname = True + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.set_alpn_protocols(["h2", "http/1.1"]) + with socket.create_connection((self.domain, int(port)), timeout=self.timeout) as sock: + with ctx.wrap_socket(sock, server_hostname=self.domain) as tls_sock: + detail["strict_ok"] = True + detail["protocol"] = tls_sock.version() + cipher = tls_sock.cipher() + detail["cipher"] = cipher[0] if cipher else None + detail["alpn"] = tls_sock.selected_alpn_protocol() + cert = tls_sock.getpeercert() + if cert: + detail["issuer"] = cert.get("issuer") + detail["subject"] = cert.get("subject") + sans = cert.get("subjectAltName", ()) + detail["san"] = [v for k, v in sans if k == "DNS"] + detail["san_count"] = len(detail["san"]) + except Exception as exc: + detail["strict_error"] = str(exc) + + # Fill metadata best-effort even on strict failure. + if not detail["subject"] or not detail["issuer"] or not detail["san"]: + try: + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + ctx.set_alpn_protocols(["h2", "http/1.1"]) + with socket.create_connection((self.domain, int(port)), timeout=self.timeout) as sock: + with ctx.wrap_socket(sock, server_hostname=self.domain) as tls_sock: + if not detail["protocol"]: + detail["protocol"] = tls_sock.version() + if not detail["cipher"]: + cipher = tls_sock.cipher() + detail["cipher"] = cipher[0] if cipher else None + if not detail["alpn"]: + detail["alpn"] = tls_sock.selected_alpn_protocol() + der_cert = tls_sock.getpeercert(binary_form=True) + x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_ASN1, der_cert) + if not detail["subject"]: + detail["subject"] = str(x509.get_subject()) + if not detail["issuer"]: + detail["issuer"] = str(x509.get_issuer()) + if not detail["san"]: + san_values: List[str] = [] + for i in range(x509.get_extension_count()): + ext = x509.get_extension(i) + if ext.get_short_name().decode("utf-8").lower() == "subjectaltname": + txt = str(ext) + for part in txt.split(","): + part = part.strip() + if part.startswith("DNS:"): + san_values.append(part.split(":", 1)[1]) + detail["san"] = san_values + detail["san_count"] = len(san_values) + try: + detail["self_signed"] = str(x509.get_subject()) == str(x509.get_issuer()) + except Exception: + pass + try: + pkey = x509.get_pubkey() + detail["key_bits"] = int(pkey.bits()) if pkey else None + key_type_id = pkey.type() if pkey else None + key_type_map = { + getattr(OpenSSL.crypto, "TYPE_RSA", 6): "RSA", + getattr(OpenSSL.crypto, "TYPE_DSA", 116): "DSA", + getattr(OpenSSL.crypto, "TYPE_EC", 408): "EC", + } + detail["key_type"] = key_type_map.get(key_type_id, str(key_type_id) if key_type_id is not None else None) + except Exception: + pass + try: + detail["signature_algorithm"] = x509.get_signature_algorithm().decode("utf-8") + except Exception: + pass + except Exception: + pass + + return detail + + return await loop.run_in_executor(self.io_executor, sync_tls) + + async def _request_detail(self, url: str, follow_redirects: bool = True, headers: Optional[Dict[str, str]] = None) -> Dict[str, Any]: + try: + started = time.perf_counter() + response = await self.client.get( + url, + timeout=self.timeout, + follow_redirects=follow_redirects, + headers=headers or self.headers, + ) + total_ms = int((time.perf_counter() - started) * 1000) + + chain: List[Dict[str, Any]] = [] + for hop in list(response.history) + [response]: + loc = hop.headers.get("Location") + app_loc = None if loc else self._extract_app_redirect(hop.headers, hop.content) + chain.append( + { + "url": str(hop.url), + "status": hop.status_code, + "location": loc, + "app_redirect": app_loc, + "elapsed_ms": int(hop.elapsed.total_seconds() * 1000) if hop.elapsed else None, + } + ) + + body = response.content or b"" + body_sample = body[:131072].decode("utf-8", errors="ignore") + return { + "ok": True, + "url": str(response.url), + "status": response.status_code, + "history": chain, + "content_type": response.headers.get("Content-Type"), + "title": self._extract_title(body), + "sha256": hashlib.sha256(body).hexdigest() if body else None, + "body_bytes": len(body), + "body_sample": body_sample, + "total_ms": total_ms, + "http_version": response.http_version, + "security_headers": self._security_headers(response.headers), + "infra_headers": self._infra_headers(response.headers), + "set_cookie": response.headers.get_list("set-cookie") if hasattr(response.headers, "get_list") else [], + "allow": response.headers.get("Allow"), + "hsts": response.headers.get("Strict-Transport-Security"), + } + except Exception as exc: + message = str(exc).strip() + err = f"{exc.__class__.__name__}: {message}" if message else exc.__class__.__name__ + return {"ok": False, "error": err} + + async def _security_txt_check(self) -> Dict[str, Any]: + url = f"https://{self.domain}/.well-known/security.txt" + try: + response = await self.client.get(url, timeout=self.timeout, follow_redirects=True, headers=self.headers) + text = response.text or "" + has_contact = "contact:" in text.lower() + has_expires = "expires:" in text.lower() + return { + "url": str(response.url), + "status": response.status_code, + "has_contact": has_contact, + "has_expires": has_expires, + "present": response.status_code == 200 and "contact:" in text.lower(), + } + except Exception as exc: + return {"url": url, "status": None, "present": False, "error": f"{exc.__class__.__name__}: {exc}"} + + async def _caa_check(self) -> Dict[str, Any]: + loop = asyncio.get_running_loop() + + def sync_caa() -> Dict[str, Any]: + resolver = dns.resolver.Resolver() + resolver.nameservers = [self.dns_server] + resolver.timeout = self.timeout + resolver.lifetime = max(self.timeout * 2, self.timeout + 1.0) + try: + answers = resolver.resolve(self.domain, "CAA") + entries: List[str] = [str(rr).strip() for rr in answers if str(rr).strip()] + return {"present": bool(entries), "entries": entries} + except dns.resolver.NoAnswer: + return {"present": False, "entries": []} + except dns.resolver.NXDOMAIN: + return {"present": False, "entries": [], "error": "NXDOMAIN"} + except Exception as exc: + return {"present": False, "entries": [], "error": f"{exc.__class__.__name__}: {exc}"} + + return await loop.run_in_executor(self.io_executor, sync_caa) + + @staticmethod + def _txt_rr_to_text(rr: Any) -> str: + chunks = getattr(rr, "strings", None) + if isinstance(chunks, (list, tuple)) and chunks: + parts: List[str] = [] + for chunk in chunks: + if isinstance(chunk, (bytes, bytearray)): + parts.append(chunk.decode("utf-8", errors="ignore")) + else: + parts.append(str(chunk)) + return "".join(parts).strip() + + text = str(rr).strip() + if text.startswith('"') and text.endswith('"') and len(text) >= 2: + text = text[1:-1] + return text.replace('" "', "").strip() + + @staticmethod + def _parse_tag_list(record: str) -> Dict[str, str]: + tags: Dict[str, str] = {} + for part in (record or "").split(";"): + token = part.strip() + if "=" not in token: + continue + key, value = token.split("=", 1) + k = key.strip().lower() + v = value.strip() + if k: + tags[k] = v + return tags + + @staticmethod + def _estimate_dkim_key_bits(p_value: Optional[str]) -> Optional[int]: + if not p_value: + return None + raw = re.sub(r"\s+", "", str(p_value)) + if not raw: + return None + if not re.fullmatch(r"[A-Za-z0-9+/=]+", raw): + return None + padding = 0 + if raw.endswith("=="): + padding = 2 + elif raw.endswith("="): + padding = 1 + key_bytes = max(0, (len(raw) * 3 // 4) - padding) + if key_bytes <= 0: + return None + return key_bytes * 8 + + async def _email_auth_detail(self) -> Dict[str, Any]: + loop = asyncio.get_running_loop() + + def sync_email_auth() -> Dict[str, Any]: + resolver = dns.resolver.Resolver() + resolver.nameservers = [self.dns_server] + resolver.timeout = self.timeout + resolver.lifetime = max(self.timeout * 2, self.timeout + 1.0) + + root = self.domain.strip().rstrip(".") + + def resolve_txt(name: str) -> Tuple[List[str], Optional[str]]: + try: + answers = resolver.resolve(name, "TXT") + values = [self._txt_rr_to_text(rr) for rr in answers] + clean = [v for v in values if v] + return clean, None + except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN): + return [], None + except Exception as exc: + return [], f"{exc.__class__.__name__}: {exc}" + + root_txt, root_err = resolve_txt(root) + dmarc_txt, dmarc_err = resolve_txt(f"_dmarc.{root}") + + spf_records = [r for r in root_txt if r.lower().startswith("v=spf1")] + spf_all = None + spf_policy = "missing" + spf_level = "warning" + spf_summary = "SPF record not found" + if spf_records: + rec = spf_records[0] + all_match = re.search(r"(?:^|\s)([+\-~?])?all(?:\s|$)", rec, flags=re.IGNORECASE) + qualifier = all_match.group(1) if all_match else None + spf_all = qualifier if qualifier is not None else ("+" if all_match else None) + if len(spf_records) > 1: + spf_policy = "ambiguous" + spf_level = "warning" + spf_summary = f"Multiple SPF records found ({len(spf_records)})" + elif spf_all == "-": + spf_policy = "strict" + spf_level = "ok" + spf_summary = "SPF configured with hard fail (-all)" + elif spf_all == "~": + spf_policy = "soft" + spf_level = "info" + spf_summary = "SPF configured with soft fail (~all)" + elif spf_all in {"+", "?"}: + spf_policy = "weak" + spf_level = "warning" + spf_summary = f"SPF policy is weak ({spf_all}all)" + elif "redirect=" in rec.lower(): + spf_policy = "redirect" + spf_level = "info" + spf_summary = "SPF uses redirect= policy" + else: + spf_policy = "incomplete" + spf_level = "warning" + spf_summary = "SPF present but missing explicit all policy" + + dmarc_records = [r for r in dmarc_txt if r.lower().startswith("v=dmarc1")] + dmarc_tags: Dict[str, str] = self._parse_tag_list(dmarc_records[0]) if dmarc_records else {} + dmarc_policy = (dmarc_tags.get("p") or "").strip().lower() + dmarc_pct = self._as_int(dmarc_tags.get("pct"), default=100) + dmarc_level = "warning" + dmarc_summary = "DMARC record not found" + if dmarc_records: + if dmarc_policy in {"reject", "quarantine"} and dmarc_pct >= 100: + dmarc_level = "ok" + dmarc_summary = f"DMARC enforced (p={dmarc_policy}, pct={dmarc_pct})" + elif dmarc_policy in {"reject", "quarantine"}: + dmarc_level = "info" + dmarc_summary = f"DMARC partially enforced (p={dmarc_policy}, pct={dmarc_pct})" + elif dmarc_policy == "none": + dmarc_level = "warning" + dmarc_summary = "DMARC monitor-only policy (p=none)" + elif dmarc_policy: + dmarc_level = "info" + dmarc_summary = f"DMARC policy set to p={dmarc_policy}" + else: + dmarc_level = "warning" + dmarc_summary = "DMARC present but missing p= policy" + + selectors = [ + "default", + "selector1", + "selector2", + "google", + "k1", + "k2", + "smtp", + "mail", + "mx", + "dkim", + "zoho", + "mandrill", + ] + found_dkim: List[Dict[str, Any]] = [] + lookup_errors: List[str] = [] + for selector in selectors: + qname = f"{selector}._domainkey.{root}" + records, err = resolve_txt(qname) + if err: + lookup_errors.append(f"{selector}:{err}") + continue + dkim_records = [r for r in records if r.lower().startswith("v=dkim1")] + if not dkim_records: + continue + for record in dkim_records: + tags = self._parse_tag_list(record) + bits = self._estimate_dkim_key_bits(tags.get("p")) + found_dkim.append( + { + "selector": selector, + "record": record, + "tags": tags, + "key_bits_est": bits, + "revoked": tags.get("p", "").strip() == "", + } + ) + + dkim_level = "info" + dkim_summary = "No DKIM key found in common selectors (non-exhaustive)" + if found_dkim: + revoked = [d for d in found_dkim if d.get("revoked")] + weak = [d for d in found_dkim if isinstance(d.get("key_bits_est"), int) and int(d["key_bits_est"]) < 2048] + if revoked: + dkim_level = "warning" + dkim_summary = f"DKIM selectors found but {len(revoked)} key(s) revoked" + elif weak: + dkim_level = "warning" + dkim_summary = f"DKIM key too short on {len(weak)} selector(s) (<2048 bits)" + else: + dkim_level = "ok" + dkim_summary = f"DKIM selector(s) found: {', '.join(sorted({d['selector'] for d in found_dkim}))}" + + max_bits = max( + [int(d.get("key_bits_est")) for d in found_dkim if isinstance(d.get("key_bits_est"), int)], + default=None, + ) + + return { + "spf": { + "present": bool(spf_records), + "records": spf_records[:3], + "all_qualifier": spf_all, + "policy": spf_policy, + "level": spf_level, + "summary": spf_summary, + "lookup_error": root_err, + }, + "dmarc": { + "present": bool(dmarc_records), + "records": dmarc_records[:3], + "policy": dmarc_policy or None, + "pct": dmarc_pct if dmarc_records else None, + "adkim": dmarc_tags.get("adkim"), + "aspf": dmarc_tags.get("aspf"), + "rua": dmarc_tags.get("rua"), + "ruf": dmarc_tags.get("ruf"), + "level": dmarc_level, + "summary": dmarc_summary, + "lookup_error": dmarc_err, + }, + "dkim": { + "present_common_selectors": bool(found_dkim), + "selectors_checked": selectors, + "selectors_found": sorted({str(d.get("selector")) for d in found_dkim if d.get("selector")}), + "records": [str(d.get("record") or "") for d in found_dkim[:8] if str(d.get("record") or "")], + "max_key_bits_est": max_bits, + "non_exhaustive": True, + "level": dkim_level, + "summary": dkim_summary, + "lookup_errors": lookup_errors[:5], + }, + } + + return await loop.run_in_executor(self.io_executor, sync_email_auth) + + async def _dns_takeover_hints(self) -> Dict[str, Any]: + loop = asyncio.get_running_loop() + + def sync_takeover() -> Dict[str, Any]: + resolver = dns.resolver.Resolver() + resolver.nameservers = [self.dns_server] + resolver.timeout = self.timeout + resolver.lifetime = max(self.timeout * 2, self.timeout + 1.0) + providers = ( + "github.io", + "herokudns.com", + "azurewebsites.net", + "cloudfront.net", + "fastly.net", + "pantheonsite.io", + "readthedocs.io", + "surge.sh", + ) + try: + answers = resolver.resolve(self.domain, "CNAME") + cnames = [str(rr).strip().rstrip(".").lower() for rr in answers if str(rr).strip()] + except Exception: + return {"suspect": False, "cnames": []} + + hints: List[str] = [] + for cname in cnames: + if not any(suffix in cname for suffix in providers): + continue + try: + has_a = bool(resolver.resolve(cname, "A")) + except Exception: + has_a = False + try: + has_aaaa = bool(resolver.resolve(cname, "AAAA")) + except Exception: + has_aaaa = False + if not has_a and not has_aaaa: + hints.append(cname) + return {"suspect": bool(hints), "cnames": cnames, "suspect_targets": hints} + + return await loop.run_in_executor(self.io_executor, sync_takeover) + + async def _open_redirect_probe(self) -> Dict[str, Any]: + probes = ["next", "url", "redirect"] + base = f"https://{self.domain}/" + out: List[Dict[str, Any]] = [] + for p in probes: + url = f"{base}?{p}=https://example.com" + try: + r = await self.client.get(url, headers=self.headers, timeout=self.timeout, follow_redirects=False) + loc = r.headers.get("Location") + maybe_open = False + if loc: + try: + parsed = urlparse(loc) + host = (parsed.hostname or "").lower() + if host and host == "example.com": + maybe_open = True + except Exception: + maybe_open = False + out.append({"param": p, "status": r.status_code, "location": loc, "possible_open_redirect": maybe_open}) + except Exception as exc: + out.append({"param": p, "status": None, "location": None, "error": f"{exc.__class__.__name__}: {exc}"}) + return {"possible": any(bool(x.get("possible_open_redirect")) for x in out), "probes": out} + + async def _http_methods_check(self) -> Dict[str, Any]: + def _split_methods(raw: str) -> List[str]: + return [m.strip().upper() for m in (raw or "").split(",") if m.strip()] + + def _collect_from_response(response: Any) -> Tuple[set[str], set[str]]: + allow_set: set[str] = set() + cors_set: set[str] = set() + if response is None: + return allow_set, cors_set + + chain = list(getattr(response, "history", []) or []) + [response] + for part in chain: + headers = getattr(part, "headers", None) + if headers is None: + continue + allow_set.update(_split_methods(headers.get("Allow") or "")) + cors_set.update(_split_methods(headers.get("Access-Control-Allow-Methods") or "")) + return allow_set, cors_set + + async def infer_safe_methods(url: str) -> List[str]: + # Probe a random non-existing path to minimize side effects while + # checking whether common methods are handled by the target stack. + token = "".join(random.choice(string.ascii_lowercase) for _ in range(12)) + target = f"{url.rstrip('/')}/.knockpy-method-probe-{token}" + supported: List[str] = [] + for method in ("GET", "HEAD", "POST", "OPTIONS"): + try: + resp = await self.client.request( + method, + target, + headers=self.headers, + timeout=self.timeout, + follow_redirects=True, + content=b"" if method == "POST" else None, + ) + except Exception: + continue + if int(resp.status_code) not in {405, 501}: + supported.append(method) + return supported + + async def check(url: str) -> Dict[str, Any]: + try: + r_follow = await self.client.options(url, headers=self.headers, timeout=self.timeout, follow_redirects=True) + allow_set, cors_set = _collect_from_response(r_follow) + + # Some servers expose method metadata only before redirects or only + # on non-followed OPTIONS responses. + r_nofollow = await self.client.options(url, headers=self.headers, timeout=self.timeout, follow_redirects=False) + allow_nf, cors_nf = _collect_from_response(r_nofollow) + allow_set.update(allow_nf) + cors_set.update(cors_nf) + inferred = await infer_safe_methods(url) + + return { + "status": r_follow.status_code, + "allow": sorted(allow_set), + "cors_allow": sorted(cors_set), + "inferred_safe": inferred, + } + except Exception as exc: + return { + "status": None, + "allow": [], + "cors_allow": [], + "inferred_safe": [], + "error": f"{exc.__class__.__name__}: {exc}", + } + + http_res, https_res = await asyncio.gather(check(f"http://{self.domain}"), check(f"https://{self.domain}")) + risky = {"PUT", "DELETE", "TRACE", "CONNECT"} + observed = ( + set(http_res.get("allow") or []) + | set(https_res.get("allow") or []) + | set(http_res.get("cors_allow") or []) + | set(https_res.get("cors_allow") or []) + ) + exposed = sorted(observed & risky) + inferred_union = sorted( + set(http_res.get("inferred_safe") or []) | set(https_res.get("inferred_safe") or []) + ) + return {"http": http_res, "https": https_res, "risky_methods": exposed, "inferred_safe": inferred_union} + + async def _rate_limit_waf_check(self) -> Dict[str, Any]: + statuses: List[Optional[int]] = [] + flags: List[str] = [] + for _ in range(3): + try: + r = await self.client.get(f"https://{self.domain}", headers=self.headers, timeout=self.timeout, follow_redirects=True) + statuses.append(r.status_code) + body = (r.text or "")[:4000].lower() + if "captcha" in body or "attention required" in body or "cloudflare" in body and "ray id" in body: + flags.append("challenge") + except Exception: + statuses.append(None) + return {"statuses": statuses, "rate_limited": any(s == 429 for s in statuses if s is not None), "waf_flags": sorted(set(flags))} + + def _redact_api_secrets(self, text: Optional[str]) -> Optional[str]: + if not text: + return text + redacted = text + for secret in (self.vt_key, self.shodan_key): + if secret: + redacted = redacted.replace(secret, "***") + return redacted + + @staticmethod + def _as_int(value: Any, default: int = 0) -> int: + try: + return int(value) + except Exception: + return default + + @staticmethod + def _as_float(value: Any) -> Optional[float]: + try: + return float(value) + except Exception: + return None + + @staticmethod + def _iso_from_epoch(value: Any) -> Optional[str]: + try: + ts = int(value) + return datetime.utcfromtimestamp(ts).isoformat() + "Z" + except Exception: + return None + + @staticmethod + def _cvss_severity(score: Optional[float]) -> str: + if score is None: + return "unknown" + if score >= 9.0: + return "critical" + if score >= 7.0: + return "high" + if score >= 4.0: + return "medium" + if score > 0: + return "low" + return "none" + + def _normalize_shodan_vulns(self, vulns_obj: Any) -> List[Dict[str, Any]]: + entries: List[Dict[str, Any]] = [] + if isinstance(vulns_obj, dict): + for cve_id_raw, meta in vulns_obj.items(): + cve_id = str(cve_id_raw).strip() + if not cve_id: + continue + score: Optional[float] = None + if isinstance(meta, dict): + for key in ("cvss", "cvss_v3", "cvss3", "cvss_score", "cvss3_score", "score"): + if key in meta: + score = self._as_float(meta.get(key)) + if score is not None: + break + entries.append( + { + "id": cve_id, + "score": score, + "severity": self._cvss_severity(score), + } + ) + elif isinstance(vulns_obj, list): + for item in vulns_obj: + cve_id = str(item).strip() + if cve_id: + entries.append({"id": cve_id, "score": None, "severity": "unknown"}) + + return sorted( + entries, + key=lambda x: ( + -(x.get("score") if isinstance(x.get("score"), (int, float)) else -1.0), + str(x.get("id") or ""), + ), + ) + + @staticmethod + def _vt_category_bucket(label: str) -> str: + text = (label or "").strip().lower() + if not text: + return "unknown" + if any(k in text for k in ("phish", "credential", "brand", "fake login")): + return "phishing" + if any(k in text for k in ("malware", "trojan", "ransom", "worm", "virus", "botnet", "c2", "backdoor")): + return "malware" + if any(k in text for k in ("spam", "scam", "fraud")): + return "fraud/scam" + if any(k in text for k in ("suspicious", "unknown")): + return "suspicious" + return text + + def _vt_dominant_threat_category(self, attrs: Dict[str, Any]) -> str: + votes: Dict[str, int] = {} + + categories_map = attrs.get("categories") or {} + if isinstance(categories_map, dict): + for raw in categories_map.values(): + bucket = self._vt_category_bucket(str(raw)) + votes[bucket] = votes.get(bucket, 0) + 1 + + analysis_results = attrs.get("last_analysis_results") or {} + if isinstance(analysis_results, dict): + for engine_data in analysis_results.values(): + if not isinstance(engine_data, dict): + continue + if str(engine_data.get("category") or "").strip().lower() not in {"malicious", "suspicious"}: + continue + label = str(engine_data.get("result") or engine_data.get("category") or "") + bucket = self._vt_category_bucket(label) + votes[bucket] = votes.get(bucket, 0) + 1 + + if not votes: + return "unknown" + return sorted(votes.items(), key=lambda x: (-x[1], x[0]))[0][0] + + def _vt_dns_anomalies(self, attrs: Dict[str, Any]) -> List[str]: + anomalies: List[str] = [] + records = attrs.get("last_dns_records") or [] + if not isinstance(records, list): + return anomalies + + provider_suffixes = ( + "github.io", + "herokudns.com", + "azurewebsites.net", + "cloudfront.net", + "fastly.net", + "pantheonsite.io", + "readthedocs.io", + "surge.sh", + ) + for rec in records[:80]: + if not isinstance(rec, dict): + continue + rtype = str(rec.get("type") or "").strip().upper() + value = str(rec.get("value") or rec.get("rdata") or "").strip().lower().rstrip(".") + if not rtype or not value: + continue + if rtype in {"A", "AAAA"}: + try: + ip_obj = ipaddress.ip_address(value) + if ip_obj.is_private or ip_obj.is_loopback or ip_obj.is_link_local or ip_obj.is_reserved: + anomalies.append(f"{rtype} points to non-public IP {value}") + except Exception: + continue + elif rtype == "CNAME": + if any(value == s or value.endswith("." + s) for s in provider_suffixes): + anomalies.append(f"CNAME to takeover-prone provider suffix: {value}") + elif rtype == "TXT": + if "v=spf1" in value and "+all" in value: + anomalies.append("TXT SPF policy contains +all (overly permissive)") + deduped = [] + seen: set[str] = set() + for item in anomalies: + if item not in seen: + seen.add(item) + deduped.append(item) + return deduped[:8] + + def _vt_cert_anomalies(self, attrs: Dict[str, Any]) -> List[str]: + anomalies: List[str] = [] + cert = attrs.get("last_https_certificate") or {} + if not isinstance(cert, dict): + return anomalies + + subject = cert.get("subject") or {} + issuer = cert.get("issuer") or {} + if isinstance(subject, dict) and isinstance(issuer, dict): + subj_cn = str(subject.get("CN") or "").strip().lower() + issuer_cn = str(issuer.get("CN") or "").strip().lower() + if subj_cn and issuer_cn and subj_cn == issuer_cn: + anomalies.append("Certificate appears self-issued (issuer CN == subject CN)") + if subj_cn.startswith("*."): + anomalies.append(f"Wildcard certificate CN detected: {subj_cn}") + + valid_from = self._as_int(cert.get("validity", {}).get("not_before"), default=0) if isinstance(cert.get("validity"), dict) else 0 + valid_to = self._as_int(cert.get("validity", {}).get("not_after"), default=0) if isinstance(cert.get("validity"), dict) else 0 + if valid_from and valid_to and valid_to > valid_from: + lifetime_days = int((valid_to - valid_from) / 86400) + if lifetime_days > 398: + anomalies.append(f"Certificate validity unusually long: {lifetime_days} days") + try: + if datetime.utcfromtimestamp(valid_to) < datetime.utcnow(): + anomalies.append("Certificate appears expired") + except Exception: + pass + + san = cert.get("extensions", {}).get("subject_alternative_name") if isinstance(cert.get("extensions"), dict) else None + san_list = san if isinstance(san, list) else [] + san_values = [str(x).strip().lower() for x in san_list if str(x).strip()] + if len(san_values) > 40: + anomalies.append(f"Very large SAN list: {len(san_values)} entries") + return anomalies[:8] + + async def _fetch_json(self, url: str, headers: Optional[Dict[str, str]] = None) -> Dict[str, Any]: + try: + response = await self.client.get(url, headers=headers or self.headers, timeout=self.timeout, follow_redirects=True) + except Exception as exc: + message = str(exc).strip() + err = f"{exc.__class__.__name__}: {message}" if message else exc.__class__.__name__ + return {"ok": False, "status": None, "json": None, "error": self._redact_api_secrets(err)} + + payload: Optional[Any] = None + if response.content: + try: + payload = response.json() + except Exception: + payload = None + + if response.status_code >= 400: + detail = None + if isinstance(payload, dict): + detail = payload.get("error") or payload.get("message") + if not detail: + detail = f"HTTP {response.status_code}" + return { + "ok": False, + "status": response.status_code, + "json": payload if isinstance(payload, dict) else None, + "error": self._redact_api_secrets(str(detail)), + } + + return {"ok": True, "status": response.status_code, "json": payload if isinstance(payload, dict) else {}, "error": None} + + async def _virustotal_security(self) -> Dict[str, Any]: + if not self.vt_key: + return {"enabled": False} + + url = f"https://www.virustotal.com/api/v3/domains/{self.domain}" + fetch = await self._fetch_json(url, headers={"x-apikey": self.vt_key, "accept": "application/json"}) + if not fetch.get("ok"): + return { + "enabled": True, + "ok": False, + "level": "info", + "summary": "VirusTotal check unavailable", + "status_code": fetch.get("status"), + "error": fetch.get("error"), + } + + payload = fetch.get("json") or {} + attrs = ((payload.get("data") or {}).get("attributes") or {}) if isinstance(payload, dict) else {} + stats = attrs.get("last_analysis_stats") or {} + malicious = self._as_int(stats.get("malicious")) + suspicious = self._as_int(stats.get("suspicious")) + harmless = self._as_int(stats.get("harmless")) + undetected = self._as_int(stats.get("undetected")) + reputation = self._as_int(attrs.get("reputation"), default=0) + + if malicious > 0: + level = "critical" + elif suspicious > 0: + level = "warning" + else: + level = "ok" + + summary = ( + f"VT detections: malicious={malicious}, suspicious={suspicious}, " + f"harmless={harmless}, undetected={undetected}" + ) + + categories_map = attrs.get("categories") or {} + categories = [] + if isinstance(categories_map, dict): + categories = sorted({str(v).strip() for v in categories_map.values() if str(v).strip()}) + dominant_category = self._vt_dominant_threat_category(attrs) + dns_anomalies = self._vt_dns_anomalies(attrs) + cert_anomalies = self._vt_cert_anomalies(attrs) + + total_votes = attrs.get("total_votes") or {} + return { + "enabled": True, + "ok": True, + "level": level, + "summary": summary, + "status_code": fetch.get("status"), + "malicious": malicious, + "suspicious": suspicious, + "harmless": harmless, + "undetected": undetected, + "reputation": reputation, + "categories": categories[:8], + "dominant_threat_category": dominant_category, + "dns_anomalies": dns_anomalies, + "cert_anomalies": cert_anomalies, + "tags": [str(t).strip() for t in (attrs.get("tags") or []) if str(t).strip()][:10], + "last_analysis_date": self._iso_from_epoch(attrs.get("last_analysis_date")), + "last_modification_date": self._iso_from_epoch(attrs.get("last_modification_date")), + "votes_harmless": self._as_int(total_votes.get("harmless")), + "votes_malicious": self._as_int(total_votes.get("malicious")), + } + + async def _shodan_security(self, ips: List[str]) -> Dict[str, Any]: + if not self.shodan_key: + return {"enabled": False} + if not ips: + return { + "enabled": True, + "ok": False, + "level": "info", + "summary": "Shodan check skipped: no resolved IP", + "error": None, + } + + target_ip = str(ips[0]).strip() + url = f"https://api.shodan.io/shodan/host/{target_ip}?key={self.shodan_key}" + fetch = await self._fetch_json(url) + if not fetch.get("ok"): + status_code = fetch.get("status") + if status_code == 404: + return { + "enabled": True, + "ok": True, + "level": "info", + "summary": "No Shodan host data available for resolved IP", + "status_code": status_code, + "ip": target_ip, + "ports": [], + "vulns": [], + } + return { + "enabled": True, + "ok": False, + "level": "info", + "summary": "Shodan check unavailable", + "status_code": status_code, + "error": fetch.get("error"), + } + + payload = fetch.get("json") or {} + ports = sorted({self._as_int(p) for p in (payload.get("ports") or []) if self._as_int(p) > 0}) + vulns_obj = payload.get("vulns") + vuln_entries = self._normalize_shodan_vulns(vulns_obj) + vulns = [str(v.get("id") or "").strip() for v in vuln_entries if str(v.get("id") or "").strip()] + + hostnames = [str(h).strip() for h in (payload.get("hostnames") or []) if str(h).strip()] + domains = [str(d).strip() for d in (payload.get("domains") or []) if str(d).strip()] + affected_targets = sorted(set(hostnames + domains)) + + risky_ports = {21, 23, 25, 445, 1433, 1521, 2375, 3306, 3389, 5432, 5900, 6379, 9200, 11211, 27017} + exposed = sorted(p for p in ports if p in risky_ports) + if vulns: + level = "critical" + summary = f"Shodan reports {len(vulns)} known vulnerability identifiers" + elif exposed: + level = "warning" + summary = f"Potentially exposed services on ports: {', '.join(str(p) for p in exposed)}" + else: + level = "ok" + summary = f"Shodan host profile present ({len(ports)} open ports)" + + return { + "enabled": True, + "ok": True, + "level": level, + "summary": summary, + "status_code": fetch.get("status"), + "ip": str(payload.get("ip_str") or target_ip), + "ports": ports[:30], + "vulns": vulns[:30], + "vuln_count": len(vulns), + "vulns_ranked": vuln_entries[:50], + "org": payload.get("org"), + "isp": payload.get("isp"), + "os": payload.get("os"), + "hostnames": hostnames[:20], + "domains": domains[:20], + "affected_targets": affected_targets[:25], + "tags": [str(t).strip() for t in (payload.get("tags") or []) if str(t).strip()][:10], + "last_update": str(payload.get("last_update") or "").strip() or None, + "risky_ports": exposed, + } + + async def _threat_intel(self, dns_detail: Dict[str, Any]) -> Dict[str, Any]: + ips = [str(ip).strip() for ip in ((dns_detail.get("a") or []) + (dns_detail.get("aaaa") or [])) if str(ip).strip()] + vt_info, shodan_info = await asyncio.gather(self._virustotal_security(), self._shodan_security(ips)) + return {"virustotal": vt_info, "shodan": shodan_info} + + def _assess_security( + self, + https_chain: Dict[str, Any], + tls_info: Dict[str, Any], + tls_versions: Optional[List[str]], + security_txt: Dict[str, Any], + caa: Dict[str, Any], + email_auth: Dict[str, Any], + takeover: Dict[str, Any], + open_redirect: Dict[str, Any], + methods: Dict[str, Any], + rate_waf: Dict[str, Any], + threat_intel: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + checks: Dict[str, Dict[str, Any]] = {} + catalog = load_server_versions_catalog(auto_update=True) + + set_cookie = https_chain.get("set_cookie") or [] + hsts = https_chain.get("hsts") + sec_headers = https_chain.get("security_headers") or {} + infra_headers = https_chain.get("infra_headers") or {} + https_server = str(infra_headers.get("Server") or "").strip() + server_assessment = assess_server_banner(https_server, catalog=catalog) + body_sample = str(https_chain.get("body_sample") or "") + mixed_count = len(re.findall(r"""(?i)\b(?:src|href)\s*=\s*["']http://""", body_sample)) + emails = sorted(set(self.EMAIL_RE.findall(body_sample))) + + # security.txt + if security_txt.get("present"): + checks["security_txt"] = {"level": "ok", "summary": "security.txt present"} + else: + checks["security_txt"] = {"level": "info", "summary": "security.txt missing or invalid", "detail": security_txt} + + # cookie flags + if not set_cookie: + checks["cookies"] = {"level": "info", "summary": "No Set-Cookie in final HTTPS response"} + else: + missing_secure = 0 + missing_httponly = 0 + missing_samesite = 0 + for c in set_cookie: + lc = c.lower() + if "secure" not in lc: + missing_secure += 1 + if "httponly" not in lc: + missing_httponly += 1 + if "samesite=" not in lc: + missing_samesite += 1 + level = "ok" + if missing_secure > 0: + level = "warning" + elif missing_httponly > 0 or missing_samesite > 0: + level = "info" + checks["cookies"] = { + "level": level, + "summary": f"cookies={len(set_cookie)} missing_secure={missing_secure} missing_httponly={missing_httponly} missing_samesite={missing_samesite}", + } + + # HSTS / headers + header_issues: List[str] = [] + if not hsts: + header_issues.append("missing HSTS") + if not sec_headers.get("Content-Security-Policy"): + header_issues.append("missing CSP") + if not sec_headers.get("X-Frame-Options"): + header_issues.append("missing X-Frame-Options") + if not sec_headers.get("X-Content-Type-Options"): + header_issues.append("missing X-Content-Type-Options") + checks["headers"] = { + "level": "ok" if not header_issues else "warning", + "summary": "Header baseline looks good" if not header_issues else ", ".join(header_issues), + } + + # TLS hygiene + tls_issues: List[str] = [] + if not tls_info.get("strict_ok"): + tls_issues.append("strict certificate verify failed") + key_bits = tls_info.get("key_bits") + key_type = tls_info.get("key_type") + if key_type == "RSA" and isinstance(key_bits, int) and key_bits < 2048: + tls_issues.append(f"weak RSA key size {key_bits}") + if tls_info.get("self_signed"): + tls_issues.append("self-signed certificate") + supported_tls = [str(v).strip() for v in (tls_versions or []) if str(v).strip()] + legacy_tls = [v for v in supported_tls if v in {"TLS 1.0", "TLS 1.1"}] + if legacy_tls: + tls_issues.append(f"legacy TLS supported: {', '.join(legacy_tls)}") + checks["tls_hygiene"] = { + "level": "ok" if not tls_issues else "warning", + "summary": "TLS certificate hygiene looks good" if not tls_issues else ", ".join(tls_issues), + } + + # CAA + if caa.get("present"): + checks["caa"] = {"level": "ok", "summary": f"CAA present ({len(caa.get('entries') or [])} entries)"} + else: + checks["caa"] = {"level": "info", "summary": "CAA not found"} + + spf = email_auth.get("spf") or {} + checks["spf"] = { + "level": str(spf.get("level") or "info"), + "summary": str(spf.get("summary") or "SPF check unavailable"), + } + dmarc = email_auth.get("dmarc") or {} + checks["dmarc"] = { + "level": str(dmarc.get("level") or "info"), + "summary": str(dmarc.get("summary") or "DMARC check unavailable"), + } + dkim = email_auth.get("dkim") or {} + checks["dkim"] = { + "level": str(dkim.get("level") or "info"), + "summary": str(dkim.get("summary") or "DKIM check unavailable"), + } + + # takeover hints + if takeover.get("suspect"): + checks["takeover"] = {"level": "warning", "summary": f"Potential takeover hint: {', '.join(takeover.get('suspect_targets') or [])}"} + else: + checks["takeover"] = {"level": "ok", "summary": "No takeover hint detected"} + + # open redirect + if open_redirect.get("possible"): + checks["open_redirect"] = {"level": "warning", "summary": "Potential open redirect behavior detected"} + else: + checks["open_redirect"] = {"level": "ok", "summary": "No open redirect behavior detected in safe probes"} + + # methods + risky = methods.get("risky_methods") or [] + if risky: + checks["methods"] = {"level": "warning", "summary": f"Potentially risky methods exposed: {', '.join(risky)}"} + else: + checks["methods"] = {"level": "ok", "summary": "No risky HTTP methods exposed"} + + # mixed content + if mixed_count > 0: + checks["mixed_content"] = {"level": "warning", "summary": f"Potential mixed content references: {mixed_count}"} + else: + checks["mixed_content"] = {"level": "ok", "summary": "No mixed content reference detected"} + + # rate-limit/WAF + if rate_waf.get("rate_limited"): + checks["rate_waf"] = {"level": "info", "summary": "Rate-limit behavior detected (HTTP 429)"} + elif rate_waf.get("waf_flags"): + checks["rate_waf"] = {"level": "info", "summary": f"WAF/challenge hint: {', '.join(rate_waf.get('waf_flags') or [])}"} + else: + checks["rate_waf"] = {"level": "ok", "summary": "No obvious WAF/rate-limit behavior in small burst"} + + # email exposure + if emails: + checks["emails"] = {"level": "info", "summary": f"Email-like strings found in response body: {len(emails)}"} + else: + checks["emails"] = {"level": "ok", "summary": "No email-like strings in sampled HTML body"} + + srv_status = str(server_assessment.get("status") or "unknown") + if srv_status == "outdated": + checks["server_version"] = { + "level": "warning", + "summary": f"Web server possibly outdated ({server_assessment.get('version')} < {server_assessment.get('latest')})", + } + elif srv_status == "up-to-date": + checks["server_version"] = {"level": "ok", "summary": "Web server version is up-to-date"} + elif srv_status == "advisory": + checks["server_version"] = {"level": "info", "summary": "Web server version check is advisory for this product"} + else: + checks["server_version"] = {"level": "info", "summary": "Web server version could not be reliably evaluated"} + + ti = threat_intel or {} + vt = ti.get("virustotal") or {} + if vt.get("enabled"): + if vt.get("ok"): + checks["virustotal"] = { + "level": str(vt.get("level") or "info"), + "summary": str(vt.get("summary") or "VirusTotal data available"), + } + else: + reason = str(vt.get("error") or vt.get("summary") or "request failed") + checks["virustotal"] = {"level": "info", "summary": f"VirusTotal unavailable: {reason}"} + + shodan = ti.get("shodan") or {} + if shodan.get("enabled"): + if shodan.get("ok"): + checks["shodan"] = { + "level": str(shodan.get("level") or "info"), + "summary": str(shodan.get("summary") or "Shodan data available"), + } + else: + reason = str(shodan.get("error") or shodan.get("summary") or "request failed") + checks["shodan"] = {"level": "info", "summary": f"Shodan unavailable: {reason}"} + + return { + "checks": checks, + "security_txt": security_txt, + "caa": caa, + "email_auth": email_auth, + "takeover": takeover, + "open_redirect": open_redirect, + "methods": methods, + "rate_waf": rate_waf, + "emails": emails[:20], + "mixed_content_count": mixed_count, + "server_assessment": server_assessment, + "server_versions_updated_at": catalog.get("updated_at"), + "threat_intel": ti, + } + + async def _useragent_compare(self) -> Dict[str, Any]: + browser_headers = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" + ) + } + default_probe, browser_probe = await asyncio.gather( + self._request_detail(f"https://{self.domain}", headers=self.headers), + self._request_detail(f"https://{self.domain}", headers=browser_headers), + ) + return {"default": default_probe, "browser_like": browser_probe} + + def _extract_https_port(self, value: Optional[str]) -> Optional[int]: + if not value: + return None + try: + parsed = urlparse(value) + except Exception: + return None + if parsed.scheme.lower() != "https": + return None + if parsed.port is None: + return 443 + if parsed.port <= 0 or parsed.port > 65535: + return None + return int(parsed.port) + + def _redirect_https_port_from_chains( + self, + http_chain: Dict[str, Any], + https_chain: Dict[str, Any], + http_first_hop: Optional[Dict[str, Any]] = None, + ) -> Optional[int]: + candidates: List[str] = [] + for chain in (http_chain, https_chain, http_first_hop or {}): + if not isinstance(chain, dict): + continue + url = chain.get("url") + if isinstance(url, str) and url: + candidates.append(url) + for hop in chain.get("history") or []: + if isinstance(hop, dict): + loc = hop.get("location") + app = hop.get("app_redirect") + if isinstance(loc, str) and loc: + candidates.append(loc) + if isinstance(app, str) and app: + candidates.append(app) + for candidate in candidates: + port = self._extract_https_port(candidate) + if port and port != 443: + return port + return None + + async def _collect_verbose(self) -> Dict[str, Any]: + """Gather extended diagnostics used by `--verbose`. + + How: + - executes multiple independent probes concurrently via `asyncio.gather` + - merges protocol/network/security data into one structured payload + Where: + - attached to scan result as `result["verbose"]` and rendered by + `knockpy/output.py`. + """ + ( + dns_detail, + tls_detail, + tls_versions, + tcp_ports, + http_chain, + https_chain, + ua_compare, + http_first_hop, + security_txt, + caa_info, + email_auth, + takeover_hints, + open_redirect, + methods_info, + rate_waf, + ) = await asyncio.gather( + self._dns_detail(), + self._tls_handshake_detail(), + self._check_tls_versions(), + self._tcp_ports(), + self._request_detail(f"http://{self.domain}", follow_redirects=True), + self._request_detail(f"https://{self.domain}", follow_redirects=True), + self._useragent_compare(), + self._request_detail(f"http://{self.domain}", follow_redirects=False), + self._security_txt_check(), + self._caa_check(), + self._email_auth_detail(), + self._dns_takeover_hints(), + self._open_redirect_probe(), + self._http_methods_check(), + self._rate_limit_waf_check(), + ) + threat_intel = await self._threat_intel(dns_detail) + verbose_data: Dict[str, Any] = { + "dns": dns_detail, + "tls": tls_detail, + "tls_versions": tls_versions, + "tcp_ports": tcp_ports, + "http_chain": http_chain, + "https_chain": https_chain, + "ua_compare": ua_compare, + "threat_intel": threat_intel, + "security": self._assess_security( + https_chain=https_chain, + tls_info=tls_detail, + tls_versions=tls_versions, + security_txt=security_txt, + caa=caa_info, + email_auth=email_auth, + takeover=takeover_hints, + open_redirect=open_redirect, + methods=methods_info, + rate_waf=rate_waf, + threat_intel=threat_intel, + ), + } + + redirect_port = self._redirect_https_port_from_chains(http_chain, https_chain, http_first_hop) + if redirect_port: + redirect_tls, redirect_tls_versions = await asyncio.gather( + self._tls_handshake_detail(port=redirect_port), + self._check_tls_versions(port=redirect_port), + ) + verbose_data["redirect_tls"] = { + "port": redirect_port, + "handshake": redirect_tls, + "versions": redirect_tls_versions, + } + + return verbose_data + + async def _cert_fetch(self, resolved_ips: Optional[List[str]] = None) -> Tuple[Optional[bool], Optional[str], Optional[str], Optional[List[str]]]: + """Collect certificate validity + metadata + supported TLS versions. + + Why two phases: + - strict phase validates trust/hostname (real security state) + - fallback phase extracts metadata even on validation failure, so the + report remains actionable. + """ + loop = asyncio.get_running_loop() + + def sync_cert() -> Tuple[Optional[bool], Optional[str], Optional[str]]: + cert_ok = False + expiry_iso: Optional[str] = None + common_name: Optional[str] = None + + # Strict verification: chain trust + hostname verification. + try: + ctx = ssl.create_default_context() + ctx.check_hostname = True + ctx.verify_mode = ssl.CERT_REQUIRED + with socket.create_connection((self.domain, 443), timeout=self.timeout) as sock: + with ctx.wrap_socket(sock, server_hostname=self.domain) as tls_sock: + cert = tls_sock.getpeercert() + if cert: + not_after = cert.get("notAfter") + if not_after: + expiry_ts = ssl.cert_time_to_seconds(not_after) + expiry_date = datetime.utcfromtimestamp(expiry_ts).date() + expiry_iso = expiry_date.isoformat() + cert_ok = expiry_date >= datetime.utcnow().date() + + subject = cert.get("subject", ()) + for rdns in subject: + for key, val in rdns: + if key == "commonName": + common_name = val + break + if common_name: + break + except Exception: + cert_ok = False + + # Best-effort metadata extraction even when strict verification fails. + if expiry_iso is None or common_name is None: + try: + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + with socket.create_connection((self.domain, 443), timeout=self.timeout) as sock: + with ctx.wrap_socket(sock, server_hostname=self.domain) as tls_sock: + der_cert = tls_sock.getpeercert(binary_form=True) + x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_ASN1, der_cert) + if common_name is None: + common_name = x509.get_subject().commonName or None + if expiry_iso is None: + not_after = x509.get_notAfter().decode("utf-8") + expiry_date = datetime.strptime(not_after, "%Y%m%d%H%M%SZ").date() + expiry_iso = expiry_date.isoformat() + except Exception: + pass + + return cert_ok, expiry_iso, common_name + + async with self.cert_semaphore: + is_valid, expiry, cn = await loop.run_in_executor(self.io_executor, sync_cert) + + tls_versions = await self._check_tls_versions(resolved_ips=resolved_ips) + # Keep certificate validation status independent from protocol hardening. + # Legacy TLS support is handled as a warning in output/report layers. + return is_valid, expiry, cn, tls_versions + + def _tls_probe_cache_key(self, port: int, resolved_ips: Optional[List[str]]) -> str: + if resolved_ips: + normalized = ",".join(sorted({str(ip).strip() for ip in resolved_ips if str(ip).strip()})) + if normalized: + return f"{port}|{normalized}" + return f"{port}|{self.domain.lower()}" + + async def _check_tls_versions(self, port: int = 443, resolved_ips: Optional[List[str]] = None) -> List[str]: + loop = asyncio.get_running_loop() + cache_key = self._tls_probe_cache_key(port, resolved_ips) + now = time.monotonic() + cached = self.TLS_PROBE_CACHE.get(cache_key) + if cached and cached[0] > now: + return list(cached[1]) + + def sync_tls_check() -> List[str]: + supported: List[str] = [] + tls_targets: List[Tuple[str, Any]] = [] + + if hasattr(ssl, "TLSVersion"): + if hasattr(ssl.TLSVersion, "TLSv1"): + tls_targets.append(("TLS 1.0", ssl.TLSVersion.TLSv1)) + if hasattr(ssl.TLSVersion, "TLSv1_1"): + tls_targets.append(("TLS 1.1", ssl.TLSVersion.TLSv1_1)) + if hasattr(ssl.TLSVersion, "TLSv1_2"): + tls_targets.append(("TLS 1.2", ssl.TLSVersion.TLSv1_2)) + if hasattr(ssl.TLSVersion, "TLSv1_3"): + tls_targets.append(("TLS 1.3", ssl.TLSVersion.TLSv1_3)) + + def _probe_fixed_version(version: Any, allow_legacy_fallback: bool) -> bool: + def _attempt(seclevel0: bool) -> bool: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + if seclevel0: + try: + # OpenSSL 3 may refuse TLS 1.0/1.1 at default security level. + # Lowering SECLEVEL for the probe lets us detect protocol support + # without changing the main scan client's TLS policy. + context.set_ciphers("DEFAULT:@SECLEVEL=0") + except Exception: + return False + context.minimum_version = version + context.maximum_version = version + with socket.create_connection((self.domain, int(port)), timeout=self.timeout) as sock: + with context.wrap_socket(sock, server_hostname=self.domain): + return True + + try: + return _attempt(seclevel0=False) + except Exception: + if allow_legacy_fallback: + try: + return _attempt(seclevel0=True) + except Exception: + return False + return False + + for name, version in tls_targets: + is_legacy = name in {"TLS 1.0", "TLS 1.1"} + if _probe_fixed_version(version, allow_legacy_fallback=is_legacy): + supported.append(name) + + return supported + + supported_versions = await loop.run_in_executor(self.io_executor, sync_tls_check) + if len(self.TLS_PROBE_CACHE) > 4096: + # Bound memory usage during very large brute-force sessions. + expired_keys = [k for k, v in self.TLS_PROBE_CACHE.items() if v[0] <= now] + for key in expired_keys: + self.TLS_PROBE_CACHE.pop(key, None) + if len(self.TLS_PROBE_CACHE) > 4096: + for key in list(self.TLS_PROBE_CACHE.keys())[:512]: + self.TLS_PROBE_CACHE.pop(key, None) + self.TLS_PROBE_CACHE[cache_key] = (now + self.TLS_PROBE_TTL, list(supported_versions)) + return supported_versions + + async def scan(self) -> Optional[Dict[str, Any]]: + """Execute full checks for the current domain and return normalized data. + + Used by `_run_async` worker tasks. Returns `None` when DNS resolution + fails, otherwise returns protocol/cert data plus optional verbose details. + """ + ips = await self._resolve_async(self.domain) + if not ips: + return None + + http, https = await asyncio.gather( + self._http_check(f"http://{self.domain}"), + self._http_check(f"https://{self.domain}"), + ) + + if http[0] and http[1] and not https[0]: + location = http[1] + if not location.startswith(("http://", "https://")): + location = "http://" + location + redirected = location.split("://", 1)[1].split("/")[0] + https = await self._http_check(f"https://{redirected}") + + cert = (None, None, None, None) + if https[0]: + cert = await self._cert_fetch(resolved_ips=ips) + + takeover = await self._takeover_check(http, https) + + result = { + "domain": self.domain, + "ip": ips, + "http": list(http), + "https": list(https), + "cert": list(cert), + } + if takeover: + result["takeover"] = takeover + takeover_status = str(takeover.get("status") or "") + provider = str(takeover.get("provider") or "unknown") + cname = str(takeover.get("cname") or "-") + if takeover_status == "likely": + result.setdefault("scan_notes", []).append( + f"Takeover risk: likely ({provider}) via CNAME {cname}." + ) + elif takeover_status == "possible": + result.setdefault("scan_notes", []).append( + f"Takeover risk: possible ({provider}) via CNAME {cname}." + ) + if self.verbose: + result["verbose"] = await self._collect_verbose() + return result + + +async def _run_async( + domains: List[str], + dns: Optional[str], + useragent: Optional[str], + timeout: Optional[float], + threads: Optional[int], + recon: bool, + bruteforce: bool, + wordlist: Optional[str], + verbose: bool = False, + enable_axfr: bool = True, + api_key_virustotal: Optional[str] = None, + api_key_shodan: Optional[str] = None, + progress_callback: Optional[Callable[[int, int], None]] = None, +) -> Union[List[dict], dict, None]: + """Main orchestrator used by both CLI and Python API. + + Flow: + 1. normalize/expand target set (recon + bruteforce) + 2. optionally run AXFR check on root domain + 3. run scanner(s) with bounded concurrency + 4. attach scan notes/AXFR metadata and return normalized result(s) + """ + if not domains: + return None + + base = domains[0] + effective_useragent = pick_user_agent(useragent) + # Phase 1: expand targets before scanner execution. + if recon or bruteforce: + expanded: List[str] = [] + if recon: + recon_engine = Recon( + base, + timeout=(timeout or 3.0), + max_concurrency=40, + useragent=effective_useragent, + virustotal_key=api_key_virustotal, + shodan_key=api_key_shodan, + ) + expanded.extend(await recon_engine.start()) + if bruteforce: + brute_engine = Bruteforce(base, wordlist) + expanded.extend(brute_engine.start()) + domains = list(OrderedDict.fromkeys(domains + expanded)) + + timeout_value = timeout or 3.0 + axfr_info: Optional[Dict[str, Any]] = None + # Phase 2: run root-domain AXFR probe once and attach metadata to results. + if enable_axfr: + axfr_timeout = max(3.0, timeout_value * 2.0 + 1.0) + try: + axfr_info = await asyncio.wait_for(_axfr_check_root(base, dns, timeout_value), timeout=axfr_timeout) + except asyncio.TimeoutError: + axfr_info = {"domain": base, "status": "timeout", "checks": [], "error": f"Global AXFR timeout after {axfr_timeout:.1f}s"} + + def _axfr_note(axfr: Dict[str, Any]) -> str: + status = str(axfr.get("status") or "unknown") + if status == "allowed": + ns = axfr.get("allowed_ns") or "-" + count = axfr.get("record_count") or "-" + return f"AXFR on root domain: allowed via {ns} (records: {count})." + checks = axfr.get("checks") or [] + return f"AXFR on root domain: {status} ({len(checks)} NS checked)." + + def _inject_axfr(result_obj: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]], axfr: Optional[Dict[str, Any]]) -> None: + if not axfr or not result_obj: + return + note = _axfr_note(axfr) + if isinstance(result_obj, dict): + notes = result_obj.get("scan_notes") or [] + if note not in notes: + notes.append(note) + result_obj["scan_notes"] = notes + result_obj["axfr"] = axfr + return + if isinstance(result_obj, list): + target = None + for item in result_obj: + if str(item.get("domain") or "").lower() == base.lower(): + target = item + break + if target is None and result_obj: + target = result_obj[0] + if isinstance(target, dict): + notes = target.get("scan_notes") or [] + if note not in notes: + notes.append(note) + target["scan_notes"] = notes + target["axfr"] = axfr + + max_workers = threads or 250 + io_workers = max(64, min(512, max_workers * 2)) + timeout_cfg = httpx.Timeout(timeout_value) + limits_cfg = httpx.Limits( + max_connections=max(100, max_workers * 2), + max_keepalive_connections=max(50, max_workers), + ) + + # Fast path for single target: keep behavior compatible with direct checks. + if len(domains) == 1: + with ThreadPoolExecutor(max_workers=io_workers) as io_executor: + async with httpx.AsyncClient( + http2=True, + verify=False, + timeout=timeout_cfg, + limits=limits_cfg, + ) as client: + scanner = AsyncScanner( + domains[0], + client=client, + dns_server=dns, + useragent=effective_useragent, + timeout=timeout_value, + root_domain=base, + io_executor=io_executor, + verbose=verbose, + api_key_virustotal=api_key_virustotal, + api_key_shodan=api_key_shodan, + ) + result = await scanner.scan() + if result is None and not recon and not bruteforce: + fallback_domain = _www_fallback_domain(domains[0]) + if fallback_domain: + fallback_scanner = AsyncScanner( + fallback_domain, + client=client, + dns_server=dns, + useragent=effective_useragent, + timeout=timeout_value, + root_domain=base, + io_executor=io_executor, + verbose=verbose, + api_key_virustotal=api_key_virustotal, + api_key_shodan=api_key_shodan, + ) + result = await fallback_scanner.scan() + if result is not None: + result["scan_notes"] = [ + f"No results for {domains[0]}; tried fallback {fallback_domain}." + ] + if progress_callback: + progress_callback(1, 1) + _inject_axfr(result, axfr_info) + return result + + cert_semaphore = asyncio.Semaphore(50) + + # Multi-target path with bounded concurrency and shared HTTP client. + with ThreadPoolExecutor(max_workers=io_workers) as io_executor: + async with httpx.AsyncClient( + http2=True, + verify=False, + timeout=timeout_cfg, + limits=limits_cfg, + ) as client: + queue: "asyncio.Queue[Optional[str]]" = asyncio.Queue() + for domain in domains: + queue.put_nowait(domain) + + worker_count = max(1, min(max_workers, len(domains))) + for _ in range(worker_count): + queue.put_nowait(None) + + results: List[Dict[str, Any]] = [] + total = len(domains) + done = 0 + done_lock = asyncio.Lock() + + async def worker() -> None: + nonlocal done + while True: + scan_domain = await queue.get() + try: + if scan_domain is None: + return + scanner = AsyncScanner( + scan_domain, + client=client, + dns_server=dns, + useragent=effective_useragent, + timeout=timeout_value, + cert_semaphore=cert_semaphore, + root_domain=base, + io_executor=io_executor, + verbose=verbose, + api_key_virustotal=api_key_virustotal, + api_key_shodan=api_key_shodan, + ) + result = await scanner.scan() + if result: + results.append(result) + finally: + if scan_domain is not None: + async with done_lock: + done += 1 + if progress_callback: + progress_callback(done, total) + queue.task_done() + + workers = [asyncio.create_task(worker()) for _ in range(worker_count)] + await queue.join() + await asyncio.gather(*workers) + + _inject_axfr(results, axfr_info) + return results + + +def _run_coro_sync(coro: Any) -> Any: + """Run async code from sync callers (CLI and public API). + + If already inside an event loop, execute in a helper thread to avoid + `RuntimeError: asyncio.run() cannot be called from a running event loop`. + """ + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + + outcome: Dict[str, Any] = {} + + def target() -> None: + try: + outcome["result"] = asyncio.run(coro) + except Exception as exc: # pragma: no cover - fallback path + outcome["error"] = exc + + thread = threading.Thread(target=target, daemon=True) + thread.start() + thread.join() + + if "error" in outcome: + raise outcome["error"] + return outcome.get("result") + + +def KNOCKPY( + domain: Union[str, Iterable[str]], + dns: Optional[str] = None, + useragent: Optional[str] = None, + timeout: Optional[float] = None, + threads: Optional[int] = None, + recon: bool = False, + bruteforce: bool = False, + wordlist: Optional[str] = None, + silent: bool = False, + verbose: bool = False, + enable_axfr: bool = True, + api_key_virustotal: Optional[str] = None, + api_key_shodan: Optional[str] = None, +) -> Union[List[dict], dict, None]: + """Public synchronous Python API entrypoint. + + Example: + `KNOCKPY("example.com", recon=True, bruteforce=True, verbose=False)` + """ + if isinstance(domain, str): + domains = [domain] + else: + domains = list(domain) + + return _run_coro_sync( + _run_async( + domains, + dns, + useragent, + timeout, + threads, + recon, + bruteforce, + wordlist, + verbose=verbose, + enable_axfr=enable_axfr, + api_key_virustotal=api_key_virustotal, + api_key_shodan=api_key_shodan, + ) + ) diff --git a/knockpy/knockpy.py b/knockpy/knockpy.py new file mode 100644 index 0000000..cf875ab --- /dev/null +++ b/knockpy/knockpy.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Compatibility module exposing imports expected by legacy users. + +This module re-exports CLI entrypoint and public APIs so external scripts can +`import knockpy.knockpy` without depending on internal file structure. +""" + +from .cli import main +from .core import AsyncScanner, Bruteforce, KNOCKPY, Recon, fmt_td +from .output import output, print_scan_status, show_reports_catalog +from .storage import export_report, get_report, list_reports, save_scan +from .version import __version__ + +__all__ = [ + "__version__", + "AsyncScanner", + "Bruteforce", + "KNOCKPY", + "Recon", + "fmt_td", + "main", + "output", + "print_scan_status", + "show_reports_catalog", + "save_scan", + "list_reports", + "get_report", + "export_report", +] diff --git a/knockpy/output.py b/knockpy/output.py new file mode 100644 index 0000000..5045575 --- /dev/null +++ b/knockpy/output.py @@ -0,0 +1,751 @@ +from __future__ import annotations + +"""Terminal rendering helpers for knockpy. + +This module contains presentation-only logic for standard and verbose output. +It does not perform network or persistence operations. +""" + +from datetime import date, timedelta +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +from rich import box +from rich.console import Console +from rich.panel import Panel +from rich.table import Table + +from .core import ROOT, fmt_td +from .storage import count_reports, get_db_path + +console = Console() +err_console = Console(stderr=True) + + +# Shared layout constants. +KV_FIELD_WIDTH = 30 +SUMMARY_DOMAIN_WIDTH = 26 +SUMMARY_IP_WIDTH = 16 +SUMMARY_CODE_WIDTH = 6 +SUMMARY_CERT_WIDTH = 8 +SUMMARY_EXPIRY_WIDTH = 12 +SUMMARY_STATUS_WIDTH = 10 +COL3_W1 = 22 +COL3_W2 = 18 + + +def _table_width() -> int: + try: + return max(80, int(console.size.width) - 2) + except Exception: + return 100 + + +def _new_table( + *, + title: Optional[str] = None, + box_style: Any = box.SIMPLE, + show_header: bool = True, + header_style: Optional[str] = None, +) -> Table: + return Table( + title=title, + box=box_style, + show_header=show_header, + header_style=header_style, + title_justify="left", + width=_table_width(), + expand=False, + pad_edge=False, + ) + + +def _kv_value_width() -> int: + # Keep a fixed start point for Value across all 2-column tables. + # Table border/spacing overhead is intentionally conservative. + return max(24, _table_width() - KV_FIELD_WIDTH - 8) + + +def _add_kv_columns(table: Table) -> None: + value_width = _kv_value_width() + table.add_column( + "Field", + style="cyan", + width=KV_FIELD_WIDTH, + min_width=KV_FIELD_WIDTH, + max_width=KV_FIELD_WIDTH, + no_wrap=True, + ) + table.add_column( + "Value", + width=value_width, + min_width=value_width, + max_width=value_width, + overflow="fold", + no_wrap=False, + ) + + +def _add_3col_columns(table: Table, c1: str, c2: str, c3: str, w1: int = COL3_W1, w2: int = COL3_W2) -> None: + rem = max(20, _table_width() - w1 - w2 - 8) + table.add_column(c1, width=w1, min_width=w1, max_width=w1, no_wrap=True, justify="left") + table.add_column(c2, width=w2, min_width=w2, max_width=w2, no_wrap=True, justify="left") + table.add_column(c3, width=rem, min_width=rem, max_width=rem, overflow="fold", no_wrap=False, justify="left") + + +def _parse_http_block(block: List[Any]) -> Dict[str, Any]: + data = list(block or []) + return { + "status": data[0] if len(data) > 0 else None, + "redirect": data[1] if len(data) > 1 else None, + "server": data[2] if len(data) > 2 else None, + "body_len": data[3] if len(data) > 3 else None, + "app_redirect": data[4] if len(data) > 4 else None, + "body_preview": data[5] if len(data) > 5 else None, + "error": data[6] if len(data) > 6 else None, + } + + +def _is_weak_tls(version: str) -> bool: + return version.startswith("SSLv") or version in ("TLS 1.0", "TLS 1.1") + + +def _fmt_status(code: Any) -> str: + if code is None: + return "[red]-[/red]" + try: + value = int(code) + except Exception: + return str(code) + if value >= 400: + return f"[red]{value}[/red]" + if value >= 300: + return f"[yellow]{value}[/yellow]" + return f"[green]{value}[/green]" + + +def _fmt_http_status(code: Any, https_ok: bool) -> str: + if code is None: + return "[red]-[/red]" + try: + value = int(code) + except Exception: + return str(code) + if value >= 400: + if https_ok: + return f"[yellow]{value}[/yellow]" + return f"[red]{value}[/red]" + if value >= 300: + return f"[yellow]{value}[/yellow]" + return f"[green]{value}[/green]" + + +def _fmt_optional(value: Any, failed: bool = False) -> str: + if value in (None, ""): + return "[red]-[/red]" if failed else "-" + return f"[red]{value}[/red]" if failed else str(value) + + +def _level_tag(level: str) -> str: + lv = (level or "").lower() + if lv == "ok": + return "[green]ok[/green]" + if lv in {"warning", "critical"}: + return f"[red]{lv}[/red]" + return "[yellow]info[/yellow]" + + +def _status_text(http_data: List[Any], https_data: List[Any], cert_data: List[Any]) -> str: + has_http = http_data[0] is not None + has_https = https_data[0] is not None + cert_ok = bool(cert_data[0]) + tls_versions = cert_data[3] if len(cert_data) > 3 and isinstance(cert_data[3], list) else [] + weak_tls = any(_is_weak_tls(v) for v in tls_versions) + + if not has_http and not has_https: + return "dns-only" + if weak_tls: + return "warning" + if (has_http or has_https) and not cert_ok: + return "warning" + return "ok" + + +def _tls_label(tls_versions: Optional[List[str]]) -> str: + if not tls_versions: + return "-" + return ", ".join(tls_versions) + + +def _format_ips_for_cell(values: List[str]) -> str: + ips = [str(v).strip() for v in values if str(v).strip()] + if not ips: + return "-" + lines: List[str] = [] + current = "" + max_len = max(8, SUMMARY_IP_WIDTH) + for ip in ips: + if not current: + current = ip + continue + candidate = f"{current}, {ip}" + if len(candidate) <= max_len: + current = candidate + continue + lines.append(current) + current = ip + if current: + lines.append(current) + return "\n".join(lines) + + +SECURITY_CHECK_ORDER = [ + ("security_txt", "security.txt"), + ("headers", "Headers"), + ("cookies", "Cookies"), + ("server_version", "Server Version"), + ("virustotal", "VirusTotal"), + ("shodan", "Shodan"), + ("tls_hygiene", "TLS Hygiene"), + ("mixed_content", "Mixed Content"), + ("caa", "CAA"), + ("spf", "SPF"), + ("dmarc", "DMARC"), + ("dkim", "DKIM"), + ("takeover", "Takeover Hint"), + ("open_redirect", "Open Redirect"), + ("methods", "HTTP Methods"), + ("rate_waf", "Rate-limit/WAF"), + ("emails", "Email Exposure"), +] + +METHOD_FOCUS_ORDER = ["GET", "HEAD", "POST", "PUT", "DELETE", "TRACE", "OPTIONS"] +DANGEROUS_HTTP_METHODS = {"PUT", "DELETE", "TRACE", "CONNECT"} + + +def _preview(values: List[Any], limit: int = 3) -> str: + items = [str(v).strip() for v in values if str(v).strip()] + if not items: + return "-" + head = items[:limit] + tail = len(items) - len(head) + if tail > 0: + return f"{', '.join(head)} (+{tail})" + return ", ".join(head) + + +def _methods_evidence_lines(methods: Dict[str, Any]) -> List[str]: + methods_http_obj = methods.get("http") or {} + methods_https_obj = methods.get("https") or {} + methods_http = methods_http_obj.get("allow") or [] + methods_https = methods_https_obj.get("allow") or [] + methods_http_cors = methods_http_obj.get("cors_allow") or [] + methods_https_cors = methods_https_obj.get("cors_allow") or [] + inferred_safe = methods.get("inferred_safe") or [] + + enabled_union = { + str(m).strip().upper() + for m in ( + list(methods_http) + + list(methods_https) + + list(methods_http_cors) + + list(methods_https_cors) + + list(inferred_safe) + ) + if str(m).strip() + } + enabled_focus = [m for m in METHOD_FOCUS_ORDER if m in enabled_union] + dangerous = [m for m in METHOD_FOCUS_ORDER + ["CONNECT"] if m in enabled_union and m in DANGEROUS_HTTP_METHODS] + + lines: List[str] = [] + if enabled_focus: + lines.append(f"Supported Methods: {' '.join(enabled_focus)}") + else: + lines.append("Supported Methods: unknown (server does not disclose Allow/CORS methods)") + + if inferred_safe: + lines.append(f"source=inferred-safe-probe ({' '.join(inferred_safe)})") + if dangerous: + lines.append(f"[red]dangerous={', '.join(dangerous)}[/red]") + + lines.append(f"probe_status=http:{methods_http_obj.get('status') or '-'} https:{methods_https_obj.get('status') or '-'}") + + if methods_http or methods_https: + lines.append(f"http={_preview(methods_http, limit=8)} | https={_preview(methods_https, limit=8)}") + if methods_http_cors or methods_https_cors: + lines.append(f"cors_http={_preview(methods_http_cors, limit=8)} | cors_https={_preview(methods_https_cors, limit=8)}") + + lines.append(f"risky={_preview(methods.get('risky_methods') or [], limit=6)}") + return lines + + +def _security_evidence_map(verbose: Dict[str, Any]) -> Dict[str, List[str]]: + security = verbose.get("security") or {} + tls_info = verbose.get("tls") or {} + evidence: Dict[str, List[str]] = {} + + def add(key: str, text: Any) -> None: + value = str(text).strip() + if not value or value == "-": + return + bucket = evidence.setdefault(key, []) + if value not in bucket: + bucket.append(value) + + sec_txt = security.get("security_txt") or {} + add("security_txt", f"url={sec_txt.get('url') or '-'}") + add("security_txt", f"status={sec_txt.get('status') or '-'}") + + srv = security.get("server_assessment") or {} + add("server_version", f"banner={srv.get('banner') or '-'}") + add("server_version", f"product={srv.get('product_label') or '-'}") + add("server_version", f"detected={srv.get('version') or '-'} latest={srv.get('latest') or '-'}") + add("server_version", f"catalog_updated={security.get('server_versions_updated_at') or '-'}") + + if tls_info.get("strict_ok") is False: + add("tls_hygiene", f"strict_error={tls_info.get('strict_error') or '-'}") + add("tls_hygiene", f"issuer={tls_info.get('issuer') or '-'}") + add("tls_hygiene", f"protocol={tls_info.get('protocol') or '-'}") + add("tls_hygiene", f"cipher={tls_info.get('cipher') or '-'}") + + add("mixed_content", f"refs={security.get('mixed_content_count') or 0}") + + caa = security.get("caa") or {} + add("caa", f"entries={_preview(caa.get('entries') or [], limit=3)}") + + email_auth = security.get("email_auth") or {} + spf = email_auth.get("spf") or {} + add("spf", f"policy={spf.get('policy') or '-'}") + add("spf", f"record={_preview(spf.get('records') or [], limit=1)}") + + dmarc = email_auth.get("dmarc") or {} + add("dmarc", f"policy={dmarc.get('policy') or '-'}") + add("dmarc", f"pct={dmarc.get('pct') if dmarc.get('pct') is not None else '-'}") + add("dmarc", f"rua={dmarc.get('rua') or '-'}") + add("dmarc", f"record={_preview(dmarc.get('records') or [], limit=1)}") + + dkim = email_auth.get("dkim") or {} + add("dkim", f"selectors={_preview(dkim.get('selectors_found') or [], limit=4)}") + add("dkim", f"max_bits={dkim.get('max_key_bits_est') or '-'}") + + takeover = security.get("takeover") or {} + add("takeover", f"suspects={_preview(takeover.get('suspect_targets') or [], limit=4)}") + + methods = security.get("methods") or {} + for line in _methods_evidence_lines(methods): + add("methods", line) + + open_redirect = security.get("open_redirect") or {} + probes = open_redirect.get("probes") or [] + flagged = [p for p in probes if isinstance(p, dict) and p.get("possible_open_redirect")] + if flagged: + add("open_redirect", f"confirmed={len(flagged)}/{len(probes)} probes") + first = flagged[0] + add("open_redirect", f"param={first.get('param') or '-'} status={first.get('status') or '-'}") + + rate_waf = security.get("rate_waf") or {} + add("rate_waf", f"statuses={_preview(rate_waf.get('statuses') or [], limit=8)}") + add("rate_waf", f"flags={_preview(rate_waf.get('waf_flags') or [], limit=6)}") + + emails = security.get("emails") or [] + add("emails", f"samples={_preview(emails, limit=3)}") + + threat = verbose.get("threat_intel") or security.get("threat_intel") or {} + vt = threat.get("virustotal") or {} + if vt.get("enabled") and vt.get("ok"): + add("virustotal", f"mal={vt.get('malicious', 0)}") + add("virustotal", f"susp={vt.get('suspicious', 0)}") + add("virustotal", f"rep={vt.get('reputation', 0)}") + + shodan = threat.get("shodan") or {} + if shodan.get("enabled") and shodan.get("ok"): + add("shodan", f"ip={shodan.get('ip') or '-'}") + add("shodan", f"ports={len(shodan.get('ports') or [])}") + add("shodan", f"vulns={int(shodan.get('vuln_count') or len(shodan.get('vulns') or []))}") + + return evidence + + +def _merge_check_summary( + key: str, + item: Dict[str, Any], + evidence_map: Dict[str, List[str]], +) -> str: + summary = str(item.get("summary") or "-") + details = evidence_map.get(key) or [] + if not details: + return summary + + level = str(item.get("level") or "").strip().lower() + max_parts = 3 if level in {"warning", "critical"} else 1 + if key in {"virustotal", "shodan", "server_version", "tls_hygiene", "methods"}: + max_parts = max(max_parts, 2) + compact = details[:max_parts] + return f"{summary} | {' | '.join(compact)}" + + +def _print_security_checks_table(verbose: Dict[str, Any]) -> None: + security = verbose.get("security") or {} + checks = security.get("checks") or {} + if not checks: + return + + evidence_map = _security_evidence_map(verbose) + table = _new_table(title="Security Checks", box_style=box.SIMPLE_HEAVY) + _add_3col_columns(table, "Check", "Level", "Summary") + + for key, label in SECURITY_CHECK_ORDER: + item = checks.get(key) + if not isinstance(item, dict): + continue + summary = _merge_check_summary(key, item, evidence_map) + table.add_row(label, _level_tag(str(item.get("level") or "info")), summary) + + console.print(table) + + +def _build_verbose_context(item: Dict[str, Any]) -> Dict[str, Any]: + http_data = _parse_http_block(item.get("http") or []) + https_data = _parse_http_block(item.get("https") or []) + cert = item.get("cert") or [None, None, None, None] + + https_status = https_data.get("status") + https_ok = isinstance(https_status, int) and 200 <= int(https_status) < 400 + + cert_valid = cert[0] if len(cert) > 0 else None + cert_expiry = cert[1] if len(cert) > 1 else None + cert_cn = cert[2] if len(cert) > 2 else None + tls_versions = cert[3] if len(cert) > 3 and cert[3] else [] + weak_tls = [v for v in tls_versions if _is_weak_tls(v)] + + expiry_expired = False + if cert_expiry: + try: + expiry_expired = date.fromisoformat(cert_expiry) < date.today() + except ValueError: + expiry_expired = False + cn_mismatch_probable = cert_valid is False and not expiry_expired + + takeover = item.get("takeover") or {} + takeover_status = str(takeover.get("status") or "").strip().lower() + takeover_provider = str(takeover.get("provider") or "-") + takeover_cname = str(takeover.get("cname") or "-") + if takeover_status == "likely": + takeover_text = f"[red]likely[/red] ({takeover_provider}) {takeover_cname}" + elif takeover_status == "possible": + takeover_text = f"[yellow]possible[/yellow] ({takeover_provider}) {takeover_cname}" + else: + takeover_text = "[green]none[/green]" + + return { + "http": http_data, + "https": https_data, + "https_ok": https_ok, + "cert_valid": cert_valid, + "cert_expiry": cert_expiry, + "cert_cn": cert_cn, + "tls_versions": tls_versions, + "weak_tls": weak_tls, + "expiry_expired": expiry_expired, + "cn_mismatch_probable": cn_mismatch_probable, + "takeover_text": takeover_text, + } + + +def _print_verbose_check( + item: Dict[str, Any], + ctx: Dict[str, Any], +) -> None: + http_data = ctx["http"] + https_data = ctx["https"] + table = _new_table(title=f"Verbose Check: {item.get('domain', '-')}", box_style=box.SIMPLE_HEAVY) + _add_kv_columns(table) + + table.add_row("IP", ", ".join(item.get("ip") or []) or "[red]-[/red]") + table.add_row("HTTP Status", _fmt_http_status(http_data.get("status"), https_ok=bool(ctx["https_ok"]))) + table.add_row("HTTPS Status", _fmt_status(https_data.get("status"))) + table.add_row("Redirect (HTTP)", _fmt_optional(http_data.get("redirect") or http_data.get("app_redirect"))) + table.add_row("Redirect (HTTPS)", _fmt_optional(https_data.get("redirect") or https_data.get("app_redirect"))) + + http_len = http_data.get("body_len") + https_len = https_data.get("body_len") + http_len_text = "-" if http_len is None else str(http_len) + https_len_text = "-" if https_len is None else str(https_len) + table.add_row("Body Bytes (HTTP/HTTPS)", f"{http_len_text} / {https_len_text}") + + http_error = http_data.get("error") + https_error = https_data.get("error") + if http_error or https_error: + merged = "; ".join(x for x in [str(http_error or "").strip(), str(https_error or "").strip()] if x) + table.add_row("Request Errors", merged) + + cert_valid = ctx["cert_valid"] + if cert_valid is False: + cert_valid_text = "[red]False[/red]" + elif cert_valid is True: + cert_valid_text = "[green]True[/green]" + else: + cert_valid_text = "[red]-[/red]" + + table.add_row("Cert Valid", cert_valid_text) + table.add_row("Cert Expiry", _fmt_optional(ctx["cert_expiry"], failed=bool(ctx["expiry_expired"]))) + table.add_row("Cert CN", _fmt_optional(ctx["cert_cn"], failed=bool(ctx["cn_mismatch_probable"]))) + + tls_versions = ctx["tls_versions"] + tls_text = ", ".join(tls_versions) if tls_versions else "[red]-[/red]" + if ctx["weak_tls"]: + tls_text = f"[red]{tls_text}[/red]" + table.add_row("TLS Supported", tls_text) + table.add_row("Takeover", str(ctx["takeover_text"])) + + console.print(table) + + +def _print_dns_axfr_group(verbose: Dict[str, Any], axfr: Dict[str, Any]) -> None: + dns_info = verbose.get("dns") or {} + + table = _new_table(title="DNS + AXFR", box_style=box.SIMPLE) + _add_kv_columns(table) + + table.add_row("DNS A", ", ".join(dns_info.get("a") or []) or "-") + table.add_row("DNS AAAA", ", ".join(dns_info.get("aaaa") or []) or "-") + table.add_row("DNS CNAME", ", ".join(dns_info.get("cname") or []) or "-") + table.add_row("DNS TTL", ", ".join(f"{k}:{v}" for k, v in (dns_info.get("ttl") or {}).items()) or "-") + table.add_row("DNS Errors", ", ".join(f"{k}:{v}" for k, v in (dns_info.get("errors") or {}).items()) or "-") + + if isinstance(axfr, dict) and axfr: + table.add_row("AXFR Status", str(axfr.get("status") or "-")) + table.add_row("AXFR Allowed NS", str(axfr.get("allowed_ns") or "-")) + table.add_row("AXFR Record Count", str(axfr.get("record_count") or "-")) + checks = axfr.get("checks") or [] + table.add_row("AXFR NS Checked", str(len(checks))) + if checks: + details = ", ".join( + f"{c.get('ns')}:{c.get('status')}" for c in checks if isinstance(c, dict) and c.get("ns") + ) + table.add_row("AXFR Per-NS", details or "-") + if axfr.get("error"): + table.add_row("AXFR Error", str(axfr.get("error"))) + + console.print(table) + + +def _print_tls_handshake(verbose: Dict[str, Any]) -> None: + tls_info = verbose.get("tls") or {} + + table = _new_table(title="TLS Handshake", box_style=box.SIMPLE) + _add_kv_columns(table) + + strict_ok = tls_info.get("strict_ok") + table.add_row("Strict Verify", "[green]True[/green]" if strict_ok else "[red]False[/red]") + table.add_row("Strict Error", tls_info.get("strict_error") or "-") + table.add_row("ALPN", tls_info.get("alpn") or "-") + table.add_row("TLS Protocol", tls_info.get("protocol") or "-") + table.add_row("Cipher", tls_info.get("cipher") or "-") + table.add_row("SAN DNS", ", ".join(tls_info.get("san") or []) or "-") + table.add_row("Issuer", str(tls_info.get("issuer") or "-")) + + console.print(table) + + +def _print_redirect_tls(verbose: Dict[str, Any]) -> None: + redirect_tls = verbose.get("redirect_tls") or {} + if not redirect_tls: + return + + r_port = redirect_tls.get("port") + r_handshake = redirect_tls.get("handshake") or {} + r_versions = redirect_tls.get("versions") or [] + + table = _new_table(title="Redirected TLS Port", box_style=box.SIMPLE) + _add_kv_columns(table) + + table.add_row("Port", str(r_port or "-")) + table.add_row("Strict Verify", "[green]True[/green]" if r_handshake.get("strict_ok") else "[red]False[/red]") + table.add_row("Strict Error", str(r_handshake.get("strict_error") or "-")) + table.add_row("TLS Protocol", str(r_handshake.get("protocol") or "-")) + table.add_row("Cipher", str(r_handshake.get("cipher") or "-")) + table.add_row("ALPN", str(r_handshake.get("alpn") or "-")) + table.add_row("Supported Versions", ", ".join(r_versions) if r_versions else "-") + + console.print(table) + + weak_redirect = [v for v in r_versions if _is_weak_tls(v)] + if weak_redirect: + err_console.print( + f"[yellow]Warning:[/yellow] Redirected port {r_port} supports legacy TLS: {', '.join(weak_redirect)}" + ) + + +def output(results: Union[dict, List[dict], None], elapsed: Optional[timedelta] = None) -> None: + """Render the compact summary table shown after a scan.""" + if not results: + err_console.print("[yellow]No results to display.[/yellow]") + return + + items = [results] if isinstance(results, dict) else results + notes: List[str] = [] + + table = _new_table(box_style=box.SIMPLE_HEAVY, show_header=True, header_style="bold cyan") + table.add_column("Domain", style="cyan", width=SUMMARY_DOMAIN_WIDTH, min_width=SUMMARY_DOMAIN_WIDTH, max_width=SUMMARY_DOMAIN_WIDTH, no_wrap=True, overflow="ellipsis") + table.add_column( + "IP", + style="white", + width=SUMMARY_IP_WIDTH, + min_width=SUMMARY_IP_WIDTH, + max_width=SUMMARY_IP_WIDTH, + no_wrap=False, + overflow="fold", + ) + table.add_column("HTTP", justify="center", width=SUMMARY_CODE_WIDTH, min_width=SUMMARY_CODE_WIDTH, max_width=SUMMARY_CODE_WIDTH, no_wrap=True) + table.add_column("HTTPS", justify="center", width=SUMMARY_CODE_WIDTH, min_width=SUMMARY_CODE_WIDTH, max_width=SUMMARY_CODE_WIDTH, no_wrap=True) + table.add_column("Cert", justify="center", width=SUMMARY_CERT_WIDTH, min_width=SUMMARY_CERT_WIDTH, max_width=SUMMARY_CERT_WIDTH, no_wrap=True) + table.add_column("Expiry", justify="center", width=SUMMARY_EXPIRY_WIDTH, min_width=SUMMARY_EXPIRY_WIDTH, max_width=SUMMARY_EXPIRY_WIDTH, no_wrap=True) + table.add_column("TLS", overflow="fold", no_wrap=False) + table.add_column("Status", justify="center", width=SUMMARY_STATUS_WIDTH, min_width=SUMMARY_STATUS_WIDTH, max_width=SUMMARY_STATUS_WIDTH, no_wrap=True) + + for item in items: + for note in (item.get("scan_notes") or []): + if isinstance(note, str) and note and note not in notes: + notes.append(note) + + ips = _format_ips_for_cell(item.get("ip") or []) + + http = item.get("http") or [None, None, None, None] + https = item.get("https") or [None, None, None, None] + cert = item.get("cert") or [None, None, None, None] + + http_code = "-" if len(http) < 1 or http[0] is None else str(http[0]) + https_code = "-" if len(https) < 1 or https[0] is None else str(https[0]) + + cert_ok = cert[0] + cert_text = "-" + if cert_ok is True: + cert_text = "[green]valid[/green]" + elif cert_ok is False: + cert_text = "[red]invalid[/red]" + + status = _status_text(http, https, cert) + takeover = item.get("takeover") or {} + takeover_status = str(takeover.get("status") or "").strip().lower() + if takeover_status in {"possible", "likely"} and status == "ok": + status = "warning" + + if status == "ok": + status_text = "[green]ok[/green]" + elif status == "warning": + status_text = "[yellow]warning[/yellow]" + else: + status_text = "[magenta]dns-only[/magenta]" + + table.add_row( + item.get("domain", "-"), + ips, + http_code, + https_code, + cert_text, + cert[1] or "-", + _tls_label(cert[3] if len(cert) > 3 else None), + status_text, + ) + + console.print(table) + for note in notes: + console.print(f"[yellow]Note:[/yellow] {note}") + console.print(Panel.fit(f"[bold]Domains:[/bold] {len(items)} [bold]Elapsed:[/bold] {fmt_td(elapsed)}", border_style="cyan")) + + +def print_scan_status( + timeout: float, + threads: Optional[int], + dns: str, + useragent: str, + recon: bool, + brute: bool, + wordlist: Optional[str] = None, + target_count: Optional[int] = None, +) -> None: + if threads is None: + if target_count is None: + threads_value = "auto (dynamic: min(300, max(20, targets)))" + else: + auto_threads = min(300, max(20, target_count)) + threads_value = f"auto ({auto_threads})" + else: + threads_value = str(threads) + + default_wordlist = ROOT / "wordlist" / "wordlist.txt" + default_wordlist_status = "present" if default_wordlist.is_file() else "missing" + wordlist_mode = "custom" if wordlist else "default" + + table = _new_table(title="Scan Status", box_style=box.MINIMAL_DOUBLE_HEAD) + _add_kv_columns(table) + + table.add_row("Timeout", str(timeout)) + table.add_row("Threads", threads_value) + table.add_row("DNS", dns) + table.add_row("User-Agent", useragent) + table.add_row("Recon", str(recon)) + table.add_row("Bruteforce", str(brute)) + table.add_row("Wordlist Mode", wordlist_mode) + table.add_row("Default Wordlist", f"{default_wordlist} ({default_wordlist_status})") + table.add_row("Reports DB", str(get_db_path())) + table.add_row("Reports Count", str(count_reports())) + + console.print(table) + + +def show_reports_catalog(reports: List[Dict[str, Any]]) -> None: + if not reports: + err_console.print("[yellow]No reports found in database.[/yellow]") + return + + table = _new_table(title="Stored Reports", box_style=box.SIMPLE_HEAVY) + table.add_column("ID", justify="right", style="cyan", width=5, min_width=5, max_width=5, no_wrap=True) + table.add_column("Created", width=19, min_width=19, max_width=19, no_wrap=True) + table.add_column("Target", overflow="fold", no_wrap=False) + table.add_column("Mode", width=12, min_width=12, max_width=12, no_wrap=True) + table.add_column("Results", justify="right", width=8, min_width=8, max_width=8, no_wrap=True) + table.add_column("Elapsed", justify="right", width=10, min_width=10, max_width=10, no_wrap=True) + + for report in reports: + elapsed = "-" + if report.get("elapsed_seconds") is not None: + elapsed = fmt_td(timedelta(seconds=float(report["elapsed_seconds"]))) + table.add_row( + str(report.get("id")), + str(report.get("created_at")), + str(report.get("target")), + str(report.get("mode")), + str(report.get("result_count")), + elapsed, + ) + + console.print(table) + + +def output_verbose(results: Union[dict, List[dict], None]) -> None: + """Render deep protocol/security diagnostics for `--verbose` mode.""" + if not results: + err_console.print("[yellow]No results to inspect.[/yellow]") + return + + items = [results] if isinstance(results, dict) else results + + for item in items: + verbose = item.get("verbose") or {} + if not verbose: + continue + + ctx = _build_verbose_context(item) + + # 3 columns with compact evidence. + _print_security_checks_table(verbose) + + # 2 columns. + _print_verbose_check(item=item, ctx=ctx) + + axfr = item.get("axfr") if isinstance(item.get("axfr"), dict) else {} + _print_dns_axfr_group(verbose, axfr or {}) + _print_tls_handshake(verbose) + _print_redirect_tls(verbose) diff --git a/knockpy/server_versions.py b/knockpy/server_versions.py new file mode 100644 index 0000000..e9e9328 --- /dev/null +++ b/knockpy/server_versions.py @@ -0,0 +1,338 @@ +from __future__ import annotations + +"""Web-server version catalog and banner assessment helpers. + +This module maintains a local catalog (`~/.knockpy/server_versions.json`) and +provides best-effort evaluation of `Server` headers (up-to-date/outdated/advisory). +""" + +import json +import os +import re +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import httpx + +CATALOG_PATH = Path.home() / ".knockpy" / "server_versions.json" +CATALOG_MAX_AGE = timedelta(days=7) + + +def _harden_user_file(path: Path) -> None: + try: + path.parent.mkdir(parents=True, exist_ok=True) + except Exception: + pass + try: + os.chmod(path, 0o600) + except Exception: + pass + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _default_catalog() -> Dict[str, Any]: + return { + "updated_at": None, + "products": { + "apache_http_server": { + "label": "Apache HTTP Server", + "latest": None, + "source": "https://downloads.apache.org/httpd/", + "check_mode": "strict", + }, + "nginx": { + "label": "Nginx", + "latest": None, + "source": "https://nginx.org/en/download.html", + "check_mode": "strict", + }, + "microsoft_iis": { + "label": "Microsoft-IIS", + "latest": "10.0", + "source": "https://learn.microsoft.com/iis/", + "check_mode": "advisory", + "note": "IIS patch state depends on Windows cumulative updates.", + }, + "microsoft_httpapi": { + "label": "Microsoft-HTTPAPI", + "latest": None, + "source": "https://learn.microsoft.com/windows/win32/http/http-api-start-page", + "check_mode": "advisory", + "note": "HTTPAPI patch state is OS-coupled; banner version is not enough.", + }, + "oracle_application_server": { + "label": "Oracle-Application-Server", + "latest": None, + "source": "https://docs.oracle.com/", + "check_mode": "advisory", + "note": "Legacy Oracle AS versions often require lifecycle/support validation, not simple version compare.", + }, + "securetransport": { + "label": "SecureTransport", + "latest": None, + "source": "https://www.progress.com/", + "check_mode": "advisory", + "note": "Version freshness should be validated against vendor advisory/support matrix.", + }, + "openresty": { + "label": "OpenResty", + "latest": None, + "source": "https://openresty.org/en/download.html", + "check_mode": "advisory", + }, + "litespeed": { + "label": "LiteSpeed", + "latest": None, + "source": "https://www.litespeedtech.com/products/litespeed-web-server/download", + "check_mode": "advisory", + }, + "caddy": { + "label": "Caddy", + "latest": None, + "source": "https://github.com/caddyserver/caddy/releases", + "check_mode": "advisory", + }, + "envoy": { + "label": "Envoy", + "latest": None, + "source": "https://github.com/envoyproxy/envoy/releases", + "check_mode": "advisory", + }, + "jetty": { + "label": "Jetty", + "latest": None, + "source": "https://github.com/jetty/jetty.project/releases", + "check_mode": "advisory", + }, + "tomcat": { + "label": "Apache Tomcat", + "latest": None, + "source": "https://tomcat.apache.org/whichversion.html", + "check_mode": "advisory", + }, + "gunicorn": { + "label": "Gunicorn", + "latest": None, + "source": "https://github.com/benoitc/gunicorn/releases", + "check_mode": "advisory", + }, + }, + } + + +def _read_catalog() -> Dict[str, Any]: + """Read catalog from disk and merge missing defaults from current release.""" + defaults = _default_catalog() + if CATALOG_PATH.is_file(): + _harden_user_file(CATALOG_PATH) + try: + data = json.loads(CATALOG_PATH.read_text(encoding="utf-8")) + if isinstance(data, dict) and isinstance(data.get("products"), dict): + merged = dict(data) + merged_products = dict(data.get("products") or {}) + default_products = dict(defaults.get("products") or {}) + for name, meta in default_products.items(): + existing = merged_products.get(name) + if not isinstance(existing, dict): + merged_products[name] = dict(meta) + continue + merged_item = dict(meta) + merged_item.update(existing) + merged_products[name] = merged_item + merged["products"] = merged_products + return merged + except Exception: + pass + return defaults + + +def _write_catalog(data: Dict[str, Any]) -> None: + CATALOG_PATH.parent.mkdir(parents=True, exist_ok=True) + CATALOG_PATH.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") + _harden_user_file(CATALOG_PATH) + + +def _parse_version(text: str) -> Tuple[int, ...]: + nums = [int(p) for p in re.findall(r"\d+", text)] + return tuple(nums) if nums else (0,) + + +def _max_version(values: List[str]) -> Optional[str]: + if not values: + return None + unique = sorted(set(values), key=_parse_version) + return unique[-1] if unique else None + + +def _fetch_apache_latest(timeout: float = 12.0) -> Optional[str]: + url = "https://downloads.apache.org/httpd/" + try: + response = httpx.get(url, timeout=timeout, follow_redirects=True) + response.raise_for_status() + versions = re.findall(r"httpd-(\d+\.\d+\.\d+)\.tar\.(?:gz|bz2|xz)", response.text or "") + return _max_version(versions) + except Exception: + return None + + +def _fetch_nginx_latest(timeout: float = 12.0) -> Optional[str]: + url = "https://nginx.org/en/download.html" + try: + response = httpx.get(url, timeout=timeout, follow_redirects=True) + response.raise_for_status() + versions = re.findall(r"nginx-(\d+\.\d+\.\d+)\.tar\.gz", response.text or "") + return _max_version(versions) + except Exception: + return None + + +def update_server_versions_catalog(timeout: float = 12.0) -> Dict[str, Any]: + """Refresh known latest versions from upstream sources and persist catalog.""" + catalog = _read_catalog() + products = dict(catalog.get("products") or {}) + + apache_latest = _fetch_apache_latest(timeout=timeout) + if apache_latest: + products.setdefault("apache_http_server", {}).update({"latest": apache_latest}) + + nginx_latest = _fetch_nginx_latest(timeout=timeout) + if nginx_latest: + products.setdefault("nginx", {}).update({"latest": nginx_latest}) + + catalog["products"] = products + catalog["updated_at"] = _now_iso() + _write_catalog(catalog) + return catalog + + +def _catalog_is_stale(catalog: Dict[str, Any]) -> bool: + raw = catalog.get("updated_at") + if not raw: + return True + try: + ts = datetime.fromisoformat(str(raw)) + if ts.tzinfo is None: + ts = ts.replace(tzinfo=timezone.utc) + except Exception: + return True + return datetime.now(timezone.utc) - ts > CATALOG_MAX_AGE + + +def load_server_versions_catalog(auto_update: bool = True) -> Dict[str, Any]: + """Load catalog, optionally auto-refreshing stale data.""" + catalog = _read_catalog() + if auto_update and _catalog_is_stale(catalog): + try: + catalog = update_server_versions_catalog() + except Exception: + pass + else: + if not CATALOG_PATH.is_file(): + _write_catalog(catalog) + return catalog + + +def _detect_product_and_version(server_banner: str) -> Tuple[Optional[str], Optional[str]]: + text = (server_banner or "").strip() + if not text: + return None, None + patterns = [ + ("apache_http_server", r"(?i)\bapache\/(\d+\.\d+(?:\.\d+)*)", None), + ("nginx", r"(?i)\bnginx\/(\d+\.\d+(?:\.\d+)*)", None), + ("microsoft_iis", r"(?i)\bmicrosoft-iis\/(\d+\.\d+(?:\.\d+)*)", None), + ("microsoft_httpapi", r"(?i)\bmicrosoft-httpapi\/(\d+\.\d+(?:\.\d+)*)", None), + ("oracle_application_server", r"(?i)\boracle-application-server-(11g|12c)\b", None), + ("securetransport", r"(?i)\bsecuretransport\s+(\d+(?:\.\d+)+)", None), + ("openresty", r"(?i)\bopenresty\/(\d+\.\d+(?:\.\d+)*)", None), + ("litespeed", r"(?i)\blitespeed\/(\d+\.\d+(?:\.\d+)*)", None), + ("caddy", r"(?i)\bcaddy(?:\/|\s+)(\d+\.\d+(?:\.\d+)*)", None), + ("envoy", r"(?i)\benvoy(?:\/|\s+)(\d+\.\d+(?:\.\d+)*)", None), + ("jetty", r"(?i)\bjetty(?:\(|/)(\d+\.\d+(?:\.\d+)*)", None), + ("tomcat", r"(?i)\bcoyote\/(\d+\.\d+(?:\.\d+)*)", None), + ("gunicorn", r"(?i)\bgunicorn\/(\d+\.\d+(?:\.\d+)*)", None), + ] + for product, pattern, fixed_version in patterns: + match = re.search(pattern, text) + if match: + version = match.group(1) if match.groups() else fixed_version + return product, version + return None, None + + +def assess_server_banner( + server_banner: Optional[str], catalog: Optional[Dict[str, Any]] = None +) -> Dict[str, Any]: + """Classify a web-server banner against the local version catalog. + + Used by verbose checks and HTML export enrichment. + """ + banner = (server_banner or "").strip() + if not banner: + return {"status": "unknown", "label": "unknown", "banner": "-", "product": None, "version": None, "latest": None} + + product, version = _detect_product_and_version(banner) + if not product or not version: + return { + "status": "unknown", + "label": "unknown", + "banner": banner, + "product": None, + "version": None, + "latest": None, + "note": "Version not exposed in Server header.", + } + + data = catalog or load_server_versions_catalog(auto_update=False) + product_meta = (data.get("products") or {}).get(product) or {} + latest = product_meta.get("latest") + check_mode = str(product_meta.get("check_mode") or "strict") + label = str(product_meta.get("label") or product) + note = product_meta.get("note") + + if check_mode != "strict": + return { + "status": "advisory", + "label": "advisory", + "banner": banner, + "product": product, + "product_label": label, + "version": version, + "latest": latest, + "note": note or "Patch status may depend on OS/vendor backports.", + } + + if not latest: + return { + "status": "unknown", + "label": "unknown", + "banner": banner, + "product": product, + "product_label": label, + "version": version, + "latest": None, + "note": "Latest reference not available.", + } + + cur = _parse_version(version) + ref = _parse_version(str(latest)) + if cur < ref: + state = "outdated" + elif cur == ref: + state = "up-to-date" + else: + state = "newer-than-reference" + + return { + "status": state, + "label": state, + "banner": banner, + "product": product, + "product_label": label, + "version": version, + "latest": latest, + "note": "Best-effort check. Distro backports may apply security fixes on older version strings.", + } diff --git a/knockpy/storage.py b/knockpy/storage.py new file mode 100644 index 0000000..14c0d1f --- /dev/null +++ b/knockpy/storage.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +"""Persistence and report export facade for knockpy. + +Public storage API remains stable while implementation is split by concern: +- `knockpy.storage_parts.db`: SQLite persistence and settings +- `knockpy.storage_parts.export`: report row normalization and HTML export +""" + +from .storage_parts.db import ( + count_reports, + delete_report, + get_db_path, + get_report, + get_setting, + get_settings, + init_db, + list_reports, + reset_reports, + save_scan, + set_setting, +) +from .storage_parts.export import export_report + +__all__ = [ + "get_db_path", + "init_db", + "save_scan", + "list_reports", + "count_reports", + "get_report", + "delete_report", + "reset_reports", + "get_setting", + "get_settings", + "set_setting", + "export_report", +] diff --git a/knockpy/storage_parts/__init__.py b/knockpy/storage_parts/__init__.py new file mode 100644 index 0000000..2cde88b --- /dev/null +++ b/knockpy/storage_parts/__init__.py @@ -0,0 +1 @@ +"""Storage support modules grouped by concern.""" diff --git a/knockpy/storage_parts/db.py b/knockpy/storage_parts/db.py new file mode 100644 index 0000000..5bd80d5 --- /dev/null +++ b/knockpy/storage_parts/db.py @@ -0,0 +1,235 @@ +from __future__ import annotations + +import json +import os +import sqlite3 +from datetime import timedelta +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + + +def _harden_user_file(path: Path) -> None: + try: + os.chmod(path, 0o600) + except Exception: + pass + + +def get_db_path() -> Path: + custom = os.getenv("KNOCKPY_DB") + if custom: + path = Path(custom).expanduser().resolve() + else: + path = Path.home() / ".knockpy" / "reports.db" + path.parent.mkdir(parents=True, exist_ok=True) + return path + + +def init_db(db_path: Optional[Path] = None) -> Path: + """Initialize DB schema and return DB path. + + Called by all storage entrypoints to ensure schema is available. + """ + path = db_path or get_db_path() + conn = sqlite3.connect(path) + try: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS scans ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + target TEXT NOT NULL, + mode TEXT NOT NULL, + settings_json TEXT NOT NULL, + elapsed_seconds REAL, + result_count INTEGER NOT NULL, + results_json TEXT NOT NULL + ) + """ + ) + conn.execute("CREATE INDEX IF NOT EXISTS idx_scans_created ON scans(created_at DESC)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_scans_target ON scans(target)") + conn.execute( + """ + CREATE TABLE IF NOT EXISTS settings ( + key TEXT PRIMARY KEY, + value TEXT, + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ) + """ + ) + conn.commit() + finally: + conn.close() + _harden_user_file(path) + return path + + +def _normalize_results(results: Union[dict, List[dict], None]) -> List[dict]: + if results is None: + return [] + if isinstance(results, dict): + return [results] + return results + + +def save_scan( + target: str, + mode: str, + settings: Dict[str, Any], + results: Union[dict, List[dict], None], + elapsed: Optional[timedelta], + db_path: Optional[Path] = None, +) -> int: + path = init_db(db_path) + rows = _normalize_results(results) + elapsed_seconds = elapsed.total_seconds() if elapsed else None + + conn = sqlite3.connect(path) + try: + cur = conn.cursor() + cur.execute( + """ + INSERT INTO scans (target, mode, settings_json, elapsed_seconds, result_count, results_json) + VALUES (?, ?, ?, ?, ?, ?) + """, + ( + target, + mode, + json.dumps(settings, ensure_ascii=False), + elapsed_seconds, + len(rows), + json.dumps(rows, ensure_ascii=False), + ), + ) + conn.commit() + return int(cur.lastrowid) + finally: + conn.close() + + +def list_reports(limit: int = 50, db_path: Optional[Path] = None) -> List[Dict[str, Any]]: + path = init_db(db_path) + conn = sqlite3.connect(path) + conn.row_factory = sqlite3.Row + try: + rows = conn.execute( + """ + SELECT id, created_at, target, mode, result_count, elapsed_seconds + FROM scans + ORDER BY id DESC + LIMIT ? + """, + (limit,), + ).fetchall() + return [dict(r) for r in rows] + finally: + conn.close() + + +def count_reports(db_path: Optional[Path] = None) -> int: + path = init_db(db_path) + conn = sqlite3.connect(path) + try: + row = conn.execute("SELECT COUNT(*) FROM scans").fetchone() + return int(row[0]) if row else 0 + finally: + conn.close() + + +def get_report(selector: Optional[str], db_path: Optional[Path] = None) -> Optional[Dict[str, Any]]: + path = init_db(db_path) + conn = sqlite3.connect(path) + conn.row_factory = sqlite3.Row + + query = "" + params: tuple[Any, ...] = () + + if selector is None or selector == "latest": + query = "SELECT * FROM scans ORDER BY id DESC LIMIT 1" + elif isinstance(selector, str) and selector.isdigit(): + query = "SELECT * FROM scans WHERE id = ?" + params = (int(selector),) + else: + selector_text = selector if isinstance(selector, str) else str(selector) + query = "SELECT * FROM scans WHERE target = ? ORDER BY id DESC LIMIT 1" + params = (selector_text,) + + try: + row = conn.execute(query, params).fetchone() + if not row: + return None + data = dict(row) + data["settings"] = json.loads(data.pop("settings_json")) + data["results"] = json.loads(data.pop("results_json")) + return data + finally: + conn.close() + + +def delete_report(report_id: int, db_path: Optional[Path] = None) -> bool: + path = init_db(db_path) + conn = sqlite3.connect(path) + try: + cur = conn.cursor() + cur.execute("DELETE FROM scans WHERE id = ?", (report_id,)) + conn.commit() + return cur.rowcount > 0 + finally: + conn.close() + + +def reset_reports(db_path: Optional[Path] = None) -> int: + """Delete all stored scan reports and return the number of removed rows.""" + path = init_db(db_path) + conn = sqlite3.connect(path) + try: + cur = conn.cursor() + cur.execute("DELETE FROM scans") + conn.commit() + return int(cur.rowcount or 0) + finally: + conn.close() + + +def get_setting(key: str, db_path: Optional[Path] = None) -> Optional[str]: + path = init_db(db_path) + conn = sqlite3.connect(path) + try: + row = conn.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone() + return str(row[0]) if row and row[0] is not None else None + finally: + conn.close() + + +def get_settings(prefix: Optional[str] = None, db_path: Optional[Path] = None) -> Dict[str, Optional[str]]: + path = init_db(db_path) + conn = sqlite3.connect(path) + try: + if prefix: + rows = conn.execute( + "SELECT key, value FROM settings WHERE key LIKE ? ORDER BY key", + (f"{prefix}%",), + ).fetchall() + else: + rows = conn.execute("SELECT key, value FROM settings ORDER BY key").fetchall() + return {str(k): (None if v is None else str(v)) for k, v in rows} + finally: + conn.close() + + +def set_setting(key: str, value: Optional[str], db_path: Optional[Path] = None) -> None: + path = init_db(db_path) + conn = sqlite3.connect(path) + try: + conn.execute( + """ + INSERT INTO settings (key, value, updated_at) + VALUES (?, ?, datetime('now')) + ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=datetime('now') + """, + (key, value), + ) + conn.commit() + finally: + conn.close() diff --git a/knockpy/storage_parts/export.py b/knockpy/storage_parts/export.py new file mode 100644 index 0000000..ec0a149 --- /dev/null +++ b/knockpy/storage_parts/export.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any, Dict, List, Optional + +from .html_report import _build_html_report, _only_ipv4_list +from ..server_versions import assess_server_banner, load_server_versions_catalog + + +def _safe_name(text: str) -> str: + return re.sub(r"[^A-Za-z0-9_.-]+", "_", text).strip("_") or "report" + + +def _rows_for_export(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + rows: List[Dict[str, Any]] = [] + catalog = load_server_versions_catalog(auto_update=True) + for item in results: + http = item.get("http") or [None, None, None, None] + https = item.get("https") or [None, None, None, None] + cert = item.get("cert") or [None, None, None, None] + ipv4_only = _only_ipv4_list(item.get("ip") or []) + http_server_assessment = assess_server_banner(http[2] if len(http) > 2 else None, catalog=catalog) + https_server_assessment = assess_server_banner(https[2] if len(https) > 2 else None, catalog=catalog) + rows.append( + { + "domain": item.get("domain", ""), + "ip": ipv4_only, + "http_status": http[0], + "http_redirect": http[1], + "http_server": http[2], + "http_server_status": http_server_assessment.get("status"), + "http_server_version": http_server_assessment.get("version"), + "http_server_latest": http_server_assessment.get("latest"), + "https_status": https[0], + "https_redirect": https[1], + "https_server": https[2], + "https_server_status": https_server_assessment.get("status"), + "https_server_version": https_server_assessment.get("version"), + "https_server_latest": https_server_assessment.get("latest"), + "cert_valid": cert[0], + "cert_expiry": cert[1], + "cert_cn": cert[2], + "tls_versions": cert[3] or [], + } + ) + return rows + + +def export_report(report: Dict[str, Any], export_format: str, output_path: Optional[str] = None) -> str: + """Export a report to disk. + + Currently supports only HTML and writes a self-contained file. + """ + if export_format != "html": + raise ValueError("Only 'html' export is supported.") + + results = report.get("results") or [] + target = _safe_name(report.get("target", "scan")) + report_id = report.get("id", "unknown") + + if output_path: + out = Path(output_path) + else: + out = Path.cwd() / f"{target}_report_{report_id}.html" + + if out.exists() and out.is_dir(): + raise IsADirectoryError(f"Output path is a directory: {out}") + + out.parent.mkdir(parents=True, exist_ok=True) + + rows = _rows_for_export(results) + out.write_text(_build_html_report(report, rows, results), encoding="utf-8") + return str(out) diff --git a/knockpy/storage_parts/html_report.py b/knockpy/storage_parts/html_report.py new file mode 100644 index 0000000..ef853fb --- /dev/null +++ b/knockpy/storage_parts/html_report.py @@ -0,0 +1,1299 @@ +from __future__ import annotations + +import html +import ipaddress +import json +from datetime import date +from typing import Any, Dict, List, Optional + +from ..version import __version__ + +def _build_html_report(report: Dict[str, Any], rows: List[Dict[str, Any]], results: List[Dict[str, Any]]) -> str: + """Build a self-contained interactive HTML report. + + Where used: + - called by `export_report()` after scan rows are normalized. + Why: + - keeps report generation deterministic and isolated from DB/runtime code. + """ + ip_set: set[str] = set() + domain_set: set[str] = set() + links: List[Dict[str, str]] = [] + for item in results: + domain = str(item.get("domain") or "").strip() + if not domain: + continue + domain_set.add(domain) + for ip in item.get("ip") or []: + ip_text = str(ip).strip() + if not ip_text: + continue + ip_set.add(ip_text) + links.append({"ip": ip_text, "domain": domain}) + + ips = sorted(ip_set) + domains = sorted(domain_set) + + results_by_domain = {str(item.get("domain") or ""): item for item in results} + findings_by_category: Dict[str, Dict[str, Dict[str, str]]] = {} + table_rows: List[str] = [] + for idx, row in enumerate(rows): + domain = str(row.get("domain") or "-") + detail_obj = results_by_domain.get(domain, {}) + issues_for_row = _detail_issues(detail_obj, row) + for level, issue_text in issues_for_row: + if str(level).strip().lower() == "ok": + continue + issue_key = str(issue_text).strip() + if not issue_key: + continue + category_key = _finding_category(issue_key) + findings_by_category.setdefault(category_key, {}) + findings_by_category[category_key].setdefault(issue_key, {}) + if domain not in findings_by_category[category_key][issue_key]: + findings_by_category[category_key][issue_key][domain] = _finding_evidence_for_issue( + issue_key, detail_obj, row + ) + detail_html = _render_human_detail(detail_obj, row, issues=issues_for_row) + row_id = f"detail-row-{idx}" + cert_label = _cert_label(row.get("cert_valid")) + status_label = _status_from_issues(row, issues_for_row) + http_status = row.get("http_status") + https_status = row.get("https_status") + table_rows.append( + "" + f"{html.escape(domain)}" + f"{_render_ip_column(row.get('ip'))}" + f"{_html_status_badge(http_status)}" + f"{_render_server_cell(row.get('http_server'), row.get('http_server_status'), row.get('http_server_version'), row.get('http_server_latest'))}" + f"{_html_status_badge(https_status)}" + f"{_render_server_cell(row.get('https_server'), row.get('https_server_status'), row.get('https_server_version'), row.get('https_server_latest'))}" + f"{_html_state_badge(cert_label, cert_label)}" + f"{html.escape(str(row.get('cert_expiry') or '-'))}" + f"{_render_tls_column(row.get('tls_versions'))}" + f"{_html_state_badge(status_label, status_label)}" + "" + "" + "{detail}" + "".format(rid=row_id, detail=detail_html) + ) + + finding_blocks: List[str] = [] + if findings_by_category: + def _category_domain_count(issue_map: Dict[str, Dict[str, str]]) -> int: + uniq: set[str] = set() + for affected in issue_map.values(): + uniq.update(affected.keys()) + return len(uniq) + + ordered_categories = sorted( + findings_by_category.items(), + key=lambda item: (_category_domain_count(item[1]), len(item[1])), + reverse=True, + ) + + for category_name, issue_map in ordered_categories: + issue_sections: List[str] = [] + ordered_issues = sorted(issue_map.items(), key=lambda item: len(item[1]), reverse=True) + for issue_text, affected in ordered_issues: + domains_sorted = sorted(affected.items(), key=lambda item: item[0]) + rows_html = "".join( + "" + f"{html.escape(domain_name)}" + f"{html.escape(evidence)}" + "" + for domain_name, evidence in domains_sorted + ) + issue_sections.append( + "
" + f"
{html.escape(issue_text)}" + f" {len(domains_sorted)} domains
" + "" + "" + f"{rows_html}" + "
DomainEvidence
" + "
" + ) + + finding_blocks.append( + "
" + "" + f"{html.escape(category_name)}" + f"{_category_domain_count(issue_map)} domains, {len(issue_map)} issues" + "" + "
" + f"{''.join(issue_sections)}" + "
" + "
" + ) + findings_html = "".join(finding_blocks) if finding_blocks else "

No critical confirmed issue detected.

" + + payload = { + "ips": ips, + "domains": domains, + "links": links, + } + payload_json = _safe_json_for_script(payload) + + return f""" + + + + + + Knockpy v{html.escape(str(__version__))} - Report #{html.escape(str(report.get("id")))} + + + +
+
+

+ Report #{html.escape(str(report.get("id")))} + Knockpy v{html.escape(str(__version__))} +

+
+
Created: {html.escape(str(report.get("created_at")))}
+
Target: {html.escape(str(report.get("target")))}
+
Mode: {html.escape(str(report.get("mode")))}
+
Results: {html.escape(str(report.get("result_count")))}
+
+
+ +
+

Findings Summary

+ {findings_html} +
+ +
+ IP โ†” Domain Correlation +
+
+ + + + +
+ +
+ IP nodes + Domain nodes +
+
Click a domain node to see associated IPs.
+
+
+ +
+

Results Table

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {"".join(table_rows) if table_rows else ''} + +
DomainIPHTTPHTTP ServerHTTPSHTTPS ServerCertExpiryTLSStatus
No rows
+
+
+
+ + + + +""" + + +def _fmt_detail_value(value: Any) -> str: + if value in (None, "", []): + return "-" + if isinstance(value, list): + if not value: + return "-" + return ", ".join(str(item) for item in value) + if isinstance(value, dict): + if not value: + return "-" + return ", ".join(f"{k}: {v}" for k, v in value.items()) + return str(value) + + +def _safe_json_for_script(data: Any) -> str: + # Prevent closing the script tag and HTML entity parsing inside inline JS. + raw = json.dumps(data, ensure_ascii=False) + return ( + raw.replace("<", "\\u003c") + .replace(">", "\\u003e") + .replace("&", "\\u0026") + .replace("\u2028", "\\u2028") + .replace("\u2029", "\\u2029") + ) + + +def _render_ip_column(value: Any) -> str: + if isinstance(value, list): + ips = [str(ip).strip() for ip in value if str(ip).strip()] + else: + text = str(value or "").strip() + ips = [text] if text else [] + + if not ips: + return "-" + if len(ips) == 1: + return f"{html.escape(ips[0])}" + + lines = "".join(f"{html.escape(ip)}" for ip in ips) + return f"
{lines}
" + + +def _render_tls_column(value: Any) -> str: + versions: List[str] = [] + if isinstance(value, list): + versions = [str(v).strip() for v in value if str(v).strip()] + else: + text = str(value or "").strip() + if text: + if "," in text: + versions = [part.strip() for part in text.split(",") if part.strip()] + else: + versions = [text] + + if not versions: + return "-" + if len(versions) == 1: + return f"{html.escape(versions[0])}" + + lines = "".join(f"{html.escape(v)}" for v in versions) + return f"
{lines}
" + + +def _badge_class_for_state(text: str) -> str: + value = (text or "").strip().lower() + if value in {"ok", "valid"}: + return f"badge-{value}" + if value in {"warning", "invalid", "dns-only"}: + return f"badge-{value}" + if value in {"-", "none", ""}: + return "badge-na" + return "badge-na" + + +def _html_state_badge(value: Any, label: Optional[str] = None) -> str: + text = "-" if value in (None, "") else str(label if label is not None else value) + return f"{html.escape(text)}" + + +def _html_status_badge(value: Any) -> str: + if value in (None, ""): + return _html_state_badge("-") + try: + code = int(value) + except Exception: + return _html_state_badge(str(value)) + if code >= 400: + cls = "badge-http-bad" + elif code >= 300: + cls = "badge-http-redirect" + else: + cls = "badge-http-good" + return f"{code}" + + +def _render_server_cell(server: Any, status: Any, version: Any, latest: Any) -> str: + server_text = html.escape(str(server or "-")) + status_value = str(status or "").strip().lower() + if status_value == "outdated": + return f"{server_text}" + if status_value in {"advisory", "newer-than-reference"}: + return f"{server_text}" + return server_text + + +def _detail_value_html(key: str, value: Any) -> str: + if key in {"HTTP Status + Body Bytes", "HTTPS Status + Body Bytes"} and isinstance(value, (list, tuple)) and len(value) == 2: + status_html = _html_status_badge(value[0]) + body_text = html.escape(_fmt_detail_value(value[1])) + return ( + "
" + "
" + "
Status
" + f"
{status_html}
" + "
" + "
" + "
Body Bytes
" + f"
{body_text}
" + "
" + "
" + ) + if key in {"Status", "Certificate"}: + return _html_state_badge(value) + if key in {"HTTP Status", "HTTPS Status"}: + return _html_status_badge(value) + return html.escape(_fmt_detail_value(value)) + + +def _render_human_detail( + detail_obj: Dict[str, Any], + row: Dict[str, Any], + issues: Optional[List[tuple[str, str]]] = None, +) -> str: + """Render expandable per-domain details for the HTML report table rows.""" + http = detail_obj.get("http") or [] + https = detail_obj.get("https") or [] + cert = detail_obj.get("cert") or [] + takeover = detail_obj.get("takeover") or {} + notes = detail_obj.get("scan_notes") or [] + all_ips = detail_obj.get("ip") or row.get("ip") + clean_notes = [str(n).strip() for n in notes if str(n).strip()] + + overview_pairs: List[tuple[str, Any]] = [] + if clean_notes: + overview_pairs.append(("Notes", clean_notes)) + + sections: List[tuple[str, List[tuple[str, Any]]]] = [ + ( + "Overview", + overview_pairs, + ), + ( + "HTTP", + [ + ("HTTP Status", http[0] if len(http) > 0 else row.get("http_status")), + ("HTTP Body Bytes", http[3] if len(http) > 3 else None), + ("HTTP Redirect", http[1] if len(http) > 1 else row.get("http_redirect")), + ("HTTP App Redirect", http[4] if len(http) > 4 else None), + ], + ), + ( + "HTTPS", + [ + ("HTTPS Status", https[0] if len(https) > 0 else row.get("https_status")), + ("HTTPS Body Bytes", https[3] if len(https) > 3 else None), + ("HTTPS Redirect", https[1] if len(https) > 1 else row.get("https_redirect")), + ("HTTPS App Redirect", https[4] if len(https) > 4 else None), + ], + ), + ( + "Certificate / TLS", + [ + ("Certificate", _cert_label(cert[0] if len(cert) > 0 else row.get("cert_valid"))), + ("Certificate CN", cert[2] if len(cert) > 2 else row.get("cert_cn")), + ], + ), + ( + "Takeover", + [ + ("Candidate", "yes" if takeover else "no"), + ("Status", takeover.get("status") if takeover else "-"), + ("Provider", takeover.get("provider") if takeover else "-"), + ("CNAME", takeover.get("cname") if takeover else "-"), + ("Matched Fingerprints", takeover.get("matched_fingerprints") if takeover else []), + ], + ), + ] + issues = issues if issues is not None else _detail_issues(detail_obj, row) + domain_text = str(row.get("domain") or "").strip() + verbose_cmd = f"knockpy -d {domain_text} --verbose" if domain_text else None + + parts = ["
"] + for section_title, pairs in sections: + parts.append( + f"
{html.escape(section_title)}
" + ) + for key, value in pairs: + label = html.escape(str(key)) + rendered = _detail_value_html(str(key), value) + parts.append( + f"
{label}
{rendered}
" + ) + parts.append("
") + parts.append("
Issues Found
") + parts.append("
") + parts.append("
    ") + for level, text in issues: + cls = "issue-ok" if level == "ok" else ("issue-warn" if level == "warn" else "issue-bad") + parts.append(f"
  • {html.escape(text)}
  • ") + parts.append("
") + if verbose_cmd: + parts.append( + "
" + "
Verbose Tip
" + "
Use --verbose for deep diagnostics on this subdomain " + "(without --recon/--bruteforce): " + f"{html.escape(verbose_cmd)}
" + "
" + ) + parts.append("
") + return "".join(parts) + + +def _cert_label(value: Any) -> str: + if value is True: + return "valid" + if value is False: + return "invalid" + return "-" + + +def _status_from_issues(row: Dict[str, Any], issues: List[tuple[str, str]]) -> str: + has_http = row.get("http_status") is not None + has_https = row.get("https_status") is not None + if not has_http and not has_https: + return "dns-only" + for level, _ in issues: + if str(level).strip().lower() != "ok": + return "warning" + return "ok" + + +def _only_ipv4_list(values: List[Any]) -> List[str]: + ipv4: List[str] = [] + for value in values: + ip = str(value).strip() + if not ip: + continue + try: + parsed = ipaddress.ip_address(ip) + except ValueError: + continue + if parsed.version == 4: + ipv4.append(ip) + return ipv4 + + +def _dns_name_matches(host: str, pattern: str) -> bool: + host = host.strip(".").lower() + pattern = pattern.strip(".").lower() + if not host or not pattern: + return False + if pattern == host: + return True + if pattern.startswith("*."): + # Wildcard matches exactly one label: *.example.com -> a.example.com + suffix = pattern[2:] + if not suffix: + return False + if host == suffix: + return False + if host.endswith("." + suffix): + left = host[: -(len(suffix) + 1)] + return "." not in left + return False + + +def _hostname_matches_cert(domain: str, cn: Optional[str], sans: Optional[List[str]]) -> Optional[bool]: + host = str(domain or "").strip().lower() + if not host: + return None + candidates: List[str] = [] + if isinstance(cn, str) and cn.strip(): + candidates.append(cn.strip()) + if isinstance(sans, list): + candidates.extend(str(v).strip() for v in sans if str(v).strip()) + if not candidates: + return None + for pattern in candidates: + try: + if _dns_name_matches(host, pattern): + return True + except Exception: + continue + return False + + +def _detail_issues(detail_obj: Dict[str, Any], row: Dict[str, Any]) -> List[tuple[str, str]]: + """Aggregate protocol/certificate/security issues for one row. + + The returned list drives both: + - `Issues Found` details in expanded row + - top-level status badge (`ok`/`warning`/`dns-only`) + """ + http = detail_obj.get("http") or [] + https = detail_obj.get("https") or [] + cert = detail_obj.get("cert") or [] + takeover = detail_obj.get("takeover") or {} + + http_status = http[0] if len(http) > 0 else row.get("http_status") + https_status = https[0] if len(https) > 0 else row.get("https_status") + http_app_redirect = http[4] if len(http) > 4 else None + https_app_redirect = https[4] if len(https) > 4 else None + + cert_valid = cert[0] if len(cert) > 0 else row.get("cert_valid") + cert_expiry = cert[1] if len(cert) > 1 else row.get("cert_expiry") + cert_cn = cert[2] if len(cert) > 2 else row.get("cert_cn") + tls_versions = cert[3] if len(cert) > 3 and cert[3] else row.get("tls_versions") + tls_verbose = (detail_obj.get("verbose") or {}).get("tls") or {} + cert_sans = tls_verbose.get("san") if isinstance(tls_verbose, dict) else None + + def _as_int(value: Any) -> Optional[int]: + try: + return int(value) if value is not None else None + except Exception: + return None + + http_code = _as_int(http_status) + https_code = _as_int(https_status) + https_ok = https_code is not None and 200 <= https_code < 400 + + issues: List[tuple[str, str]] = [] + seen_msgs: set[str] = set() + weak_tls_best: set[str] = set() + weak_tls_msg: Optional[str] = None + + def _collect_weak_tls_from_text(text: str) -> set[str]: + msg_l = str(text or "").strip().lower() + if "weak tls versions detected" not in msg_l: + return set() + out: set[str] = set() + for item in ("TLS 1.0", "TLS 1.1"): + if item.lower() in msg_l: + out.add(item) + if "sslv" in msg_l: + # Keep coarse SSLv marker if present in message. + out.add("SSLv*") + return out + + def _add_issue(level: str, text: str) -> None: + msg = str(text).strip() + if not msg: + return + weak_set = _collect_weak_tls_from_text(msg) + if weak_set: + nonlocal weak_tls_best, weak_tls_msg + if weak_set.issubset(weak_tls_best): + return + weak_tls_best |= weak_set + ordered = [v for v in ("SSLv*", "TLS 1.0", "TLS 1.1") if v in weak_tls_best] + weak_tls_msg = f"Weak TLS versions detected: {', '.join(ordered)}." + # Replace prior weak-tls messages so only the most complete one remains. + issues[:] = [(lv, m) for lv, m in issues if "weak tls versions detected" not in m.lower()] + seen_msgs_copy = {m for _, m in issues} + seen_msgs.clear() + seen_msgs.update(seen_msgs_copy) + if weak_tls_msg in seen_msgs: + return + seen_msgs.add(weak_tls_msg) + issues.append((level, weak_tls_msg)) + return + if msg in seen_msgs: + return + seen_msgs.add(msg) + issues.append((level, msg)) + + if http_code is None and https_code is None: + _add_issue("warn", "No HTTP/HTTPS response received. Check DNS, firewall, or timeout.") + # Treat common application-level denials/misses as informational (not warning): + # - 403: often expected access control + # - 404: often expected missing route + soft_http_statuses = {403, 404} + soft_https_statuses = {403, 404} + if http_code is not None and http_code >= 400 and http_code not in soft_http_statuses and https_ok: + _add_issue("warn", f"HTTP returned {http_code}, but HTTPS is reachable. Browser may auto-upgrade to HTTPS.") + if http_code is not None and http_code >= 400 and http_code not in soft_http_statuses and not https_ok: + _add_issue("warn", f"HTTP returned error status {http_code}.") + if https_code is not None and https_code >= 400 and https_code not in soft_https_statuses: + _add_issue("warn", f"HTTPS returned error status {https_code}.") + if http_app_redirect: + _add_issue("warn", f"HTTP application-level redirect detected: {http_app_redirect}") + if https_app_redirect: + _add_issue("warn", f"HTTPS application-level redirect detected: {https_app_redirect}") + if https_code is None: + _add_issue("warn", "HTTPS endpoint not reachable or not responding.") + + takeover_status = str(takeover.get("status") or "").strip().lower() + takeover_provider = str(takeover.get("provider") or "unknown") + takeover_cname = str(takeover.get("cname") or "-") + if takeover_status == "likely": + _add_issue("bad", f"Possible subdomain takeover likely ({takeover_provider}) via CNAME {takeover_cname}.") + elif takeover_status == "possible": + _add_issue("warn", f"Possible subdomain takeover to verify ({takeover_provider}) via CNAME {takeover_cname}.") + + expiry_expired = False + if cert_expiry: + try: + expiry_expired = date.fromisoformat(str(cert_expiry)) < date.today() + except Exception: + expiry_expired = False + + weak_tls: List[str] = [] + if isinstance(tls_versions, list): + weak_tls = [v for v in tls_versions if isinstance(v, str) and (v.startswith("SSLv") or v in ("TLS 1.0", "TLS 1.1"))] + elif isinstance(tls_versions, str): + weak_tls = [v.strip() for v in tls_versions.split(",") if v.strip() in ("TLS 1.0", "TLS 1.1") or v.strip().startswith("SSLv")] + + domain_name = str(row.get("domain") or detail_obj.get("domain") or "").strip().lower() + cn_match = _hostname_matches_cert(domain_name, str(cert_cn) if cert_cn else None, cert_sans if isinstance(cert_sans, list) else None) + cn_mismatch_probable = cert_valid is False and cn_match is False + if cert_valid is False: + _add_issue("bad", "Certificate validation failed.") + if cn_mismatch_probable and cert_cn: + _add_issue("bad", f"Likely hostname mismatch: certificate CN/SAN ({cert_cn}) does not match domain.") + elif not expiry_expired and not weak_tls: + _add_issue("bad", "Certificate trust/chain verification failed (not an obvious CN mismatch).") + if expiry_expired: + _add_issue("bad", f"Certificate expired on {cert_expiry}.") + if weak_tls: + _add_issue("bad", f"Weak TLS versions detected: {', '.join(weak_tls)}.") + elif cert_valid is None and https_code is not None: + _add_issue("warn", "Certificate check unavailable despite HTTPS response.") + + # Web server version checks (from export row enrichment). + for proto in ("http", "https"): + status = str(row.get(f"{proto}_server_status") or "").strip().lower() + version = row.get(f"{proto}_server_version") + latest = row.get(f"{proto}_server_latest") + if status == "outdated": + _add_issue("bad", f"{proto.upper()} server potentially outdated ({version} < {latest}).") + elif status == "advisory": + _add_issue("warn", f"{proto.upper()} server version check is advisory; verify patch level at OS/vendor level.") + + # Include all warning/critical findings from verbose security checks when available. + verbose = detail_obj.get("verbose") or {} + security = verbose.get("security") or {} + checks = security.get("checks") or {} + if isinstance(checks, dict): + for check_name, check in checks.items(): + if not isinstance(check, dict): + continue + level = str(check.get("level") or "").strip().lower() + summary = str(check.get("summary") or "").strip() + if not summary: + continue + prefix = check_name.replace("_", " ").strip().capitalize() + if level in {"warning", "critical"}: + _add_issue("warn" if level == "warning" else "bad", f"{prefix}: {summary}") + + # Surface scan-level notes if they describe a problem. + # AXFR note handling: + # - "allowed" is a real issue (zone transfer exposed) + # - failed/refused/denied/closed are informational and should not raise warnings. + for note in detail_obj.get("scan_notes") or []: + text = str(note).strip() + if not text: + continue + low = text.lower() + if "axfr on root domain:" in low: + if "allowed" in low: + _add_issue("bad", text) + continue + if any(x in low for x in ("failed", "error", "timeout", "allowed", "warning", "invalid", "outdated")): + _add_issue("warn", text) + + # Report only critical and high-confidence findings. + critical_issues = [item for item in issues if str(item[0]).strip().lower() == "bad"] + if critical_issues: + return critical_issues + return [("ok", "No critical confirmed issue detected.")] + + +def _finding_evidence_for_issue(issue_text: str, detail_obj: Dict[str, Any], row: Dict[str, Any]) -> str: + """Build issue-specific evidence text for findings summary rows.""" + issue = str(issue_text or "").strip().lower() + takeover = detail_obj.get("takeover") or {} + takeover_status = str(takeover.get("status") or "").strip().lower() + takeover_provider = str(takeover.get("provider") or "-").strip() + takeover_cname = str(takeover.get("cname") or "-").strip() + matched_fp = takeover.get("matched_fingerprints") or [] + http = detail_obj.get("http") or [] + https = detail_obj.get("https") or [] + cert = detail_obj.get("cert") or [] + verbose = detail_obj.get("verbose") or {} + tls_verbose = verbose.get("tls") if isinstance(verbose, dict) else {} + tls_verbose = tls_verbose if isinstance(tls_verbose, dict) else {} + + domain_name = str(row.get("domain") or detail_obj.get("domain") or "-").strip() + http_status = http[0] if len(http) > 0 else row.get("http_status") + https_status = https[0] if len(https) > 0 else row.get("https_status") + http_server = str(http[2] if len(http) > 2 else row.get("http_server") or "-").strip() + https_server = str(https[2] if len(https) > 2 else row.get("https_server") or "-").strip() + cert_expiry = cert[1] if len(cert) > 1 else row.get("cert_expiry") + cert_cn = str(cert[2] if len(cert) > 2 else row.get("cert_cn") or "-").strip() + tls_versions = cert[3] if len(cert) > 3 else row.get("tls_versions") + san_values = tls_verbose.get("san") or [] + strict_error = str(tls_verbose.get("strict_error") or "-").strip() + issuer = str(tls_verbose.get("issuer") or "-").strip() + protocol = str(tls_verbose.get("protocol") or "-").strip() + cipher = str(tls_verbose.get("cipher") or "-").strip() + + http_version = row.get("http_server_version") + http_latest = row.get("http_server_latest") + https_version = row.get("https_server_version") + https_latest = row.get("https_server_latest") + + evidence_parts: List[str] = [] + if "server potentially outdated" in issue: + is_https = issue.startswith("https ") + if is_https: + evidence_parts.append("service=HTTPS") + evidence_parts.append(f"banner={https_server}") + evidence_parts.append(f"status={https_status if https_status is not None else '-'}") + if https_version or https_latest: + evidence_parts.append(f"version={https_version or '-'} latest_ref={https_latest or '-'}") + else: + evidence_parts.append("service=HTTP") + evidence_parts.append(f"banner={http_server}") + evidence_parts.append(f"status={http_status if http_status is not None else '-'}") + if http_version or http_latest: + evidence_parts.append(f"version={http_version or '-'} latest_ref={http_latest or '-'}") + elif "hostname mismatch" in issue: + evidence_parts.append("service=HTTPS/TLS certificate") + evidence_parts.append(f"domain={domain_name}") + evidence_parts.append(f"cert_cn={cert_cn}") + evidence_parts.append(f"https_banner={https_server}") + evidence_parts.append(f"issuer={issuer}") + if isinstance(san_values, list) and san_values: + evidence_parts.append(f"san={', '.join(str(v) for v in san_values[:6])}") + elif "certificate validation failed" in issue or "certificate trust/chain verification failed" in issue: + evidence_parts.append("service=HTTPS/TLS") + evidence_parts.append(f"strict_error={strict_error}") + evidence_parts.append(f"issuer={issuer}") + elif "certificate expired" in issue: + evidence_parts.append("service=HTTPS/TLS") + evidence_parts.append(f"cert_expiry={cert_expiry or '-'}") + evidence_parts.append(f"issuer={issuer}") + elif "weak tls versions detected" in issue: + evidence_parts.append("service=HTTPS/TLS") + if isinstance(tls_versions, list) and tls_versions: + evidence_parts.append(f"tls={', '.join(str(v) for v in tls_versions)}") + evidence_parts.append(f"protocol={protocol}") + evidence_parts.append(f"cipher={cipher}") + elif "subdomain takeover" in issue: + evidence_parts.append("service=DNS CNAME + HTTP/HTTPS fingerprint") + evidence_parts.append(f"provider={takeover_provider}") + evidence_parts.append(f"cname={takeover_cname}") + evidence_parts.append(f"takeover_status={takeover_status or '-'}") + if matched_fp: + evidence_parts.append(f"matched_fp={', '.join(str(v) for v in matched_fp)}") + evidence_parts.append(f"http={http_status if http_status is not None else '-'} https={https_status if https_status is not None else '-'}") + elif "axfr" in issue or "zone transfer" in issue: + evidence_parts.append("service=DNS AXFR") + evidence_parts.append(f"note={issue_text}") + elif "http returned" in issue or "https returned" in issue or "redirect" in issue: + evidence_parts.append("service=HTTP/HTTPS") + evidence_parts.append(f"http={http_status if http_status is not None else '-'} https={https_status if https_status is not None else '-'}") + evidence_parts.append(f"http_banner={http_server}") + evidence_parts.append(f"https_banner={https_server}") + else: + if http_status is not None or https_status is not None: + evidence_parts.append( + f"HTTP={http_status if http_status is not None else '-'} HTTPS={https_status if https_status is not None else '-'}" + ) + if takeover_status in {"likely", "possible"}: + evidence_parts.append(f"takeover={takeover_status} provider={takeover_provider} cname={takeover_cname}") + if cert_expiry: + evidence_parts.append(f"cert_expiry={cert_expiry}") + if isinstance(tls_versions, list) and tls_versions: + evidence_parts.append(f"tls={', '.join(str(v) for v in tls_versions)}") + return " | ".join(evidence_parts) if evidence_parts else "-" + + +def _finding_category(issue_text: str) -> str: + """Return a category label used to group findings in HTML summary.""" + text = str(issue_text or "").strip().lower() + if not text: + return "Security finding" + if "takeover" in text: + return "Subdomain takeover" + if "certificate" in text or "tls" in text or "hostname mismatch" in text: + return "TLS / certificate" + if "server potentially outdated" in text or "server version" in text: + return "Web server version" + if "axfr" in text or "zone transfer" in text: + return "DNS AXFR exposure" + if "http returned" in text or "https returned" in text or "redirect" in text: + return "HTTP/HTTPS behavior" + if "header" in text: + return "Security headers" + return "Security finding" diff --git a/knockpy/version.py b/knockpy/version.py new file mode 100644 index 0000000..152fc9d --- /dev/null +++ b/knockpy/version.py @@ -0,0 +1,78 @@ +"""Version helpers for knockpy.""" + +from __future__ import annotations + +import re +from typing import Any, Dict, Tuple + +import httpx + +__version__ = "9.0.0" +PYPI_PROJECT = "knock-subdomains" +PYPI_URL = f"https://pypi.org/pypi/{PYPI_PROJECT}/json" + + +def _version_key(raw: str) -> Tuple[int, ...]: + """Convert a version string to a numeric tuple for conservative compare. + + Examples: + - "9.0.0" -> (9, 0, 0) + - "9.0.0rc1" -> (9, 0, 0, 1) + """ + parts: list[int] = [] + for token in re.findall(r"\d+", str(raw or "")): + try: + parts.append(int(token)) + except Exception: + continue + return tuple(parts) if parts else (0,) + + +def is_newer_version(latest: str, current: str) -> bool: + a = _version_key(latest) + b = _version_key(current) + width = max(len(a), len(b)) + return a + (0,) * (width - len(a)) > b + (0,) * (width - len(b)) + + +def check_latest_version(current: str = __version__, timeout: float = 2.5) -> Dict[str, Any]: + """Check latest published version on PyPI. + + Returns a dict to keep caller logic stable and easy to render: + - ok: bool + - current: str + - latest: Optional[str] + - update_available: bool + - url: str + - error: Optional[str] + """ + result: Dict[str, Any] = { + "ok": False, + "current": str(current), + "latest": None, + "update_available": False, + "url": PYPI_URL, + "error": None, + } + try: + response = httpx.get( + PYPI_URL, + timeout=timeout, + follow_redirects=True, + headers={"User-Agent": f"knockpy/{current}"}, + ) + if int(response.status_code) >= 400: + result["error"] = f"HTTP {response.status_code}" + return result + payload = response.json() + latest = str((payload.get("info") or {}).get("version") or "").strip() + if not latest: + result["error"] = "Missing version in PyPI response" + return result + result["ok"] = True + result["latest"] = latest + result["update_available"] = is_newer_version(latest, str(current)) + return result + except Exception as exc: + result["error"] = f"{exc.__class__.__name__}: {exc}" + return result diff --git a/knock/wordlist/wordlist.txt b/knockpy/wordlist/wordlist.txt similarity index 99% rename from knock/wordlist/wordlist.txt rename to knockpy/wordlist/wordlist.txt index 536d67f..7339bdb 100644 --- a/knock/wordlist/wordlist.txt +++ b/knockpy/wordlist/wordlist.txt @@ -139,6 +139,8 @@ 98 99 a +a-dtap +a.auth-ns a01 a02 a1 @@ -153,7 +155,6 @@ aaa2 aai aap aaron -a.auth-ns ab aba abacus @@ -185,9 +186,9 @@ acceptatie acces acceso access -accessibilita access1 access2 +accessibilita accessories accommodation account @@ -208,6 +209,7 @@ acs act acta action +actions activate activation active @@ -270,7 +272,6 @@ adserv adserver adsl adt -a-dtap adtest adult adv @@ -299,6 +300,8 @@ africa afrodita afs ag +ag-hinrichs +ag-kopf-moertz aga agate age @@ -308,10 +311,8 @@ agenda agent agents agenzia -ag-hinrichs agile agk -ag-kopf-moertz agora agri agriculture @@ -342,11 +343,12 @@ aj ajax ajuda ak +ak-gw akademia akademik akamai -ak-gw akira +aks al alabama aladdin @@ -492,13 +494,15 @@ ape apex aphrodite api +api-dev +api-gw +api-test api1 api2 api3 -api-dev apidev +apigw apis -api-test apitest apk apl @@ -574,6 +578,7 @@ arena ares argentina argo +argocd argon argos argus @@ -711,6 +716,8 @@ australia austria aut auth +auth-dev +auth-test auth1 auth2 author @@ -765,6 +772,7 @@ azmoon azs azure b +b.auth-ns b01 b02 b1 @@ -861,7 +869,6 @@ batch batman battle bau -b.auth-ns bayarea baza bazaar @@ -1023,10 +1030,10 @@ blitz block blocked blog +blog-dev blog1 blog2 blog3 -blog-dev blogdev blogg blogger @@ -1175,6 +1182,7 @@ build buildbot builder building +buildkite bulgaria bulk bulkmail @@ -1207,6 +1215,7 @@ bydgoszcz bz c c-00 +c.auth-ns c1 c10 c11 @@ -1348,7 +1357,6 @@ catalyst catering cats cau -c.auth-ns cb cbc cbf1 @@ -1378,6 +1386,7 @@ ccs cct cctv cd +cd-dev cdb cdburner cdc @@ -1468,14 +1477,14 @@ chart charts chase chat +chat-service +chat-service2 chat1 chat2 chat3 chat4 chats chatserver -chat-service -chat-service2 che cheboksary check @@ -1521,6 +1530,7 @@ chrysler chs church ci +ci-dev cia cib cic @@ -1533,13 +1543,14 @@ cine cinema cio cip +circleci cirrus cis cisco -cisco1 -cisco2 cisco-capwap-controller cisco-lwapp-controller +cisco1 +cisco2 ciscoworks cit citi @@ -1619,12 +1630,12 @@ cmi cmp cmr cms +cms-test cms1 cms2 cms3 cmsadmin cmsdev -cms-test cmstest cmt cn @@ -1725,8 +1736,8 @@ con concentrator concord concorde -concorso concorsi +concorso concours concurso concursos @@ -1749,6 +1760,7 @@ consola console construction construtor +consul consult consulta consultant @@ -1798,6 +1810,7 @@ copy copyright coral core +core-platform core0 core01 core1 @@ -2114,6 +2127,10 @@ cz czat czech d +d-app +d-click +d-image +d-view d0 d1 d10 @@ -2152,7 +2169,6 @@ daniel dante dap daphne -d-app dar dark darkorbit @@ -2216,7 +2232,6 @@ dc3 dcc dce dchub -d-click dcp dcs dd @@ -2241,6 +2256,7 @@ debian debug dec deco +decommissioned dedicated deep deepolis @@ -2321,6 +2337,9 @@ deti detroit deutsch dev +dev-chat +dev-chat-service +dev-www dev0 dev01 dev02 @@ -2350,8 +2369,6 @@ deva devadmin devapi devblog -dev-chat -dev-chat-service devdb devel devel2 @@ -2375,7 +2392,6 @@ devtest devweb devwiki devwowza -dev-www devwww dewey dex @@ -2416,7 +2432,6 @@ digital digitalmedia dilbert dima -d-image dimdim dingo dining @@ -2456,21 +2471,21 @@ distanza distanze distanziati distanziato -distribuito distribuiti -distributore -distributori +distribuito distributer distributers distribution distributor +distributore +distributori distributors distribuzione distribuzioni +diversi diversita diversity diverso -diversi diy diz dj @@ -2499,6 +2514,7 @@ dna dnews dnn dns +dns-2 dns0 dns01 dns02 @@ -2510,7 +2526,6 @@ dns11 dns12 dns13 dns14 -dns-2 dns2 dns3 dns4 @@ -2526,6 +2541,7 @@ dnstest do doc docker +docker-registry docs doctor document @@ -2546,15 +2562,14 @@ dokuwiki dolphin dom domain +domain-controller domainadmin domaincontrol -domain-controller domaincontroller domaincontrolpanel domaincp domaindnszones domainmanagement -domain domains domen domeny @@ -2652,7 +2667,6 @@ dummy dump dv dvd -d-view dvr dw dwb @@ -2668,6 +2682,12 @@ dynip dz dzb e +e-com +e-commerce +e-learning +e-mail +e-resultats +e-shop e0 e1 e2 @@ -2716,10 +2736,8 @@ ecm ecms eco ecology -e-com ecom ecomm -e-commerce ecommerce econ econom @@ -2800,6 +2818,7 @@ ekaterinburg ekb eko ekonomi +eks ektron el elab @@ -2808,7 +2827,6 @@ elara elastix elc elearn -e-learning elearning elearning2 elec @@ -2839,7 +2857,6 @@ els elsa elvis em -e-mail email email1 email2 @@ -2957,7 +2974,6 @@ era erasmus erato erc -e-resultats eric eris ernie @@ -2980,7 +2996,6 @@ eservice eservices eset esf -e-shop eshop eski esl @@ -3152,9 +3167,9 @@ fax fax2 faxserver fb +fb-canvas fbapp fbapps -fb-canvas fbdev fbe fbl @@ -3410,6 +3425,7 @@ fst ft ftp ftp- +ftp-eu ftp0 ftp01 ftp02 @@ -3431,7 +3447,6 @@ ftp8 ftp9 ftpadmin ftpd -ftp-eu ftpmini ftps ftpsearch @@ -3452,9 +3467,9 @@ fusion futbol future fw +fw-1 fw01 fw02 -fw-1 fw1 fw2 fw3 @@ -3596,6 +3611,7 @@ girls gis gis2 git +git-runner github gitlab gitweb @@ -3608,6 +3624,7 @@ gjc gjs gjxy gk +gke gl gladiator glass @@ -3659,6 +3676,7 @@ gps gpweb gq gr +gr-mbpc1 gra graal grace @@ -3674,6 +3692,7 @@ graphic graphics graphics2 graphite +graphql graphs grc great @@ -3685,7 +3704,6 @@ greenfox greetings grey grid -gr-mbpc1 group groupon groups @@ -3797,6 +3815,7 @@ hannibal hans hao happy +harbor hardcore hardware hardy @@ -3857,6 +3876,7 @@ hg hgfgdf hh hi +hi-tech hidden hideip hideip-uk @@ -3874,7 +3894,6 @@ his historia history hit -hi-tech hitech hive hj @@ -4183,6 +4202,7 @@ imc imchat imcservices img +img-m img0 img01 img02 @@ -4212,7 +4232,6 @@ imga imgb imgc imgf -img-m imgm imgn imgs @@ -4319,6 +4338,7 @@ interior intern internacional internal +internal-api international internet interno @@ -4357,6 +4377,11 @@ ios iota iowa ip +ip-ca +ip-hk +ip-uk +ip-us +ip-usa ip1 ip118 ip176-194 @@ -4371,14 +4396,12 @@ ipade ipam ipb ipc -ip-ca ipfixe -ip-hk iphone ipkvm ipl-a -iplanet ipl-m +iplanet ipm ipmi ipmonitor @@ -4395,9 +4418,6 @@ ipsec-gw ipsi ipt iptv -ip-uk -ip-us -ip-usa ipv ipv4 ipv6 @@ -4497,6 +4517,7 @@ jackson jacksonville jacob jade +jaeger jaguar jakarta jam @@ -4619,6 +4640,7 @@ k1 k2 k3 k4 +k8s ka kabinet kairos @@ -4681,6 +4703,7 @@ kernel kevin key keyan +keycloak keynote keys keyserver @@ -4763,6 +4786,11 @@ ksp kt ku ku6 +kube +kube-api +kube-dashboard +kubelet +kubernetes kuku kultura kunde @@ -4885,6 +4913,7 @@ lee leeds leela legacy +legacy-old legal legend legion @@ -5134,6 +5163,8 @@ lyra lyrics lyris m +m-dev +m-test m0 m01 m1 @@ -5190,6 +5221,13 @@ magnolia mahara maia mail +mail-1 +mail-2 +mail-backup +mail-gw +mail-old +mail-out +mail-relay mail0 mail01 mail02 @@ -5198,7 +5236,6 @@ mail04 mail05 mail06 mail07 -mail-1 mail1 mail10 mail11 @@ -5209,7 +5246,6 @@ mail15 mail16 mail17 mail18 -mail-2 mail2 mail20 mail21 @@ -5236,7 +5272,6 @@ maila mailadmin mailarchive mailb -mail-backup mailbackup mailbck mailbox @@ -5254,7 +5289,6 @@ mailgate2 mailgate3 mailgateway mailguard -mail-gw mailgw mailgw1 mailgw2 @@ -5271,12 +5305,9 @@ maillog mailman mailmems mailmx -mail-old mailold -mail-out mailout mailout2 -mail-relay mailrelay mailroom mails @@ -5419,7 +5450,6 @@ md1 mdaemon mdb mdc -m-dev mdev mdm mds @@ -5430,8 +5460,8 @@ mech mechatronics med media -media01 media-1 +media01 media1 media2 media3 @@ -5640,6 +5670,7 @@ mobi mobiel mobil mobile +mobile-test mobile1 mobile2 mobileapp @@ -5647,7 +5678,6 @@ mobileapps mobiledev mobileiron mobilemail -mobile-test mobiletest mobility moc @@ -5692,11 +5722,11 @@ montreal moo mooc moodle +moodle-dev +moodle-test moodle1 moodle2 -moodle-dev moodledev -moodle-test moodletest moon moose @@ -5756,6 +5786,8 @@ mrtg mrtg1 mrtg2 ms +ms-exchange +ms-sql ms1 ms2 ms3 @@ -5765,7 +5797,6 @@ msc msdn msdnaa mse -ms-exchange msexchange msg msi @@ -5775,7 +5806,6 @@ msn msoid msp mss -ms-sql mssql mssql0 mssql01 @@ -5802,7 +5832,6 @@ mta5 mta6 mtb mtc -m-test mtest mti mtm @@ -5827,8 +5856,8 @@ mus muse museum music -musica music-hn +musica musik musique mustang @@ -5840,6 +5869,9 @@ mvs mw mweb mx +mx-1 +mx-a +mx-b mx0 mx00 mx01 @@ -5847,7 +5879,6 @@ mx02 mx03 mx04 mx05 -mx-1 mx1 mx10 mx11 @@ -5864,8 +5895,6 @@ mx6 mx7 mx8 mx9 -mx-a -mx-b mxbackup mxs my @@ -5948,12 +5977,12 @@ nascosti nascosto nashville nat +nat-pool nat1 nat2 nat3 nat4 national -nat-pool nats nature nauka @@ -5961,8 +5990,8 @@ nautilus nav navi navigator -nazionali nazionale +nazionali nb nba nc @@ -6018,6 +6047,7 @@ neuro nevada nevis new +new-www new1 new2 new3 @@ -6050,7 +6080,6 @@ newtest newton newweb newwebmail -new-www newwww newyear newyork @@ -6171,6 +6200,8 @@ nps nr ns ns- +ns-1 +ns-2 ns0 ns01 ns02 @@ -6178,7 +6209,6 @@ ns03 ns04 ns05 ns06 -ns-1 ns1 ns10 ns100 @@ -6205,7 +6235,6 @@ ns17 ns18 ns19 ns1a -ns-2 ns2 ns20 ns201 @@ -6338,6 +6367,7 @@ oas oascentral oasis oauth +oauth2 ob obchod obelix @@ -6396,6 +6426,7 @@ ohio oic oid oidb +oidc oil ois oita @@ -6410,6 +6441,7 @@ ol ola olap old +old-www old1 old2 old3 @@ -6423,7 +6455,6 @@ oldman oldsite oldweb oldwebmail -old-www oldwww ole oleg @@ -6477,6 +6508,7 @@ openid openmeetings opennms opensource +opentelemetry openview openvpn openx @@ -6522,7 +6554,6 @@ orientation origen origen-www origin -origin2 origin-cdn origin-images origin-live @@ -6530,6 +6561,7 @@ origin-m origin-staging origin-user origin-www +origin2 orion orion2 orlando @@ -6551,6 +6583,7 @@ osx ot ota otc +otel other otp otrs @@ -6592,10 +6625,10 @@ owncloud ows ox ox-d -oxford ox-i -oxnard ox-ui +oxford +oxnard oxygen oyp oyun @@ -7164,6 +7197,9 @@ praxis prc prd pre +pre-prod +pre-production +pre-www pregnant prelive prelive-admin @@ -7177,9 +7213,7 @@ prenotazioni prensa prep prepaid -pre-prod preprod -pre-production pres presence present @@ -7195,7 +7229,6 @@ prev preview preview1 preview2 -pre-www prewww pri price @@ -7220,6 +7253,7 @@ prisma priv privacy private +private-api prm pro proba @@ -7333,6 +7367,7 @@ pts pub pub2 public +public-api public1 public2 publica @@ -7591,10 +7626,10 @@ relay3 relay4 relay5 release -release0000 release-chat release-chat-service release-commondata +release0000 releasephp relief religion @@ -7756,11 +7791,11 @@ rostov roundcube route router -router1 -router2 router-b router-uk router-us +router1 +router2 routing roy royal @@ -7810,6 +7845,7 @@ rubin ruby rugby rune +runner rural rus russia @@ -7823,6 +7859,8 @@ ryan ryazan rz s +s-dtap +s-dtap2 s0 s01 s02 @@ -8133,8 +8171,6 @@ sdm sdo sdp sds -s-dtap -s-dtap2 se sea seafight @@ -8210,6 +8246,7 @@ sergio seri serial serv +serv-refi serv1 serv2 server @@ -8291,7 +8328,6 @@ serviziweb serviziweb1 serviziweb2 serviziweb3 -serv-refi serwer serwis ses @@ -8502,6 +8538,11 @@ smsgateway smsgw smt smtp +smtp-gw +smtp-in +smtp-out +smtp-out-01 +smtp-relay smtp0 smtp01 smtp02 @@ -8527,18 +8568,13 @@ smtp8 smtp9 smtpa smtpauth -smtp-gw smtpgw smtphost -smtp-in smtpin smtpmail smtpmax -smtp-out smtpout -smtp-out-01 smtpout2 -smtp-relay smtprelay smtps smtptest @@ -8623,6 +8659,7 @@ southeast southwest sovet sp +sp-test sp1 sp2 spa @@ -8696,7 +8733,6 @@ sprint sps spss spt -sp-test sptest sputnik spy @@ -8805,6 +8841,7 @@ ssh1 ssh2 ssi ssl +ssl-vpn ssl0 ssl01 ssl1 @@ -8812,11 +8849,12 @@ ssl2 ssl3 ssl4 ssltest -ssl-vpn sslvpn ssm ssmtp sso +sso-dev +sso-test sso2 ssotest ssp @@ -8839,12 +8877,12 @@ stage1 stage2 stages staging -staging1 -staging2 -staging40 staging-chat staging-chat-service staging-commondata +staging1 +staging2 +staging40 stagingphp stalker stamp @@ -8866,6 +8904,7 @@ stat stat1 stat2 static +static-m static0 static01 static1 @@ -8876,7 +8915,6 @@ static5 static6 static7 static8 -static-m statics station statistica @@ -9124,6 +9162,7 @@ szkolenia sztz szukaj t +t-dtap t1 t2 t3 @@ -9187,7 +9226,6 @@ tdc tde tdm tds -t-dtap te tea teach @@ -9214,6 +9252,7 @@ teens tehran teknik teknobyen-gw2 +tekton tel tele tele2 @@ -9257,6 +9296,8 @@ terry tes tesla test +test-admin +test-www test01 test02 test03 @@ -9285,7 +9326,6 @@ test7 test8 test9 test99 -test-admin testadmin testapi testapp @@ -9323,7 +9363,6 @@ testvb testweb testwiki testwp -test-www testwww testxp testy @@ -9341,12 +9380,12 @@ tftp tg tgp th +th-core thai thailand thanhtra thankyou thc -th-core the theater theatre @@ -9847,6 +9886,7 @@ vid1 vid2 vid3 video +video-m video1 video2 video3 @@ -9854,7 +9894,6 @@ video4 videochat videoconf videoconferencia -video-m videos videos2 videoteca @@ -9881,9 +9920,9 @@ virgin virginia virgo virt +virt-gw virt1 virt2 -virt-gw virtual virtual1 virtual2 @@ -10095,6 +10134,7 @@ wds we weather web +web-dev web0 web01 web02 @@ -10164,7 +10204,6 @@ webdav webdb webdemo webdesign -web-dev webdev webdev2 webdisk @@ -10193,6 +10232,7 @@ weblogic weblogs webm webmail +webmail-old webmail01 webmail02 webmail1 @@ -10200,7 +10240,6 @@ webmail2 webmail3 webmail4 webmail5 -webmail-old webmailtest webmaker webmakerl @@ -10458,17 +10497,37 @@ wws wwu www www- -www0 www-01 -www01 www-02 +www-1 +www-2 +www-3 +www-4 +www-5 +www-a +www-admin +www-b +www-c +www-cache +www-dev +www-devel +www-int +www-new +www-old +www-org +www-origin +www-prod +www-staging +www-test +www-uat +www0 +www01 www02 www03 www04 www05 www06 www07 -www-1 www1 www10 www11 @@ -10480,7 +10539,6 @@ www16 www17 www18 www19 -www-2 www2 www20 www21 @@ -10492,15 +10550,12 @@ www26 www27 www28 www29 -www-3 www3 www30 www31 -www-4 www4 www40 www42 -www-5 www5 www6 www66 @@ -10508,37 +10563,21 @@ www7 www8 www9 www99 -www-a wwwa -www-admin wwwalt -www-b wwwb -www-c -www-cache wwwcache wwwchat -www-dev wwwdev -www-devel wwwftp -www-int wwwm wwwmail -www-new wwwnew -www-old wwwold -www-org -www-origin -www-prod wwws -www-staging wwwstg wwwt -www-test wwwtest -www-uat wwww wwwww wwwx @@ -10551,6 +10590,7 @@ wyoming wyx wz x +x-ray x1 x2 x3 @@ -10604,7 +10644,6 @@ xms xp xpam xq -x-ray xray xs xsc @@ -10678,6 +10717,7 @@ yxy yy yz z +z-log z1 z2 z3 @@ -10717,9 +10757,9 @@ zeta zeus zf zh +zh-cn zhaopin zhaosheng -zh-cn zhidao zhuanti zim @@ -10729,7 +10769,6 @@ zion zip zj zk -z-log zlog zm zmail @@ -10754,4 +10793,4 @@ zy zyz zz zzb -zzz +zzz \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 24a1a33..15f142b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,5 @@ -requests>=2.32.5 dnspython>=2.8.0 -pyOpenSSL>=25.3.0 -beautifulsoup4>=4.14.2 -tqdm>=4.67.1 httpx[http2]>=0.28.1 -dotenv>=0.9.9 \ No newline at end of file +pyOpenSSL>=25.3.0 +python-dotenv>=1.0.1 +rich>=13.9.4 diff --git a/setup.py b/setup.py index 2a9c88f..fab1b13 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -from setuptools import setup +from setuptools import find_packages, setup import os def load_requirements(path): @@ -9,19 +9,20 @@ def load_requirements(path): setup( name='knock-subdomains', - version='8.0.0', + version='9.0.0', description='Knockpy Subdomains Scan', - url='https://github.com/guelfoweb/knock', + url='https://github.com/guelfoweb/knockpy', author='Gianni Amato', author_email='guelfoweb@gmail.com', license='GPL-3.0', - packages=['knock'], - package_data={"knock": [wordlist, 'report'],}, + packages=find_packages(include=["knockpy", "knockpy.*"]), + package_data={"knockpy": [wordlist]}, include_package_data=True, install_requires=load_requirements("requirements.txt"), entry_points={ 'console_scripts': [ - 'knockpy=knock.knockpy:main', + 'knockpy=knockpy.knockpy:main', + 'knockpy.py=knockpy.knockpy:main', ], } ) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..812f61d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +import os +import sys + + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +if ROOT not in sys.path: + sys.path.insert(0, ROOT) diff --git a/tests/test_cli_helpers.py b/tests/test_cli_helpers.py new file mode 100644 index 0000000..6d5bd14 --- /dev/null +++ b/tests/test_cli_helpers.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from pathlib import Path + +from knockpy.cli import _load_domains_from_file, _normalize_domain_input, _normalize_domain_list, _should_save_scan + + +def test_load_domains_from_file_strips_empty_lines(tmp_path: Path): + source = tmp_path / "domains.txt" + source.write_text("\nexample.com\n\nwww.test.org\n", encoding="utf-8") + assert _load_domains_from_file(str(source)) == ["example.com", "www.test.org"] + + +def test_normalize_domain_input_accepts_url_and_plain_domain(): + assert _normalize_domain_input("https://www.Example.com/path?q=1") == "example.com" + assert _normalize_domain_input("sub.example.com") == "sub.example.com" + assert _normalize_domain_input("not a domain") is None + + +def test_normalize_domain_list_deduplicates_and_filters_invalid(): + values = [ + "https://www.example.com", + "example.com", + "sub.example.com", + "bad domain", + "", + ] + assert _normalize_domain_list(values) == ["example.com", "sub.example.com"] + + +def test_should_save_scan_only_for_recon_or_bruteforce(): + assert _should_save_scan(recon=False, bruteforce=False) is False + assert _should_save_scan(recon=True, bruteforce=False) is True + assert _should_save_scan(recon=False, bruteforce=True) is True diff --git a/tests/test_output_helpers.py b/tests/test_output_helpers.py new file mode 100644 index 0000000..20b096b --- /dev/null +++ b/tests/test_output_helpers.py @@ -0,0 +1,145 @@ +from datetime import timedelta +import os +import sys + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from knockpy.core import fmt_td +from knockpy.output import _merge_check_summary, _security_evidence_map, _status_text, _tls_label + + +def test_fmt_td_formats_hhmmss(): + assert fmt_td(timedelta(hours=1, minutes=2, seconds=3)) == "01:02:03" + + +def test_fmt_td_none(): + assert fmt_td(None) == "-" + + +def test_tls_label_empty(): + assert _tls_label(None) == "-" + assert _tls_label([]) == "-" + + +def test_tls_label_values(): + assert _tls_label(["TLS 1.2", "TLS 1.3"]) == "TLS 1.2, TLS 1.3" + + +def test_status_dns_only(): + assert _status_text([None], [None], [None]) == "dns-only" + + +def test_status_warning_for_http_without_cert(): + assert _status_text([200], [None], [False]) == "warning" + + +def test_status_ok(): + assert _status_text([200], [200], [True]) == "ok" + + +def test_security_evidence_map_compacts_key_details(): + verbose = { + "tls": {"strict_ok": False, "strict_error": "CERT_FAIL", "protocol": "TLSv1.3", "cipher": "AES256"}, + "security": { + "mixed_content_count": 2, + "server_versions_updated_at": "2026-02-11", + "server_assessment": { + "banner": "Apache/2.4.49", + "product_label": "Apache HTTP Server", + "version": "2.4.49", + "latest": "2.4.62", + }, + "methods": { + "http": {"allow": ["GET", "HEAD"]}, + "https": {"allow": ["GET", "HEAD", "OPTIONS"]}, + "risky_methods": [], + "inferred_safe": [], + }, + }, + } + evidence = _security_evidence_map(verbose) + assert "mixed_content" in evidence + assert evidence["mixed_content"][0] == "refs=2" + assert "server_version" in evidence + assert any("detected=2.4.49 latest=2.4.62" in part for part in evidence["server_version"]) + assert "methods" in evidence + assert evidence["methods"][0] == "Supported Methods: GET HEAD OPTIONS" + + +def test_merge_check_summary_limits_details_by_level(): + evidence_map = {"headers": ["a=1", "b=2", "c=3", "d=4"]} + warning_item = {"level": "warning", "summary": "Header issues"} + info_item = {"level": "info", "summary": "Header baseline"} + assert _merge_check_summary("headers", warning_item, evidence_map) == "Header issues | a=1 | b=2 | c=3" + assert _merge_check_summary("headers", info_item, evidence_map) == "Header baseline | a=1" + + +def test_merge_check_summary_methods_keeps_enabled_and_risky(): + evidence_map = {"methods": ["Supported Methods: GET HEAD OPTIONS", "risky=-", "x=ignored"]} + item = {"level": "ok", "summary": "No risky HTTP methods exposed"} + assert ( + _merge_check_summary("methods", item, evidence_map) + == "No risky HTTP methods exposed | Supported Methods: GET HEAD OPTIONS | risky=-" + ) + + +def test_security_evidence_map_methods_flags_dangerous(): + verbose = { + "security": { + "methods": { + "http": {"allow": ["GET", "TRACE"]}, + "https": {"allow": ["POST", "DELETE"]}, + "risky_methods": ["DELETE", "TRACE"], + "inferred_safe": [], + }, + } + } + evidence = _security_evidence_map(verbose) + assert evidence["methods"][0] == "Supported Methods: GET POST DELETE TRACE" + assert evidence["methods"][1] == "[red]dangerous=DELETE, TRACE[/red]" + + +def test_security_evidence_map_methods_unknown_without_allow(): + verbose = { + "security": { + "methods": { + "http": {"status": 405, "allow": []}, + "https": {"status": 405, "allow": []}, + "risky_methods": [], + "inferred_safe": [], + }, + } + } + evidence = _security_evidence_map(verbose) + assert evidence["methods"][0] == "Supported Methods: unknown (server does not disclose Allow/CORS methods)" + assert evidence["methods"][1] == "probe_status=http:405 https:405" + + +def test_security_evidence_map_methods_from_cors_allow(): + verbose = { + "security": { + "methods": { + "http": {"status": 200, "allow": [], "cors_allow": ["GET", "HEAD", "POST", "OPTIONS"]}, + "https": {"status": 200, "allow": [], "cors_allow": []}, + "risky_methods": [], + "inferred_safe": [], + }, + } + } + evidence = _security_evidence_map(verbose) + assert evidence["methods"][0] == "Supported Methods: GET HEAD POST OPTIONS" + + +def test_security_evidence_map_methods_from_inferred_probe(): + verbose = { + "security": { + "methods": { + "http": {"status": 200, "allow": [], "cors_allow": []}, + "https": {"status": 200, "allow": [], "cors_allow": []}, + "risky_methods": [], + "inferred_safe": ["GET", "HEAD", "POST", "OPTIONS"], + }, + } + } + evidence = _security_evidence_map(verbose) + assert evidence["methods"][0] == "Supported Methods: GET HEAD POST OPTIONS" diff --git a/tests/test_runtime_helpers.py b/tests/test_runtime_helpers.py new file mode 100644 index 0000000..0c99ca2 --- /dev/null +++ b/tests/test_runtime_helpers.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from datetime import timedelta + +import knockpy.engine.runtime as runtime + + +def test_pick_user_agent_respects_explicit_value(): + assert runtime.pick_user_agent("MyAgent/1.0") == "MyAgent/1.0" + + +def test_pick_user_agent_random_uses_pool(monkeypatch): + monkeypatch.setattr(runtime.random, "choice", lambda items: items[0]) + assert runtime.pick_user_agent("random") == runtime.USER_AGENTS[0] + assert runtime.pick_user_agent(None) == runtime.USER_AGENTS[0] + + +def test_www_fallback_domain_only_when_needed(): + assert runtime._www_fallback_domain("example.com") == "www.example.com" + assert runtime._www_fallback_domain("www.example.com") is None + assert runtime._www_fallback_domain("not a domain") is None + + +def test_fmt_td_expected_format(): + assert runtime.fmt_td(timedelta(hours=2, minutes=3, seconds=4)) == "02:03:04" + assert runtime.fmt_td(None) == "-" diff --git a/tests/test_storage_api.py b/tests/test_storage_api.py new file mode 100644 index 0000000..d43bf23 --- /dev/null +++ b/tests/test_storage_api.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from datetime import timedelta +from pathlib import Path + +from knockpy.storage import ( + count_reports, + delete_report, + get_report, + get_setting, + get_settings, + init_db, + list_reports, + reset_reports, + save_scan, + set_setting, +) + + +def test_storage_scan_lifecycle_roundtrip(tmp_path: Path): + db_path = tmp_path / "reports.db" + init_db(db_path) + assert db_path.exists() + + report_id = save_scan( + target="example.com", + mode="domain", + settings={"timeout": 3.0, "threads": 50}, + results={"domain": "example.com", "ip": ["93.184.216.34"]}, + elapsed=timedelta(seconds=4), + db_path=db_path, + ) + assert report_id > 0 + assert count_reports(db_path) == 1 + + reports = list_reports(limit=10, db_path=db_path) + assert len(reports) == 1 + assert reports[0]["id"] == report_id + assert reports[0]["target"] == "example.com" + + by_latest = get_report("latest", db_path=db_path) + assert by_latest is not None + assert by_latest["id"] == report_id + assert by_latest["target"] == "example.com" + assert by_latest["mode"] == "domain" + assert isinstance(by_latest["results"], list) + assert by_latest["results"][0]["domain"] == "example.com" + + by_id = get_report(str(report_id), db_path=db_path) + assert by_id is not None + assert by_id["id"] == report_id + + by_target = get_report("example.com", db_path=db_path) + assert by_target is not None + assert by_target["id"] == report_id + + assert delete_report(report_id, db_path=db_path) is True + assert count_reports(db_path) == 0 + assert delete_report(report_id, db_path=db_path) is False + + +def test_storage_settings_roundtrip(tmp_path: Path): + db_path = tmp_path / "reports.db" + init_db(db_path) + + set_setting("runtime.timeout", "4", db_path=db_path) + set_setting("runtime.threads", "120", db_path=db_path) + set_setting("api.shodan", None, db_path=db_path) + + assert get_setting("runtime.timeout", db_path=db_path) == "4" + assert get_setting("runtime.threads", db_path=db_path) == "120" + assert get_setting("api.shodan", db_path=db_path) is None + assert get_setting("missing", db_path=db_path) is None + + all_settings = get_settings(db_path=db_path) + assert all_settings["runtime.timeout"] == "4" + assert all_settings["runtime.threads"] == "120" + assert "api.shodan" in all_settings + + runtime_only = get_settings(prefix="runtime.", db_path=db_path) + assert sorted(runtime_only.keys()) == ["runtime.threads", "runtime.timeout"] + + +def test_storage_reset_reports_returns_deleted_count(tmp_path: Path): + db_path = tmp_path / "reports.db" + init_db(db_path) + for idx in range(3): + save_scan( + target=f"example-{idx}.com", + mode="domain", + settings={}, + results=[], + elapsed=None, + db_path=db_path, + ) + assert count_reports(db_path) == 3 + assert reset_reports(db_path) == 3 + assert count_reports(db_path) == 0 diff --git a/tests/test_version_helpers.py b/tests/test_version_helpers.py new file mode 100644 index 0000000..7b44eb8 --- /dev/null +++ b/tests/test_version_helpers.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import knockpy.version as version + + +class _DummyResponse: + def __init__(self, status_code: int, payload): + self.status_code = status_code + self._payload = payload + + def json(self): + return self._payload + + +def test_version_key_and_compare(): + assert version._version_key("9.0.0") == (9, 0, 0) + assert version._version_key("9.0.0rc1") == (9, 0, 0, 1) + assert version.is_newer_version("9.1.0", "9.0.9") is True + assert version.is_newer_version("9.0.0", "9.0.0") is False + assert version.is_newer_version("9.0", "9.0.0") is False + + +def test_check_latest_version_update_available(monkeypatch): + monkeypatch.setattr( + version.httpx, + "get", + lambda *args, **kwargs: _DummyResponse(200, {"info": {"version": "9.1.0"}}), + ) + info = version.check_latest_version(current="9.0.0") + assert info["ok"] is True + assert info["latest"] == "9.1.0" + assert info["update_available"] is True + + +def test_check_latest_version_no_update(monkeypatch): + monkeypatch.setattr( + version.httpx, + "get", + lambda *args, **kwargs: _DummyResponse(200, {"info": {"version": "9.0.0"}}), + ) + info = version.check_latest_version(current="9.0.0") + assert info["ok"] is True + assert info["update_available"] is False + + +def test_check_latest_version_handles_http_error(monkeypatch): + monkeypatch.setattr( + version.httpx, + "get", + lambda *args, **kwargs: _DummyResponse(503, {"info": {"version": "9.1.0"}}), + ) + info = version.check_latest_version(current="9.0.0") + assert info["ok"] is False + assert "HTTP 503" in str(info["error"]) + + +def test_check_latest_version_handles_exception(monkeypatch): + def _boom(*args, **kwargs): + raise RuntimeError("network down") + + monkeypatch.setattr(version.httpx, "get", _boom) + info = version.check_latest_version(current="9.0.0") + assert info["ok"] is False + assert "RuntimeError" in str(info["error"])