diff --git a/packages/banip/Makefile b/packages/banip/Makefile index 39e6c7ecb..95c27b4ea 100644 --- a/packages/banip/Makefile +++ b/packages/banip/Makefile @@ -1,12 +1,12 @@ # banIP - ban incoming and outgoing IPs via named nftables Sets -# Copyright (c) 2018-2024 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2018-2026 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. include $(TOPDIR)/rules.mk PKG_NAME:=banip -PKG_VERSION:=1.0.1 -PKG_RELEASE:=4 +PKG_VERSION:=1.8.10 +PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0-or-later PKG_MAINTAINER:=Dirk Brenken @@ -16,7 +16,7 @@ define Package/banip SECTION:=net CATEGORY:=Network TITLE:=banIP blocks IPs via named nftables Sets - DEPENDS:=+jshn +jsonfilter +firewall4 +ca-bundle +rpcd +rpcd-mod-rpcsys + DEPENDS:=+jshn +jsonfilter +firewall4 +gawk +ca-bundle +rpcd +rpcd-mod-rpcsys PKGARCH:=all endef @@ -51,7 +51,7 @@ define Package/banip/install $(INSTALL_BIN) ./files/banip.init $(1)/etc/init.d/banip $(INSTALL_DIR) $(1)/usr/lib - $(INSTALL_CONF) ./files/banip-functions.sh $(1)/usr/lib + $(INSTALL_DATA) ./files/banip-functions.sh $(1)/usr/lib $(INSTALL_DIR) $(1)/etc/config $(INSTALL_CONF) ./files/banip.conf $(1)/etc/config/banip @@ -66,6 +66,9 @@ define Package/banip/install $(INSTALL_DIR) $(1)/www/cgi-bin $(INSTALL_BIN) ./files/banip.cgi $(1)/www/cgi-bin/banip + + $(INSTALL_DIR) $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/95-banip-housekeeping $(1)/etc/uci-defaults endef $(eval $(call BuildPackage,banip)) diff --git a/packages/banip/files/95-banip-housekeeping b/packages/banip/files/95-banip-housekeeping new file mode 100644 index 000000000..5dbc8648e --- /dev/null +++ b/packages/banip/files/95-banip-housekeeping @@ -0,0 +1,42 @@ +#!/bin/sh +# Copyright (c) 2015-2026 Dirk Brenken (dev@brenken.org) +# This is free software, licensed under the GNU General Public License v3. + +# (s)hellcheck exceptions +# shellcheck disable=all + +export LC_ALL=C +export PATH="/usr/sbin:/usr/bin:/sbin:/bin" + +config="banip" +old_options="ban_loginput ban_logforwardwan ban_logforwardlan ban_blockinput ban_blockforwardwan ban_blockforwardlan ban_blocktype ban_blockpolicy" + +for option in ${old_options}; do + old_values="$(uci -q get "${config}.global.${option}" 2>/dev/null)" + for value in ${old_values}; do + case "${option}" in + "ban_loginput" | "ban_logforwardwan") + uci -q set "${config}".global.ban_loginbound="${value}" + ;; + "ban_logforwardlan") + uci -q set "${config}".global.ban_logoutbound="${value}" + ;; + "ban_blockpolicy") + if printf "%s" "${old_values}" | grep -qw "input\|forwardwan\|forwardlan"; then + break + else + continue 2 + fi + ;; + esac + done + uci -q delete "${config}.global.${option}" +done +[ -n "$(uci -q changes "${config}")" ] && uci -q commit "${config}" + +custom_feed="/etc/banip/banip.custom.feeds" +if grep -q '"rule_4"' "${custom_feed}"; then + mv -f "${custom_feed}" "${custom_feed}.backup.$(date "+%Y%m%d%H%M%S")" + : > "${custom_feed}" +fi +exit 0 diff --git a/packages/banip/files/README.md b/packages/banip/files/README.md index 5b5820ec1..8109069b0 100644 --- a/packages/banip/files/README.md +++ b/packages/banip/files/README.md @@ -2,69 +2,97 @@ # banIP - ban incoming and outgoing IP addresses/subnets via Sets in nftables +## Table of Contents +* [Description](#description) +* [Quick Start](#quick-start) +* [Main Features](#main-features) +* [Prerequisites](#prerequisites) +* [Installation and Usage](#installation-and-usage) +* [banIP CLI interface](#banip-cli-interface) +* [banIP config options](#banip-config-options) +* [Examples](#examples) +* [Best practise and tweaks](#best-practise-and-tweaks) +* [Troubleshooting & debug options](#troubleshooting-and-debug-options) +* [Support](#support) +* [Removal](#removal) +* [Donations](#donations) + + ## Description -IP address blocking is commonly used to protect against brute force attacks, prevent disruptive or unauthorized address(es) from access or it can be used to restrict access to or from a particular geographic area — for example. Further more banIP scans the log file via logread and bans IPs that make too many password failures, e.g. via ssh. +IP address blocking is commonly used to protect against brute force attacks, prevent disruptive or unauthorized address(es) from access or it can be used to restrict access to or from a particular geographic area — for example. Further more banIP scans the log file via logread and bans IPs that make too many password failures, e.g. via ssh. + + +## Quick Start +For a typical setup these few steps are enough to get banIP up and running — see the sections below for details: +1. Install the LuCI companion package: `apk update && apk add luci-app-banip` (this pulls in the `banip` backend as a dependency). +2. Open LuCI under `Services → banIP`, tick `Enabled` and (recommended) set a `Startup Trigger Interface` to your WAN interface (avoid IPv6/wan6). +3. Activate a small, sensible feed selection to start with, e.g. `cinsscore`, `debl`, `turris` and `doh` in their default chains (≈20K IPs). +4. Start and verify the service: + +```sh +/etc/init.d/banip start +/etc/init.d/banip status +``` + +**Please note:** don't blindly enable (too) many feeds at once — on low memory devices this will sooner or later lead to OOM conditions. + ## Main Features -* banIP supports the following fully pre-configured domain blocklist feeds (free for private usage, for commercial use please check their individual licenses). -**Please note:** By default every feed blocks packet traversal in all supported chains, the table columns "WAN-INP", "WAN-FWD" and "LAN-FWD" show for which chains the feeds are suitable in common scenarios: - * WAN-INP chain applies to packets from internet to your router - * WAN-FWD chain applies to packets from internet to other local devices (not your router) - * LAN-FWD chain applies to local packets going out to the internet (not your router) - For instance the first entry should be limited to the LAN forward chain - just set the 'LAN-Forward Chain' option under the 'Feed/Set Seetings' config tab accordingly. - -| Feed | Focus | WAN-INP | WAN-FWD | LAN-FWD | Port-Limit | Information | -| :------------------ | :----------------------------- | :-----: | :-----: | :-----: | :----------: | :----------------------------------------------------------- | -| adaway | adaway IPs | | | x | tcp: 80, 443 | [Link](https://github.com/dibdot/banIP-IP-blocklists) | -| adguard | adguard IPs | | | x | tcp: 80, 443 | [Link](https://github.com/dibdot/banIP-IP-blocklists) | -| adguardtrackers | adguardtracker IPs | | | x | tcp: 80, 443 | [Link](https://github.com/dibdot/banIP-IP-blocklists) | -| antipopads | antipopads IPs | | | x | tcp: 80, 443 | [Link](https://github.com/dibdot/banIP-IP-blocklists) | -| asn | ASN segments | x | x | x | | [Link](https://asn.ipinfo.app) | -| backscatterer | backscatterer IPs | x | x | | | [Link](https://www.uceprotect.net/en/index.php) | -| becyber | malicious attacker IPs | x | x | | | [Link](https://github.com/duggytuxy/malicious_ip_addresses) | -| binarydefense | binary defense banlist | x | x | | | [Link](https://iplists.firehol.org/?ipset=bds_atif) | -| bogon | bogon prefixes | x | x | x | | [Link](https://team-cymru.com) | -| bruteforceblock | bruteforceblocker IPs | x | x | | | [Link](https://danger.rulez.sk/index.php/bruteforceblocker/) | -| country | country blocks | x | x | | | [Link](https://www.ipdeny.com/ipblocks) | -| cinsscore | suspicious attacker IPs | x | x | | | [Link](https://cinsscore.com/#list) | -| debl | fail2ban IP blacklist | x | x | | | [Link](https://www.blocklist.de) | -| doh | public DoH-Provider | | | x | tcp: 80, 443 | [Link](https://github.com/dibdot/DoH-IP-blocklists) | -| drop | spamhaus drop compilation | x | x | | | [Link](https://www.spamhaus.org) | -| dshield | dshield IP blocklist | x | x | | | [Link](https://www.dshield.org) | -| etcompromised | ET compromised hosts | x | x | | | [Link](https://iplists.firehol.org/?ipset=et_compromised) | -| feodo | feodo tracker | x | x | | | [Link](https://feodotracker.abuse.ch) | -| firehol1 | firehol level 1 compilation | x | x | | | [Link](https://iplists.firehol.org/?ipset=firehol_level1) | -| firehol2 | firehol level 2 compilation | x | x | | | [Link](https://iplists.firehol.org/?ipset=firehol_level2) | -| firehol3 | firehol level 3 compilation | x | x | | | [Link](https://iplists.firehol.org/?ipset=firehol_level3) | -| firehol4 | firehol level 4 compilation | x | x | | | [Link](https://iplists.firehol.org/?ipset=firehol_level4) | -| greensnow | suspicious server IPs | x | x | | | [Link](https://greensnow.co) | -| hagezi | Threat IP blocklist | | | x | tcp: 80, 443 | [Link](https://github.com/hagezi/dns-blocklists) | -| ipblackhole | blackhole IPs | x | x | | | [Link](https://github.com/BlackHoleMonster/IP-BlackHole) | -| ipsum | malicious IPs | x | x | | | [Link](https://github.com/stamparm/ipsum) | -| ipthreat | hacker and botnet TPs | x | x | | | [Link](https://ipthreat.net) | -| myip | real-time IP blocklist | x | x | | | [Link](https://myip.ms) | -| oisdbig | OISD-big IPs | | | x | tcp: 80, 443 | [Link](https://github.com/dibdot/banIP-IP-blocklists) | -| oisdnsfw | OISD-nsfw IPs | | | x | tcp: 80, 443 | [Link](https://github.com/dibdot/banIP-IP-blocklists) | -| oisdsmall | OISD-small IPs | | | x | tcp: 80, 443 | [Link](https://github.com/dibdot/banIP-IP-blocklists) | -| pallebone | curated IP blocklist | x | x | | | [Link](https://github.com/pallebone/StrictBlockPAllebone) | -| proxy | open proxies | x | x | | | [Link](https://iplists.firehol.org/?ipset=proxylists) | -| ssbl | SSL botnet IPs | x | x | | | [Link](https://sslbl.abuse.ch) | -| stevenblack | stevenblack IPs | | | x | tcp: 80, 443 | [Link](https://github.com/dibdot/banIP-IP-blocklists) | -| threat | emerging threats | x | x | | | [Link](https://rules.emergingthreats.net) | -| threatview | malicious IPs | x | x | | | [Link](https://threatview.io) | -| tor | tor exit nodes | x | x | x | | [Link](https://www.dan.me.uk) | -| turris | turris sentinel blocklist | x | x | | | [Link](https://view.sentinel.turris.cz) | -| uceprotect1 | spam protection level 1 | x | x | | | [Link](https://www.uceprotect.net/en/index.php) | -| uceprotect2 | spam protection level 2 | x | x | | | [Link](https://www.uceprotect.net/en/index.php) | -| uceprotect3 | spam protection level 3 | x | x | | | [Link](https://www.uceprotect.net/en/index.php) | -| urlhaus | urlhaus IDS IPs | x | x | | | [Link](https://urlhaus.abuse.ch) | -| urlvir | malware related IPs | x | x | | | [Link](https://iplists.firehol.org/?ipset=urlvir) | -| webclient | malware related IPs | x | x | | | [Link](https://iplists.firehol.org/?ipset=firehol_webclient) | -| voip | VoIP fraud blocklist | x | x | | | [Link](https://voipbl.org) | -| yoyo | yoyo IPs | | | x | tcp: 80, 443 | [Link](https://github.com/dibdot/banIP-IP-blocklists) | +* banIP supports the following fully pre-configured IP blocklist feeds (free for private usage, for commercial use please check their individual licenses). +**Please note:** By default, each feed blocks the packet flow in the chain(s) shown in the table below. _Inbound_ combines the chains WAN-Input and WAN-Forward, _Outbound_ represents the LAN-FWD chain: + * WAN-INP chain applies to packets from internet to your router + * WAN-FWD chain applies to packets from internet to other local devices (not your router) + * LAN-FWD chain applies to local packets going out to the internet (not your router) + + How to read the default direction: most reputation feeds list the **source** IPs of attackers, scanners and spammers — those are blocked _inbound_, because the unwanted connection is initiated **from the internet towards you**. A second group of feeds lists IPs that your **own clients should never talk to** (malware hosting, command-and-control servers, threat/DoH/DNS endpoints); those are blocked _outbound_ in the LAN-FWD chain, because the connection is initiated **from your LAN**. Feeds where both risks apply (e.g. spamhaus, emerging threats, tor, proxy, vpn) default to **both** directions. This distinction matters: a feed of malware-download or C2 IPs in the _inbound_ chain alone provides little protection, because your LAN client is the one initiating the connection — see the `country`/`asn` note in the [Best practise](#best-practise-and-tweaks) section for the common "block a country" case. + + The listed standard assignments can be changed to your needs under the `Feed/Set Settings` config tab (options `ban_feedin`, `ban_feedout`, `ban_feedinout` and `ban_feedreset`). + +| Feed | Focus | Inbound | Outbound | Proto/Port | Information | +| :------------------ | :----------------------------- | :-----: | :------: | :---------------: | :----------------------------------------------------------- | +| asn | ASN segments | x | | | [Link](https://asn.ipinfo.app) | +| backscatterer | backscatterer IPs | x | | | [Link](https://www.uceprotect.net/en/index.php) | +| becyber | malicious attacker IPs | x | | | [Link](https://github.com/duggytuxy/malicious_ip_addresses) | +| binarydefense | binary defense banlist | x | | | [Link](https://iplists.firehol.org/?ipset=bds_atif) | +| bogon | bogon prefixes | x | | | [Link](https://team-cymru.com) | +| bruteforceblock | bruteforceblocker IPs | x | | | [Link](https://danger.rulez.sk/index.php/bruteforceblocker/) | +| country | country blocks | x | | | [Link](https://www.ipdeny.com/ipblocks) | +| cinsscore | suspicious attacker IPs | x | | | [Link](https://cinsscore.com/#list) | +| debl | fail2ban IP blacklist | x | | | [Link](https://www.blocklist.de) | +| dns | public DNS-Server | | x | tcp, udp: 53, 853 | [Link](https://public-dns.info) | +| doh | public DoH-Server | | x | tcp, udp: 80, 443 | [Link](https://github.com/dibdot/DoH-IP-blocklists) | +| dshield | dshield IP blocklist | x | | | [Link](https://www.dshield.org) | +| etcompromised | ET compromised hosts | x | | | [Link](https://iplists.firehol.org/?ipset=et_compromised) | +| feodo | feodo tracker | x | x | | [Link](https://feodotracker.abuse.ch) | +| firehol1 | firehol level 1 compilation | x | | | [Link](https://iplists.firehol.org/?ipset=firehol_level1) | +| firehol2 | firehol level 2 compilation | x | | | [Link](https://iplists.firehol.org/?ipset=firehol_level2) | +| firehol3 | firehol level 3 compilation | x | | | [Link](https://iplists.firehol.org/?ipset=firehol_level3) | +| firehol4 | firehol level 4 compilation | x | | | [Link](https://iplists.firehol.org/?ipset=firehol_level4) | +| greensnow | suspicious server IPs | x | | | [Link](https://greensnow.co) | +| hagezi | Threat IP blocklist | | x | tcp, udp: 80, 443 | [Link](https://github.com/hagezi/dns-blocklists) | +| ipblackhole | blackhole IPs | x | | | [Link](https://github.com/BlackHoleMonster/IP-BlackHole) | +| ipexdbl | IPEX dynamic blocklists | x | | | [Link](https://github.com/ZEROF/ipextractor) | +| ipsum | malicious IPs | x | | | [Link](https://github.com/stamparm/ipsum) | +| ipthreat | hacker and botnet IPs | x | | | [Link](https://ipthreat.net) | +| myip | real-time IP blocklist | x | | | [Link](https://myip.ms) | +| proxy | open proxies | x | x | | [Link](https://iplists.firehol.org/?ipset=proxylists) | +| spamhaus | Spamhaus DROP | x | x | | [Link](https://www.spamhaus.org/blocklists/) | +| threat | emerging threats | x | x | | [Link](https://rules.emergingthreats.net) | +| threatview | malicious IPs | x | x | | [Link](https://threatview.io) | +| tor | tor exit nodes | x | x | | [Link](https://www.dan.me.uk) | +| turris | turris sentinel blocklist | x | | | [Link](https://view.sentinel.turris.cz) | +| uceprotect1 | spam protection level 1 | x | | | [Link](https://www.uceprotect.net/en/index.php) | +| uceprotect2 | spam protection level 2 | x | | | [Link](https://www.uceprotect.net/en/index.php) | +| uceprotect3 | spam protection level 3 | x | | | [Link](https://www.uceprotect.net/en/index.php) | +| urlhaus | urlhaus IDS IPs | | x | tcp, udp: 80, 443 | [Link](https://urlhaus.abuse.ch) | +| urlvir | malware related IPs | | x | tcp, udp: 80, 443 | [Link](https://iplists.firehol.org/?ipset=urlvir) | +| webclient | malware related IPs | | x | tcp, udp: 80, 443 | [Link](https://iplists.firehol.org/?ipset=firehol_webclient) | +| voip | VoIP fraud blocklist | x | | | [Link](https://voipbl.org) | +| vpn | vpn IPs | x | x | | [Link](https://github.com/X4BNet/lists_vpn) | +| vpndc | vpn datacenter IPs | x | x | | [Link](https://github.com/X4BNet/lists_vpn) | * Zero-conf like automatic installation & setup, usually no manual changes needed -* All Sets are handled in a separate nft table/namespace 'banIP' +* All Sets are handled in a separate nft table/namespace `banIP` * Full IPv4 and IPv6 support * Supports nft atomic Set loading * Supports blocking by ASN numbers and by iso country codes @@ -74,52 +102,60 @@ IP address blocking is commonly used to protect against brute force attacks, pre * All local input types support ranges in CIDR notation * Auto-add the uplink subnet or uplink IP to the local allowlist * Prevent common ICMP, UDP and SYN flood attacks and drop spoofed tcp flags & invalid conntrack packets (DoS attacks) in an additional prerouting chain -* Provides a small background log monitor to ban unsuccessful login attempts in real-time (like fail2ban, crowdsec etc.) -* Auto-add unsuccessful LuCI, nginx, Asterisk or ssh login attempts to the local blocklist +* Provides a background log monitor to ban unsuccessful login attempts in real-time (like fail2ban, crowdsec etc.) with three-tier IP deduplication, dynamic cache management and optional RDAP-based subnet blocking +* Auto-add unsuccessful LuCI, Asterisk or ssh login attempts to the local blocklist * Auto-add entire subnets to the blocklist Set based on an additional RDAP request with the monitored suspicious IP * Fast feed processing as they are handled in parallel as background jobs (on capable multi-core hardware) -* Per feed it can be defined whether the wan-input chain, the wan-forward chain or the lan-forward chain should be blocked (default: all chains) +* Per feed it can be defined whether the inbound chain (wan-input, wan-forward) or the outbound chain (lan-forward) should be blocked * Automatic blocklist backup & restore, the backups will be used in case of download errors or during startup -* Automatically selects one of the following download utilities with ssl support: aria2c, curl, uclient-fetch or full wget -* Provides HTTP ETag support to download only ressources that have been updated on the server side, to speed up banIP reloads and to save bandwith -* Supports an 'allowlist only' mode, this option skips all blocklists and restricts the internet access only to specific, explicitly allowed IP segments +* Automatically selects one of the following download utilities with ssl support: curl, uclient-fetch or full wget +* Provides HTTP ETag support to download only resources that have been updated on the server side, to speed up banIP reloads and to save bandwidth +* Supports an `allowlist only` mode, this option restricts the internet access only to specific, explicitly allowed IP segments * Supports external allowlist URLs to reference additional IPv4/IPv6 feeds -* Optionally always allow certain protocols/destination ports in wan-input and wan-forward chains -* Deduplicate IPs accross all Sets (single IPs only, no intervals) +* Optionally always allow certain protocols/destination ports in the inbound chain +* Deduplicate IPs across all Sets (single IPs only, no intervals) +* Implements BCP38 ingress filtering to prevent IP address spoofing * Provides comprehensive runtime information -* Provides a detailed Set report +* Provides a detailed Set report, incl. a map that shows the geolocation of your own uplink addresses (in green) and the location of potential attackers (in red) * Provides a Set search engine for certain IPs * Feed parsing by fast & flexible regex rulesets * Minimal status & error logging to syslog, enable debug logging to receive more output -* Procd based init system support (start/stop/restart/reload/status/report/search/survey/lookup) +* Procd based init system support (start/stop/restart/reload/status/report/search/content) * Procd network interface trigger support * Add new or edit existing banIP feeds on your own with the LuCI integrated custom feed editor -* Supports destination port & protocol limitations for external feeds (see the feed list above). To change the default assignments just use the feed editor +* Supports destination port & protocol limitations for external feeds (see the feed list above). To change the default assignments just use the custom feed editor * Supports allowing / blocking of certain VLAN forwards * Provides an option to transfer logging events on remote servers via cgi interface + ## Prerequisites -* **[OpenWrt](https://openwrt.org)**, latest stable release or a snapshot with nft/firewall 4 support -* A download utility with SSL support: 'aria2c', 'curl', full 'wget' or 'uclient-fetch' with one of the 'libustream-*' SSL libraries, the latter one doesn't provide support for ETag HTTP header -* A certificate store like 'ca-bundle', as banIP checks the validity of the SSL certificates of all download sites by default -* For E-Mail notifications you need to install and setup the additional 'msmtp' package - -**Please note:** -* Devices with less than 256Mb of RAM are **_not_** supported -* Any previous installation of ancient banIP 0.7.x must be uninstalled, and the /etc/banip folder and the /etc/config/banip configuration file must be deleted (they are recreated when this version is installed) - -## Installation & Usage -* Update your local apk repository (_apk update_) -* Install banIP (_apk add banip_) - the banIP service is disabled by default -* Install the LuCI companion package 'luci-app-banip' (apk add luci-app-banip) -* It's strongly recommended to use the LuCI frontend to easily configure all aspects of banIP, the application is located in LuCI under the 'Services' menu +* **[OpenWrt](https://openwrt.org)**, latest stable release or a development snapshot with nft/firewall 4 support +* A download utility with SSL support: `curl`, full `wget` or `uclient-fetch` with one of the `libustream-*` SSL libraries, the latter one doesn't provide support for ETag HTTP header +* A certificate store like `ca-bundle`, as banIP checks the validity of the SSL certificates of all download sites by default +* For E-Mail notifications you need to install and setup the additional `msmtp` package + +**Please note:** +* Devices with less than 256MB of RAM are **_not_** supported +* After system upgrades it's recommended to start with a fresh banIP default config +* Only `reload` actually refreshes the feeds (ETag check plus download of changed feeds). `start`, `restart` — and `boot`/`resume` — restore the existing blocklist backups and only download feeds that have **no** backup yet; they do **not** re-fetch already cached feeds. To update your blocklists (e.g. from a cron job) always use `reload`. + + + +## Installation and Usage +* Update your router's apk repository (apk update) +* Install the LuCI companion package `luci-app-banip` which also installs the main banIP package as a dependency +* Enable the banIP system service (System -> Startup) and enable banIP itself (banIP -> General Settings) +* It's strongly recommended to use the LuCI frontend to easily configure all aspects of banIP, the application is located in LuCI under the `Services` menu +* It's also recommended to configure a `Startup Trigger Interface` to depend on your WAN ifup events during boot or restart of your router. Avoid IPv6 (wan6) interfaces here, as IPv6/netifd is chatty and would trigger frequent unnecessary banIP restarts * To be able to use banIP in a meaningful way, you must activate the service and possibly also activate a few blocklist feeds -* If you're using a complex network setup, e.g. special tunnel interfaces, than untick the 'Auto Detection' option under the 'General Settings' tab and set the required options manually -* Start the service with '/etc/init.d/banip start' and check everything is working by running '/etc/init.d/banip status' and also check the 'Firewall Log' and 'Processing Log' tabs +* If you're using a complex network setup, e.g. special tunnel interfaces, then untick the `Auto Detection` option under the `General Settings` tab and set the required options manually +* Start the service with `/etc/init.d/banip start` and check everything is working by running `/etc/init.d/banip status`, also check the `Processing Log` tab + ## banIP CLI interface -* All important banIP functions are accessible via CLI, too. If you're going to configure banIP via CLI, edit the config file '/etc/config/banip' and enable the service, add pre-configured feeds and add/change other options to your needs, see the options reference table below. -``` +* All important banIP functions are accessible via CLI, too. If you're going to configure banIP via CLI, edit the config file `/etc/config/banip` and enable the service, add pre-configured feeds and add/change other options to your needs, see the options reference table below. + +```sh ~# /etc/init.d/banip Syntax: /etc/init.d/banip [command] @@ -131,16 +167,18 @@ Available commands: enable Enable service autostart disable Disable service autostart enabled Check if service is started on boot - report [text|json|mail] Print banIP related Set statistics + report [text|json|mail|gen] Print banIP related Set statistics search [|] Check if an element exists in a banIP Set - survey [] List all elements of a given banIP Set - lookup Lookup the IPs of domain names in the local lists and update them + content [] [true|false] Listing of all or only elements with hits of a given banIP Set running Check if service is running status Service status trace Start with syscall trace info Dump procd service info ``` +The `report` sub-command accepts an output mode: `text` (default, human-readable table), `json` (machine-readable output, incl. GeoIP map data when `ban_map=1`), `mail` (send the report via `msmtp`) and `gen` (regenerate the report data files in the background, used by the LuCI frontend). + + ## banIP config options | Option | Type | Default | Description | @@ -148,205 +186,282 @@ Available commands: | ban_enabled | option | 0 | enable the banIP service | | ban_nicelimit | option | 0 | ulimit nice level of the banIP service (range 0-19) | | ban_filelimit | option | 1024 | ulimit max open/number of files (range 1024-4096) | -| ban_loglimit | option | 100 | scan only the last n log entries permanently. A value of '0' disables the monitor | -| ban_logcount | option | 1 | how many times the IP must appear in the log to be considered as suspicious | -| ban_logterm | list | regex | various regex for logfile parsing (default: dropbear, sshd, luci, nginx, asterisk and cgi-remote events) | -| ban_logreadfile | option | /var/log/messages | alternative location for parsing a log file via tail, to deactivate the standard parsing via logread | +| ban_loglimit | option | 100 | scan only the last n log entries permanently. A value of `0` disables the monitor | +| ban_logcount | option | 1 | how many times the IP must appear in the log per blocking cycle to trigger auto-blocking | +| ban_logterm | list | regex | various regex for logfile parsing (default: dropbear, sshd, luci, asterisk and cgi-remote events) | +| ban_logreadfile | option | - / logread | parse this log file via tail instead of the default logread; if left empty (default) banIP reads the system log via logread | | ban_autodetect | option | 1 | auto-detect wan interfaces, devices and subnets | | ban_debug | option | 0 | enable banIP related debug logging | -| ban_icmplimit | option | 10 | threshold in number of packets to detect icmp DoS in prerouting chain. A value of '0' disables this safeguard | -| ban_synlimit | option | 10 | threshold in number of packets to detect syn DoS in prerouting chain. A value of '0' disables this safeguard | -| ban_udplimit | option | 100 | threshold in number of packets to detect udp DoS in prerouting chain. A value of '0' disables this safeguard | -| ban_logprerouting | option | 0 | log supsicious packets in the prerouting chain | -| ban_loginput | option | 0 | log supsicious packets in the wan-input chain | -| ban_logforwardwan | option | 0 | log supsicious packets in the wan-forward chain | -| ban_logforwardlan | option | 0 | log supsicious packets in the lan-forward chain | +| ban_icmplimit | option | 25 | threshold in number of packets to detect icmp DoS in prerouting chain. A value of `0` disables this safeguard | +| ban_synlimit | option | 10 | threshold in number of packets to detect syn DoS in prerouting chain. A value of `0` disables this safeguard | +| ban_udplimit | option | 100 | threshold in number of packets to detect udp DoS in prerouting chain. A value of `0` disables this safeguard | +| ban_logprerouting | option | 0 | log suspicious packets in the prerouting chain | +| ban_loginbound | option | 0 | log suspicious packets in the inbound chain (wan-input and wan-forward) | +| ban_logoutbound | option | 0 | log suspicious packets in the outbound chain (lan-forward) | +| ban_logratelimit | option | 10 | rate (per second) for the shared nft log limit, applied globally across all logged rules | +| ban_logburstlimit | option | 5 | burst size in packets for the shared nft log limit | | ban_autoallowlist | option | 1 | add wan IPs/subnets and resolved domains automatically to the local allowlist (not only to the Sets) | | ban_autoblocklist | option | 1 | add suspicious attacker IPs and resolved domains automatically to the local blocklist (not only to the Sets) | -| ban_autoblocksubnet | option | 0 | add entire subnets to the blocklist Sets based on an additional RDAP request with the suspicious IP | -| ban_autoallowuplink | option | subnet | limit the uplink autoallow function to: 'subnet', 'ip' or 'disable' it at all | -| ban_allowlistonly | option | 0 | skip all blocklists and restrict the internet access only to specific, explicitly allowed IP segments | -| ban_allowflag | option | - | always allow certain protocols(tcp or udp) plus destination ports or port ranges, e.g.: 'tcp 80 443-445' | +| ban_autoblocksubnet | option | 0 | add entire subnets to the blocklist Sets based on a rate-limited, non-blocking RDAP lookup for the suspicious IP | +| ban_autoallowuplink | option | subnet | limit the uplink autoallow function to: `subnet`, `ip` or `disable` it at all | +| ban_allowlistonly | option | 0 | restrict the internet access only to specific, explicitly allowed IP segments | +| ban_allowflag | option | - | always allow certain protocols(tcp or udp) plus destination ports or port ranges, e.g.: `tcp 80 443-444` | | ban_allowurl | list | - | external allowlist feed URLs, one or more references to simple remote IP lists | | ban_basedir | option | /tmp | base working directory while banIP processing | -| ban_reportdir | option | /tmp/banIP-report | directory where banIP stores the report files | -| ban_backupdir | option | /tmp/banIP-backup | directory where banIP stores the compressed backup files | +| ban_reportdir | option | /tmp/banIP-report | directory where banIP stores report files | +| ban_backupdir | option | /tmp/banIP-backup | directory where banIP stores compressed backup files | +| ban_errordir | option | /tmp/banIP-error | directory where banIP stores processing error files | | ban_protov4 | option | - / autodetect | enable IPv4 support | | ban_protov6 | option | - / autodetect | enable IPv6 support | -| ban_ifv4 | list | - / autodetect | logical wan IPv4 interfaces, e.g. 'wan' | -| ban_ifv6 | list | - / autodetect | logical wan IPv6 interfaces, e.g. 'wan6' | -| ban_dev | list | - / autodetect | wan device(s), e.g. 'eth2' | +| ban_ifv4 | list | - / autodetect | logical wan IPv4 interfaces, e.g. `wan` | +| ban_ifv6 | list | - / autodetect | logical wan IPv6 interfaces, e.g. `wan6` | +| ban_dev | list | - / autodetect | wan device(s), e.g. `eth2` | | ban_vlanallow | list | - | always allow certain VLAN forwards, e.g. br-lan.20 | | ban_vlanblock | list | - | always block certain VLAN forwards, e.g. br-lan.10 | -| ban_trigger | list | - | logical reload trigger interface(s), e.g. 'wan' | +| ban_trigger | list | - | logical reload trigger interface(s), e.g. `wan` (avoid IPv6 interfaces) | | ban_triggerdelay | option | 20 | trigger timeout during interface reload and boot | -| ban_deduplicate | option | 1 | deduplicate IP addresses across all active Sets | +| ban_deduplicate | option | 1 | deduplicate IP addresses across all active Sets (see optional feed flag `dup` below) | | ban_splitsize | option | 0 | split the processing/loading of Sets in chunks of n lines/members (saves RAM) | | ban_cores | option | - / autodetect | limit the cpu cores used by banIP (saves RAM) | | ban_nftloglevel | option | warn | nft loglevel, values: emerg, alert, crit, err, warn, notice, info, debug | | ban_nftpriority | option | -100 | nft priority for the banIP table (the prerouting table is fixed to priority -150) | | ban_nftpolicy | option | memory | nft policy for banIP-related Sets, values: memory, performance | -| ban_nftexpiry | option | - | expiry time for auto added blocklist members, e.g. '5m', '2h' or '1d' | -| ban_feed | list | - | external download feeds, e.g. 'yoyo', 'doh', 'country' or 'talos' (see feed table) | -| ban_asn | list | - | ASNs for the 'asn' feed, e.g.'32934' | +| ban_nftexpiry | option | - | expiry time (ms|s|m|h|d|w) for auto added blocklist members (also controls the monitor cache refresh interval) | +| ban_nftretry | option | 3 | number of Set load attempts in case of an error | +| ban_nftcount | option | 0 | enable nft counter for every Set element | +| ban_bcp38 | option | 0 | block packets with spoofed source IP addresses in all supported chains | +| ban_map | option | 0 | enable a GeoIP Map with suspicious Set elements | +| ban_feed | list | - | external download feeds, e.g. `yoyo`, `doh`, `country` or `talos` (see feed table) | +| ban_asn | list | - | ASNs for the `asn` feed, e.g.`32934` | +| ban_asnsplit | option | - | the selected ASNs are stored in separate Sets | | ban_region | list | - | Regional Internet Registry (RIR) country selection. Supported regions are: AFRINIC, ARIN, APNIC, LACNIC and RIPE | -| ban_country | list | - | country iso codes for the 'country' feed, e.g. 'ru' | -| ban_blockpolicy | option | - | limit the default block policy to a certain chain, e.g. 'input', 'forwardwan' or 'forwardlan' | -| ban_blocktype | option | drop | 'drop' packets silently on input and forwardwan chains or actively 'reject' the traffic | -| ban_blockinput | list | - | limit a feed to the wan-input chain, e.g. 'country' | -| ban_blockforwardwan | list | - | limit a feed to the wan-forward chain, e.g. 'debl' | -| ban_blockforwardlan | list | - | limit a feed to the lan-forward chain, e.g. 'doh' | -| ban_fetchcmd | option | - / autodetect | 'uclient-fetch', 'wget', 'curl' or 'aria2c' | +| ban_country | list | - | country iso codes for the `country` feed, e.g. `ru` | +| ban_countrysplit | option | - | the selected countries are stored in separate Sets | +| ban_blockpolicy | option | drop | `drop` packets silently on input and forwardwan chains or actively `reject` the traffic | +| ban_feedin | list | - | limit the selected feeds to the inbound chain (wan-input and wan-forward) | +| ban_feedout | list | - | limit the selected feeds to the outbound chain (lan-forward) | +| ban_feedinout | list | - | set the selected feeds to the inbound and outbound chain (lan-forward) | +| ban_feedreset | list | - | override the default feed configuration and remove existing port/protocol limitations | +| ban_feedcomplete | list | - | opt out the selected feeds from the deduplication process | +| ban_fetchcmd | option | - / autodetect | `uclient-fetch`, `wget` or `curl` | | ban_fetchparm | option | - / autodetect | set the config options for the selected download utility | | ban_fetchretry | option | 5 | number of download attempts in case of an error (not supported by uclient-fetch) | | ban_fetchinsecure | option | 0 | don't check SSL server certificates during download | | ban_mailreceiver | option | - | receiver address for banIP related notification E-Mails | | ban_mailsender | option | no-reply@banIP | sender address for banIP related notification E-Mails | | ban_mailtopic | option | banIP notification | topic for banIP related notification E-Mails | -| ban_mailprofile | option | ban_notify | mail profile used in 'msmtp' for banIP related notification E-Mails | +| ban_mailprofile | option | ban_notify | mail profile used in `msmtp` for banIP related notification E-Mails | | ban_mailnotification | option | 0 | receive E-Mail notifications with every banIP run | -| ban_reportelements | option | 1 | count Set elements in the report, disable this option to speed up the report significantly | | ban_resolver | option | - | external resolver used for DNS lookups, by default the local resolver/forwarder will be used | | ban_remotelog | option | 0 | enable the cgi interface to receive remote logging events | | ban_remotetoken | option | - | unique token to communicate with the cgi interface | + ## Examples -**banIP report information** +**banIP report information** + ``` ~# /etc/init.d/banip report ::: ::: banIP Set Statistics ::: - Timestamp: 2024-04-17 23:02:15 + Timestamp: 2026-01-12 19:33:11 ------------------------------ - blocked syn-flood packets : 5 - blocked udp-flood packets : 11 - blocked icmp-flood packets : 6 - blocked invalid ct packets : 277 + blocked syn-flood packets : 0 + blocked udp-flood packets : 10 + blocked icmp-flood packets : 11480 + blocked invalid ct packets : 1653 blocked invalid tcp packets: 0 + blocked bcp38 packets : 0 --- auto-added IPs to allowlist: 0 auto-added IPs to blocklist: 0 - Set | Elements | WAN-Input (packets) | WAN-Forward (packets) | LAN-Forward (packets) | Port/Protocol Limit + Set | Count | Inbound (packets) | Outbound (packets) | Port/Protocol | Elements (max. 50) ---------------------+--------------+-----------------------+-----------------------+-----------------------+------------------------ - allowlistv4MAC | 0 | - | - | ON: 0 | - - allowlistv6MAC | 0 | - | - | ON: 0 | - - allowlistv4 | 1 | ON: 0 | ON: 0 | ON: 0 | - - allowlistv6 | 2 | ON: 0 | ON: 0 | ON: 0 | - - adguardtrackersv6 | 105 | - | - | ON: 0 | tcp: 80, 443 - adguardtrackersv4 | 816 | - | - | ON: 0 | tcp: 80, 443 - becyberv4 | 229006 | ON: 2254 | ON: 0 | - | - - cinsscorev4 | 7135 | ON: 1630 | ON: 2 | - | - - deblv4 | 10191 | ON: 23 | ON: 0 | - | - - countryv6 | 38233 | ON: 7 | ON: 0 | - | - - countryv4 | 37169 | ON: 2323 | ON: 0 | - | - - deblv6 | 65 | ON: 0 | ON: 0 | - | - - dropv6 | 66 | ON: 0 | ON: 0 | - | - - dohv4 | 1219 | - | - | ON: 0 | tcp: 80, 443 - dropv4 | 895 | ON: 75 | ON: 0 | - | - - dohv6 | 832 | - | - | ON: 0 | tcp: 80, 443 - threatv4 | 20 | ON: 0 | ON: 0 | - | - - firehol1v4 | 753 | ON: 1 | ON: 0 | - | - - ipthreatv4 | 1369 | ON: 20 | ON: 0 | - | - - firehol2v4 | 2216 | ON: 1 | ON: 0 | - | - - turrisv4 | 5613 | ON: 179 | ON: 0 | - | - - blocklistv4MAC | 0 | - | - | ON: 0 | - - blocklistv6MAC | 0 | - | - | ON: 0 | - - blocklistv4 | 0 | ON: 0 | ON: 0 | ON: 0 | - - blocklistv6 | 0 | ON: 0 | ON: 0 | ON: 0 | - + allowlist.v4 | 1 | ON: 0 | ON: 0 | - | + allowlist.v4MAC | 0 | - | ON: 0 | - | + allowlist.v6 | 1 | ON: 0 | ON: 0 | - | + allowlist.v6MAC | 0 | - | ON: 0 | - | + blocklist.v4 | 7 | ON: 358 | ON: 812 | - | 5.187.35.0, 20.160.0.0, + | | | | | 45.135.232.0, 91.202.233 + | | | | | .0 + blocklist.v4MAC | 0 | - | ON: 0 | - | + blocklist.v6 | 0 | ON: 4 | ON: 0 | - | + blocklist.v6MAC | 0 | - | ON: 0 | - | + dns.v4 | 95493 | - | ON: 2039 | tcp, udp: 53, 853 | 8.8.8.8 + dns.v6 | 251 | - | ON: 0 | tcp, udp: 53, 853 | + doh.v4 | 1663 | - | ON: 0 | tcp, udp: 80, 443 | + doh.v6 | 1204 | - | ON: 0 | tcp, udp: 80, 443 | + hagezi.v4 | 39535 | - | ON: 0 | tcp, udp: 80, 443 | ---------------------+--------------+-----------------------+-----------------------+-----------------------+------------------------ - 25 | 335706 | 17 (6513) | 17 (2) | 12 (0) + 13 | 138155 | 4 (362) | 13 (2851) | 10 | 5 ``` **banIP runtime information** -``` + +```sh +~# /etc/init.d/banip status ::: banIP runtime information + status : active (nft: ✔, monitor: ✔) - + version : 0.9.6-r1 - + element_count : 108036 - + active_feeds : allowlistv4MAC, allowlistv6MAC, allowlistv4, allowlistv6, cinsscorev4, deblv4, countryv6, countryv4, deblv6, dohv4, dohv6, turrisv4, blocklistv4MAC, blocklistv6MAC, blocklistv4, blocklistv6 + + frontend_ver : 1.8.9-r1 + + backend_ver : 1.8.9-r1 + + element_count : 138 148 (chains: 7, sets: 13, rules: 50) + + active_feeds : allowlist.v4MAC, allowlist.v6MAC, allowlist.v4, allowlist.v6, dns.v4, blocklist.v4MAC, blocklist.v6MAC, doh.v6, blocklist.v4, doh.v4, blocklist.v6, dns.v6, hagezi.v4 + active_devices : wan: pppoe-wan / wan-if: wan, wan_6 / vlan-allow: - / vlan-block: - - + active_uplink : 217.83.205.130, fe80::9cd6:12e9:c4df:75d3, 2003:ed:b5ff:43bd:9cd5:12e7:c3ef:75d8 - + nft_info : priority: -100, policy: performance, loglevel: warn, expiry: 2h, limit (icmp/syn/udp): 10/10/100 - + run_info : base: /mnt/data/banIP, backup: /mnt/data/banIP/backup, report: /mnt/data/banIP/report - + run_flags : auto: ✔, proto (4/6): ✔/✔, log (pre/inp/fwd/lan): ✔/✘/✘/✘, dedup: ✔, split: ✘, custom feed: ✘, allowed only: ✘ - + last_run : action: reload, log: logread, fetch: curl, duration: 1m 21s, date: 2024-05-27 05:56:29 - + system_info : cores: 4, memory: 1661, device: Bananapi BPI-R3, OpenWrt SNAPSHOT r26353-a96354bcfb + + active_uplink : 5.73.162.23, 2a13:4800:204:319e:b26d:238b:d7fe:8213 + + nft_info : ver: 1.1.6-r1, priority: -100, policy: performance, loglevel: warn, expiry: 2h, limit (icmp/syn/udp): 25/10/100 + + run_info : base: /mnt/data/banIP, backup: /mnt/data/banIP/backup, report: /mnt/data/banIP/report, error: /mnt/data/banIP/error + + run_flags : auto: ✔, proto (4/6): ✔/✔, bcp38: ✔, log (pre/in/out): ✘/✘/✔, count: ✔, dedup: ✔, split: ✘, custom feed: ✘, allowed only: ✘ + + last_run : mode: restart, 2026-01-12 06:16:19, duration: 0m 36s, memory: 1446.84 MB available + + system_info : cores: 4, log: logread, fetch: curl, Bananapi BPI-R3, mediatek/filogic, OpenWrt SNAPSHOT (r32542-bf46d119a2) ``` **banIP search information** -``` -~# /etc/init.d/banip search 221.228.105.173 + +```sh +~# /etc/init.d/banip search 8.8.8.8 ::: ::: banIP Search ::: - Looking for IP '221.228.105.173' on 2023-02-08 22:12:48 + Looking for IP '8.8.8.8' on 2025-01-13 22:13:36 --- - IP found in Set 'oisdbasicv4' + IP found in Set 'country.v4' + IP found in Set 'doh.v4' ``` -**banIP survey information** -``` -~# /etc/init.d/banip survey cinsscorev4 +**banIP Set content information** +List all elements of a given Set with hit counters, e.g.: + +```sh +~# /etc/init.d/banip content turris.v4 ::: -::: banIP Survey +::: banIP Set Content ::: - List of elements in the Set 'cinsscorev4' on 2023-03-06 14:07:58 + List elements of the Set 'turris.v4' on 2025-06-08 23:28:55 --- -1.10.187.179 -1.10.203.30 -1.10.255.58 -1.11.67.53 -1.11.114.211 +1.4.228.135, packets: 0 +1.23.16.3, packets: 0 +1.33.35.42, packets: 0 +1.33.231.132, packets: 0 +1.34.29.158, packets: 0 +1.34.231.106, packets: 0 +1.52.91.174, packets: 0 +1.64.149.142, packets: 0 +1.69.243.13, packets: 0 +1.70.139.250, packets: 0 +1.70.171.246, packets: 0 +1.82.191.114, packets: 0 [...] ``` -## Best practise & tweaks -**Recommendation for low memory systems** -nftables supports the atomic loading of firewall rules (incl. elements), which is cool but unfortunately is also very memory intensive. To reduce the memory pressure on low memory systems (i.e. those with 256-512Mb RAM), you should optimize your configuration with the following options: +List only elements with hits of a given Set with hit counters, e.g.: +```sh +~# /etc/init.d/banip content turris.v4 true +::: +::: banIP Set Content +::: + List elements of the Set 'turris.v4' on 2025-06-08 23:30:59 + --- +74.50.211.178, packets: 1 +109.205.213.115, packets: 18 +109.205.213.123, packets: 35 +109.205.213.248, packets: 29 +109.205.213.250, packets: 20 +109.205.213.252, packets: 30 +122.222.152.65, packets: 1 +186.91.25.141, packets: 2 +190.203.106.113, packets: 2 +200.123.238.20, packets: 1 +``` + + +## Best practise and tweaks +**Recommendation for low memory systems** +nftables supports the atomic loading of firewall rules (incl. elements), which is cool but unfortunately is also very memory intensive. To reduce the memory pressure on low memory systems (i.e. those with 256-512MB RAM), you should optimize your configuration with the following options: -* point 'ban_basedir', 'ban_reportdir' and 'ban_backupdir' to an external usb drive -* set 'ban_cores' to '1' (only useful on a multicore system) to force sequential feed processing -* set 'ban_splitsize' e.g. to '1024' to split the load of an external Set after every 1024 lines/elements -* set 'ban_reportelements' to '0' to disable the CPU intensive counting of Set elements +* point `ban_basedir`, `ban_reportdir`, `ban_backupdir` and `ban_errordir` to an external usb drive or ssd +* set `ban_cores` to `1` (only useful on a multicore system) to force sequential feed processing +* set `ban_splitsize` e.g. to `1024` to split the load of an external Set after every 1024 lines/elements +* set `ban_nftcount` to `0` to deactivate the CPU- and memory-intensive creation of counter elements at chain / Set level. With this setting, all packet counters are disabled, the Set Reporting will show zero values for these even when the protection rules are actively dropping traffic. Only the DoS protection counters (`syn-flood`, `udp-flood`, `icmp-flood`, etc.) are always enabled. **Sensible choice of blocklists** -The following feeds are just my personal recommendation as an initial setup: -* cinsscore, debl, turris in WAN-Input and WAN-Forward chain -* doh in LAN-Forward chain +The following feeds are just my personal recommendation as an initial setup: +* cinsscore, debl, turris and doh in their default chains + +In total, this feed selection blocks about 20K IP addresses. It may also be useful to include some countries to the country feed. +Please note: don't just blindly activate (too) many feeds at once, sooner or later this will lead to OOM conditions. + +**Feed direction (inbound vs. outbound) — and the country/ASN case** +This is the single most common source of confusion, so it's worth understanding the model before you pick feeds. + +banIP filters on the **connection level** (source/destination IP), not on content. The default chain of a feed therefore has to match **who initiates the unwanted connection**: + +* **Inbound** (WAN-Input + WAN-Forward): blocks connections that come **from the internet towards you**. This is the right direction for the large majority of reputation feeds, which list the **source** IPs of attackers, scanners, brute-forcers and spam senders (`cinsscore`, `debl`, `dshield`, `firehol*`, `bruteforceblock`, `uceprotect*`, …). A fail2ban-style feed only makes sense inbound. +* **Outbound** (LAN-Forward): blocks connections your **own LAN clients initiate towards** a listed IP. This is the right direction for feeds that list destinations your devices should never reach: malware-download hosts, command-and-control servers, DoH/DNS bypass endpoints and threat IPs (`doh`, `dns`, `hagezi`, `feodo`, `urlhaus`, `urlvir`, `webclient`). +* **Both**: feeds where the risk genuinely exists in both directions default to `inout` (`spamhaus`, `threat`, `threatview`, `tor`, `proxy`, `vpn`, `vpndc`). + +The classic mistake: enabling a malware-URL / C2 feed like `urlhaus` in the **inbound** chain and expecting it to stop a LAN client from downloading malware. It won't — the client is the one opening the connection, so only the **outbound** chain can intercept it. These feeds ship with sensible defaults (outbound or both), but if you maintain your own custom feeds, keep this rule of thumb in mind. + +**The country/ASN case:** the `country` and `asn` feeds default to the **inbound** chain. This blocks traffic *from* the selected country/ASN towards you — the typical "keep attackers from region X out" use case. If your goal is instead to stop your **own clients from reaching** a country or ASN (the recurring "block all connections *to* country X" request), inbound alone does nothing for you. Switch the feed to the outbound or both chain: -In total, this feed selection blocks about 20K IP addresses. It may also be useful to include some countries to the country feed in WAN-Input and WAN-Forward chain. -Please note: don't just blindly activate (too) many feeds at once, sooner or later this will lead to OOM conditions. +* via LuCI: `Feed/Set Settings` → move the feed into `Outbound` or `Inbound + Outbound` +* via CLI in `/etc/config/banip`: + +``` +# block outbound connections to the selected countries as well as inbound +list ban_feedinout 'country' + +# or restrict the country feed exclusively to the outbound (LAN-FWD) chain +list ban_feedout 'country' +``` + +Be aware that bidirectional country blocking on large countries quickly inflates element counts and memory usage; consider `ban_countrysplit`/`ban_asnsplit` and the low-memory tweaks above. **Log Terms for logfile parsing** -Like fail2ban and crowdsec, banIP supports logfile scanning and automatic blocking of suspicious attacker IPs. -In the default config only the log terms to detect failed login attempts via dropbear and LuCI are in place. The following search pattern has been tested as well - just transfer the required regular expression via cut and paste to your config (without quotation marks): +Like fail2ban and crowdsec, banIP supports logfile scanning and automatic blocking of suspicious attacker IPs. +In the default config only the log terms to detect failed login attempts via dropbear and LuCI are in place. The following search patterns have been tested as well: + ``` dropbear : 'Exit before auth from' LuCI : 'luci: failed login' sshd1 : 'error: maximum authentication attempts exceeded' sshd2 : 'sshd.*Connection closed by.*\[preauth\]' asterisk : 'SecurityEvent=\"InvalidAccountID\".*RemoteAddress=' -nginx : 'received a suspicious remote IP '\''.*'\''' openvpn : 'TLS Error: could not determine wrapping from \[AF_INET\]' AdGuard : 'AdGuardHome.*\[error\].*/control/login: from ip' +Remote : 'received a suspicious remote IP' +``` + +**Source IP position** +For every matched line banIP picks the attacker's address out of the log entry. By default it uses the *last* IP in the line, which is correct for all login-style formats above: the service always appends the real remote address as the final field, after any user-supplied data (such as a login name). This stays robust even when an attacker puts an IP-like string into a username or similar field. + +Some formats instead lead with the source IP — most notably web-server access logs, where the client address is the first field. For those, prefix the term with `first:`: + +``` +nginx : 'first:"[A-Z]+ /wp-login\.php[^"]*" 40[13]' ``` -You find the 'Log Terms' option in LuCI under the 'Log Settings' tab. Feel free to add more log terms to meet your needs and protect additional services. + +The `first:`/`last:` prefix is parsed off before matching and applies only to that single term, so terms with different anchors can be mixed freely. + +Please note: `first:` only fits formats where the source IP is genuinely the first address in the line. Do **not** use it for Suricata fast.log (its leading `[1:2013:7]` signature id) or netfilter `LOG` lines (their leading `MAC=` field), as that leading field would be picked instead of the source — keep those on the default `last`. + +You find the `Log Terms` option in LuCI under the `Log Settings` tab. Feel free to add more log terms to meet your needs and protect additional services. **Allow-/Blocklist handling** -banIP supports local allow- and block-lists, MAC/IPv4/IPv6 addresses (incl. ranges in CIDR notation) or domain names. These files are located in /etc/banip/banip.allowlist and /etc/banip/banip.blocklist. -Unsuccessful login attempts or suspicious requests will be tracked and added to the local blocklist (see the 'ban_autoblocklist' option). The blocklist behaviour can be further tweaked with the 'ban_nftexpiry' option. -Depending on the options 'ban_autoallowlist' and 'ban_autoallowuplink' the uplink subnet or the uplink IP will be added automatically to local allowlist. -Furthermore, you can reference external Allowlist URLs with additional IPv4 and IPv6 feeds (see 'ban_allowurl'). -Both local lists also accept domain names as input to allow IP filtering based on these names. The corresponding IPs (IPv4 & IPv6) will be extracted and added to the Sets. You can also start the domain lookup separately via /etc/init.d/banip lookup at any time. +banIP supports local allow- and block-lists, MAC/IPv4/IPv6 addresses (incl. ranges in CIDR notation) or domain names. These files are located in /etc/banip/banip.allowlist and /etc/banip/banip.blocklist. +Unsuccessful login attempts or suspicious requests will be tracked and added to the local blocklist (see the `ban_autoblocklist` option). The blocklist behaviour can be further tweaked with the `ban_nftexpiry` option. +Depending on the options `ban_autoallowlist` and `ban_autoallowuplink` the uplink subnet or the uplink IP will be added automatically to local allowlist. +Furthermore, you can reference external Allowlist URLs with additional IPv4 and IPv6 feeds (see `ban_allowurl`). +Both local lists also accept domain names as input to allow IP filtering based on these names. The corresponding IPs (IPv4 & IPv6) will be extracted and added to the Sets. **Allowlist-only mode** -banIP supports an "allowlist only" mode. This option skips all blocklists and restricts Internet access only to certain, explicitly permitted IP segments - and blocks access to the rest of the Internet. All IPs that are _not_ listed in the allowlist or in the external allowlist URLs are blocked. In this mode it might be useful to limit the allowlist feed to the wan-input / wan-forward chain, to still allow lan-forward communication to the rest of the world. +banIP supports an "allowlist only" mode. This option restricts Internet access only to certain, explicitly permitted IP segments - and blocks access to the rest of the Internet. All IPs that are _not_ listed in the allowlist or in the external allowlist URLs are blocked. In this mode it might be useful to limit the allowlist feed to the inbound chain, to still allow outbound communication to the rest of the world. -**MAC/IP-binding** -banIP supports concatenation of local MAC addresses/ranges with IPv4/IPv6 addresses, e.g. to enforce dhcp assignments. +**MAC/IP-binding** +banIP supports concatenation of local MAC addresses/ranges with IPv4/IPv6 addresses, e.g. to enforce dhcp assignments or to free connected clients from outbound blocking. The following notations in the local allow- and block-list are supported: + ``` MAC-address only: C8:C2:9B:F7:80:12 => this will be populated to the v4MAC- and v6MAC-Sets with the IP-wildcards 0.0.0.0/0 and ::/0 @@ -369,32 +484,82 @@ C8:C2:9B:F7:80:12 192.168.1.10 => this will be populated to C8:C2:9B:F7:80:12 => this will be populated to v6MAC-Set with the IP-wildcard ::/0 ``` +**MAC-address logging in nftables** +The MAC-address logging format in nftables is a little bit unusual. It is generated by the kernel's NF_LOG module and places all MAC-related data into one flat field, without separators or labels. For example, the field MAC=7e:1a:2f:fc:ee:29:68:34:21:1f:a7:b1:08:00 is actually a concatenation of the following: + +``` +[Source MAC (6 bytes)] + [Destination MAC (6 bytes)] + [EtherType (2 bytes)] +7e:1a:2f:fc:ee:29 → the source MAC address +68:34:21:1f:a7:b1 → the destination MAC address +08:00 → the EtherType for IPv4 (0x0800) +``` +**BCP38** +BCP38 (**B**est **C**urrent **P**ractice, RFC 2827) defines ingress filtering to prevent IP address spoofing. In practice, this means: +* dropping packets arriving on the WAN whose source address is not valid or routable via that interface +* dropping packets leaving LAN => WAN whose source address does not belong to the local/internal prefixes + +In banIP, the BCP38 implementation uses nftables’ FIB lookup to enforce this. It checks whether the packet’s source address is not valid for the incoming interface or whether the routing table reports no route for this source on this interface. Packets that fail this check are dropped. + +**Reporting Counter behavior** +The `ban_nftcount` option globally controls all Reporting counters in the banIP table — both per-Set element counters and chain rule counters. Default is `0` (disabled) to keep memory usage and CPU overhead low on constrained devices. Only the DoS protection counters (`cnt_synflood`, `cnt_udpflood`, `cnt_icmpflood`, `cnt_tcpinvalid`, `cnt_ctinvalid`, `cnt_bcp38`) become always populated and visible in `/etc/init.d/banip report`. + +When enabled (`ban_nftcount=1`): +* every Set element gets its own packet/byte counter (memory cost: ~16 bytes per element, so ~1.6 MB for a 100k-element feed) +* all chain rules count packets and bytes +* the GeoIP Map (`ban_map=1`) becomes available, as it requires per-element counters to identify hit IPs + +When disabled (`ban_nftcount=0`): +* memory footprint is minimal, suitable for low memory routers +* the Set Reporting still works structurally, but packet counts will all read as zero +* the GeoIP Map cannot be enabled + +For most modern routers with ≥512 MB RAM, enabling `ban_nftcount` is recommended for full visibility into what banIP is actually blocking. + +**Log Limit / Options** +All log rules (prerouting flood protection, inbound and outbound feeds) share a single nft limit named `loglimit`. The default of 10/second with burst 5 is intentionally aligned with typical kernel printk rate limits, so banIP log messages are not dropped by the kernel. Tune via `ban_logratelimit` and `ban_logburstlimit` accordingly if you have adjusted the kernel defaults. Set the `ban_logratelimit` to `0` to disable rate limiting entirely, e.g. when using ulogd or other userspace log handlers without printk constraints. + +**Set reporting, enable the GeoIP Map** +banIP includes a powerful reporting tool on the Set Reporting tab which shows the latest NFT banIP Set statistics. To get the latest statistics always press the "Refresh" button. +In addition to a tabular overview banIP reporting includes a GeoIP map in a modal popup window/iframe that shows the geolocation of your own uplink addresses (in green) and the locations of potential attackers (in red). To enable the GeoIP Map set the following options (in "Feed/Set Settings" config tab): + +* set `ban_nftcount` to `1` to enable the nft counter for every Set element +* set `ban_map` to `1` to include the external components listed below and activate the GeoIP map + +To make this work, banIP uses the following external components: +* [Leaflet](https://leafletjs.com/) is a lightweight open-source JavaScript library for interactive maps +* [OpenStreetMap](https://www.openstreetmap.org/) provides the map data under an open-source license +* [CARTO basemap styles](https://github.com/CartoDB/basemap-styles) based on [OpenMapTiles](https://openmaptiles.org/schema) +* The free and quite fast [IP Geolocation API](https://ip-api.com/) to resolve the required IP/geolocation information + **CGI interface to receive remote logging events** -banIP ships a basic cgi interface in '/www/cgi-bin/banip' to receive remote logging events (disabled by default). The cgi interface evaluates logging events via GET or POST request (see examples below). To enable the cgi interface set the following options: +banIP ships a basic cgi interface in `/www/cgi-bin/banip` to receive remote logging events (disabled by default). The cgi interface evaluates logging events via GET or POST request (see examples below). To enable the cgi interface set the following options: - * set 'ban_remotelog' to '1' to enbale the cgi interface - * set 'ban_remotetoken' to a secret transfer token, allowed token characters consist of '[A-Za-z]', '[0-9]', '.' and ':' +* set `ban_remotelog` to `1` to enable the cgi interface +* set `ban_remotetoken` to a secret transfer token, allowed token characters consist of '[A-Za-z]', '[0-9]', '.' and ':' +* add the remote logging event to the logterm - Examples to transfer remote logging events from an internal server to banIP via cgi interface: +Examples to transfer remote logging events from an internal server to banIP via cgi interface: - * POST request: curl --insecure --data "=" https://192.168.1.1/cgi-bin/banip - * GET request: wget --no-check-certificate https://192.168.1.1/cgi-bin/banip?= +* POST request: `curl --insecure --data "=" https://192.168.1.1/cgi-bin/banip` +* GET request: `wget --no-check-certificate https://192.168.1.1/cgi-bin/banip?=` Please note: for security reasons use this cgi interface only internally and only encrypted via https transfer protocol. **Download options** By default banIP uses the following pre-configured download options: + ``` - * aria2c: --timeout=20 --retry-wait=10 --max-tries=5 --max-file-not-found=5 --allow-overwrite=true --auto-file-renaming=false --log-level=warn --dir=/ -o - * curl: --connect-timeout 20 --retry-delay 10 --retry 5 --retry-all-errors --fail --silent --show-error --location -o - * wget: --no-cache --no-cookies --timeout=20 --waitretry=10 --tries=5 --retry-connrefused --max-redirect=0 -O + * curl: --connect-timeout 20 --retry-delay 10 --retry 4 --retry-max-time 80 --retry-all-errors --fail --silent --show-error --location -o + * wget: --no-cache --no-cookies --timeout=20 --waitretry=10 --tries=5 --retry-connrefused -O * uclient-fetch: --timeout=20 -O ``` -To override the default set 'ban_fetchretry', 'ban_fetchinsecure' or globally 'ban_fetchparm' to your needs. -**Configure E-Mail notifications via 'msmtp'** -To use the email notification you must install and configure the package 'msmtp'. -Modify the file '/etc/msmtprc', e.g.: +The retry-related values shown above are derived from `ban_fetchretry` (default `5`): for curl `--retry` is `ban_fetchretry - 1` and `--retry-max-time` is `(ban_fetchretry - 1) * 20`, for wget `--tries` equals `ban_fetchretry`. To override the defaults set `ban_fetchretry`, `ban_fetchinsecure` or globally `ban_fetchparm` to your needs. + +**Configure E-Mail notifications via `msmtp`** +To use the email notification you must install and configure the package `msmtp`. +Modify the file `/etc/msmtprc`, e.g.: + ``` [...] defaults @@ -411,64 +576,122 @@ from
@gmail.com user password ``` + Finally add a valid E-Mail receiver address in banIP. **Send status E-Mails and update the banIP lists via cron job** -For a regular, automatic status mailing and update of the used lists on a daily basis set up a cron job, e.g. +For a regular, automatic update of the used feeds or other regular banIP tasks (e.g. status mailing) set up a cron job. Use `reload` here — `start`/`restart` would only restore the backups instead of fetching fresh feeds. In LuCI you find the cron settings under `System` => `Scheduled Tasks`. On the command line the cron file is located at `/etc/crontabs/root`: + ``` 55 03 * * * /etc/init.d/banip report mail 00 04 * * * /etc/init.d/banip reload ``` -**Redirect asterisk security logs to lodg/logread** -By default banIP scans the logfile via logread, so to monitor attacks on asterisk, its security log must be available via logread. To do this, edit '/etc/asterisk/logger.conf' and add the line 'syslog.local0 = security', then run 'asterisk -rx reload logger' to update the running asterisk configuration. -**Change/add banIP feeds and port limitations** -The banIP default blocklist feeds are stored in an external JSON file '/etc/banip/banip.feeds'. All custom changes should be stored in an external JSON file '/etc/banip/banip.custom.feeds' (empty by default). It's recommended to use the LuCI based Custom Feed Editor to make changes to this file. -A valid JSON source object contains the following information, e.g.: -``` +**Redirect asterisk security logs to syslog/logread** +By default banIP scans the logfile via logread, so to monitor attacks on asterisk, its security log must be available via logread. To do this, edit `/etc/asterisk/logger.conf` and add the line `syslog.local0 = security`, then run `asterisk -rx reload logger` to update the running asterisk configuration. + +**Change/add banIP feeds and set optional feed flags** +The banIP default blocklist feeds are stored in an external JSON file `/etc/banip/banip.feeds`. This file is shipped with the package and is **overwritten on every package update**, so never edit it directly. All of your custom changes belong in the separate JSON file `/etc/banip/banip.custom.feeds` (empty by default), which is preserved across updates. It's recommended to use the LuCI based Custom Feed Editor (`Custom Feed Editor`tab), which validates the JSON for you. + +A feed is a single JSON object, keyed by a unique feed name (no spaces, no special characters). Example: + +```json [...] -"stevenblack":{ - "url_4": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/stevenblack-ipv4.txt", - "url_6": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/stevenblack-ipv6.txt", - "rule_4": "/^127\\./{next}/^(([1-9][0-9]{0,2}\\.){1}([0-9]{1,3}\\.){2}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)[[:space:]]/{printf \"%s,\\n\",$1}", - "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)[[:space:]]/{printf \"%s,\\n\",$1}", - "descr": "stevenblack IPs", - "flag": "tcp 80 443" + "doh":{ + "url_4": "https://raw.githubusercontent.com/dibdot/DoH-IP-blocklists/master/doh-ipv4.txt", + "url_6": "https://raw.githubusercontent.com/dibdot/DoH-IP-blocklists/master/doh-ipv6.txt", + "rule": "feed 1", + "chain": "out", + "descr": "public DoH-Server", + "flag": "tcp udp 80 443" }, [...] ``` -Add an unique feed name (no spaces, no special chars) and make the required changes: adapt at least the URL, the regex and the description for a new feed. -Please note: the flag field is optional, it's a space separated list of options: supported are 'gz' as an archive format, protocols 'tcp' or 'udp' with port numbers/port ranges for destination port limitations - multiple definitions are possible. - -**Debug options** -Whenever you encounter banIP related processing problems, please check the "Processing Log" tab. -Typical symptoms: -* The nftables initialization failed: untick the 'Auto Detection' option in the 'General Settings' config section and set the required options manually -* A blocklist feed does not work: maybe a temporary server problem or the download URL has been changed. In the latter case, just use the Custom Feed Editor to point this feed to a new URL -To get much more processing information, please enable "Verbose Debug Logging" and restart banIP. - -Whenever you encounter firewall problems, enable the logging of certain chains in the "Log Settings" config section, restart banIP and check the "Firewall Log" tab. -Typical symptoms: -* A feed blocks a legit IP: disable the entire feed or add this IP to your local allowlist and reload banIP -* A feed (e.g. doh) interrupts almost all client connections: check the feed table above for reference and limit the feed to a certain chain in the "Feed/Set Settings" config section -* The allowlist doesn't free a certain IP/MAC address: check the current content of the allowlist with the "Set Survey" under the "Set Reporting" tab to make sure that the desired IP/MAC is listed - if not, reload banIP +The object supports the following fields: + +| Field | Required | Description | +| :------ | :------: | :--------------------------------------------------------------------------------------------------------------------- | +| url_4 | yes\* | download URL of the IPv4 list. \*at least one of `url_4`/`url_6` must be present | +| url_6 | yes\* | download URL of the IPv6 list. May point to the **same** URL as `url_4` if the source mixes IPv4 and IPv6 in one file | +| rule | yes | the parsing ruleset, max. 4 space separated parameters (see below) | +| chain | yes | the default chain/direction: `in`, `out` or `inout` (see below) | +| descr | yes | a short human-readable description shown in LuCI and the feed table | +| flag | no | optional, space separated list of extra options: archive format and/or protocol/port limitations (see below) | + +**The `url_4` / `url_6` fields** +Each address family is fetched and processed independently. Three cases: +* IPv4-only source: set `url_4` only, omit `url_6` +* separate IPv4 and IPv6 files: set both to their respective URLs (e.g. `doh`, `spamhaus`) +* a single dual-stack file that mixes v4 and v6 entries: point both `url_4` and `url_6` at that same URL. banIP fetches it only once and process it for each family and the per-family regex extracts the matching addresses; the non-matching lines are simply ignored. (e.g. `threatview`, which ships v4 and v6 in one file). + +**The `rule` field** +The rule consists of max. 4 individual, space separated parameters: +1. **type**: `feed` or `suricata` (required) + * `feed`: a plain IP/CIDR list, one entry per line (the common case) + * `suricata`: a Suricata/Snort-style ruleset; banIP extracts the IPs out of the rule lines +2. **prefix**: an optional search term (a literal string, not a regex) that a line must contain to be considered a valid entry. Use it to skip comment/header lines or to pick only the relevant rows. Omit it if every data line is a bare IP. +3. **column**: the 1-based column that holds the IP within a matching line, e.g. `1` for a bare list or `13` for the dshield block file (required) +4. **separator**: an optional field separator; default is the whitespace character class `[[:space:]]`. Pass a literal character such as `,` for comma-separated sources (e.g. turris). + +**The `chain` field** +Defines the default blocking direction. See the [Best practise feed-direction note](#best-practise-and-tweaks) for the reasoning behind each value: +* `in`: inbound only (WAN-Input + WAN-Forward). Use for source-IP reputation feeds (attackers, scanners, spam senders). +* `out`: outbound only (LAN-Forward). Use for destination feeds your own clients should never reach (malware hosts, C2, DoH/DNS endpoints). +* `inout`: both directions. Use when the risk genuinely exists on both sides (e.g. spamhaus, tor, proxy, vpn). + +This is only the **default**; a user can always override it at runtime per feed via `ban_feedin` / `ban_feedout` / `ban_feedinout`, or strip a feed's port/protocol limitation via `ban_feedreset` — without touching the feed JSON. + +**The `flag` field (optional)** +A space separated list of extra options: +* `gz`: the source is gzip-compressed and will be decompressed before parsing (e.g. `backscatterer`, `ipthreat`, `uceprotect*`) +* protocol/port limitation — one or both of `tcp` / `udp` followed by one or more destination ports or port ranges. This restricts the feed's rules to those destination ports, which is mainly useful for outbound feeds to cut false positives. +Examples: + * `tcp udp 80 443`: limit to HTTP/HTTPS (e.g. `doh`, `hagezi`, `feodo`, `urlhaus`, `urlvir`, `webclient`) + * `tcp udp 53 853`: limit to plain DNS and DNS-over-TLS (e.g. `dns`) + * `tcp 22`: limit to SSH + * `tcp 5060-5061 udp 5060`: mix of single ports and ranges + * `gz` and a port limitation can be combined, e.g. `gz tcp udp 80 443` + +After editing `/etc/banip/banip.custom.feeds`, reload banIP (`/etc/init.d/banip reload`) and check the `Processing Log` tab — a malformed JSON object or a wrong column/separator typically shows up there as a feed that loads zero elements. + + +## Troubleshooting & debug options +banIP provides an optional debug mode that writes diagnostic information to the system log and captures internal error output in a dedicated error logfile - by default located in the banIP base directory as `/tmp/ban_error.log`. The log file is automatically cleared at the beginning of each run. Under normal conditions, all error messages are discarded to keep regular runs clean and silent. + +Whenever you encounter banIP related processing problems, please enable `Verbose Debug Logging`, restart banIP and check the `Processing Log` tab. +Typical symptoms: +* The nftables initialization failed: untick the `Auto Detection` option in the `General Settings` config section and set the required device and tools options manually +* A blocklist feed does not work: maybe a temporary server problem or the download URL has been changed. In the latter case, just use the Custom Feed Editor to point this feed to a new URL + +In case of a nft processing error, banIP creates an error directory (by default `/tmp/banIP-error`) with the faulty nft load files. +For further troubleshooting, you can try to load such an error file manually to determine the exact cause of the error, e.g.: `nft -f error.file.nft`. + +Whenever you encounter firewall problems, enable the logging of certain chains in the `Log Settings` config section, restart banIP and check the `Firewall Log` tab. +Typical symptoms: +* A feed blocks a legit IP: disable the entire feed or add this IP to your local allowlist and reload banIP +* A feed (e.g. doh) interrupts almost all client connections: check the feed table above for reference and reset the feed to the defaults in the "Feed/Set Settings" config tab section +* The allowlist doesn't free a certain IP/MAC address: check the current content of the allowlist with the "Set Content" under the "Set Reporting" tab to make sure that the desired IP/MAC is listed - if not, reload banIP + + ## Support -Please join the banIP discussion in this [forum thread](https://forum.openwrt.org/t/banip-support-thread/16985) or contact me by mail -If you want to report an error, please describe it in as much detail as possible - with (debug) logs, the current banIP status, your banIP configuration, etc. +Please join the banIP discussion in this [forum thread](https://forum.openwrt.org/t/banip-support-thread/16985) or contact me by mail +If you want to report an error, please describe it in as much detail as possible - with (debug) logs, the current banIP status, your banIP configuration, etc. + ## Removal -Stop all banIP related services with _/etc/init.d/banip stop_ and remove the banip package if necessary. +Stop all banIP related services with `/etc/init.d/banip stop` and remove the banip package if necessary. + ## Donations -You like this project - is there a way to donate? Generally speaking "No" - I have a well-paying full-time job and my OpenWrt projects are just a hobby of mine in my spare time. +You like this project - is there a way to donate? Generally speaking "No" - I have a well-paying full-time job and my OpenWrt projects are just a hobby of mine in my spare time. -If you still insist to donate some bucks ... +If you still insist to donate some bucks ... * I would be happy if you put your money in kind into other, social projects in your area, e.g. a children's hospice * Let's meet and invite me for a coffee if you are in my area, the “Markgräfler Land” in southern Germany or in Switzerland (Basel) * Send your money to my [PayPal account](https://www.paypal.me/DirkBrenken) and I will collect your donations over the year to support various social projects in my area -No matter what you decide - thank you very much for your support! +No matter what you decide - thank you very much for your support! -Have fun! -Dirk +Have fun! +Dirk \ No newline at end of file diff --git a/packages/banip/files/banip-functions.sh b/packages/banip/files/banip-functions.sh old mode 100644 new mode 100755 index f3f9e3550..945ae9b35 --- a/packages/banip/files/banip-functions.sh +++ b/packages/banip/files/banip-functions.sh @@ -1,5 +1,5 @@ # banIP shared function library/include - ban incoming and outgoing IPs via named nftables Sets -# Copyright (c) 2018-2024 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2018-2026 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. # (s)hellcheck exceptions @@ -15,44 +15,57 @@ export PATH="/usr/sbin:/usr/bin:/sbin:/bin" ban_basedir="/tmp" ban_backupdir="/tmp/banIP-backup" ban_reportdir="/tmp/banIP-report" +ban_errordir="/tmp/banIP-error" +ban_rundir="/var/run/banIP" ban_feedfile="/etc/banip/banip.feeds" ban_countryfile="/etc/banip/banip.countries" ban_customfeedfile="/etc/banip/banip.custom.feeds" ban_allowlist="/etc/banip/banip.allowlist" ban_blocklist="/etc/banip/banip.blocklist" ban_mailtemplate="/etc/banip/banip.tpl" -ban_pidfile="/var/run/banip.pid" -ban_rtfile="/var/run/banip_runtime.json" -ban_rdapfile="/var/run/banip_rdap.json" +ban_pidfile="${ban_rundir}/banIP.pid" +ban_rtfile="${ban_rundir}/banIP.runtime.json" +ban_rdapfile="${ban_rundir}/banIP.rdap.json" +ban_lock="${ban_rundir}/banIP.lock" +ban_etaglock="${ban_rundir}/banIP.etag.lock" +ban_deduplock="${ban_rundir}/banIP.deduplicate.lock" ban_rdapurl="https://rdap.db.ripe.net/ip/" -ban_lock="/var/run/banip.lock" -ban_logreadfile="/var/log/messages" +ban_geourl="http://ip-api.com/batch" +ban_errorlog="/dev/null" +ban_logreadfile="" ban_logreadcmd="" ban_mailsender="no-reply@banIP" ban_mailreceiver="" ban_mailtopic="banIP notification" ban_mailprofile="ban_notify" ban_mailnotification="0" -ban_reportelements="1" ban_remotelog="0" ban_remotetoken="" ban_nftloglevel="warn" ban_nftpriority="-100" ban_nftpolicy="memory" ban_nftexpiry="" -ban_loglimit="100" -ban_icmplimit="10" +ban_nftretry="3" +ban_nftcount="0" +ban_map="0" +ban_bcp38="0" +ban_icmplimit="25" ban_synlimit="10" ban_udplimit="100" +ban_loglimit="100" +ban_logratelimit="10" +ban_logburstlimit="5" ban_logcount="1" ban_logterm="" +ban_logterm_map="" ban_region="" ban_country="" +ban_countrysplit="0" ban_asn="" +ban_asnsplit="0" ban_logprerouting="0" -ban_loginput="0" -ban_logforwardwan="0" -ban_logforwardlan="0" +ban_loginbound="0" +ban_logoutbound="0" ban_allowurl="" ban_allowflag="" ban_allowlistonly="0" @@ -64,11 +77,12 @@ ban_deduplicate="1" ban_splitsize="0" ban_autodetect="1" ban_feed="" -ban_blockpolicy="" -ban_blocktype="drop" -ban_blockinput="" -ban_blockforwardwan="" -ban_blockforwardlan="" +ban_feedin="" +ban_feedout="" +ban_feedinout="" +ban_feedcomplete="" +ban_feedreset="" +ban_blockpolicy="drop" ban_protov4="0" ban_protov6="0" ban_ifv4="" @@ -83,54 +97,102 @@ ban_fetchinsecure="" ban_fetchretry="5" ban_rdapparm="" ban_etagparm="" +ban_geoparm="" ban_cores="" +ban_srtmem="8" +ban_srtopts="" ban_packages="" ban_trigger="" ban_resolver="" ban_enabled="0" +ban_confload="0" ban_debug="0" -# gather system information -# -f_system() { - local cpu core - - if [ -z "${ban_dev}" ]; then - ban_debug="$(uci_get banip global ban_debug "0")" - ban_cores="$(uci_get banip global ban_cores)" - fi - ban_packages="$("${ban_ubuscmd}" -S call rpc-sys packagelist '{ "all": true }' 2>/dev/null)" - ban_ver="$(printf "%s" "${ban_packages}" | "${ban_jsoncmd}" -ql1 -e '@.packages.banip')" - ban_sysver="$("${ban_ubuscmd}" -S call system board 2>/dev/null | "${ban_jsoncmd}" -ql1 -e '@.model' -e '@.release.target' -e '@.release.distribution' -e '@.release.version' -e '@.release.revision' | - "${ban_awkcmd}" 'BEGIN{RS="";FS="\n"}{printf "%s, %s, %s %s %s %s",$1,$2,$3,$4,$5,$6}')" - if [ -z "${ban_cores}" ]; then - cpu="$("${ban_grepcmd}" -c '^processor' /proc/cpuinfo 2>/dev/null)" - core="$("${ban_grepcmd}" -cm1 '^core id' /proc/cpuinfo 2>/dev/null)" - [ "${cpu}" = "0" ] && cpu="1" - [ "${core}" = "0" ] && core="1" - ban_cores="$((cpu * core))" - [ "${ban_cores}" -gt "16" ] && ban_cores="16" - fi -} - # command selector # f_cmd() { local cmd pri_cmd="${1}" sec_cmd="${2}" + # check primary command, + # if not found check secondary command if provided, otherwise log error + # cmd="$(command -v "${pri_cmd}" 2>/dev/null)" - if [ ! -x "${cmd}" ]; then + if [ -z "${cmd}" ]; then if [ -n "${sec_cmd}" ]; then [ "${sec_cmd}" = "optional" ] && return - cmd="$(command -v "${sec_cmd}" 2>/dev/null)" + cmd="$(command -v "${sec_cmd}" 2>>"${ban_errorlog}")" fi - if [ -x "${cmd}" ]; then - printf "%s" "${cmd}" + if [ -n "${cmd}" ]; then + printf '%s' "${cmd}" else f_log "emerg" "command '${pri_cmd:-"-"}'/'${sec_cmd:-"-"}' not found" fi else - printf "%s" "${cmd}" + printf '%s' "${cmd}" + fi +} + +# determine available system memory (MemAvailable) in MB +# mode "float" returns MiB with two decimals, default is integer MiB +# +f_mem() { + local mem mode="${1}" + + if [ "${mode}" = "float" ]; then + mem="$("${ban_awkcmd}" '/^MemAvailable/{printf "%.2f", $2/1024}' "/proc/meminfo" 2>>"${ban_errorlog}")" + else + mem="$("${ban_awkcmd}" '/^MemAvailable/{printf "%s", int($2/1024)}' "/proc/meminfo" 2>>"${ban_errorlog}")" + fi + printf '%s' "${mem:-"0"}" +} + +# gather system information +# +f_system() { + local free_mem mem_cores + + ban_debug="$(uci_get banip global ban_debug "0")" + ban_cores="$(uci_get banip global ban_cores)" + ban_basedir="$(uci_get banip global ban_basedir "/tmp")" + + # set debug log file + # + if [ "${ban_debug}" = "1" ] && [ -d "${ban_basedir}" ]; then + ban_errorlog="${ban_basedir}/ban_error.log" + else + ban_errorlog="/dev/null" + fi + + # create runtime directory + # + f_mkdir "${ban_rundir}" + + # get banIP version and system information + # + ban_packages="$("${ban_ubuscmd}" -S call rpc-sys packagelist '{ "all": true }' 2>>"${ban_errorlog}")" + ban_bver="$(printf '%s' "${ban_packages}" | "${ban_jsoncmd}" -ql1 -e '@.packages.banip')" + ban_fver="$(printf '%s' "${ban_packages}" | "${ban_jsoncmd}" -ql1 -e '@.packages["luci-app-banip"]')" + ban_sysver="$("${ban_ubuscmd}" -S call system board 2>>"${ban_errorlog}" | "${ban_jsoncmd}" -ql1 -e '@.model' -e '@.release.target' -e '@.release.distribution' -e '@.release.version' -e '@.release.revision' | + "${ban_awkcmd}" 'BEGIN{RS="";FS="\n"}{printf "%s, %s, %s %s (%s)",$1,$2,$3,$4,$5}')" + + # detect cpu cores and cap them by available memory for memory-aware + # parallel processing (>= 48 MiB per job, floored to 1 core); a user-set + # ban_cores is only ever lowered by the cap, never raised + # + [ -z "${ban_cores}" ] && ban_cores="$("${ban_grepcmd}" -cm16 '^processor' /proc/cpuinfo 2>>"${ban_errorlog}")" + case "${ban_cores}" in "" | 0 | *[!0-9]*) ban_cores="1" ;; esac + free_mem="$(f_mem)" + mem_cores="$((free_mem / 48))" + [ "${mem_cores}" -lt "1" ] && mem_cores="1" + [ "${ban_cores}" -gt "1" ] && [ "${mem_cores}" -lt "${ban_cores}" ] && ban_cores="${mem_cores}" + + # derive the GNU sort buffer from available memory (>= 8 MiB per core); + # only applied when a coreutils sort is present (busybox sort has no --buffer-size) + # + ban_srtmem="$((free_mem / 2 / ban_cores))" + [ "${ban_srtmem}" -lt "8" ] && ban_srtmem="8" + if "${ban_sortcmd}" --version 2>/dev/null | "${ban_grepcmd}" -q "coreutils"; then + ban_srtopts="--buffer-size=${ban_srtmem}M" fi } @@ -140,9 +202,9 @@ f_mkdir() { local dir="${1}" if [ ! -d "${dir}" ]; then - rm -f "${dir}" + "${ban_rmcmd}" -f "${dir}" mkdir -p "${dir}" - f_log "debug" "f_mkdir ::: directory: ${dir}" + f_log "debug" "f_mkdir ::: directory: ${dir}" fi } @@ -163,8 +225,9 @@ f_tmp() { f_mkdir "${ban_basedir}" ban_tmpdir="$(mktemp -p "${ban_basedir}" -d)" ban_tmpfile="$(mktemp -p "${ban_tmpdir}" -tu)" + [ "${ban_debug}" = "1" ] && : >"${ban_errorlog}" - f_log "debug" "f_tmp ::: base_dir: ${ban_basedir:-"-"}, tmp_dir: ${ban_tmpdir:-"-"}" + f_log "debug" "f_tmp ::: base_dir: ${ban_basedir:-"-"}, tmp_dir: ${ban_tmpdir:-"-"}" } # remove directories @@ -173,22 +236,8 @@ f_rmdir() { local dir="${1}" if [ -d "${dir}" ]; then - rm -rf "${dir}" - f_log "debug" "f_rmdir ::: directory: ${dir}" - fi -} - -# convert chars -# -f_char() { - local char="${1}" - - if [ "${char}" = "1" ]; then - printf "%s" "✔" - elif [ "${char}" = "0" ] || [ -z "${char}" ]; then - printf "%s" "✘" - else - printf "%s" "${char}" + "${ban_rmcmd}" -rf "${dir}" + f_log "debug" "f_rmdir ::: directory: ${dir}" fi } @@ -199,25 +248,42 @@ f_trim() { string="${string#"${string%%[![:space:]]*}"}" string="${string%"${string##*[![:space:]]}"}" - printf "%s" "${string}" + printf '%s' "${string}" } # remove log monitor # f_rmpid() { - local ppid pid pids + local ppid pid pids_next pids_all childs newchilds - ppid="$("${ban_catcmd}" "${ban_pidfile}" 2>/dev/null)" + # kill all descendant processes of the pid in pidfile + # + ppid="$("${ban_catcmd}" "${ban_pidfile}" 2>>"${ban_errorlog}")" if [ -n "${ppid}" ]; then - pids="$("${ban_pgrepcmd}" -P "${ppid}" 2>/dev/null)" - for pid in ${pids}; do - pids="${pids} $("${ban_pgrepcmd}" -P "${pid}" 2>/dev/null)" + pids_next="$("${ban_pgrepcmd}" -P "${ppid}" 2>>"${ban_errorlog}")" + pids_all="" + while [ -n "${pids_next}" ]; do + for pid in ${pids_next}; do + case " ${pids_all} " in + *" ${pid} "*) ;; + + *) + pids_all="${pids_all} ${pid}" + ;; + esac + done + newchilds="" + for pid in ${pids_next}; do + childs="$("${ban_pgrepcmd}" -P "${pid}" 2>>"${ban_errorlog}")" + [ -n "${childs}" ] && newchilds="${newchilds} ${childs}" + done + pids_next="$(f_trim "${newchilds}")" done - for pid in ${pids}; do + for pid in ${pids_all}; do kill -INT "${pid}" >/dev/null 2>&1 done fi - : >"${ban_rdapfile}" >"${ban_pidfile}" + : >"${ban_pidfile}" } # write log messages @@ -227,24 +293,24 @@ f_log() { if [ -n "${log_msg}" ] && { [ "${class}" != "debug" ] || [ "${ban_debug}" = "1" ]; }; then if [ -x "${ban_logcmd}" ]; then - "${ban_logcmd}" -p "${class}" -t "banIP-${ban_ver}[${$}]" "${log_msg::512}" + "${ban_logcmd}" -p "${class}" -t "banIP-${ban_bver:-"-"}[${$}]" "${log_msg::512}" else - printf "%s %s %s\n" "${class}" "banIP-${ban_ver}[${$}]" "${log_msg::512}" + printf '%s %s %s\n' "${class}" "banIP-${ban_bver:-"-"}[${$}]" "${log_msg::512}" >&2 fi fi if [ "${class}" = "err" ] || [ "${class}" = "emerg" ]; then if [ "${class}" = "err" ]; then "${ban_nftcmd}" delete table inet banIP >/dev/null 2>&1 if [ "$(uci_get banip global ban_enabled)" = "1" ]; then - f_genstatus "error" + [ -s "${ban_rtfile}" ] && f_genstatus "error" [ "${ban_mailnotification}" = "1" ] && [ -n "${ban_mailreceiver}" ] && [ -x "${ban_mailcmd}" ] && f_mail else - f_genstatus "disabled" + [ -s "${ban_rtfile}" ] && f_genstatus "disabled" fi fi f_rmdir "${ban_tmpdir}" f_rmpid - rm -rf "${ban_lock}" + "${ban_rmcmd}" -rf "${ban_lock}" exit 1 fi } @@ -254,166 +320,312 @@ f_log() { f_conf() { local rir ccode region country - unset ban_dev ban_vlanallow ban_vlanblock ban_ifv4 ban_ifv6 ban_feed ban_allowurl ban_blockinput ban_blockforwardwan ban_blockforwardlan ban_logterm ban_region ban_country ban_asn + [ "${ban_confload}" = "1" ] && return 0 + config_cb() { option_cb() { - local option="${1}" - local value="${2}" - eval "${option}=\"${value}\"" + local option="${1}" value="${2//\"/\\\"}" + + case "${option}" in + *[!a-zA-Z0-9_]*) ;; + + *) + [ -n "${value}" ] && eval "${option}=\"\${value}\"" + ;; + esac } list_cb() { - local option="${1}" - local value="${2}" + local anchor pat append option="${1}" value="${2//\"/\\\"}" + case "${option}" in - "ban_ifv4") - eval "${option}=\"$(printf "%s" "${ban_ifv4}")${value} \"" - ;; - "ban_ifv6") - eval "${option}=\"$(printf "%s" "${ban_ifv6}")${value} \"" - ;; - "ban_dev") - eval "${option}=\"$(printf "%s" "${ban_dev}")${value} \"" - ;; - "ban_vlanallow") - eval "${option}=\"$(printf "%s" "${ban_vlanallow}")${value} \"" - ;; - "ban_vlanblock") - eval "${option}=\"$(printf "%s" "${ban_vlanblock}")${value} \"" - ;; - "ban_trigger") - eval "${option}=\"$(printf "%s" "${ban_trigger}")${value} \"" - ;; - "ban_feed") - eval "${option}=\"$(printf "%s" "${ban_feed}")${value} \"" - ;; - "ban_allowurl") - eval "${option}=\"$(printf "%s" "${ban_allowurl}")${value} \"" - ;; - "ban_blockinput") - eval "${option}=\"$(printf "%s" "${ban_blockinput}")${value} \"" - ;; - "ban_blockforwardwan") - eval "${option}=\"$(printf "%s" "${ban_blockforwardwan}")${value} \"" - ;; - "ban_blockforwardlan") - eval "${option}=\"$(printf "%s" "${ban_blockforwardlan}")${value} \"" - ;; - "ban_logterm") - eval "${option}=\"$(printf "%s" "${ban_logterm}")${value}\\|\"" - ;; - "ban_region") - eval "${option}=\"$(printf "%s" "${ban_region}")${value} \"" + *[!a-zA-Z0-9_]*) ;; + + "ban_logterm") + case "${value}" in + first:*) + anchor="first" + pat="${value#first:}" ;; - "ban_country") - eval "${option}=\"$(printf "%s" "${ban_country}")${value} \"" + last:*) + anchor="last" + pat="${value#last:}" ;; - "ban_asn") - eval "${option}=\"$(printf "%s" "${ban_asn}")${value} \"" + *) + anchor="last" + pat="${value}" ;; + esac + eval "append=\"\${${option}}\"" + if [ -n "${append}" ]; then + eval "${option}=\"\${append}\\|${pat}\"" + else + eval "${option}=\"${pat}\"" + fi + ban_logterm_map="${ban_logterm_map}${anchor}$(printf '\037')${pat}$(printf '\036')" + ;; + *) + eval "append=\"\${${option}}\"" + eval "${option}=\"\${append}\${value} \"" + ;; esac } } config_load banip - [ -f "${ban_logreadfile}" ] && ban_logreadcmd="$(command -v tail)" || ban_logreadcmd="$(command -v logread)" + ban_confload="1" + + if [ -f "${ban_logreadfile}" ]; then + ban_logreadcmd="$(command -v tail)" + else + ban_logreadcmd="$(command -v logread)" + fi for rir in ${ban_region}; do while read -r ccode region country; do - if [ "${rir}" = "${region}" ] && ! printf "%s" "${ban_country}" | "${ban_grepcmd}" -qw "${ccode}"; then - ban_country="${ban_country} ${ccode}" + if [ "${rir}" = "${region}" ]; then + case " ${ban_country} " in + *" ${ccode} "*) ;; + + *) + ban_country="${ban_country} ${ccode}" + ;; + esac fi done <"${ban_countryfile}" done } +# IPv4/IPv6 validation +# +f_chkip() { + local ipv type prefix separator col1 col2 feed="${feed}" + + ipv="${1}" + type="${2}" + case "${type}" in + "feed" | "local") + case "${3}" in + [0-9][0-9]) + prefix="" + col1="${3:0:1}" + col2="${3:1:1}" + separator="${4:-[[:space:]]+}" + ;; + [0-9]) + prefix="" + col1="${3}" + col2="" + separator="${4:-[[:space:]]+}" + ;; + *) + prefix="${3}" + col1="${4}" + col2="" + separator="${5:-[[:space:]]+}" + ;; + esac + ;; + "suricata") + prefix="" + col1="${3}" + col2="" + separator="${4:-[[:space:]]+}" + ;; + esac + "${ban_awkcmd}" -v ipv="${ipv}" -v type="${type}" -v pre="${prefix}" -v col1="${col1}" -v col2="${col2}" -F "${separator}" ' + { + # suricata pre-processing + if (type == "suricata") { + delete M + if (ipv == "4") { + match($0, /content:"(([0-9]{1,3}\.){3}[0-9]{1,3})"/, M) + } else if (ipv == "6") { + match($0, /content:"(([A-Fa-f0-9]{0,4}:){2,7}[A-Fa-f0-9]{0,4})"/, M) + } + if (M[1] == "") next + $col1 = M[1] + } + ip = $col1 + gsub(/\r|^[[:space:]]+|[[:space:]]+$/, "", ip) + # prefix filter + if (pre != "" && index($0, pre) != 1) next + # skip empty lines or comments + if (ip == "" || ip ~ /^#/) next + # reject invalid lengths + len = length(ip) + if (len < 3 || len > 43) next + # reject MAC addresses when ipv=6 + if (ipv == "6" && ip ~ /^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$/) next + # reject IPv4 when ipv=6 + if (ipv == "6" && ip ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) next + # reject IPv4-mapped IPv6 addresses + if (ipv == "6" && tolower(ip) ~ /^::ffff:/) next + # reject IPv6 when ipv=4 + if (ipv == "4" && ip ~ /:/) next + # apply mask + if (col2 != "") { + mask = $col2 + lowip = (ipv == "4") ? ip "/" mask : tolower(ip "/" mask) + } else { + lowip = (ipv == "4") ? ip : tolower(ip) + } + # CIDR check + if (lowip ~ /\//) { + if (split(lowip, C, "/") != 2) next + base = C[1] + mask = C[2] + if (mask !~ /^[0-9]+$/) next + # IPv4 CIDR + if (ipv == "4") { + if (base ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { + if (mask > 32) next + n = split(base, A, ".") + # reject loopback and unspecified addresses + if (A[1] == 127 || base == "0.0.0.0") next + # reject leading zeros and octets > 255 + for (i=1; i<=4; i++) { + if (length(A[i]) > 1 && substr(A[i], 1, 1) == "0") next + if (A[i] > 255) next + } + print lowip ", " + next + } + } + # IPv6 CIDR + if (ipv == "6") { + if (base ~ /^[0-9a-f:]+$/ && base ~ /:/) { + if (mask > 128) next + if (base == "::1" || base == "::") next + if (base ~ /^fe80:/) next + print lowip ", " + next + } + } + } + # IPv4 check + if (ipv == "4") { + if (lowip ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { + n = split(lowip, A, ".") + # reject loopback and unspecified addresses + if (A[1] == 127 || lowip == "0.0.0.0") next + # reject leading zeros and octets > 255 + for (i=1; i<=4; i++) { + if (length(A[i]) > 1 && substr(A[i], 1, 1) == "0") next + if (A[i] > 255) next + } + print lowip ", " + next + } + } + # IPv6 check + if (ipv == "6") { + if (lowip ~ /^[0-9a-f:]+$/ && lowip ~ /:/) { + # reject loopback and unspecified addresses + if (lowip == "::1" || lowip == "::") next + # reject link-local addresses + if (lowip ~ /^fe80:/) next + print lowip ", " + next + } + } + }' + + f_log "debug" "f_chkip ::: feed: ${feed}, ipver: ${ipv}, type: ${type}, prefix: ${prefix:-"-"}, col1: ${col1:-"-"}, col2: ${col2:-"-"}, separator: ${separator:-"-"}" +} + # get nft/monitor actuals # f_actual() { local nft monitor ppid pids pid - if "${ban_nftcmd}" -t list set inet banIP allowlistv4MAC >/dev/null 2>&1; then - nft="$(f_char "1")" + if "${ban_nftcmd}" -t list table inet banIP >/dev/null 2>&1; then + nft="✔" else - nft="$(f_char "0")" + nft="✘" fi - ppid="$("${ban_catcmd}" "${ban_pidfile}" 2>/dev/null)" + monitor="✘" + ppid="$("${ban_catcmd}" "${ban_pidfile}" 2>>"${ban_errorlog}")" if [ -n "${ppid}" ]; then - pids="$("${ban_pgrepcmd}" -P "${ppid}" 2>/dev/null)" + pids="${ppid} $("${ban_pgrepcmd}" -P "${ppid}" 2>>"${ban_errorlog}")" for pid in ${pids}; do if "${ban_pgrepcmd}" -f "${ban_logreadcmd##*/}" -P "${pid}" >/dev/null 2>&1; then - monitor="$(f_char "1")" + monitor="✔" break - else - monitor="$(f_char "0")" fi done - else - monitor="$(f_char "0")" fi - printf "%s" "nft: ${nft}, monitor: ${monitor}" + printf '%s' "nft: ${nft}, monitor: ${monitor}" } # get fetch utility # -f_getfetch() { - local util utils insecure - - ban_fetchcmd="$(command -v "${ban_fetchcmd}")" - if { [ "${ban_autodetect}" = "1" ] && [ -z "${ban_fetchcmd}" ]; } || [ ! -x "${ban_fetchcmd}" ]; then - utils="aria2 curl wget-ssl libustream-openssl libustream-wolfssl libustream-mbedtls" - for util in ${utils}; do - if printf "%s" "${ban_packages}" | "${ban_jsoncmd}" -ql1 -e "@.packages[\"${util}\"]" >/dev/null 2>&1; then - case "${util}" in - "aria2") - util="aria2c" - ;; - "wget-ssl") - util="wget" - ;; - "libustream-openssl" | "libustream-wolfssl" | "libustream-mbedtls") - util="uclient-fetch" - ;; - esac +f_getdl() { + local fetch fetch_list insecure update="0" - if [ -x "$(command -v "${util}")" ]; then - ban_fetchcmd="$(command -v "${util}")" - uci_set banip global ban_fetchcmd "${util}" + # check if the configured fetch utility is available and has SSL support, + # if not try to find an alternative with SSL support or log an error if not found + # + ban_fetchcmd="$(command -v "${ban_fetchcmd}" 2>/dev/null)" + if [ -z "${ban_fetchcmd}" ]; then + fetch_list="curl wget-ssl libustream-openssl libustream-wolfssl libustream-mbedtls" + for fetch in ${fetch_list}; do + case "${ban_packages}" in *"\"${fetch}\""*) + case "${fetch}" in + "wget-ssl") + fetch="wget" + ;; + "libustream-openssl" | "libustream-wolfssl" | "libustream-mbedtls") + fetch="uclient-fetch" + ;; + esac + ban_fetchcmd="$(command -v "${fetch}" 2>/dev/null)" + if [ -n "${ban_fetchcmd}" ]; then + update="1" + uci_set banip global ban_fetchcmd "${fetch}" uci_commit "banip" break fi - fi + ;; + esac done fi - [ ! -x "${ban_fetchcmd}" ] && f_log "err" "download utility with SSL support not found, please set 'ban_fetchcmd' manually" + [ -z "${ban_fetchcmd}" ] && f_log "err" "download utility with SSL support not found, please set 'ban_fetchcmd' manually" + + # check the fetch retry value + # + case "${ban_fetchretry}" in + 0* | *[!0-9]*) + ban_fetchretry="5" + ;; + esac + # set fetch parameters based on the fetch utility and check if insecure fetching is enabled + # case "${ban_fetchcmd##*/}" in - "aria2c") - [ "${ban_fetchinsecure}" = "1" ] && insecure="--check-certificate=false" - ban_fetchparm="${ban_fetchparm:-"${insecure} --timeout=20 --retry-wait=10 --max-tries=${ban_fetchretry} --max-file-not-found=${ban_fetchretry} --allow-overwrite=true --auto-file-renaming=false --log-level=warn --dir=/ -o"}" - ban_rdapparm="--timeout=5 --allow-overwrite=true --auto-file-renaming=false --dir=/ -o" - ban_etagparm="--timeout=5 --allow-overwrite=true --auto-file-renaming=false --dir=/ --dry-run --log -" - ;; - "curl") - [ "${ban_fetchinsecure}" = "1" ] && insecure="--insecure" - ban_fetchparm="${ban_fetchparm:-"${insecure} --connect-timeout 20 --retry-delay 10 --retry ${ban_fetchretry} --retry-max-time $((ban_fetchretry * 20)) --retry-all-errors --fail --silent --show-error --location -o"}" - ban_rdapparm="--connect-timeout 5 --silent --location -o" - ban_etagparm="--connect-timeout 5 --silent --location --head" - ;; - "wget") - [ "${ban_fetchinsecure}" = "1" ] && insecure="--no-check-certificate" - ban_fetchparm="${ban_fetchparm:-"${insecure} --no-cache --no-cookies --timeout=20 --waitretry=10 --tries=${ban_fetchretry} --retry-connrefused -O"}" - ban_rdapparm="--timeout=5 -O" - ban_etagparm="--timeout=5 --spider --server-response" - ;; - "uclient-fetch") - [ "${ban_fetchinsecure}" = "1" ] && insecure="--no-check-certificate" - ban_fetchparm="${ban_fetchparm:-"${insecure} --timeout=20 -O"}" - ban_rdapparm="--timeout=5 -O" - ;; + "curl") + [ "${ban_fetchinsecure}" = "1" ] && insecure="--insecure" + ban_fetchparm="${ban_fetchparm:-"${insecure} --connect-timeout 20 --retry-delay 10 --retry $((ban_fetchretry - 1)) --retry-max-time $(((ban_fetchretry - 1) * 20)) --retry-all-errors --fail --silent --show-error --location -o"}" + ban_rdapparm="--connect-timeout 5 --silent --location -o" + ban_etagparm="--connect-timeout 5 --silent --location --head" + ban_geoparm="--connect-timeout 5 --silent --location --data" + ;; + "wget") + [ "${ban_fetchinsecure}" = "1" ] && insecure="--no-check-certificate" + ban_fetchparm="${ban_fetchparm:-"${insecure} --no-cache --no-cookies --timeout=20 --waitretry=10 --tries=${ban_fetchretry} --retry-connrefused -O"}" + ban_rdapparm="--timeout=5 -O" + ban_etagparm="--timeout=5 --spider --server-response" + ban_geoparm="--timeout=5 --quiet -O- --post-data" + ;; + "uclient-fetch") + [ "${ban_fetchinsecure}" = "1" ] && insecure="--no-check-certificate" + ban_fetchparm="${ban_fetchparm:-"${insecure} --timeout=20 -O"}" + ban_rdapparm="--timeout=5 -O" + ban_geoparm="--timeout=5 --quiet -O- --post-data" + ;; esac - f_log "debug" "f_getfetch ::: auto: ${ban_autodetect}, cmd: ${ban_fetchcmd:-"-"}, fetch_parm: ${ban_fetchparm:-"-"}, rdap_parm: ${ban_rdapparm:-"-"}, etag_parm: ${ban_etagparm:-"-"}" + f_log "debug" "f_getdl ::: auto/update: ${ban_autodetect}/${update}, cmd: ${ban_fetchcmd:-"-"}, parm: ${ban_fetchparm:-"-"}, rdapparm: ${ban_rdapparm:-"-"}, etagparm: ${ban_etagparm:-"-"}, geoparm: ${ban_geoparm:-"-"}" } # get wan interfaces @@ -462,7 +674,7 @@ f_getif() { ban_ifv6="$(f_trim "${ban_ifv6}")" [ -z "${ban_ifv4}" ] && [ -z "${ban_ifv6}" ] && f_log "err" "no wan interfaces" - f_log "debug" "f_getif ::: auto/update: ${ban_autodetect}/${update}, interfaces (4/6): ${ban_ifv4}/${ban_ifv6}, protocols (4/6): ${ban_protov4}/${ban_protov6}" + f_log "debug" "f_getif ::: auto/update: ${ban_autodetect}/${update}, interfaces (4/6): ${ban_ifv4}/${ban_ifv6}, protocols (4/6): ${ban_protov4}/${ban_protov6}" } # get wan devices @@ -477,11 +689,15 @@ f_getdev() { network_get_device dev "${iface}" if [ -n "${dev}" ]; then dev_del="${dev_del/${dev} / }" - if ! printf " %s " "${ban_dev}" | "${ban_grepcmd}" -q " ${dev} "; then + case " ${ban_dev} " in + *" ${dev} "*) ;; + + *) ban_dev="${ban_dev}${dev} " uci_add_list banip global ban_dev "${dev}" f_log "info" "add device '${dev}' to config" - fi + ;; + esac fi done for dev in ${dev_del}; do @@ -497,13 +713,13 @@ f_getdev() { ban_dev="$(f_trim "${ban_dev}")" [ -z "${ban_dev}" ] && f_log "err" "no wan devices" - f_log "debug" "f_getdev ::: auto/update: ${ban_autodetect}/${update}, wan_devices: ${ban_dev}" + f_log "debug" "f_getdev ::: auto/update: ${ban_autodetect}/${update}, wan_devices: ${ban_dev}" } # get local uplink # -f_getuplink() { - local uplink iface ip update="0" +f_getup() { + local uplink iface timestamp ip if [ "${ban_autoallowlist}" = "1" ] && [ "${ban_autoallowuplink}" != "disable" ]; then for iface in ${ban_ifv4} ${ban_ifv6}; do @@ -513,35 +729,49 @@ f_getuplink() { elif [ "${ban_autoallowuplink}" = "ip" ]; then network_get_ipaddr uplink "${iface}" fi - if [ -n "${uplink}" ] && ! printf " %s " "${ban_uplink}" | "${ban_grepcmd}" -q " ${uplink} "; then - ban_uplink="${ban_uplink}${uplink} " + if [ -n "${uplink}" ]; then + case " ${ban_uplink} " in + *" ${uplink} "*) ;; + + *) + ban_uplink="${ban_uplink}${uplink} " + ;; + esac fi if [ "${ban_autoallowuplink}" = "subnet" ]; then network_get_subnet6 uplink "${iface}" elif [ "${ban_autoallowuplink}" = "ip" ]; then network_get_ipaddr6 uplink "${iface}" fi - if [ -n "${uplink}" ] && ! printf " %s " "${ban_uplink}" | "${ban_grepcmd}" -q " ${uplink} "; then - ban_uplink="${ban_uplink}${uplink} " + if [ -n "${uplink%fe80::*}" ]; then + case " ${ban_uplink} " in + *" ${uplink} "*) ;; + + *) + ban_uplink="${ban_uplink}${uplink} " + ;; + esac + fi + done + ban_uplink="$(f_trim "${ban_uplink}")" + for ip in ${ban_uplink}; do + if ! "${ban_grepcmd}" -q "${ip} " "${ban_allowlist}"; then + "${ban_sedcmd}" -i "/# uplink added on /d" "${ban_allowlist}" + break fi done + timestamp="$(date "+%Y-%m-%d %H:%M:%S")" for ip in ${ban_uplink}; do if ! "${ban_grepcmd}" -q "${ip} " "${ban_allowlist}"; then - if [ "${update}" = "0" ]; then - "${ban_sedcmd}" -i "/# uplink added on /d" "${ban_allowlist}" - fi - printf "%-45s%s\n" "${ip}" "# uplink added on $(date "+%Y-%m-%d %H:%M:%S")" >>"${ban_allowlist}" + printf '%-45s%s\n' "${ip}" "# uplink added on ${timestamp}" >>"${ban_allowlist}" f_log "info" "add uplink '${ip}' to local allowlist" - update="1" fi done - ban_uplink="$(f_trim "${ban_uplink}")" elif [ "${ban_autoallowlist}" = "1" ] && [ "${ban_autoallowuplink}" = "disable" ]; then "${ban_sedcmd}" -i "/# uplink added on /d" "${ban_allowlist}" - update="1" fi - f_log "debug" "f_getuplink ::: auto/update: ${ban_autoallowlist}/${update}, uplink: ${ban_uplink:-"-"}" + f_log "debug" "f_getup ::: auto-allow/auto-uplink: ${ban_autoallowlist}/${ban_autoallowuplink}, uplink: ${ban_uplink:-"-"}" } # get feed information @@ -567,145 +797,298 @@ f_getfeed() { f_getelements() { local file="${1}" - [ -s "${file}" ] && printf "%s" "elements={ $("${ban_catcmd}" "${file}" 2>/dev/null) };" + [ -s "${file}" ] && printf '%s' "elements={ $("${ban_catcmd}" "${file}" 2>>"${ban_errorlog}") };" } # handle etag http header # f_etag() { - local http_head http_code etag_id etag_rc out_rc="4" feed="${1}" feed_url="${2}" feed_suffix="${3}" + local http_head http_code etag_id etag_cnt etag_match result out_rc="4" feed="${1}" feed_url="${2}" feed_suffix="${3}" feed_cnt="${4:-"1"}" if [ -n "${ban_etagparm}" ]; then + + # ensure etag file exists + # [ ! -f "${ban_backupdir}/banIP.etag" ] && : >"${ban_backupdir}/banIP.etag" + + # fetch http headers and extract http code and etag/last-modified header + # http_head="$("${ban_fetchcmd}" ${ban_etagparm} "${feed_url}" 2>&1)" - http_code="$(printf "%s" "${http_head}" | "${ban_awkcmd}" 'tolower($0)~/^http\/[0123\.]+ /{printf "%s",$2}')" - etag_id="$(printf "%s" "${http_head}" | "${ban_awkcmd}" 'tolower($0)~/^[[:space:]]*etag: /{gsub("\"","");printf "%s",$2}')" - etag_rc="${?}" + http_code="$(printf '%s' "${http_head}" | "${ban_awkcmd}" 'tolower($0)~/^[[:space:]]*http\/[0123\.]+ /{code=$2} END{printf "%s",code}')" + etag_id="$(printf '%s' "${http_head}" | "${ban_awkcmd}" 'tolower($0)~/^[[:space:]]*etag: /{gsub(/[\r"]/,"");id=$2} END{printf "%s",id}')" + + # if etag header is not present, try to use last-modified header as fallback for change detection + # + if [ -z "${etag_id}" ]; then + etag_id="$(printf '%s' "${http_head}" | "${ban_awkcmd}" 'tolower($0)~/^[[:space:]]*last-modified: /{gsub(/[Ll]ast-[Mm]odified:|[[:space:]]|,|:/,"");lm=$1} END{printf "%s",lm}')" + fi - if [ "${http_code}" = "404" ] || { [ "${etag_rc}" = "0" ] && [ -n "${etag_id}" ] && "${ban_grepcmd}" -q "^${feed}${feed_suffix}[[:space:]]\+${etag_id}\$" "${ban_backupdir}/banIP.etag"; }; then + # acquire exclusive lock on etag file to serialize concurrent read-modify-write from parallel feeds + # + exec 9>"${ban_etaglock}" + "${ban_flockcmd}" -x 9 + + # compare http code and etag id with stored values, update etag file and return code accordingly + # + result="$("${ban_awkcmd}" -v f="${feed}" -v s="${feed_suffix}" -v e="${etag_id}" ' + BEGIN { p = f " " s; pl = length(p); m = 1 } + $1 == f { n++ } + index($0, p) == 1 { + rest = substr($0, pl + 1) + sub(/^[[:space:]]+/, "", rest) + if (rest == e) m = 0 + } + END { print n+0, m }' "${ban_backupdir}/banIP.etag")" + etag_cnt="${result% *}" + etag_match="${result#* }" + + if [ "${http_code}" = "200" ] && [ "${etag_cnt}" = "${feed_cnt}" ] && [ -n "${etag_id}" ] && [ "${etag_match}" = "0" ]; then out_rc="0" - elif [ "${etag_rc}" = "0" ] && [ -n "${etag_id}" ] && ! "${ban_grepcmd}" -q "^${feed}${feed_suffix}[[:space:]]\+${etag_id}\$" "${ban_backupdir}/banIP.etag"; then - "${ban_sedcmd}" -i "/^${feed}${feed_suffix}/d" "${ban_backupdir}/banIP.etag" - printf "%-20s%s\n" "${feed}${feed_suffix}" "${etag_id}" >>"${ban_backupdir}/banIP.etag" + elif [ -n "${etag_id}" ]; then + + # if feed count is less than etag count, it means the feed source has been removed or disabled, so remove all entries for this feed, + # otherwise only remove the entry with the matching feed suffix (feed url) to allow multiple sources for the same feed + # + if [ "${feed_cnt}" -lt "${etag_cnt}" ]; then + "${ban_awkcmd}" -v f="${feed}" '$1 != f' \ + "${ban_backupdir}/banIP.etag" >"${ban_backupdir}/banIP.etag.new" + else + "${ban_awkcmd}" -v f="${feed}" -v s="${feed_suffix}" ' + BEGIN { p = f " " s } + index($0, p) != 1' "${ban_backupdir}/banIP.etag" >"${ban_backupdir}/banIP.etag.new" + fi + "${ban_mvcmd}" -f "${ban_backupdir}/banIP.etag.new" "${ban_backupdir}/banIP.etag" + printf '%s\t%s\n' "${feed} ${feed_suffix}" "${etag_id}" >>"${ban_backupdir}/banIP.etag" out_rc="2" fi + + # release lock + # + exec 9>&- fi - f_log "debug" "f_etag ::: feed: ${feed}, suffix: ${feed_suffix:-"-"}, http_code: ${http_code:-"-"}, etag_id: ${etag_id:-"-"} , etag_rc: ${etag_rc:-"-"}, rc: ${out_rc}" + f_log "debug" "f_etag ::: feed: ${feed}, suffix: ${feed_suffix:-"-"}, http_code: ${http_code:-"-"}, feed/etag: ${feed_cnt}/${etag_cnt:-"0"}, rc: ${out_rc}" return "${out_rc}" } +# load file in nftset +# +f_nftload() { + local cnt="1" max_cnt="${ban_nftretry:-"3"}" load_rc="4" file="${1}" errmsg="${2}" + + while [ "${load_rc}" != "0" ]; do + "${ban_nftcmd}" -f "${file}" >/dev/null 2>&1 + load_rc="${?}" + if [ "${load_rc}" = "0" ]; then + break + elif [ "${cnt}" = "${max_cnt}" ]; then + [ ! -d "${ban_errordir}" ] && f_mkdir "${ban_errordir}" + "${ban_catcmd}" "${file}" 2>>"${ban_errorlog}" >"${ban_errordir}/err.${file##*/}" + f_log "info" "${errmsg}" + break + fi + cnt="$((cnt + 1))" + done + + f_log "debug" "f_nftload ::: file: ${file##*/}, load_rc: ${load_rc}, cnt/max_cnt: ${cnt}/${max_cnt}" + return "${load_rc}" +} + # build initial nft file with base table, chains and rules # f_nftinit() { - local wan_dev vlan_allow vlan_block log_ct log_icmp log_syn log_udp log_tcp feed_log feed_rc flag tmp_proto tmp_port allow_dport file="${1}" + local wan_dev vlan_allow vlan_block log_ct log_icmp log_syn log_udp log_tcp nft_cnt flag tmp_proto tmp_port allow_dport feed_rc="0" file="${1}" - wan_dev="$(printf "%s" "${ban_dev}" | "${ban_sedcmd}" 's/^/\"/;s/$/\"/;s/ /\", \"/g')" - [ -n "${ban_vlanallow}" ] && vlan_allow="$(printf "%s" "${ban_vlanallow%%?}" | "${ban_sedcmd}" 's/^/\"/;s/$/\"/;s/ /\", \"/g')" - [ -n "${ban_vlanblock}" ] && vlan_block="$(printf "%s" "${ban_vlanblock%%?}" | "${ban_sedcmd}" 's/^/\"/;s/$/\"/;s/ /\", \"/g')" + # format wan devices, allowed and blocked vlans as nft sets + # + wan_dev="$(printf '%s' "${ban_dev}" | "${ban_sedcmd}" 's/^/\"/;s/$/\"/;s/ /\", \"/g')" + [ -n "${ban_vlanallow}" ] && vlan_allow="$(printf '%s' "${ban_vlanallow%%?}" | "${ban_sedcmd}" 's/^/\"/;s/$/\"/;s/ /\", \"/g')" + [ -n "${ban_vlanblock}" ] && vlan_block="$(printf '%s' "${ban_vlanblock%%?}" | "${ban_sedcmd}" 's/^/\"/;s/$/\"/;s/ /\", \"/g')" + + # set nft counter flag + # + if [ "${ban_nftcount}" = "1" ]; then + nft_cnt="counter" + fi + # format allowed protocols and ports + # for flag in ${ban_allowflag}; do - if [ "${flag}" = "tcp" ] || [ "${flag}" = "udp" ]; then + case "${flag}" in + "tcp" | "udp") if [ -z "${tmp_proto}" ]; then tmp_proto="${flag}" - elif ! printf "%s" "${tmp_proto}" | "${ban_grepcmd}" -qw "${flag}"; then - tmp_proto="${tmp_proto}, ${flag}" + else + case ", ${tmp_proto}, " in + *", ${flag}, "*) ;; + + *) + tmp_proto="${tmp_proto}, ${flag}" + ;; + esac fi - elif [ -n "${flag//[![:digit:]-]/}" ]; then + ;; + "${flag//[![:digit:]-]/}") if [ -z "${tmp_port}" ]; then tmp_port="${flag}" - elif ! printf "%s" "${tmp_port}" | "${ban_grepcmd}" -qw "${flag}"; then - tmp_port="${tmp_port}, ${flag}" + else + case ", ${tmp_port}, " in + *", ${flag}, "*) ;; + + *) + tmp_port="${tmp_port}, ${flag}" + ;; + esac fi - fi + ;; + esac done if [ -n "${tmp_proto}" ] && [ -n "${tmp_port}" ]; then allow_dport="meta l4proto { ${tmp_proto} } th dport { ${tmp_port} }" fi + # build log rules for pre-routing chains if enabled, with dynamic log level and prefix + # if [ "${ban_logprerouting}" = "1" ]; then - log_icmp="log level ${ban_nftloglevel} prefix \"banIP/pre-icmp/drop: \"" - log_syn="log level ${ban_nftloglevel} prefix \"banIP/pre-syn/drop: \"" - log_udp="log level ${ban_nftloglevel} prefix \"banIP/pre-udp/drop: \"" - log_tcp="log level ${ban_nftloglevel} prefix \"banIP/pre-tcp/drop: \"" - log_ct="log level ${ban_nftloglevel} prefix \"banIP/pre-ct/drop: \"" + if [ "${ban_logratelimit}" = "0" ]; then + log_icmp="log level ${ban_nftloglevel} prefix \"banIP/pre-icmp/drop: \"" + log_syn="log level ${ban_nftloglevel} prefix \"banIP/pre-syn/drop: \"" + log_udp="log level ${ban_nftloglevel} prefix \"banIP/pre-udp/drop: \"" + log_tcp="log level ${ban_nftloglevel} prefix \"banIP/pre-tcp/drop: \"" + log_ct="log level ${ban_nftloglevel} prefix \"banIP/pre-ct/drop: \"" + else + log_icmp="limit name \"loglimit\" log level ${ban_nftloglevel} prefix \"banIP/pre-icmp/drop: \"" + log_syn="limit name \"loglimit\" log level ${ban_nftloglevel} prefix \"banIP/pre-syn/drop: \"" + log_udp="limit name \"loglimit\" log level ${ban_nftloglevel} prefix \"banIP/pre-udp/drop: \"" + log_tcp="limit name \"loglimit\" log level ${ban_nftloglevel} prefix \"banIP/pre-tcp/drop: \"" + log_ct="limit name \"loglimit\" log level ${ban_nftloglevel} prefix \"banIP/pre-ct/drop: \"" + fi fi { - # nft header (tables and chains) + # nft header, optional table cleanup, base table, chains, counters and default rules # - printf "%s\n\n" "#!${ban_nftcmd} -f" - if "${ban_nftcmd}" -t list set inet banIP allowlistv4MAC >/dev/null 2>&1; then - printf "%s\n" "delete table inet banIP" - fi - printf "%s\n" "add table inet banIP" - printf "%s\n" "add counter inet banIP cnt-icmpflood" - printf "%s\n" "add counter inet banIP cnt-udpflood" - printf "%s\n" "add counter inet banIP cnt-synflood" - printf "%s\n" "add counter inet banIP cnt-tcpinvalid" - printf "%s\n" "add counter inet banIP cnt-ctinvalid" - printf "%s\n" "add chain inet banIP pre-routing { type filter hook prerouting priority -150; policy accept; }" - printf "%s\n" "add chain inet banIP wan-input { type filter hook input priority ${ban_nftpriority}; policy accept; }" - printf "%s\n" "add chain inet banIP wan-forward { type filter hook forward priority ${ban_nftpriority}; policy accept; }" - printf "%s\n" "add chain inet banIP lan-forward { type filter hook forward priority ${ban_nftpriority}; policy accept; }" - printf "%s\n" "add chain inet banIP reject-chain" - - # default reject chain rules + printf '%s\n' "#!${ban_nftcmd} -f" "" + "${ban_nftcmd}" -t list table inet banIP >/dev/null 2>&1 && printf '%s\n' "delete table inet banIP" + printf '%s\n' "add table inet banIP" + [ "${ban_logratelimit}" != "0" ] && printf '%s\n' "add limit inet banIP loglimit { rate ${ban_logratelimit}/second burst ${ban_logburstlimit} packets }" + printf '%s\n' \ + "add chain inet banIP pre-routing { type filter hook prerouting priority -175; policy accept; }" \ + "add chain inet banIP wan-input { type filter hook input priority ${ban_nftpriority}; policy accept; }" \ + "add chain inet banIP wan-forward { type filter hook forward priority ${ban_nftpriority}; policy accept; }" \ + "add chain inet banIP lan-forward { type filter hook forward priority ${ban_nftpriority}; policy accept; }" \ + "add chain inet banIP _inbound" \ + "add chain inet banIP _outbound" \ + "add chain inet banIP _reject" \ + "add counter inet banIP cnt_icmpflood" \ + "add counter inet banIP cnt_udpflood" \ + "add counter inet banIP cnt_synflood" \ + "add counter inet banIP cnt_tcpinvalid" \ + "add counter inet banIP cnt_ctinvalid" \ + "add counter inet banIP cnt_bcp38" \ + "add rule inet banIP _reject iifname != { ${wan_dev} } meta l4proto tcp reject with tcp reset" \ + "add rule inet banIP _reject reject with icmpx host-unreachable" \ + "add rule inet banIP pre-routing iifname != { ${wan_dev} } ${nft_cnt} accept" + + # ct state invalid # - printf "%s\n" "add rule inet banIP reject-chain meta l4proto tcp reject with tcp reset" - printf "%s\n" "add rule inet banIP reject-chain reject" + [ "${ban_logprerouting}" = "1" ] && + printf '%s\n' "add rule inet banIP pre-routing ct state invalid ${log_ct}" + printf '%s\n' "add rule inet banIP pre-routing ct state invalid counter name cnt_ctinvalid drop" - # default pre-routing rules + # ICMP Flood # - printf "%s\n" "add rule inet banIP pre-routing iifname != { ${wan_dev} } counter accept" - printf "%s\n" "add rule inet banIP pre-routing ct state invalid ${log_ct} counter name cnt-ctinvalid drop" if [ "${ban_icmplimit}" -gt "0" ]; then - printf "%s\n" "add rule inet banIP pre-routing ip protocol icmp limit rate over ${ban_icmplimit}/second ${log_icmp} counter name cnt-icmpflood drop" - printf "%s\n" "add rule inet banIP pre-routing ip6 nexthdr icmpv6 limit rate over ${ban_icmplimit}/second ${log_icmp} counter name cnt-icmpflood drop" + [ "${ban_logprerouting}" = "1" ] && + printf '%s\n' "add rule inet banIP pre-routing meta nfproto . meta l4proto { ipv4 . icmp , ipv6 . icmpv6 } limit rate over ${ban_icmplimit}/second ${log_icmp}" + printf '%s\n' "add rule inet banIP pre-routing meta nfproto . meta l4proto { ipv4 . icmp , ipv6 . icmpv6 } limit rate over ${ban_icmplimit}/second counter name cnt_icmpflood drop" + fi + + # UDP Flood + # + if [ "${ban_udplimit}" -gt "0" ]; then + [ "${ban_logprerouting}" = "1" ] && + printf '%s\n' "add rule inet banIP pre-routing meta l4proto udp ct state new limit rate over ${ban_udplimit}/second ${log_udp}" + printf '%s\n' "add rule inet banIP pre-routing meta l4proto udp ct state new limit rate over ${ban_udplimit}/second counter name cnt_udpflood drop" + fi + + # SYN Flood + # + if [ "${ban_synlimit}" -gt "0" ]; then + [ "${ban_logprerouting}" = "1" ] && + printf '%s\n' "add rule inet banIP pre-routing tcp flags & (fin|syn|rst|ack) == syn limit rate over ${ban_synlimit}/second ${log_syn}" + printf '%s\n' "add rule inet banIP pre-routing tcp flags & (fin|syn|rst|ack) == syn limit rate over ${ban_synlimit}/second counter name cnt_synflood drop" + fi + + # TCP Invalid + # + if [ "${ban_logprerouting}" = "1" ]; then + printf '%s\n' \ + "add rule inet banIP pre-routing tcp flags & (fin|syn) == (fin|syn) ${log_tcp}" \ + "add rule inet banIP pre-routing tcp flags & (syn|rst) == (syn|rst) ${log_tcp}" \ + "add rule inet banIP pre-routing tcp flags & (fin|syn|rst|psh|ack|urg) < (fin) ${log_tcp}" \ + "add rule inet banIP pre-routing tcp flags & (fin|syn|rst|psh|ack|urg) == (fin|psh|urg) ${log_tcp}" fi - [ "${ban_udplimit}" -gt "0" ] && printf "%s\n" "add rule inet banIP pre-routing meta l4proto udp ct state new limit rate over ${ban_udplimit}/second ${log_udp} counter name cnt-udpflood drop" - [ "${ban_synlimit}" -gt "0" ] && printf "%s\n" "add rule inet banIP pre-routing tcp flags & (fin|syn|rst|ack) == syn limit rate over ${ban_synlimit}/second ${log_syn} counter name cnt-synflood drop" - printf "%s\n" "add rule inet banIP pre-routing tcp flags & (fin|syn) == (fin|syn) ${log_tcp} counter name cnt-tcpinvalid drop" - printf "%s\n" "add rule inet banIP pre-routing tcp flags & (syn|rst) == (syn|rst) ${log_tcp} counter name cnt-tcpinvalid drop" - printf "%s\n" "add rule inet banIP pre-routing tcp flags & (fin|syn|rst|psh|ack|urg) < (fin) ${log_tcp} counter name cnt-tcpinvalid drop" - printf "%s\n" "add rule inet banIP pre-routing tcp flags & (fin|syn|rst|psh|ack|urg) == (fin|psh|urg) ${log_tcp} counter name cnt-tcpinvalid drop" + printf '%s\n' \ + "add rule inet banIP pre-routing tcp flags & (fin|syn) == (fin|syn) counter name cnt_tcpinvalid drop" \ + "add rule inet banIP pre-routing tcp flags & (syn|rst) == (syn|rst) counter name cnt_tcpinvalid drop" \ + "add rule inet banIP pre-routing tcp flags & (fin|syn|rst|psh|ack|urg) < (fin) counter name cnt_tcpinvalid drop" \ + "add rule inet banIP pre-routing tcp flags & (fin|syn|rst|psh|ack|urg) == (fin|psh|urg) counter name cnt_tcpinvalid drop" # default wan-input rules # - printf "%s\n" "add rule inet banIP wan-input iifname != { ${wan_dev} } counter accept" - printf "%s\n" "add rule inet banIP wan-input ct state established,related counter accept" - printf "%s\n" "add rule inet banIP wan-input meta nfproto ipv4 udp sport 67-68 udp dport 67-68 counter accept" - printf "%s\n" "add rule inet banIP wan-input meta nfproto ipv6 udp sport 547 udp dport 546 counter accept" - printf "%s\n" "add rule inet banIP wan-input meta nfproto ipv6 icmpv6 type { nd-neighbor-advert, nd-neighbor-solicit, nd-router-advert} ip6 hoplimit 1 counter accept" - printf "%s\n" "add rule inet banIP wan-input meta nfproto ipv6 icmpv6 type { nd-neighbor-advert, nd-neighbor-solicit, nd-router-advert} ip6 hoplimit 255 counter accept" - [ -n "${allow_dport}" ] && printf "%s\n" "add rule inet banIP wan-input ${allow_dport} counter accept" + printf '%s\n' \ + "add rule inet banIP wan-input ct state established,related ${nft_cnt} accept" \ + "add rule inet banIP wan-input iifname != { ${wan_dev} } ${nft_cnt} accept" \ + "add rule inet banIP wan-input meta nfproto ipv4 udp sport 67-68 udp dport 67-68 ${nft_cnt} accept" \ + "add rule inet banIP wan-input meta nfproto ipv6 udp sport 547 udp dport 546 ${nft_cnt} accept" \ + "add rule inet banIP wan-input meta nfproto ipv6 icmpv6 type { nd-neighbor-solicit, nd-neighbor-advert, nd-router-advert } ip6 hoplimit 255 ${nft_cnt} accept" + [ -n "${allow_dport}" ] && + printf '%s\n' "add rule inet banIP wan-input ${allow_dport} ${nft_cnt} accept" + [ "${ban_bcp38}" = "1" ] && + printf '%s\n' "add rule inet banIP wan-input fib saddr . iif oif missing counter name cnt_bcp38 drop" + if [ "${ban_loginbound}" = "1" ]; then + printf '%s\n' "add rule inet banIP wan-input meta mark set 1 ${nft_cnt} jump _inbound" + else + printf '%s\n' "add rule inet banIP wan-input ${nft_cnt} jump _inbound" + fi # default wan-forward rules # - printf "%s\n" "add rule inet banIP wan-forward iifname != { ${wan_dev} } counter accept" - printf "%s\n" "add rule inet banIP wan-forward ct state established,related counter accept" - [ -n "${allow_dport}" ] && printf "%s\n" "add rule inet banIP wan-forward ${allow_dport} counter accept" + printf '%s\n' \ + "add rule inet banIP wan-forward iifname != { ${wan_dev} } ${nft_cnt} accept" \ + "add rule inet banIP wan-forward ct state established,related ${nft_cnt} accept" + [ -n "${allow_dport}" ] && + printf '%s\n' "add rule inet banIP wan-forward ${allow_dport} ${nft_cnt} accept" + [ "${ban_bcp38}" = "1" ] && + printf '%s\n' "add rule inet banIP wan-forward fib saddr . iif oif missing counter name cnt_bcp38 drop" + if [ "${ban_loginbound}" = "1" ]; then + printf '%s\n' "add rule inet banIP wan-forward meta mark set 2 ${nft_cnt} jump _inbound" + else + printf '%s\n' "add rule inet banIP wan-forward ${nft_cnt} jump _inbound" + fi # default lan-forward rules # - printf "%s\n" "add rule inet banIP lan-forward oifname != { ${wan_dev} } counter accept" - printf "%s\n" "add rule inet banIP lan-forward ct state established,related counter accept" - [ -n "${vlan_allow}" ] && printf "%s\n" "add rule inet banIP lan-forward iifname { ${vlan_allow} } counter accept" - [ -n "${vlan_block}" ] && printf "%s\n" "add rule inet banIP lan-forward iifname { ${vlan_block} } counter goto reject-chain" + printf '%s\n' \ + "add rule inet banIP lan-forward ct state established,related ${nft_cnt} accept" \ + "add rule inet banIP lan-forward oifname != { ${wan_dev} } ${nft_cnt} accept" + [ -n "${vlan_allow}" ] && + printf '%s\n' "add rule inet banIP lan-forward iifname { ${vlan_allow} } ${nft_cnt} accept" + [ -n "${vlan_block}" ] && + printf '%s\n' "add rule inet banIP lan-forward iifname { ${vlan_block} } ${nft_cnt} goto _reject" + [ "${ban_bcp38}" = "1" ] && + printf '%s\n' "add rule inet banIP lan-forward fib saddr . iif oif missing counter name cnt_bcp38 drop" + printf '%s\n' "add rule inet banIP lan-forward ${nft_cnt} jump _outbound" } >"${file}" - # load initial banIP table within nft (atomic load) + # load initial banIP table/rules to nftset # - feed_log="$("${ban_nftcmd}" -f "${file}" 2>&1)" + f_nftload "${file}" "can't initialize banIP nftables namespace" feed_rc="${?}" + [ "${feed_rc}" = "0" ] && f_log "info" "initialize banIP nftables namespace" - if [ "${feed_rc}" = "0" ]; then - f_log "info" "initialize banIP nftables namespace" - else - f_log "err" "can't initialize banIP nftables namespace (rc: ${feed_rc}, log: ${feed_log})" - fi - - f_log "debug" "f_nftinit ::: wan_dev: ${wan_dev}, vlan_allow: ${vlan_allow:-"-"}, vlan_block: ${vlan_block:-"-"}, allowed_dports: ${allow_dport:-"-"}, priority: ${ban_nftpriority}, policy: ${ban_nftpolicy}, icmp_limit: ${ban_icmplimit}, syn_limit: ${ban_synlimit}, udp_limit: ${ban_udplimit}, loglevel: ${ban_nftloglevel}, rc: ${feed_rc:-"-"}, log: ${feed_log:-"-"}" + f_log "debug" "f_nftinit ::: wan_dev: ${wan_dev}, vlan_allow: ${vlan_allow:-"-"}, vlan_block: ${vlan_block:-"-"}, allowed_dports: ${allow_dport:-"-"}, priority: ${ban_nftpriority}, policy: ${ban_nftpolicy}, icmp_limit: ${ban_icmplimit}, syn_limit: ${ban_synlimit}, udp_limit: ${ban_udplimit}, loglevel: ${ban_nftloglevel}, rc: ${feed_rc:-"-"}" : >"${file}" return "${feed_rc}" } @@ -713,150 +1096,241 @@ f_nftinit() { # handle downloads # f_down() { - local log_input log_forwardwan log_forwardlan start_ts end_ts tmp_raw tmp_load tmp_file split_file ruleset_raw handle rc etag_rc - local expr cnt_set cnt_dl restore_rc feed_direction feed_rc feed_log feed_comp feed_target feed_dport tmp_proto tmp_port flag - local feed="${1}" proto="${2}" feed_url="${3}" feed_rule="${4}" feed_flag="${5}" + local log_inbound log_outbound start_ts end_ts tmp_raw tmp_load tmp_file split_file table_json handles handle etag_rc etag_cnt nft_cnt nft_setcnt tmp_allow feed_name + local expr cnt_set cnt_dl restore_rc feed_direction feed_policy feed_rc feed_comp feed_complete feed_target feed_dport chain flag tmp_flush tmp_nft + local tmp_split tmp_proto tmp_port asn country feed="${1}" feed_ipv="${2}" feed_url="${3}" feed_rule="${4}" feed_chain="${5}" feed_flag="${6}" - start_ts="$(date +%s)" - feed="${feed}v${proto}" + # get feed start timestamp and prepare temporary file names based on feed name and type + # + read -r start_ts _ <"/proc/uptime" + start_ts="${start_ts%%.*}" + feed="${feed}.v${feed_ipv}" tmp_load="${ban_tmpfile}.${feed}.load" tmp_raw="${ban_tmpfile}.${feed}.raw" tmp_split="${ban_tmpfile}.${feed}.split" tmp_file="${ban_tmpfile}.${feed}.file" tmp_flush="${ban_tmpfile}.${feed}.flush" tmp_nft="${ban_tmpfile}.${feed}.nft" - tmp_allow="${ban_tmpfile}.${feed%v*}" + tmp_allow="${ban_tmpfile}.${feed%.*}" - [ "${ban_loginput}" = "1" ] && log_input="log level ${ban_nftloglevel} prefix \"banIP/inp-wan/${ban_blocktype}/${feed}: \"" - [ "${ban_logforwardwan}" = "1" ] && log_forwardwan="log level ${ban_nftloglevel} prefix \"banIP/fwd-wan/${ban_blocktype}/${feed}: \"" - [ "${ban_logforwardlan}" = "1" ] && log_forwardlan="log level ${ban_nftloglevel} prefix \"banIP/fwd-lan/reject/${feed}: \"" + # set log target + # + if [ "${ban_logratelimit}" = "0" ]; then + [ "${ban_loginbound}" = "1" ] && log_inbound="log level ${ban_nftloglevel} prefix \"banIP/inbound/${ban_blockpolicy}/${feed}: \"" + [ "${ban_logoutbound}" = "1" ] && log_outbound="log level ${ban_nftloglevel} prefix \"banIP/outbound/reject/${feed}: \"" + else + [ "${ban_loginbound}" = "1" ] && log_inbound="limit name \"loglimit\" log level ${ban_nftloglevel} prefix \"banIP/inbound/${ban_blockpolicy}/${feed}: \"" + [ "${ban_logoutbound}" = "1" ] && log_outbound="limit name \"loglimit\" log level ${ban_nftloglevel} prefix \"banIP/outbound/reject/${feed}: \"" + fi # set feed target # - if [ "${ban_blocktype}" = "reject" ]; then - feed_target="goto reject-chain" + if [ "${ban_blockpolicy}" = "reject" ]; then + feed_target="goto _reject" else feed_target="drop" fi - # set feed block direction + # set nft counter flag # - if [ "${ban_blockpolicy}" = "input" ]; then - if ! printf "%s" "${ban_blockinput}" | "${ban_grepcmd}" -q "${feed%v*}" && - ! printf "%s" "${ban_blockforwardwan}" | "${ban_grepcmd}" -q "${feed%v*}" && - ! printf "%s" "${ban_blockforwardlan}" | "${ban_grepcmd}" -q "${feed%v*}"; then - ban_blockinput="${ban_blockinput} ${feed%v*}" - fi - elif [ "${ban_blockpolicy}" = "forwardwan" ]; then - if ! printf "%s" "${ban_blockinput}" | "${ban_grepcmd}" -q "${feed%v*}" && - ! printf "%s" "${ban_blockforwardwan}" | "${ban_grepcmd}" -q "${feed%v*}" && - ! printf "%s" "${ban_blockforwardlan}" | "${ban_grepcmd}" -q "${feed%v*}"; then - ban_blockforwardwan="${ban_blockforwardwan} ${feed%v*}" - fi - elif [ "${ban_blockpolicy}" = "forwardlan" ]; then - if ! printf "%s" "${ban_blockinput}" | "${ban_grepcmd}" -q "${feed%v*}" && - ! printf "%s" "${ban_blockforwardwan}" | "${ban_grepcmd}" -q "${feed%v*}" && - ! printf "%s" "${ban_blockforwardlan}" | "${ban_grepcmd}" -q "${feed%v*}"; then - ban_blockforwardlan="${ban_blockforwardlan} ${feed%v*}" - fi + if [ "${ban_nftcount}" = "1" ]; then + nft_cnt="counter" + nft_setcnt="; counter" fi - if printf "%s" "${ban_blockinput}" | "${ban_grepcmd}" -q "${feed%v*}"; then - feed_direction="input" - fi - if printf "%s" "${ban_blockforwardwan}" | "${ban_grepcmd}" -q "${feed%v*}"; then - feed_direction="${feed_direction} forwardwan" - fi - if printf "%s" "${ban_blockforwardlan}" | "${ban_grepcmd}" -q "${feed%v*}"; then - feed_direction="${feed_direction} forwardlan" + + # set feed complete flag + # + case " ${ban_feedcomplete} " in + *" ${feed%%.*} "*) + feed_complete="true" + ;; + esac + + # set feed direction + # + feed_name="${feed%%.*}" + if case " ${ban_feedin} " in + *" ${feed_name} "*) + true + ;; + *) + false + ;; + esac; then + feed_policy="in" + feed_direction="inbound" + elif case " ${ban_feedout} " in + *" ${feed_name} "*) + true + ;; + *) + false + ;; + esac; then + feed_policy="out" + feed_direction="outbound" + elif case " ${ban_feedinout} " in + *" ${feed_name} "*) + true + ;; + *) + false + ;; + esac; then + feed_policy="inout" + feed_direction="inbound outbound" + else + feed_policy="${feed_chain}" + case "${feed_chain}" in + "in") + feed_direction="inbound" + ;; + "out") + feed_direction="outbound" + ;; + "inout") + feed_direction="inbound outbound" + ;; + *) + feed_direction="inbound" + ;; + esac fi # prepare feed flags # for flag in ${feed_flag}; do - if [ "${flag}" = "gz" ]; then + case "${flag}" in + "gz") feed_comp="${flag}" - elif [ "${flag}" = "tcp" ] || [ "${flag}" = "udp" ]; then + ;; + "tcp" | "udp") if [ -z "${tmp_proto}" ]; then tmp_proto="${flag}" - elif ! printf "%s" "${tmp_proto}" | "${ban_grepcmd}" -qw "${flag}"; then - tmp_proto="${tmp_proto}, ${flag}" + else + case ", ${tmp_proto}, " in + *", ${flag}, "*) ;; + + *) + tmp_proto="${tmp_proto}, ${flag}" + ;; + esac fi - elif [ -n "${flag//[![:digit:]-]/}" ]; then + ;; + "${flag//[![:digit:]-]/}") if [ -z "${tmp_port}" ]; then tmp_port="${flag}" - elif ! printf "%s" "${tmp_port}" | "${ban_grepcmd}" -qw "${flag}"; then - tmp_port="${tmp_port}, ${flag}" + else + case ", ${tmp_port}, " in + *", ${flag}, "*) ;; + + *) + tmp_port="${tmp_port}, ${flag}" + ;; + esac fi - fi + ;; + esac done - if [ -n "${tmp_proto}" ] && [ -n "${tmp_port}" ]; then - feed_dport="meta l4proto { ${tmp_proto} } th dport { ${tmp_port} }" - fi + + case " ${ban_feedreset} " in + *" ${feed%%.*} "*) ;; + + *) + if [ -n "${tmp_proto}" ] && [ -n "${tmp_port}" ]; then + feed_dport="meta l4proto { ${tmp_proto} } th dport { ${tmp_port} }" + fi + ;; + esac # chain/rule maintenance # if [ "${ban_action}" = "reload" ] && "${ban_nftcmd}" -t list set inet banIP "${feed}" >/dev/null 2>&1; then - ruleset_raw="$("${ban_nftcmd}" -tj list ruleset 2>/dev/null)" + table_json="$("${ban_nftcmd}" -tja list table inet banIP 2>>"${ban_errorlog}")" { - printf "%s\n" "flush set inet banIP ${feed}" - for expr in 0 1; do - handle="$(printf "%s\n" "${ruleset_raw}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.table=\"banIP\"&&@.rule.chain=\"wan-input\"][@.expr[${expr}].match.right=\"@${feed}\"].handle")" - [ -n "${handle}" ] && printf "%s\n" "delete rule inet banIP wan-input handle ${handle}" - handle="$(printf "%s\n" "${ruleset_raw}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.table=\"banIP\"&&@.rule.chain=\"wan-forward\"][@.expr[${expr}].match.right=\"@${feed}\"].handle")" - [ -n "${handle}" ] && printf "%s\n" "delete rule inet banIP wan-forward handle ${handle}" - handle="$(printf "%s\n" "${ruleset_raw}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.table=\"banIP\"&&@.rule.chain=\"lan-forward\"][@.expr[${expr}].match.right=\"@${feed}\"].handle")" - [ -n "${handle}" ] && printf "%s\n" "delete rule inet banIP lan-forward handle ${handle}" + for chain in _inbound _outbound; do + for expr in 0 1 2; do + handles="$(printf '%s\n' "${table_json}" | "${ban_jsoncmd}" -q -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[${expr}].match.right=\"@${feed}\"].handle" | "${ban_xargscmd}")" + for handle in ${handles}; do + printf '%s\n' "delete rule inet banIP ${chain} handle ${handle}" + done + done done + printf '%s\n' "flush set inet banIP ${feed}" + printf '%s\n\n' "delete set inet banIP ${feed}" } >"${tmp_flush}" fi # restore local backups # - if [ "${feed%v*}" != "blocklist" ]; then - if [ -n "${ban_etagparm}" ] && [ "${ban_action}" = "reload" ] && [ "${feed_url}" != "local" ] && [ "${feed%v*}" != "allowlist" ]; then + if [ "${feed%%.*}" != "blocklist" ]; then + if [ -n "${ban_etagparm}" ] && [ "${ban_action}" = "reload" ] && [ "${feed_url}" != "local" ] && [ "${feed%%.*}" != "allowlist" ]; then etag_rc="0" - if [ "${feed%v*}" = "country" ]; then - for country in ${ban_country}; do + case "${feed%%.*}" in + "country") + if [ "${ban_countrysplit}" = "1" ]; then + country="${feed%.*}" + country="${country#*.}" f_etag "${feed}" "${feed_url}${country}-aggregated.zone" ".${country}" - rc="${?}" - [ "${rc}" = "4" ] && break - etag_rc="$((etag_rc + rc))" - done - elif [ "${feed%v*}" = "asn" ]; then - for asn in ${ban_asn}; do + etag_rc="${?}" + else + etag_rc="0" + etag_cnt="$(printf '%s' "${ban_country}" | "${ban_wccmd}" -w)" + for country in ${ban_country}; do + if ! f_etag "${feed}" "${feed_url}${country}-aggregated.zone" ".${country}" "${etag_cnt}"; then + etag_rc="$((etag_rc + 1))" + fi + done + fi + ;; + "asn") + if [ "${ban_asnsplit}" = "1" ]; then + asn="${feed%.*}" + asn="${asn#*.}" f_etag "${feed}" "${feed_url}AS${asn}" ".${asn}" - rc="${?}" - [ "${rc}" = "4" ] && break - etag_rc="$((etag_rc + rc))" - done - else + etag_rc="${?}" + else + etag_rc="0" + etag_cnt="$(printf '%s' "${ban_asn}" | "${ban_wccmd}" -w)" + for asn in ${ban_asn}; do + if ! f_etag "${feed}" "${feed_url}AS${asn}" ".${asn}" "${etag_cnt}"; then + etag_rc="$((etag_rc + 1))" + fi + done + fi + ;; + *) f_etag "${feed}" "${feed_url}" etag_rc="${?}" - fi + ;; + esac fi if [ "${etag_rc}" = "0" ] || [ "${ban_action}" != "reload" ] || [ "${feed_url}" = "local" ]; then - if [ "${feed%v*}" = "allowlist" ] && [ ! -f "${tmp_allow}" ]; then + if [ "${feed%%.*}" = "allowlist" ] && [ ! -f "${tmp_allow}" ]; then f_restore "allowlist" "-" "${tmp_allow}" "${etag_rc}" + restore_rc="${?}" else f_restore "${feed}" "${feed_url}" "${tmp_load}" "${etag_rc}" + restore_rc="${?}" fi - restore_rc="${?}" feed_rc="${restore_rc}" fi fi # prepare local/remote allowlist # - if [ "${feed%v*}" = "allowlist" ] && [ ! -f "${tmp_allow}" ]; then - "${ban_catcmd}" "${ban_allowlist}" 2>/dev/null >"${tmp_allow}" + if [ "${feed%%.*}" = "allowlist" ] && [ ! -f "${tmp_allow}" ]; then + "${ban_catcmd}" "${ban_allowlist}" 2>>"${ban_errorlog}" >"${tmp_allow}" feed_rc="${?}" for feed_url in ${ban_allowurl}; do - feed_log="$("${ban_fetchcmd}" ${ban_fetchparm} "${tmp_load}" "${feed_url}" 2>&1)" - feed_rc="${?}" - if [ "${feed_rc}" = "0" ] && [ -s "${tmp_load}" ]; then - "${ban_catcmd}" "${tmp_load}" 2>/dev/null >>"${tmp_allow}" + if "${ban_fetchcmd}" ${ban_fetchparm} "${tmp_load}" "${feed_url}" 2>>"${ban_errorlog}"; then + if [ -s "${tmp_load}" ]; then + "${ban_catcmd}" "${tmp_load}" 2>>"${ban_errorlog}" >>"${tmp_allow}" + feed_rc="${?}" + fi else - f_log "info" "download for feed '${feed%v*}' failed (rc: ${feed_rc:-"-"}/log: ${feed_log})" + f_log "info" "download for feed '${feed%%.*}' failed" + feed_rc="4" break fi done @@ -871,118 +1345,131 @@ f_down() { # handle local feeds # - if [ "${feed%v*}" = "allowlist" ]; then + if [ "${feed%%.*}" = "allowlist" ]; then { - printf "%s\n\n" "#!${ban_nftcmd} -f" + printf '%s\n\n' "#!${ban_nftcmd} -f" [ -s "${tmp_flush}" ] && "${ban_catcmd}" "${tmp_flush}" - if [ "${proto}" = "4MAC" ]; then - "${ban_awkcmd}" '/^([0-9A-f]{2}:){5}[0-9A-f]{2}(\/([0-9]|[1-3][0-9]|4[0-8]))?([[:space:]]+([1-9][0-9]?[0-9]?\.){1}([0-9]{1,3}\.){2}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\/(1?[0-9]|2?[0-9]|3?[0-2]))?([[:space:]]+#.*$|[[:space:]]*$)|[[:space:]]+#.*$|$)/{if(!$2||$2~/#/)$2="0.0.0.0/0";if(!seen[$1]++)printf "%s . %s, ",tolower($1),$2}' "${tmp_allow}" >"${tmp_file}" - printf "%s\n" "add set inet banIP ${feed} { type ether_addr . ipv4_addr; flags interval; auto-merge; policy ${ban_nftpolicy}; $(f_getelements "${tmp_file}") }" - [ -z "${feed_direction##*forwardlan*}" ] && printf "%s\n" "add rule inet banIP lan-forward ether saddr . ip saddr @${feed} counter accept" - elif [ "${proto}" = "6MAC" ]; then - "${ban_awkcmd}" '/^([0-9A-f]{2}:){5}[0-9A-f]{2}(\/([0-9]|[1-3][0-9]|4[0-8]))?([[:space:]]+([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\/(1?[0-2][0-8]|[0-9][0-9]))?([[:space:]]+#.*$|[[:space:]]*$)|[[:space:]]+#.*$|$)/{if(!$2||$2~/#/)$2="::/0";if(!seen[$1]++)printf "%s . %s, ",tolower($1),$2}' "${tmp_allow}" >"${tmp_file}" - printf "%s\n" "add set inet banIP ${feed} { type ether_addr . ipv6_addr; flags interval; auto-merge; policy ${ban_nftpolicy}; $(f_getelements "${tmp_file}") }" - [ -z "${feed_direction##*forwardlan*}" ] && printf "%s\n" "add rule inet banIP lan-forward ether saddr . ip6 saddr @${feed} counter accept" - elif [ "${proto}" = "4" ]; then - "${ban_awkcmd}" '/^127\./{next}/^(([1-9][0-9]?[0-9]?\.){1}([0-9]{1,3}\.){2}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\/(1?[0-9]|2?[0-9]|3?[0-2]))?)([[:space:]].*|$)/{printf "%s, ",$1}' "${tmp_allow}" >"${tmp_file}" - printf "%s\n" "add set inet banIP ${feed} { type ipv4_addr; flags interval; auto-merge; policy ${ban_nftpolicy}; $(f_getelements "${tmp_file}") }" - if [ -z "${feed_direction##*input*}" ]; then + case "${feed_ipv}" in + "4MAC") + "${ban_awkcmd}" '{gsub(/\r/,"")}/^([0-9A-f]{2}:){5}[0-9A-f]{2}(\/([0-9]|[1-3][0-9]|4[0-8]))?([[:space:]]+([1-9][0-9]?[0-9]?\.){1}([0-9]{1,3}\.){2}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\/(1?[0-9]|2?[0-9]|3?[0-2]))?([[:space:]]+#.*$|[[:space:]]*$)|[[:space:]]+#.*$|$)/{if(!$2||$2~/#/)$2="0.0.0.0/0";if(!seen[$1]++)printf "%s . %s, ",tolower($1),$2}' "${tmp_allow}" >"${tmp_file}" + printf '%s\n' "add set inet banIP ${feed} { type ether_addr . ipv4_addr; flags interval; auto-merge; policy ${ban_nftpolicy}${nft_setcnt}; $(f_getelements "${tmp_file}") }" + [ -z "${feed_direction##*outbound*}" ] && printf '%s\n' "add rule inet banIP _outbound ether saddr . ip saddr @${feed} ${nft_cnt} accept" + ;; + "6MAC") + "${ban_awkcmd}" '{gsub(/\r/,"")}/^([0-9A-f]{2}:){5}[0-9A-f]{2}(\/([0-9]|[1-3][0-9]|4[0-8]))?([[:space:]]+([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\/(1?[0-2][0-8]|[0-9][0-9]))?([[:space:]]+#.*$|[[:space:]]*$)|[[:space:]]+#.*$|$)/{if(!$2||$2~/#/)$2="::/0";if(!seen[$1]++)printf "%s . %s, ",tolower($1),$2}' "${tmp_allow}" >"${tmp_file}" + printf '%s\n' "add set inet banIP ${feed} { type ether_addr . ipv6_addr; flags interval; auto-merge; policy ${ban_nftpolicy}${nft_setcnt}; $(f_getelements "${tmp_file}") }" + [ -z "${feed_direction##*outbound*}" ] && printf '%s\n' "add rule inet banIP _outbound ether saddr . ip6 saddr @${feed} ${nft_cnt} accept" + ;; + "4") + f_chkip ${feed_ipv} local 1 <"${tmp_allow}" >"${tmp_file}" + printf '%s\n' "add set inet banIP ${feed} { type ipv4_addr; flags interval; auto-merge; policy ${ban_nftpolicy}${nft_setcnt}; $(f_getelements "${tmp_file}") }" + if [ -z "${feed_direction##*inbound*}" ]; then if [ "${ban_allowlistonly}" = "1" ]; then - printf "%s\n" "add rule inet banIP wan-input ip saddr != @${feed} ${log_input} counter ${feed_target}" + if [ "${ban_loginbound}" = "1" ]; then + printf '%s\n' "add rule inet banIP _inbound ip saddr != @${feed} ${log_inbound}" + fi + printf '%s\n' "add rule inet banIP _inbound ip saddr != @${feed} ${nft_cnt} ${feed_target}" else - printf "%s\n" "add rule inet banIP wan-input ip saddr @${feed} counter accept" + printf '%s\n' "add rule inet banIP _inbound ip saddr @${feed} ${nft_cnt} accept" fi fi - if [ -z "${feed_direction##*forwardwan*}" ]; then + if [ -z "${feed_direction##*outbound*}" ]; then if [ "${ban_allowlistonly}" = "1" ]; then - printf "%s\n" "add rule inet banIP wan-forward ip saddr != @${feed} ${log_forwardwan} counter ${feed_target}" + if [ "${ban_logoutbound}" = "1" ]; then + printf '%s\n' "add rule inet banIP _outbound ip daddr != @${feed} ${log_outbound}" + fi + printf '%s\n' "add rule inet banIP _outbound ip daddr != @${feed} ${nft_cnt} ${feed_target}" else - printf "%s\n" "add rule inet banIP wan-forward ip saddr @${feed} counter accept" + printf '%s\n' "add rule inet banIP _outbound ip daddr @${feed} ${nft_cnt} accept" fi fi - if [ -z "${feed_direction##*forwardlan*}" ]; then + ;; + "6") + f_chkip ${feed_ipv} local 1 <"${tmp_allow}" >"${tmp_file}" + printf '%s\n' "add set inet banIP ${feed} { type ipv6_addr; flags interval; auto-merge; policy ${ban_nftpolicy}${nft_setcnt}; $(f_getelements "${tmp_file}") }" + if [ -z "${feed_direction##*inbound*}" ]; then if [ "${ban_allowlistonly}" = "1" ]; then - printf "%s\n" "add rule inet banIP lan-forward ip daddr != @${feed} ${log_forwardlan} counter goto reject-chain" + if [ "${ban_loginbound}" = "1" ]; then + printf '%s\n' "add rule inet banIP _inbound ip6 saddr != @${feed} ${log_inbound}" + fi + printf '%s\n' "add rule inet banIP _inbound ip6 saddr != @${feed} ${nft_cnt} ${feed_target}" else - printf "%s\n" "add rule inet banIP lan-forward ip daddr @${feed} counter accept" + printf '%s\n' "add rule inet banIP _inbound ip6 saddr @${feed} ${nft_cnt} accept" fi fi - elif [ "${proto}" = "6" ]; then - "${ban_awkcmd}" '!/^([0-9A-f]{2}:){5}[0-9A-f]{2}.*/{printf "%s\n",$1}' "${tmp_allow}" | - "${ban_awkcmd}" '/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\/(1?[0-2][0-8]|[0-9][0-9]))?)([[:space:]].*|$)/{printf "%s, ",tolower($1)}' >"${tmp_file}" - printf "%s\n" "add set inet banIP ${feed} { type ipv6_addr; flags interval; auto-merge; policy ${ban_nftpolicy}; $(f_getelements "${tmp_file}") }" - if [ -z "${feed_direction##*input*}" ]; then + if [ -z "${feed_direction##*outbound*}" ]; then if [ "${ban_allowlistonly}" = "1" ]; then - printf "%s\n" "add rule inet banIP wan-input ip6 saddr != @${feed} ${log_input} counter ${feed_target}" + if [ "${ban_logoutbound}" = "1" ]; then + printf '%s\n' "add rule inet banIP _outbound ip6 daddr != @${feed} ${log_outbound}" + fi + printf '%s\n' "add rule inet banIP _outbound ip6 daddr != @${feed} ${nft_cnt} ${feed_target}" else - printf "%s\n" "add rule inet banIP wan-input ip6 saddr @${feed} counter accept" + printf '%s\n' "add rule inet banIP _outbound ip6 daddr @${feed} ${nft_cnt} accept" fi fi - if [ -z "${feed_direction##*forwardwan*}" ]; then - if [ "${ban_allowlistonly}" = "1" ]; then - printf "%s\n" "add rule inet banIP wan-forward ip6 saddr != @${feed} ${log_forwardwan} counter ${feed_target}" - else - printf "%s\n" "add rule inet banIP wan-forward ip6 saddr @${feed} counter accept" - fi - fi - if [ -z "${feed_direction##*forwardlan*}" ]; then - if [ "${ban_allowlistonly}" = "1" ]; then - printf "%s\n" "add rule inet banIP lan-forward ip6 daddr != @${feed} ${log_forwardlan} counter ${feed_target}" - else - printf "%s\n" "add rule inet banIP lan-forward ip6 daddr @${feed} counter accept" - fi - fi - fi + ;; + esac } >"${tmp_nft}" : >"${tmp_flush}" >"${tmp_raw}" >"${tmp_file}" feed_rc="0" - elif [ "${feed%v*}" = "blocklist" ]; then + elif [ "${feed%%.*}" = "blocklist" ]; then { - printf "%s\n\n" "#!${ban_nftcmd} -f" + printf '%s\n\n' "#!${ban_nftcmd} -f" [ -s "${tmp_flush}" ] && "${ban_catcmd}" "${tmp_flush}" - if [ "${proto}" = "4MAC" ]; then - "${ban_awkcmd}" '/^([0-9A-f]{2}:){5}[0-9A-f]{2}(\/([0-9]|[1-3][0-9]|4[0-8]))?([[:space:]]+([1-9][0-9]?[0-9]?\.){1}([0-9]{1,3}\.){2}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\/(1?[0-9]|2?[0-9]|3?[0-2]))?([[:space:]]+#.*$|[[:space:]]*$)|[[:space:]]+#.*$|$)/{if(!$2||$2~/#/)$2="0.0.0.0/0";if(!seen[$1]++)printf "%s . %s, ",tolower($1),$2}' "${ban_blocklist}" >"${tmp_file}" - printf "%s\n" "add set inet banIP ${feed} { type ether_addr . ipv4_addr; flags interval; auto-merge; policy ${ban_nftpolicy}; $(f_getelements "${tmp_file}") }" - [ -z "${feed_direction##*forwardlan*}" ] && printf "%s\n" "add rule inet banIP lan-forward ether saddr . ip saddr @${feed} counter goto reject-chain" - elif [ "${proto}" = "6MAC" ]; then - "${ban_awkcmd}" '/^([0-9A-f]{2}:){5}[0-9A-f]{2}(\/([0-9]|[1-3][0-9]|4[0-8]))?([[:space:]]+([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\/(1?[0-2][0-8]|[0-9][0-9]))?([[:space:]]+#.*$|[[:space:]]*$)|[[:space:]]+#.*$|$)/{if(!$2||$2~/#/)$2="::/0";if(!seen[$1]++)printf "%s . %s, ",tolower($1),$2}' "${ban_blocklist}" >"${tmp_file}" - printf "%s\n" "add set inet banIP ${feed} { type ether_addr . ipv6_addr; flags interval; auto-merge; policy ${ban_nftpolicy}; $(f_getelements "${tmp_file}") }" - [ -z "${feed_direction##*forwardlan*}" ] && printf "%s\n" "add rule inet banIP lan-forward ether saddr . ip6 saddr @${feed} counter goto reject-chain" - elif [ "${proto}" = "4" ]; then - if [ "${ban_deduplicate}" = "1" ]; then - "${ban_awkcmd}" '/^127\./{next}/^(([1-9][0-9]?[0-9]?\.){1}([0-9]{1,3}\.){2}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\/(1?[0-9]|2?[0-9]|3?[0-2]))?)([[:space:]].*|$)/{printf "%s,\n",$1}' "${ban_blocklist}" >"${tmp_raw}" - "${ban_awkcmd}" 'NR==FNR{member[$0];next}!($0 in member)' "${ban_tmpfile}.deduplicate" "${tmp_raw}" 2>/dev/null >"${tmp_split}" - "${ban_awkcmd}" 'BEGIN{FS="[ ,]"}NR==FNR{member[$1];next}!($1 in member)' "${ban_tmpfile}.deduplicate" "${ban_blocklist}" 2>/dev/null >"${tmp_raw}" - if ! cmp -s "${tmp_raw}" "${ban_blocklist}" 2>/dev/null; then - "${ban_catcmd}" "${tmp_raw}" 2>/dev/null >"${ban_blocklist}" + case "${feed_ipv}" in + "4MAC") + "${ban_awkcmd}" '{gsub(/\r/,"")}/^([0-9A-f]{2}:){5}[0-9A-f]{2}(\/([0-9]|[1-3][0-9]|4[0-8]))?([[:space:]]+([1-9][0-9]?[0-9]?\.){1}([0-9]{1,3}\.){2}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\/(1?[0-9]|2?[0-9]|3?[0-2]))?([[:space:]]+#.*$|[[:space:]]*$)|[[:space:]]+#.*$|$)/{if(!$2||$2~/#/)$2="0.0.0.0/0";if(!seen[$1]++)printf "%s . %s, ",tolower($1),$2}' "${ban_blocklist}" >"${tmp_file}" + printf '%s\n' "add set inet banIP ${feed} { type ether_addr . ipv4_addr; flags interval; auto-merge; policy ${ban_nftpolicy}${nft_setcnt}; $(f_getelements "${tmp_file}") }" + if [ -z "${feed_direction##*outbound*}" ]; then + if [ "${ban_logoutbound}" = "1" ]; then + printf '%s\n' "add rule inet banIP _outbound ether saddr . ip saddr @${feed} ${log_outbound}" fi - else - "${ban_awkcmd}" '/^127\./{next}/^(([1-9][0-9]?[0-9]?\.){1}([0-9]{1,3}\.){2}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\/(1?[0-9]|2?[0-9]|3?[0-2]))?)([[:space:]].*|$)/{printf "%s,\n",$1}' "${ban_blocklist}" >"${tmp_split}" + printf '%s\n' "add rule inet banIP _outbound ether saddr . ip saddr @${feed} ${nft_cnt} goto _reject" fi - "${ban_awkcmd}" '{ORS=" ";print}' "${tmp_split}" 2>/dev/null >"${tmp_file}" - printf "%s\n" "add set inet banIP ${feed} { type ipv4_addr; flags interval, timeout; auto-merge; policy ${ban_nftpolicy}; $(f_getelements "${tmp_file}") }" - [ -z "${feed_direction##*input*}" ] && printf "%s\n" "add rule inet banIP wan-input ip saddr @${feed} ${log_input} counter ${feed_target}" - [ -z "${feed_direction##*forwardwan*}" ] && printf "%s\n" "add rule inet banIP wan-forward ip saddr @${feed} ${log_forwardwan} counter ${feed_target}" - [ -z "${feed_direction##*forwardlan*}" ] && printf "%s\n" "add rule inet banIP lan-forward ip daddr @${feed} ${log_forwardlan} counter goto reject-chain" - elif [ "${proto}" = "6" ]; then - if [ "${ban_deduplicate}" = "1" ]; then - "${ban_awkcmd}" '!/^([0-9A-f]{2}:){5}[0-9A-f]{2}.*/{printf "%s\n",$1}' "${ban_blocklist}" | - "${ban_awkcmd}" '/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\/(1?[0-2][0-8]|[0-9][0-9]))?)([[:space:]].*|$)/{printf "%s,\n",tolower($1)}' >"${tmp_raw}" - "${ban_awkcmd}" 'NR==FNR{member[$0];next}!($0 in member)' "${ban_tmpfile}.deduplicate" "${tmp_raw}" 2>/dev/null >"${tmp_split}" - "${ban_awkcmd}" 'BEGIN{FS="[ ,]"}NR==FNR{member[$1];next}!($1 in member)' "${ban_tmpfile}.deduplicate" "${ban_blocklist}" 2>/dev/null >"${tmp_raw}" - if ! cmp -s "${tmp_raw}" "${ban_blocklist}" 2>/dev/null; then - "${ban_catcmd}" "${tmp_raw}" 2>/dev/null >"${ban_blocklist}" + ;; + "6MAC") + "${ban_awkcmd}" '{gsub(/\r/,"")}/^([0-9A-f]{2}:){5}[0-9A-f]{2}(\/([0-9]|[1-3][0-9]|4[0-8]))?([[:space:]]+([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\/(1?[0-2][0-8]|[0-9][0-9]))?([[:space:]]+#.*$|[[:space:]]*$)|[[:space:]]+#.*$|$)/{if(!$2||$2~/#/)$2="::/0";if(!seen[$1]++)printf "%s . %s, ",tolower($1),$2}' "${ban_blocklist}" >"${tmp_file}" + printf '%s\n' "add set inet banIP ${feed} { type ether_addr . ipv6_addr; flags interval; auto-merge; policy ${ban_nftpolicy}${nft_setcnt}; $(f_getelements "${tmp_file}") }" + if [ -z "${feed_direction##*outbound*}" ]; then + if [ "${ban_logoutbound}" = "1" ]; then + printf '%s\n' "add rule inet banIP _outbound ether saddr . ip6 saddr @${feed} ${log_outbound}" fi - else - "${ban_awkcmd}" '!/^([0-9A-f]{2}:){5}[0-9A-f]{2}.*/{printf "%s\n",$1}' "${ban_blocklist}" | - "${ban_awkcmd}" '/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\/(1?[0-2][0-8]|[0-9][0-9]))?)([[:space:]].*|$)/{printf "%s,\n",tolower($1)}' >"${tmp_split}" + printf '%s\n' "add rule inet banIP _outbound ether saddr . ip6 saddr @${feed} ${nft_cnt} goto _reject" fi - "${ban_awkcmd}" '{ORS=" ";print}' "${tmp_split}" 2>/dev/null >"${tmp_file}" - printf "%s\n" "add set inet banIP ${feed} { type ipv6_addr; flags interval, timeout; auto-merge; policy ${ban_nftpolicy}; $(f_getelements "${tmp_file}") }" - [ -z "${feed_direction##*input*}" ] && printf "%s\n" "add rule inet banIP wan-input ip6 saddr @${feed} ${log_input} counter ${feed_target}" - [ -z "${feed_direction##*forwardwan*}" ] && printf "%s\n" "add rule inet banIP wan-forward ip6 saddr @${feed} ${log_forwardwan} counter ${feed_target}" - [ -z "${feed_direction##*forwardlan*}" ] && printf "%s\n" "add rule inet banIP lan-forward ip6 daddr @${feed} ${log_forwardlan} counter goto reject-chain" - fi + ;; + "4") + f_chkip ${feed_ipv} local 1 <"${ban_blocklist}" >"${tmp_file}" + printf '%s\n' "add set inet banIP ${feed} { type ipv4_addr; flags interval, timeout; auto-merge; policy ${ban_nftpolicy}${nft_setcnt}; $(f_getelements "${tmp_file}") }" + if [ -z "${feed_direction##*inbound*}" ]; then + if [ "${ban_loginbound}" = "1" ]; then + printf '%s\n' "add rule inet banIP _inbound ip saddr @${feed} ${log_inbound}" + fi + printf '%s\n' "add rule inet banIP _inbound ip saddr @${feed} ${nft_cnt} ${feed_target}" + fi + if [ -z "${feed_direction##*outbound*}" ]; then + if [ "${ban_logoutbound}" = "1" ]; then + printf '%s\n' "add rule inet banIP _outbound ip daddr @${feed} ${log_outbound}" + fi + printf '%s\n' "add rule inet banIP _outbound ip daddr @${feed} ${nft_cnt} goto _reject" + fi + ;; + "6") + f_chkip ${feed_ipv} local 1 <"${ban_blocklist}" >"${tmp_file}" + printf '%s\n' "add set inet banIP ${feed} { type ipv6_addr; flags interval, timeout; auto-merge; policy ${ban_nftpolicy}${nft_setcnt}; $(f_getelements "${tmp_file}") }" + if [ -z "${feed_direction##*inbound*}" ]; then + if [ "${ban_loginbound}" = "1" ]; then + printf '%s\n' "add rule inet banIP _inbound ip6 saddr @${feed} ${log_inbound}" + fi + printf '%s\n' "add rule inet banIP _inbound ip6 saddr @${feed} ${nft_cnt} ${feed_target}" + fi + if [ -z "${feed_direction##*outbound*}" ]; then + if [ "${ban_logoutbound}" = "1" ]; then + printf '%s\n' "add rule inet banIP _outbound ip6 daddr @${feed} ${log_outbound}" + fi + printf '%s\n' "add rule inet banIP _outbound ip6 daddr @${feed} ${nft_cnt} goto _reject" + fi + ;; + esac } >"${tmp_nft}" : >"${tmp_flush}" >"${tmp_raw}" >"${tmp_file}" feed_rc="0" @@ -990,42 +1477,80 @@ f_down() { # handle external feeds # elif [ "${restore_rc}" != "0" ] && [ "${feed_url}" != "local" ]; then + # handle country downloads # - if [ "${feed%v*}" = "country" ]; then - for country in ${ban_country}; do - feed_log="$("${ban_fetchcmd}" ${ban_fetchparm} "${tmp_raw}" "${feed_url}${country}-aggregated.zone" 2>&1)" - feed_rc="${?}" - [ "${feed_rc}" = "0" ] && "${ban_catcmd}" "${tmp_raw}" 2>/dev/null >>"${tmp_load}" - done - : >"${tmp_raw}" + if [ "${feed%%.*}" = "country" ]; then + if [ "${ban_countrysplit}" = "0" ]; then + for country in ${ban_country}; do + if "${ban_fetchcmd}" ${ban_fetchparm} "${tmp_raw}" "${feed_url}${country}-aggregated.zone" 2>>"${ban_errorlog}"; then + if [ -s "${tmp_raw}" ]; then + "${ban_catcmd}" "${tmp_raw}" 2>>"${ban_errorlog}" >>"${tmp_load}" + feed_rc="${?}" + fi + else + f_log "info" "download for feed '${feed}/${country}' failed" + fi + done + : >"${tmp_raw}" + else + country="${feed%.*}" + country="${country#*.}" + if "${ban_fetchcmd}" ${ban_fetchparm} "${tmp_load}" "${feed_url}${country}-aggregated.zone" 2>>"${ban_errorlog}"; then + feed_rc="${?}" + else + feed_rc="4" + fi + fi # handle asn downloads # - elif [ "${feed%v*}" = "asn" ]; then - for asn in ${ban_asn}; do - feed_log="$("${ban_fetchcmd}" ${ban_fetchparm} "${tmp_raw}" "${feed_url}AS${asn}" 2>&1)" - feed_rc="${?}" - [ "${feed_rc}" = "0" ] && "${ban_catcmd}" "${tmp_raw}" 2>/dev/null >>"${tmp_load}" - done - : >"${tmp_raw}" + elif [ "${feed%%.*}" = "asn" ]; then + if [ "${ban_asnsplit}" = "0" ]; then + for asn in ${ban_asn}; do + if "${ban_fetchcmd}" ${ban_fetchparm} "${tmp_raw}" "${feed_url}AS${asn}" 2>>"${ban_errorlog}"; then + if [ -s "${tmp_raw}" ]; then + "${ban_catcmd}" "${tmp_raw}" 2>>"${ban_errorlog}" >>"${tmp_load}" + feed_rc="${?}" + fi + else + f_log "info" "download for feed '${feed}/${asn}' failed" + fi + done + : >"${tmp_raw}" + else + asn="${feed%.*}" + asn="${asn#*.}" + if "${ban_fetchcmd}" ${ban_fetchparm} "${tmp_load}" "${feed_url}AS${asn}" 2>>"${ban_errorlog}"; then + feed_rc="${?}" + else + feed_rc="4" + fi + fi # handle compressed downloads # elif [ "${feed_comp}" = "gz" ]; then - feed_log="$("${ban_fetchcmd}" ${ban_fetchparm} "${tmp_raw}" "${feed_url}" 2>&1)" - feed_rc="${?}" - [ "${feed_rc}" = "0" ] && "${ban_zcatcmd}" "${tmp_raw}" 2>/dev/null >"${tmp_load}" + if "${ban_fetchcmd}" ${ban_fetchparm} "${tmp_raw}" "${feed_url}" 2>>"${ban_errorlog}"; then + if [ -s "${tmp_raw}" ]; then + "${ban_zcatcmd}" "${tmp_raw}" 2>>"${ban_errorlog}" >"${tmp_load}" + feed_rc="${?}" + fi + else + feed_rc="4" + fi : >"${tmp_raw}" # handle normal downloads # else - feed_log="$("${ban_fetchcmd}" ${ban_fetchparm} "${tmp_load}" "${feed_url}" 2>&1)" - feed_rc="${?}" + if "${ban_fetchcmd}" ${ban_fetchparm} "${tmp_load}" "${feed_url}" 2>>"${ban_errorlog}"; then + feed_rc="${?}" + else + feed_rc="4" + fi fi fi - [ "${feed_rc}" != "0" ] && f_log "info" "download for feed '${feed}' failed (rc: ${feed_rc:-"-"}/log: ${feed_log})" # backup/restore # @@ -1036,18 +1561,23 @@ f_down() { f_restore "${feed}" "${feed_url}" "${tmp_load}" "${feed_rc}" feed_rc="${?}" fi + [ "${feed_rc}" != "0" ] && f_log "info" "processing for feed '${feed}' failed, rc: ${feed_rc:-"-"}" # final file & Set preparation for regular downloads # if [ "${feed_rc}" = "0" ] && [ ! -s "${tmp_nft}" ]; then + # deduplicate Sets # - if [ "${ban_deduplicate}" = "1" ] && [ "${feed_url}" != "local" ]; then - "${ban_awkcmd}" '{sub("\r$", "");print}' "${tmp_load}" 2>/dev/null | "${ban_awkcmd}" "${feed_rule}" 2>/dev/null >"${tmp_raw}" - "${ban_awkcmd}" 'NR==FNR{member[$0];next}!($0 in member)' "${ban_tmpfile}.deduplicate" "${tmp_raw}" 2>/dev/null | tee -a "${ban_tmpfile}.deduplicate" >"${tmp_split}" - feed_rc="${?}" + if [ "${ban_deduplicate}" = "1" ] && [ "${feed_url}" != "local" ] && [ -z "${feed_complete}" ]; then + f_chkip ${feed_ipv} ${feed_rule} <"${tmp_load}" >"${tmp_raw}" + { + "${ban_flockcmd}" -x 8 + "${ban_awkcmd}" 'NR==FNR{member[$0];next}!($0 in member)' "${ban_tmpfile}.deduplicate" "${tmp_raw}" 2>>"${ban_errorlog}" | tee -a "${ban_tmpfile}.deduplicate" >"${tmp_split}" + feed_rc="${?}" + } 8>"${ban_deduplock}" else - "${ban_awkcmd}" '{sub("\r$", "");print}' "${tmp_load}" 2>/dev/null | "${ban_awkcmd}" "${feed_rule}" 2>/dev/null >"${tmp_split}" + f_chkip ${feed_ipv} ${feed_rule} <"${tmp_load}" >"${tmp_split}" feed_rc="${?}" fi : >"${tmp_raw}" >"${tmp_load}" @@ -1056,13 +1586,13 @@ f_down() { # if [ "${feed_rc}" = "0" ]; then if [ -n "${ban_splitsize//[![:digit:]]/}" ] && [ "${ban_splitsize//[![:digit:]]/}" -ge "512" ]; then - if ! "${ban_awkcmd}" "NR%${ban_splitsize//[![:digit:]]/}==1{file=\"${tmp_file}.\"++i;}{ORS=\" \";print > file}" "${tmp_split}" 2>/dev/null; then + if ! "${ban_awkcmd}" "NR%${ban_splitsize//[![:digit:]]/}==1{file=\"${tmp_file}.\"++i;}{ORS=\" \";print > file}" "${tmp_split}" 2>>"${ban_errorlog}"; then feed_rc="${?}" - rm -f "${tmp_file}".* - f_log "info" "can't split Set '${feed}' to size '${ban_splitsize//[![:digit:]]/}'" + "${ban_rmcmd}" -f "${tmp_file}".* + f_log "info" "can't split nfset '${feed}' to size '${ban_splitsize//[![:digit:]]/}'" fi else - "${ban_awkcmd}" '{ORS=" ";print}' "${tmp_split}" 2>/dev/null >"${tmp_file}.1" + "${ban_awkcmd}" '{ORS=" ";print}' "${tmp_split}" 2>>"${ban_errorlog}" >"${tmp_file}.1" feed_rc="${?}" fi fi @@ -1070,27 +1600,45 @@ f_down() { # build nft file # if [ "${feed_rc}" = "0" ] && [ -s "${tmp_file}.1" ]; then - if [ "${proto}" = "4" ]; then + if [ "${feed_ipv}" = "4" ]; then { - # nft header (IPv4 Set) input and forward rules + # nft header (IPv4 Set) incl. inbound and outbound rules # - printf "%s\n\n" "#!${ban_nftcmd} -f" + printf '%s\n\n' "#!${ban_nftcmd} -f" [ -s "${tmp_flush}" ] && "${ban_catcmd}" "${tmp_flush}" - printf "%s\n" "add set inet banIP ${feed} { type ipv4_addr; flags interval; auto-merge; policy ${ban_nftpolicy}; $(f_getelements "${tmp_file}.1") }" - [ -z "${feed_direction##*input*}" ] && printf "%s\n" "add rule inet banIP wan-input ${feed_dport} ip saddr @${feed} ${log_input} counter ${feed_target}" - [ -z "${feed_direction##*forwardwan*}" ] && printf "%s\n" "add rule inet banIP wan-forward ${feed_dport} ip saddr @${feed} ${log_forwardwan} counter ${feed_target}" - [ -z "${feed_direction##*forwardlan*}" ] && printf "%s\n" "add rule inet banIP lan-forward ${feed_dport} ip daddr @${feed} ${log_forwardlan} counter goto reject-chain" + printf '%s\n' "add set inet banIP ${feed} { type ipv4_addr; flags interval; auto-merge; policy ${ban_nftpolicy}${nft_setcnt}; $(f_getelements "${tmp_file}.1") }" + if [ -z "${feed_direction##*inbound*}" ]; then + if [ "${ban_loginbound}" = "1" ]; then + printf '%s\n' "add rule inet banIP _inbound ${feed_dport} ip saddr @${feed} ${log_inbound}" + fi + printf '%s\n' "add rule inet banIP _inbound ${feed_dport} ip saddr @${feed} ${nft_cnt} ${feed_target}" + fi + if [ -z "${feed_direction##*outbound*}" ]; then + if [ "${ban_logoutbound}" = "1" ]; then + printf '%s\n' "add rule inet banIP _outbound ${feed_dport} ip daddr @${feed} ${log_outbound}" + fi + printf '%s\n' "add rule inet banIP _outbound ${feed_dport} ip daddr @${feed} ${nft_cnt} goto _reject" + fi } >"${tmp_nft}" - elif [ "${proto}" = "6" ]; then + elif [ "${feed_ipv}" = "6" ]; then { - # nft header (IPv6 Set) plus input and forward rules + # nft header (IPv6 Set) incl. inbound and outbound rules # - printf "%s\n\n" "#!${ban_nftcmd} -f" + printf '%s\n\n' "#!${ban_nftcmd} -f" [ -s "${tmp_flush}" ] && "${ban_catcmd}" "${tmp_flush}" - printf "%s\n" "add set inet banIP ${feed} { type ipv6_addr; flags interval; auto-merge; policy ${ban_nftpolicy}; $(f_getelements "${tmp_file}.1") }" - [ -z "${feed_direction##*input*}" ] && printf "%s\n" "add rule inet banIP wan-input ${feed_dport} ip6 saddr @${feed} ${log_input} counter ${feed_target}" - [ -z "${feed_direction##*forwardwan*}" ] && printf "%s\n" "add rule inet banIP wan-forward ${feed_dport} ip6 saddr @${feed} ${log_forwardwan} counter ${feed_target}" - [ -z "${feed_direction##*forwardlan*}" ] && printf "%s\n" "add rule inet banIP lan-forward ${feed_dport} ip6 daddr @${feed} ${log_forwardlan} counter goto reject-chain" + printf '%s\n' "add set inet banIP ${feed} { type ipv6_addr; flags interval; auto-merge; policy ${ban_nftpolicy}${nft_setcnt}; $(f_getelements "${tmp_file}.1") }" + if [ -z "${feed_direction##*inbound*}" ]; then + if [ "${ban_loginbound}" = "1" ]; then + printf '%s\n' "add rule inet banIP _inbound ${feed_dport} ip6 saddr @${feed} ${log_inbound}" + fi + printf '%s\n' "add rule inet banIP _inbound ${feed_dport} ip6 saddr @${feed} ${nft_cnt} ${feed_target}" + fi + if [ -z "${feed_direction##*outbound*}" ]; then + if [ "${ban_logoutbound}" = "1" ]; then + printf '%s\n' "add rule inet banIP _outbound ${feed_dport} ip6 daddr @${feed} ${log_outbound}" + fi + printf '%s\n' "add rule inet banIP _outbound ${feed_dport} ip6 daddr @${feed} ${nft_cnt} goto _reject" + fi } >"${tmp_nft}" fi fi @@ -1100,14 +1648,19 @@ f_down() { # load generated nft file in banIP table # if [ "${feed_rc}" = "0" ]; then - if [ "${feed%v*}" = "allowlist" ]; then - cnt_dl="$("${ban_awkcmd}" 'END{printf "%d",NR}' "${tmp_allow}" 2>/dev/null)" + if [ "${feed%%.*}" = "allowlist" ]; then + cnt_dl="$("${ban_awkcmd}" 'END{printf "%d",NR}' "${tmp_allow}" 2>>"${ban_errorlog}")" + elif [ "${feed%%.*}" = "blocklist" ]; then + cnt_dl="$("${ban_awkcmd}" 'END{printf "%d",NR}' "${ban_blocklist}" 2>>"${ban_errorlog}")" else - cnt_dl="$("${ban_awkcmd}" 'END{printf "%d",NR}' "${tmp_split}" 2>/dev/null)" + cnt_dl="$("${ban_awkcmd}" 'END{printf "%d",NR}' "${tmp_split}" 2>>"${ban_errorlog}")" : >"${tmp_split}" fi - if [ "${cnt_dl:-"0"}" -gt "0" ] || [ "${feed_url}" = "local" ] || [ "${feed%v*}" = "allowlist" ] || [ "${feed%v*}" = "blocklist" ]; then - feed_log="$("${ban_nftcmd}" -f "${tmp_nft}" 2>&1)" + if [ "${cnt_dl:-"0"}" -gt "0" ] || [ "${feed%%.*}" = "allowlist" ] || [ "${feed%%.*}" = "blocklist" ]; then + + # load initial file to nftset + # + f_nftload "${tmp_nft}" "can't load initial file to nfset '${feed}'" feed_rc="${?}" # load additional split files @@ -1115,28 +1668,27 @@ f_down() { if [ "${feed_rc}" = "0" ]; then for split_file in "${tmp_file}".*; do if [ -s "${split_file}" ]; then - "${ban_sedcmd}" -i "1 i #!${ban_nftcmd} -f\nadd element inet banIP "${feed}" { " "${split_file}" - printf "%s\n" "}" >>"${split_file}" - if ! "${ban_nftcmd}" -f "${split_file}" >/dev/null 2>&1; then - f_log "info" "can't add split file '${split_file##*.}' to Set '${feed}'" - fi + "${ban_sedcmd}" -i "1 i #!${ban_nftcmd} -f\nadd element inet banIP ${feed} { " "${split_file}" + printf '%s\n' "}" >>"${split_file}" + + # load split file to nftset + # + f_nftload "${split_file}" "can't load split file '${split_file##*.}' to nfset '${feed}'" + feed_rc="${?}" : >"${split_file}" fi done - if [ "${ban_debug}" = "1" ] && [ "${ban_reportelements}" = "1" ]; then - cnt_set="$("${ban_nftcmd}" -j list set inet banIP "${feed}" 2>/dev/null | "${ban_jsoncmd}" -qe '@.nftables[*].set.elem[*]' | wc -l 2>/dev/null)" - fi - else - f_log "info" "can't initialize Set for feed '${feed}' (rc: ${feed_rc}, log: ${feed_log})" + cnt_set="$("${ban_nftcmd}" -j list set inet banIP "${feed}" 2>/dev/null | "${ban_jsoncmd}" -qe '@.nftables[*].set.elem[*]' | "${ban_wccmd}" -l 2>/dev/null)" fi else f_log "info" "skip empty feed '${feed}'" fi fi : >"${tmp_nft}" - end_ts="$(date +%s)" + read -r end_ts _ <"/proc/uptime" + end_ts="${end_ts%%.*}" - f_log "debug" "f_down ::: feed: ${feed}, cnt_dl: ${cnt_dl:-"-"}, cnt_set: ${cnt_set:-"-"}, split_size: ${ban_splitsize:-"-"}, time: $((end_ts - start_ts)), rc: ${feed_rc:-"-"}, log: ${feed_log:-"-"}" + f_log "debug" "f_down ::: feed: ${feed}, policy: ${feed_policy}, complete: ${feed_complete:-"-"}, cnt_dl: ${cnt_dl:-"-"}, cnt_set: ${cnt_set:-"-"}, split_size: ${ban_splitsize:-"-"}, time: $((end_ts - start_ts)), rc: ${feed_rc:-"-"}" } # backup feeds @@ -1149,7 +1701,7 @@ f_backup() { backup_rc="${?}" fi - f_log "debug" "f_backup ::: feed: ${feed}, file: banIP.${feed}.gz, rc: ${backup_rc}" + f_log "debug" "f_backup ::: feed: ${feed}, file: banIP.${feed}.gz, rc: ${backup_rc}" return "${backup_rc}" } @@ -1158,87 +1710,169 @@ f_backup() { f_restore() { local tmp_feed restore_rc="4" feed="${1}" feed_url="${2}" feed_file="${3}" in_rc="${4}" - [ "${feed_url}" = "local" ] && tmp_feed="${feed%v*}v4" || tmp_feed="${feed}" + [ "${feed_url}" = "local" ] && tmp_feed="${feed%.*}.v4" || tmp_feed="${feed}" if [ -s "${ban_backupdir}/banIP.${tmp_feed}.gz" ]; then - "${ban_zcatcmd}" "${ban_backupdir}/banIP.${tmp_feed}.gz" 2>/dev/null >"${feed_file}" + "${ban_zcatcmd}" "${ban_backupdir}/banIP.${tmp_feed}.gz" 2>>"${ban_errorlog}" >"${feed_file}" restore_rc="${?}" fi - f_log "debug" "f_restore ::: feed: ${feed}, file: banIP.${tmp_feed}.gz, in_rc: ${in_rc:-"-"}, rc: ${restore_rc}" + f_log "debug" "f_restore ::: feed: ${feed}, file: banIP.${tmp_feed}.gz, in_rc: ${in_rc:-"-"}, rc: ${restore_rc}" return "${restore_rc}" } -# remove disabled Sets +# remove staled Sets # f_rmset() { - local expr feedlist tmp_del ruleset_raw item table_sets handle del_set feed_log feed_rc + local feedlist tmp_del table_json feed country asn table_sets handles handle expr del_set feed_rc f_getfeed json_get_keys feedlist tmp_del="${ban_tmpfile}.final.delete" - ruleset_raw="$("${ban_nftcmd}" -tj list ruleset 2>/dev/null)" - table_sets="$(printf "%s\n" "${ruleset_raw}" | "${ban_jsoncmd}" -qe '@.nftables[@.set.table="banIP"&&@.set.family="inet"].set.name')" + table_json="$("${ban_nftcmd}" -tj list table inet banIP 2>>"${ban_errorlog}")" + table_sets="$(printf '%s\n' "${table_json}" | "${ban_jsoncmd}" -qe '@.nftables[@.set.family="inet"].set.name')" { - printf "%s\n\n" "#!${ban_nftcmd} -f" - for item in ${table_sets}; do - if ! printf "%s" "allowlist blocklist ${ban_feed}" | "${ban_grepcmd}" -q "${item%v*}" || - ! printf "%s" "allowlist blocklist ${feedlist}" | "${ban_grepcmd}" -q "${item%v*}"; then - [ -z "${del_set}" ] && del_set="${item}" || del_set="${del_set}, ${item}" - rm -f "${ban_backupdir}/banIP.${item}.gz" - printf "%s\n" "flush set inet banIP ${item}" - for expr in 0 1; do - handle="$(printf "%s\n" "${ruleset_raw}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.table=\"banIP\"&&@.rule.chain=\"wan-input\"][@.expr[${expr}].match.right=\"@${item}\"].handle")" - [ -n "${handle}" ] && printf "%s\n" "delete rule inet banIP wan-input handle ${handle}" - handle="$(printf "%s\n" "${ruleset_raw}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.table=\"banIP\"&&@.rule.chain=\"wan-forward\"][@.expr[${expr}].match.right=\"@${item}\"].handle")" - [ -n "${handle}" ] && printf "%s\n" "delete rule inet banIP wan-forward handle ${handle}" - handle="$(printf "%s\n" "${ruleset_raw}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.table=\"banIP\"&&@.rule.chain=\"lan-forward\"][@.expr[${expr}].match.right=\"@${item}\"].handle")" - [ -n "${handle}" ] && printf "%s\n" "delete rule inet banIP lan-forward handle ${handle}" - done - printf "%s\n\n" "delete set inet banIP ${item}" + printf '%s\n\n' "#!${ban_nftcmd} -f" + for feed in ${table_sets}; do + + # keep: active country split sets + # + if [ "${feed%%.*}" = "country" ] && [ "${ban_countrysplit}" = "1" ]; then + country="${feed%.*}" + country="${country#*.}" + case " ${ban_feed} " in + *" country "*) + case " ${ban_country} " in + *" ${country} "*) continue ;; + esac + ;; + esac fi + + # keep: active asn split sets + # + if [ "${feed%%.*}" = "asn" ] && [ "${ban_asnsplit}" = "1" ]; then + asn="${feed%.*}" + asn="${asn#*.}" + case " ${ban_feed} " in + *" asn "*) + case " ${ban_asn} " in + *" ${asn} "*) continue ;; + esac + ;; + esac + fi + + # keep: set is active in config and feed file, respecting split/allowlistonly exclusions + # + case " allowlist blocklist ${ban_feed} " in + *" ${feed%.*} "*) + case " allowlist blocklist ${feedlist} " in + *" ${feed%.*} "*) + if [ "${feed%.*}" != "country" ] || [ "${ban_countrysplit}" != "1" ]; then + if [ "${feed%.*}" != "asn" ] || [ "${ban_asnsplit}" != "1" ]; then + if [ "${feed%.*}" = "allowlist" ] || [ "${feed%.*}" = "blocklist" ] || [ "${ban_allowlistonly}" != "1" ]; then + continue + fi + case " ${ban_feedin} ${ban_feedout} " in + *" allowlist "*) continue ;; + esac + fi + fi + ;; + esac + ;; + esac + + # delete: set is stale or no longer active + # + [ -z "${del_set}" ] && del_set="${feed}" || del_set="${del_set}, ${feed}" + "${ban_rmcmd}" -f "${ban_backupdir}/banIP.${feed}.gz" + for chain in _inbound _outbound; do + for expr in 0 1 2; do + handles="$(printf '%s\n' "${table_json}" | "${ban_jsoncmd}" -q -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[${expr}].match.right=\"@${feed}\"].handle" | "${ban_xargscmd}")" + for handle in ${handles}; do + printf '%s\n' "delete rule inet banIP ${chain} handle ${handle}" + done + done + done + printf '%s\n' "flush set inet banIP ${feed}" + printf '%s\n\n' "delete set inet banIP ${feed}" done } >"${tmp_del}" if [ -n "${del_set}" ]; then - feed_log="$("${ban_nftcmd}" -f "${tmp_del}" 2>&1)" - feed_rc="${?}" + if "${ban_nftcmd}" -f "${tmp_del}" >/dev/null 2>&1; then + feed_rc="${?}" + else + feed_rc="4" + fi fi : >"${tmp_del}" - f_log "debug" "f_rmset ::: Set: ${del_set:-"-"}, rc: ${feed_rc:-"-"}, log: ${feed_log:-"-"}" + f_log "debug" "f_rmset ::: feed: ${del_set:-"-"}, rc: ${feed_rc:-"-"}" } # generate status information # f_genstatus() { - local mem_free mem_max object end_time duration table_sets cnt_elements="0" custom_feed="0" split="0" status="${1}" + local s_auto s_v4 s_v6 s_bcp s_lpre s_lin s_lout s_cnt s_dedup s_split s_cf s_allow s_dbg runtime + local mem_free nft_ver chain_cnt set_cnt rule_cnt object end_time duration table table_sets element_cnt="0" custom_feed="0" split="0" status="${1}" - mem_free="$("${ban_awkcmd}" '/^MemAvailable/{printf "%s",int($2/1024)}' "/proc/meminfo" 2>/dev/null)" - mem_max="$("${ban_awkcmd}" '/^VmHWM/{printf "%s",int($2)}' /proc/${$}/status 2>/dev/null)" + # memory and nftables version information + # + mem_free="$(f_mem float)" + nft_ver="$(printf '%s' "${ban_packages}" | "${ban_jsoncmd}" -ql1 -e '@.packages["nftables-json"]')" + # read config information if not already available + # [ -z "${ban_dev}" ] && f_conf + + # only gather detailed information if banIP is active + # if [ "${status}" = "active" ]; then - if [ -n "${ban_starttime}" ] && [ "${ban_action}" != "boot" ]; then - end_time="$(date "+%s")" + table="$("${ban_nftcmd}" -tj list table inet banIP 2>>"${ban_errorlog}")" + table_sets="$(printf '%s' "${table}" | "${ban_jsoncmd}" -qe '@.nftables[@.set.family="inet"].set.name')" + for object in ${table_sets}; do + element_cnt="$((element_cnt + $("${ban_nftcmd}" -j list set inet banIP "${object}" 2>>"${ban_errorlog}" | "${ban_jsoncmd}" -qe '@.nftables[*].set.elem[*]' | "${ban_wccmd}" -l 2>>"${ban_errorlog}")))" + done + chain_cnt="$(printf '%s' "${table}" | "${ban_jsoncmd}" -qe '@.nftables[*].chain.name' | "${ban_wccmd}" -l 2>>"${ban_errorlog}")" + set_cnt="$(printf '%s' "${table}" | "${ban_jsoncmd}" -qe '@.nftables[*].set.name' | "${ban_wccmd}" -l 2>>"${ban_errorlog}")" + rule_cnt="$(printf '%s' "${table}" | "${ban_jsoncmd}" -qe '@.nftables[*].rule' | "${ban_wccmd}" -l 2>>"${ban_errorlog}")" + element_cnt="$("${ban_awkcmd}" -v cnt="${element_cnt}" 'BEGIN{res="";pos=0;for(i=length(cnt);i>0;i--){res=substr(cnt,i,1)res;pos++;if(pos==3&&i>1){res=" "res;pos=0;}}; printf"%s",res}')" + if [ -n "${ban_starttime}" ]; then + read -r end_time _ <"/proc/uptime" + end_time="${end_time%%.*}" duration="$(((end_time - ban_starttime) / 60))m $(((end_time - ban_starttime) % 60))s" fi - table_sets="$("${ban_nftcmd}" -tj list ruleset 2>/dev/null | "${ban_jsoncmd}" -qe '@.nftables[@.set.table="banIP"&&@.set.family="inet"].set.name')" - if [ "${ban_reportelements}" = "1" ]; then - for object in ${table_sets}; do - cnt_elements="$((cnt_elements + $("${ban_nftcmd}" -j list set inet banIP "${object}" 2>/dev/null | "${ban_jsoncmd}" -qe '@.nftables[*].set.elem[*]' | wc -l 2>/dev/null)))" - done - fi - runtime="mode: ${ban_action:-"-"}, period: ${duration:-"-"}, memory: ${mem_free} MB available, ${mem_max} KB max. used, cores: ${ban_cores}, log: ${ban_logreadcmd##*/}, fetch: ${ban_fetchcmd##*/}" + runtime="mode: ${ban_action}, date / time: $(date "+%d/%m/%Y %H:%M:%S"), duration: ${duration:-"-"}, memory: ${mem_free} MB available" fi [ -s "${ban_customfeedfile}" ] && custom_feed="1" [ "${ban_splitsize:-"0"}" -gt "0" ] && split="1" + # map flag values to status characters + # + case "${ban_autodetect}" in 1) s_auto="✔" ;; *) s_auto="✘" ;; esac + case "${ban_protov4}" in 1) s_v4="✔" ;; *) s_v4="✘" ;; esac + case "${ban_protov6}" in 1) s_v6="✔" ;; *) s_v6="✘" ;; esac + case "${ban_bcp38}" in 1) s_bcp="✔" ;; *) s_bcp="✘" ;; esac + case "${ban_logprerouting}" in 1) s_lpre="✔" ;; *) s_lpre="✘" ;; esac + case "${ban_loginbound}" in 1) s_lin="✔" ;; *) s_lin="✘" ;; esac + case "${ban_logoutbound}" in 1) s_lout="✔" ;; *) s_lout="✘" ;; esac + case "${ban_nftcount}" in 1) s_cnt="✔" ;; *) s_cnt="✘" ;; esac + case "${ban_deduplicate}" in 1) s_dedup="✔" ;; *) s_dedup="✘" ;; esac + case "${split}" in 1) s_split="✔" ;; *) s_split="✘" ;; esac + case "${custom_feed}" in 1) s_cf="✔" ;; *) s_cf="✘" ;; esac + case "${ban_allowlistonly}" in 1) s_allow="✔" ;; *) s_allow="✘" ;; esac + case "${ban_debug}" in 1) s_dbg="✔" ;; *) s_dbg="✘" ;; esac + + # generate JSON status file + # : >"${ban_rtfile}" json_init - json_load_file "${ban_rtfile}" >/dev/null 2>&1 json_add_string "status" "${status}" - json_add_string "version" "${ban_ver}" - json_add_string "element_count" "${cnt_elements}" + json_add_string "frontend_ver" "${ban_fver}" + json_add_string "backend_ver" "${ban_bver}" + json_add_string "element_count" "${element_cnt} (chains: ${chain_cnt:-"0"}, sets: ${set_cnt:-"0"}, rules: ${rule_cnt:-"0"})" json_add_array "active_feeds" for object in ${table_sets:-"-"}; do json_add_string "${object}" "${object}" @@ -1269,11 +1903,11 @@ f_genstatus() { json_add_string "${object}" "${object}" done json_close_array - json_add_string "nft_info" "priority: ${ban_nftpriority}, policy: ${ban_nftpolicy}, loglevel: ${ban_nftloglevel}, expiry: ${ban_nftexpiry:-"-"}, limit (icmp/syn/udp): ${ban_icmplimit}/${ban_synlimit}/${ban_udplimit}" - json_add_string "run_info" "base: ${ban_basedir}, backup: ${ban_backupdir}, report: ${ban_reportdir}" - json_add_string "run_flags" "auto: $(f_char ${ban_autodetect}), proto (4/6): $(f_char ${ban_protov4})/$(f_char ${ban_protov6}), log (pre/inp/fwd/lan): $(f_char ${ban_logprerouting})/$(f_char ${ban_loginput})/$(f_char ${ban_logforwardwan})/$(f_char ${ban_logforwardlan}), dedup: $(f_char ${ban_deduplicate}), split: $(f_char ${split}), custom feed: $(f_char ${custom_feed}), allowed only: $(f_char ${ban_allowlistonly})" + json_add_string "nft_info" "ver: ${nft_ver:-"-"}, priority: ${ban_nftpriority}, policy: ${ban_nftpolicy}, loglevel: ${ban_nftloglevel}, expiry: ${ban_nftexpiry:-"-"}, limit (icmp/syn/udp): ${ban_icmplimit}/${ban_synlimit}/${ban_udplimit}, loglimit (rate/burst): ${ban_logratelimit}/${ban_logburstlimit}" + json_add_string "run_info" "base: ${ban_basedir}, backup: ${ban_backupdir}, report: ${ban_reportdir}, error: ${ban_errordir}" + json_add_string "run_flags" "auto: ${s_auto}, proto (4/6): ${s_v4}/${s_v6}, bcp38: ${s_bcp}, log (pre/in/out): ${s_lpre}/${s_lin}/${s_lout}, count: ${s_cnt}, dedup: ${s_dedup}, split: ${s_split}, custom feed: ${s_cf}, allowed only: ${s_allow}, debug: ${s_dbg}" json_add_string "last_run" "${runtime:-"-"}" - json_add_string "system_info" "$(date "+%Y-%m-%d %H:%M:%S"), ${ban_sysver}" + json_add_string "system_info" "cores: ${ban_cores}, log: ${ban_logreadcmd##*/}, fetch: ${ban_fetchcmd##*/}, ${ban_sysver}" json_dump >"${ban_rtfile}" } @@ -1285,7 +1919,7 @@ f_getstatus() { [ -z "${ban_dev}" ] && f_conf json_load_file "${ban_rtfile}" >/dev/null 2>&1 if json_get_keys keylist; then - printf "%s\n" "::: banIP runtime information" + printf '%s\n' "::: banIP runtime information" for key in ${keylist}; do if [ "${key}" = "active_feeds" ] || [ "${key}" = "active_uplink" ]; then json_get_values values "${key}" >/dev/null 2>&1 @@ -1303,329 +1937,654 @@ f_getstatus() { else json_get_var value "${key}" >/dev/null 2>&1 if [ "${key}" = "status" ]; then - [ "${value}" = "active" ] && value="${value} ($(f_actual))" || value="${value}" + [ "${value}" = "active" ] && value="${value} ($(f_actual))" fi fi if [ "${key}" != "wan_interfaces" ] && [ "${key}" != "vlan_allow" ] && [ "${key}" != "vlan_block" ]; then - printf " + %-17s : %s\n" "${key}" "${value:-"-"}" + printf ' + %-17s : %s\n' "${key}" "${value:-"-"}" fi done else - printf "%s\n" "::: no banIP runtime information available" + printf '%s\n' "::: no banIP runtime information available" fi } # domain lookup # f_lookup() { - local cnt list domain lookup ip elementsv4 elementsv6 start_time end_time duration cnt_domain="0" cnt_ip="0" feed="${1}" + local cnt list domain lookup ip dom ts proto elementsv4 elementsv6 start_time end_time duration cnt_domain="0" cnt_ip="0" feed="${1}" + local record_file tmp_dir target_file auto_flag - [ -z "${ban_dev}" ] && f_conf - start_time="$(date "+%s")" + # measure runtime of lookup function for performance insights + # + read -r start_time _ <"/proc/uptime" + start_time="${start_time%%.*}" + + # prepare list of domains to lookup, target file for auto-adding new entries and auto-add flag based on feed type + # if [ "${feed}" = "allowlist" ]; then - list="$("${ban_awkcmd}" '/^([[:alnum:]_-]{1,63}\.)+[[:alpha:]]+([[:space:]]|$)/{printf "%s ",tolower($1)}' "${ban_allowlist}" 2>/dev/null)" + list="$("${ban_awkcmd}" '{gsub(/\r/,"")}/^([[:alnum:]_-]{1,63}\.)+[[:alpha:]]+([[:space:]]|$)/{printf "%s ",tolower($1)}' "${ban_allowlist}" 2>>"${ban_errorlog}")" + target_file="${ban_allowlist}" + auto_flag="${ban_autoallowlist}" elif [ "${feed}" = "blocklist" ]; then - list="$("${ban_awkcmd}" '/^([[:alnum:]_-]{1,63}\.)+[[:alpha:]]+([[:space:]]|$)/{printf "%s ",tolower($1)}' "${ban_blocklist}" 2>/dev/null)" + list="$("${ban_awkcmd}" '{gsub(/\r/,"")}/^([[:alnum:]_-]{1,63}\.)+[[:alpha:]]+([[:space:]]|$)/{printf "%s ",tolower($1)}' "${ban_blocklist}" 2>>"${ban_errorlog}")" + target_file="${ban_blocklist}" + auto_flag="${ban_autoblocklist}" fi + # prepare temporary directory for parallel lookups + # + tmp_dir="${ban_tmpfile}.lookup.${feed}" + f_mkdir "${tmp_dir}" + + # parallel DNS lookups: one record file per domain, network-bound work runs concurrently + # + cnt="1" for domain in ${list}; do - lookup="$("${ban_lookupcmd}" "${domain}" ${ban_resolver} 2>/dev/null | "${ban_awkcmd}" '/^Address[ 0-9]*: /{if(!seen[$NF]++)printf "%s ",$NF}' 2>/dev/null)" - for ip in ${lookup}; do - if [ "${ip%%.*}" = "127" ] || [ "${ip%%.*}" = "0" ] || [ -z "${ip%%::*}" ]; then - continue - else - [ "${ip##*:}" = "${ip}" ] && elementsv4="${elementsv4} ${ip}," || elementsv6="${elementsv6} ${ip}," - if [ "${feed}" = "allowlist" ] && [ "${ban_autoallowlist}" = "1" ] && ! "${ban_grepcmd}" -q "^${ip}[[:space:]]*#" "${ban_allowlist}"; then - printf "%-45s%s\n" "${ip}" "# '${domain}' added on $(date "+%Y-%m-%d %H:%M:%S")" >>"${ban_allowlist}" - elif [ "${feed}" = "blocklist" ] && [ "${ban_autoblocklist}" = "1" ] && ! "${ban_grepcmd}" -q "^${ip}[[:space:]]*#" "${ban_blocklist}"; then - printf "%-45s%s\n" "${ip}" "# '${domain}' added on $(date "+%Y-%m-%d %H:%M:%S")" >>"${ban_blocklist}" + ( + lookup="$("${ban_lookupcmd}" "${domain}" ${ban_resolver} 2>>"${ban_errorlog}" | "${ban_awkcmd}" '/^Address[ 0-9]*: /{if(!seen[$NF]++)printf "%s ",$NF}' 2>>"${ban_errorlog}")" + [ -z "${lookup}" ] && exit 0 + ts="$(date "+%Y-%m-%d %H:%M:%S")" + for ip in ${lookup}; do + if [ "${ip%%.*}" = "127" ] || [ "${ip%%.*}" = "0" ] || [ -z "${ip%%::*}" ]; then + continue + fi + if [ "${ip##*:}" = "${ip}" ]; then + printf 'v4 %s %s %s\n' "${ip}" "${domain}" "${ts}" + else + printf 'v6 %s %s %s\n' "${ip}" "${domain}" "${ts}" fi - cnt_ip="$((cnt_ip + 1))" + done >"${tmp_dir}/${cnt}" + ) & + [ "${cnt}" -gt "${ban_cores}" ] && wait -n + cnt_domain="${cnt}" + cnt="$((cnt + 1))" + done + wait + + # collect results: aggregate IPs, persist new entries serially (no append race) + # + for record_file in "${tmp_dir}"/*; do + [ -s "${record_file}" ] || continue + while read -r proto ip dom ts; do + cnt_ip="$((cnt_ip + 1))" + if [ "${proto}" = "v4" ]; then + elementsv4="${elementsv4} ${ip}," + else + elementsv6="${elementsv6} ${ip}," fi - done - cnt_domain="$((cnt_domain + 1))" + if [ "${auto_flag}" = "1" ] && ! "${ban_grepcmd}" -q "^${ip}[[:space:]]*#" "${target_file}"; then + printf "%-45s%s\n" "${ip}" "# '${dom}' added on ${ts}" >>"${target_file}" + fi + done <"${record_file}" done + f_rmdir "${tmp_dir}" + + # add resolved IPs to nftables Sets + # if [ -n "${elementsv4}" ]; then - if ! "${ban_nftcmd}" add element inet banIP "${feed}v4" { ${elementsv4} } >/dev/null 2>&1; then - f_log "info" "can't add lookup file to Set '${feed}v4'" + if ! "${ban_nftcmd}" add element inet banIP "${feed}.v4" { ${elementsv4} } 2>>"${ban_errorlog}"; then + f_log "info" "can't add lookup file to nfset '${feed}.v4'" fi fi if [ -n "${elementsv6}" ]; then - if ! "${ban_nftcmd}" add element inet banIP "${feed}v6" { ${elementsv6} } >/dev/null 2>&1; then - f_log "info" "can't add lookup file to Set '${feed}v6'" + if ! "${ban_nftcmd}" add element inet banIP "${feed}.v6" { ${elementsv6} } 2>>"${ban_errorlog}"; then + f_log "info" "can't add lookup file to nfset '${feed}.v6'" fi fi - end_time="$(date "+%s")" + + # measure end time and log performance insights + # + read -r end_time _ <"/proc/uptime" + end_time="${end_time%%.*}" duration="$(((end_time - start_time) / 60))m $(((end_time - start_time) % 60))s" - f_log "info" "domain lookup finished in ${duration} (${feed}, ${cnt_domain} domains, ${cnt_ip} IPs)" + f_log "debug" "f_lookup ::: feed: ${feed}, domains: ${cnt_domain}, IPs: ${cnt_ip}, duration: ${duration}" } # table statistics # f_report() { - local report_jsn report_txt tmp_val ruleset_raw item table_sets set_cnt set_input set_forwardwan set_forwardlan set_cntinput set_cntforwardwan set_cntforwardlan set_proto set_dport set_details - local expr detail jsnval timestamp autoadd_allow autoadd_block sum_sets sum_setinput sum_setforwardwan sum_setforwardlan sum_setelements sum_cntinput sum_cntforwardwan sum_cntforwardlan - local sum_synflood sum_udpflood sum_icmpflood sum_ctinvalid sum_tcpinvalid output="${1}" + local report_jsn report_txt tmp_val table_json item sep table_sets set_cnt set_inbound set_outbound set_cntinbound set_cntoutbound set_proto set_dport set_details + local cnt ip expr detail jsnval timestamp autoadd_allow autoadd_block sum_sets sum_setinbound sum_setoutbound sum_cntelements sum_cntinbound sum_cntoutbound quantity + local chunk jsn table_jsn set_jsn map_jsn chain set_elements uplink_ip uplink_list sum_setelements sum_synflood sum_udpflood sum_icmpflood sum_ctinvalid sum_tcpinvalid sum_setports sum_bcp38 output="${1}" - [ -z "${ban_dev}" ] && f_conf + f_conf f_mkdir "${ban_reportdir}" report_jsn="${ban_reportdir}/ban_report.jsn" report_txt="${ban_reportdir}/ban_report.txt" + map_jsn="${ban_reportdir}/ban_map.jsn" - # json output preparation - # - ruleset_raw="$("${ban_nftcmd}" -tj list ruleset 2>/dev/null)" - table_sets="$(printf "%s" "${ruleset_raw}" | "${ban_jsoncmd}" -qe '@.nftables[@.set.table="banIP"&&@.set.family="inet"].set.name')" - sum_sets="0" - sum_setinput="0" - sum_setforwardwan="0" - sum_setforwardlan="0" - sum_setelements="0" - sum_cntinput="0" - sum_cntforwardwan="0" - sum_cntforwardlan="0" - sum_synflood="$(printf "%s" "${ruleset_raw}" | "${ban_jsoncmd}" -qe '@.nftables[@.counter.name="cnt-synflood"].*.packets')" - sum_udpflood="$(printf "%s" "${ruleset_raw}" | "${ban_jsoncmd}" -qe '@.nftables[@.counter.name="cnt-udpflood"].*.packets')" - sum_icmpflood="$(printf "%s" "${ruleset_raw}" | "${ban_jsoncmd}" -qe '@.nftables[@.counter.name="cnt-icmpflood"].*.packets')" - sum_ctinvalid="$(printf "%s" "${ruleset_raw}" | "${ban_jsoncmd}" -qe '@.nftables[@.counter.name="cnt-ctinvalid"].*.packets')" - sum_tcpinvalid="$(printf "%s" "${ruleset_raw}" | "${ban_jsoncmd}" -qe '@.nftables[@.counter.name="cnt-tcpinvalid"].*.packets')" - timestamp="$(date "+%Y-%m-%d %H:%M:%S")" - : >"${report_jsn}" - { - printf "%s\n" "{" - printf "\t%s\n" '"sets":{' + if [ "${output}" != "json" ]; then + + # json output preparation + # + : >"${report_txt}" >"${report_jsn}" >"${map_jsn}" + [ "${output}" = "gen" ] && printf '%s\n' "0" >"${ban_rundir}/banIP.report" + table_jsn="${ban_rundir}/report.table.jsn" + "${ban_nftcmd}" -tj list table inet banIP 2>>"${ban_errorlog}" >"${table_jsn}" + table_sets="$("${ban_jsoncmd}" -i "${table_jsn}" -qe '@.nftables[@.set.family="inet"].set.name')" + sum_sets="0" + sum_cntelements="0" + sum_setinbound="0" + sum_setoutbound="0" + sum_cntinbound="0" + sum_cntoutbound="0" + sum_setports="0" + sum_setelements="0" + sum_synflood="$("${ban_jsoncmd}" -i "${table_jsn}" -qe '@.nftables[@.counter.name="cnt_synflood"].*.packets')" + sum_udpflood="$("${ban_jsoncmd}" -i "${table_jsn}" -qe '@.nftables[@.counter.name="cnt_udpflood"].*.packets')" + sum_icmpflood="$("${ban_jsoncmd}" -i "${table_jsn}" -qe '@.nftables[@.counter.name="cnt_icmpflood"].*.packets')" + sum_ctinvalid="$("${ban_jsoncmd}" -i "${table_jsn}" -qe '@.nftables[@.counter.name="cnt_ctinvalid"].*.packets')" + sum_tcpinvalid="$("${ban_jsoncmd}" -i "${table_jsn}" -qe '@.nftables[@.counter.name="cnt_tcpinvalid"].*.packets')" + sum_bcp38="$("${ban_jsoncmd}" -i "${table_jsn}" -qe '@.nftables[@.counter.name="cnt_bcp38"].*.packets')" + timestamp="$(date "+%Y-%m-%d %H:%M:%S")" + + cnt="1" for item in ${table_sets}; do - set_cntinput="" - set_cntforwardwan="" - set_cntforwardlan="" - set_proto="" - set_dport="" - for expr in 0 1; do - [ -z "${set_cntinput}" ] && set_cntinput="$(printf "%s" "${ruleset_raw}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.table=\"banIP\"&&@.rule.chain=\"wan-input\"][@.expr[${expr}].match.right=\"@${item}\"].expr[*].counter.packets")" - [ "${expr}" = "1" ] && [ -z "${set_dport}" ] && set_dport="$(printf "%s" "${ruleset_raw}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.table=\"banIP\"&&@.rule.chain=\"wan-input\"][@.expr[${expr}].match.right=\"@${item}\"].expr[*].match.right.set")" - [ "${expr}" = "1" ] && [ -z "${set_proto}" ] && set_proto="$(printf "%s" "${ruleset_raw}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.table=\"banIP\"&&@.rule.chain=\"wan-input\"][@.expr[${expr}].match.right=\"@${item}\"].expr[*].match.left.payload.protocol")" - [ -z "${set_cntforwardwan}" ] && set_cntforwardwan="$(printf "%s" "${ruleset_raw}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.table=\"banIP\"&&@.rule.chain=\"wan-forward\"][@.expr[${expr}].match.right=\"@${item}\"].expr[*].counter.packets")" - [ "${expr}" = "1" ] && [ -z "${set_dport}" ] && set_dport="$(printf "%s" "${ruleset_raw}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.table=\"banIP\"&&@.rule.chain=\"wan-forward\"][@.expr[${expr}].match.right=\"@${item}\"].expr[*].match.right.set")" - [ "${expr}" = "1" ] && [ -z "${set_proto}" ] && set_proto="$(printf "%s" "${ruleset_raw}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.table=\"banIP\"&&@.rule.chain=\"wan-forward\"][@.expr[${expr}].match.right=\"@${item}\"].expr[*].match.left.payload.protocol")" - [ -z "${set_cntforwardlan}" ] && set_cntforwardlan="$(printf "%s" "${ruleset_raw}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.table=\"banIP\"&&@.rule.chain=\"lan-forward\"][@.expr[${expr}].match.right=\"@${item}\"].expr[*].counter.packets")" - [ "${expr}" = "1" ] && [ -z "${set_dport}" ] && set_dport="$(printf "%s" "${ruleset_raw}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.table=\"banIP\"&&@.rule.chain=\"lan-forward\"][@.expr[${expr}].match.right=\"@${item}\"].expr[*].match.right.set")" - [ "${expr}" = "1" ] && [ -z "${set_proto}" ] && set_proto="$(printf "%s" "${ruleset_raw}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.table=\"banIP\"&&@.rule.chain=\"lan-forward\"][@.expr[${expr}].match.right=\"@${item}\"].expr[*].match.left.payload.protocol")" - done - if [ "${ban_reportelements}" = "1" ]; then - set_cnt="$("${ban_nftcmd}" -j list set inet banIP "${item}" 2>/dev/null | "${ban_jsoncmd}" -qe '@.nftables[*].set.elem[*]' | wc -l 2>/dev/null)" - sum_setelements="$((sum_setelements + set_cnt))" - else - set_cnt="" - sum_setelements="n/a" - fi - if [ -n "${set_dport}" ]; then - set_dport="${set_dport//[\{\}\":]/}" - set_dport="${set_dport#\[ *}" - set_dport="${set_dport%* \]}" - set_dport="${set_proto}: $(f_trim "${set_dport}")" - fi - if [ -n "${set_cntinput}" ]; then - set_input="ON" - sum_setinput="$((sum_setinput + 1))" - sum_cntinput="$((sum_cntinput + set_cntinput))" - else - set_input="-" - set_cntinput="" - fi - if [ -n "${set_cntforwardwan}" ]; then - set_forwardwan="ON" - sum_setforwardwan="$((sum_setforwardwan + 1))" - sum_cntforwardwan="$((sum_cntforwardwan + set_cntforwardwan))" - else - set_forwardwan="-" - set_cntforwardwan="" - fi - if [ -n "${set_cntforwardlan}" ]; then - set_forwardlan="ON" - sum_setforwardlan="$((sum_setforwardlan + 1))" - sum_cntforwardlan="$((sum_cntforwardlan + set_cntforwardlan))" - else - set_forwardlan="-" - set_cntforwardlan="" + ( + set_jsn="${ban_rundir}/report.set.jsn.${item}" + "${ban_nftcmd}" -j list set inet banIP "${item}" 2>>"${ban_errorlog}" >"${set_jsn}" + set_cnt="$("${ban_jsoncmd}" -i "${set_jsn}" -qe '@.nftables[*].set.elem[*]' | "${ban_wccmd}" -l 2>>"${ban_errorlog}")" + set_cntinbound="" + set_cntoutbound="" + set_inbound="" + set_outbound="" + set_proto="" + set_dport="" + set_elements="" + for chain in _inbound _outbound; do + for expr in 0 1 2; do + if [ "${chain}" = "_inbound" ] && [ -z "${set_cntinbound}" ]; then + set_cntinbound="$("${ban_jsoncmd}" -i "${table_jsn}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[${expr}].match.right=\"@${item}\"].expr[*].counter.packets")" + elif [ "${chain}" = "_outbound" ] && [ -z "${set_cntoutbound}" ]; then + set_cntoutbound="$("${ban_jsoncmd}" -i "${table_jsn}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[${expr}].match.right=\"@${item}\"].expr[*].counter.packets")" + fi + [ -z "${set_proto}" ] && set_proto="$("${ban_jsoncmd}" -i "${table_jsn}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[2].match.right=\"@${item}\"].expr[0].match.right.set")" + [ -z "${set_proto}" ] && set_proto="$("${ban_jsoncmd}" -i "${table_jsn}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[1].match.right=\"@${item}\"].expr[0].match.left.payload.protocol")" + [ -z "${set_dport}" ] && set_dport="$("${ban_jsoncmd}" -i "${table_jsn}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[2].match.right=\"@${item}\"].expr[1].match.right.set")" + [ -z "${set_dport}" ] && set_dport="$("${ban_jsoncmd}" -i "${table_jsn}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[2].match.right=\"@${item}\"].expr[1].match.right")" + [ -z "${set_dport}" ] && set_dport="$("${ban_jsoncmd}" -i "${table_jsn}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[1].match.right=\"@${item}\"].expr[0].match.right.set")" + [ -z "${set_dport}" ] && set_dport="$("${ban_jsoncmd}" -i "${table_jsn}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[1].match.right=\"@${item}\"].expr[0].match.right")" + done + [ -n "${set_cntinbound}" ] && [ -n "${set_cntoutbound}" ] && [ -n "${set_proto}" ] && [ -n "${set_dport}" ] && break + done + if [ -n "${set_proto}" ] && [ -n "${set_dport}" ]; then + set_proto="${set_proto//[\{\}\":]/}" + set_proto="${set_proto#\[ *}" + set_proto="${set_proto%* \]}" + set_dport="${set_dport//[\{\}\":]/}" + set_dport="${set_dport#\[ *}" + set_dport="${set_dport%* \]}" + set_dport="${set_proto}: $(f_trim "${set_dport}")" + fi + if [ "${ban_nftcount}" = "1" ]; then + "${ban_jsoncmd}" -i "${set_jsn}" -qe '@.nftables[*].set.elem[*][@.counter.packets>0].counter.packets' >"${set_jsn}.cnt" + "${ban_jsoncmd}" -i "${set_jsn}" -qe '@.nftables[*].set.elem[*][@.counter.packets>0].val' >"${set_jsn}.val" + set_elements="$("${ban_awkcmd}" 'NR==FNR{p[FNR]=$0;next}{print p[FNR]"\t"$0}' "${set_jsn}.cnt" "${set_jsn}.val" | + "${ban_sortcmd}" -k1,1nr ${ban_srtopts} | + "${ban_awkcmd}" -F '\t' 'NR<=50{split($2,a,/[ ,]/);ORS=" ";if(a[2]=="\"range\":"||a[2]=="\"concat\":")printf"%s, ",a[4];else if(a[2]=="\"prefix\":")printf"%s, ",a[5];else printf"\"%s\", ",a[1]}')" + fi + if [ -n "${set_cntinbound}" ]; then + set_inbound="ON" + else + set_inbound="-" + set_cntinbound="" + fi + if [ -n "${set_cntoutbound}" ]; then + set_outbound="ON" + else + set_outbound="-" + set_cntoutbound="" + fi + printf '%s\n' "\"${item}\":{ \"cnt_elements\": \"${set_cnt}\", \ + \"cnt_inbound\": \"${set_cntinbound}\", \ + \"inbound\": \"${set_inbound}\", \ + \"cnt_outbound\": \"${set_cntoutbound}\", \ + \"outbound\": \"${set_outbound}\", \ + \"port\": \"${set_dport:-"-"}\", \ + \"set_elements\": [ ${set_elements%%??} ] \ + }" >"${report_jsn}.${item}" + "${ban_rmcmd}" -f "${set_jsn}"* + ) & + [ "${cnt}" -gt "${ban_cores}" ] && wait -n + cnt="$((cnt + 1))" + done + wait + "${ban_rmcmd}" -f "${table_jsn}" + + # assemble JSON from per-set fragments + # + printf '%s' "{ \"sets\":{ " >"${report_jsn}" + sep="" + for item in ${table_sets}; do + if [ -s "${report_jsn}.${item}" ]; then + printf '%s' "${sep}" >>"${report_jsn}" + "${ban_catcmd}" "${report_jsn}.${item}" >>"${report_jsn}" + sep=", " fi - [ "${sum_sets}" -gt "0" ] && printf "%s\n" "," - printf "\t\t%s\n" "\"${item}\":{" - printf "\t\t\t%s\n" "\"cnt_elements\": \"${set_cnt}\"," - printf "\t\t\t%s\n" "\"cnt_input\": \"${set_cntinput}\"," - printf "\t\t\t%s\n" "\"input\": \"${set_input}\"," - printf "\t\t\t%s\n" "\"cnt_forwardwan\": \"${set_cntforwardwan}\"," - printf "\t\t\t%s\n" "\"wan_forward\": \"${set_forwardwan}\"," - printf "\t\t\t%s\n" "\"cnt_forwardlan\": \"${set_cntforwardlan}\"," - printf "\t\t\t%s\n" "\"lan_forward\": \"${set_forwardlan}\"", - printf "\t\t\t%s\n" "\"port\": \"${set_dport:-"-"}\"" - printf "\t\t%s" "}" - sum_sets="$((sum_sets + 1))" + "${ban_rmcmd}" -f "${report_jsn}.${item}" done - printf "\n\t%s\n" "}," - printf "\t%s\n" "\"timestamp\": \"${timestamp}\"," - printf "\t%s\n" "\"autoadd_allow\": \"$("${ban_grepcmd}" -c "added on ${timestamp% *}" "${ban_allowlist}")\"," - printf "\t%s\n" "\"autoadd_block\": \"$("${ban_grepcmd}" -c "added on ${timestamp% *}" "${ban_blocklist}")\"," - printf "\t%s\n" "\"sum_synflood\": \"${sum_synflood}\"," - printf "\t%s\n" "\"sum_udpflood\": \"${sum_udpflood}\"," - printf "\t%s\n" "\"sum_icmpflood\": \"${sum_icmpflood}\"," - printf "\t%s\n" "\"sum_ctinvalid\": \"${sum_ctinvalid}\"," - printf "\t%s\n" "\"sum_tcpinvalid\": \"${sum_tcpinvalid}\"," - printf "\t%s\n" "\"sum_sets\": \"${sum_sets}\"," - printf "\t%s\n" "\"sum_setinput\": \"${sum_setinput}\"," - printf "\t%s\n" "\"sum_setforwardwan\": \"${sum_setforwardwan}\"," - printf "\t%s\n" "\"sum_setforwardlan\": \"${sum_setforwardlan}\"," - printf "\t%s\n" "\"sum_setelements\": \"${sum_setelements}\"," - printf "\t%s\n" "\"sum_cntinput\": \"${sum_cntinput}\"," - printf "\t%s\n" "\"sum_cntforwardwan\": \"${sum_cntforwardwan}\"," - printf "\t%s\n" "\"sum_cntforwardlan\": \"${sum_cntforwardlan}\"" - printf "%s\n" "}" - } >>"${report_jsn}" - - # text output preparation - # - if [ "${output}" != "json" ] && [ -s "${report_jsn}" ]; then - : >"${report_txt}" + printf '\n%s\n' "} }" >>"${report_jsn}" + + # add sum statistics + # json_init if json_load_file "${report_jsn}" >/dev/null 2>&1; then - json_get_var timestamp "timestamp" >/dev/null 2>&1 - json_get_var autoadd_allow "autoadd_allow" >/dev/null 2>&1 - json_get_var autoadd_block "autoadd_block" >/dev/null 2>&1 - json_get_var sum_synflood "sum_synflood" >/dev/null 2>&1 - json_get_var sum_udpflood "sum_udpflood" >/dev/null 2>&1 - json_get_var sum_icmpflood "sum_icmpflood" >/dev/null 2>&1 - json_get_var sum_ctinvalid "sum_ctinvalid" >/dev/null 2>&1 - json_get_var sum_tcpinvalid "sum_tcpinvalid" >/dev/null 2>&1 - json_get_var sum_sets "sum_sets" >/dev/null 2>&1 - json_get_var sum_setinput "sum_setinput" >/dev/null 2>&1 - json_get_var sum_setforwardwan "sum_setforwardwan" >/dev/null 2>&1 - json_get_var sum_setforwardlan "sum_setforwardlan" >/dev/null 2>&1 - json_get_var sum_setelements "sum_setelements" >/dev/null 2>&1 - json_get_var sum_cntinput "sum_cntinput" >/dev/null 2>&1 - json_get_var sum_cntforwardwan "sum_cntforwardwan" >/dev/null 2>&1 - json_get_var sum_cntforwardlan "sum_cntforwardlan" >/dev/null 2>&1 - { - printf "%s\n%s\n%s\n" ":::" "::: banIP Set Statistics" ":::" - printf "%s\n" " Timestamp: ${timestamp}" - printf "%s\n" " ------------------------------" - printf "%s\n" " blocked syn-flood packets : ${sum_synflood}" - printf "%s\n" " blocked udp-flood packets : ${sum_udpflood}" - printf "%s\n" " blocked icmp-flood packets : ${sum_icmpflood}" - printf "%s\n" " blocked invalid ct packets : ${sum_ctinvalid}" - printf "%s\n" " blocked invalid tcp packets: ${sum_tcpinvalid}" - printf "%s\n" " ---" - printf "%s\n" " auto-added IPs to allowlist: ${autoadd_allow}" - printf "%s\n\n" " auto-added IPs to blocklist: ${autoadd_block}" - json_select "sets" >/dev/null 2>&1 - json_get_keys table_sets >/dev/null 2>&1 - if [ -n "${table_sets}" ]; then - printf "%-25s%-15s%-24s%-24s%-24s%s\n" " Set" "| Elements" "| WAN-Input (packets)" "| WAN-Forward (packets)" "| LAN-Forward (packets)" "| Port/Protocol Limit" - printf "%s\n" " ---------------------+--------------+-----------------------+-----------------------+-----------------------+------------------------" - for item in ${table_sets}; do - printf " %-21s" "${item}" - json_select "${item}" - json_get_keys set_details - for detail in ${set_details}; do + json_select "sets" >/dev/null 2>&1 + json_get_keys table_sets >/dev/null 2>&1 + if [ -n "${table_sets}" ]; then + for item in ${table_sets}; do + sum_sets="$((sum_sets + 1))" + json_select "${item}" + json_get_keys set_details + for detail in ${set_details}; do + case "${detail}" in + "cnt_elements") json_get_var jsnval "${detail}" >/dev/null 2>&1 - case "${detail}" in + sum_cntelements="$((sum_cntelements + jsnval))" + ;; + "set_elements") + json_get_values jsnval "${detail}" >/dev/null 2>&1 + if [ -n "${jsnval}" ]; then + jsnval="$(printf '%s' "${jsnval}" | "${ban_wccmd}" -w)" + sum_setelements="$((sum_setelements + jsnval))" + fi + ;; + "inbound") + json_get_var jsnval "${detail}" >/dev/null 2>&1 + if [ "${jsnval}" = "ON" ]; then + sum_setinbound="$((sum_setinbound + 1))" + fi + ;; + "outbound") + json_get_var jsnval "${detail}" >/dev/null 2>&1 + if [ "${jsnval}" = "ON" ]; then + sum_setoutbound="$((sum_setoutbound + 1))" + fi + ;; + "cnt_inbound") + json_get_var jsnval "${detail}" >/dev/null 2>&1 + if [ -n "${jsnval}" ]; then + sum_cntinbound="$((sum_cntinbound + jsnval))" + fi + ;; + "cnt_outbound") + json_get_var jsnval "${detail}" >/dev/null 2>&1 + if [ -n "${jsnval}" ]; then + sum_cntoutbound="$((sum_cntoutbound + jsnval))" + fi + ;; + "port") + json_get_var jsnval "${detail}" >/dev/null 2>&1 + if [ "${jsnval}" != "-" ]; then + jsnval="${jsnval//[^0-9 ]/}" + jsnval="$(printf '%s' "${jsnval}" | "${ban_wccmd}" -w)" + sum_setports="$((sum_setports + jsnval))" + fi + ;; + esac + done + json_select ".." + done + "${ban_sedcmd}" -i '$d' "${report_jsn}" + printf '%s\n' "}, \ + \"timestamp\": \"${timestamp}\", \ + \"autoadd_allow\": \"$("${ban_grepcmd}" -c "added on ${timestamp% *}" "${ban_allowlist}")\", \ + \"autoadd_block\": \"$("${ban_grepcmd}" -c "added on ${timestamp% *}" "${ban_blocklist}")\", \ + \"sum_synflood\": \"${sum_synflood}\", \ + \"sum_udpflood\": \"${sum_udpflood}\", \ + \"sum_icmpflood\": \"${sum_icmpflood}\", \ + \"sum_ctinvalid\": \"${sum_ctinvalid}\", \ + \"sum_tcpinvalid\": \"${sum_tcpinvalid}\", \ + \"sum_bcp38\": \"${sum_bcp38}\", \ + \"sum_sets\": \"${sum_sets}\", \ + \"sum_setinbound\": \"${sum_setinbound}\", \ + \"sum_setoutbound\": \"${sum_setoutbound}\", \ + \"sum_cntelements\": \"${sum_cntelements}\", \ + \"sum_cntinbound\": \"${sum_cntinbound}\", \ + \"sum_cntoutbound\": \"${sum_cntoutbound}\", \ + \"sum_setports\": \"${sum_setports}\", \ + \"sum_setelements\": \"${sum_setelements}\" \ + }" >>"${report_jsn}" + fi + fi + + # retrieve/prepare map data + # + if [ "${ban_nftcount}" = "1" ] && [ "${ban_map}" = "1" ] && [ -s "${report_jsn}" ]; then + cnt="1" + f_getdl + printf '%s' ",[{}" >>"${map_jsn}" + json_init + if json_load_file "${ban_rtfile}" >/dev/null 2>&1; then + json_get_values jsnval "active_uplink" >/dev/null 2>&1 + for uplink_ip in ${jsnval}; do + uplink_ip="${uplink_ip%%/*}" + if [ -n "${uplink_ip}" ] && [ "${uplink_ip}" != "-" ]; then + uplink_list="${uplink_list}${uplink_list:+, }\"${uplink_ip}\"" + fi + done + fi + if [ -n "${uplink_list}" ]; then + "${ban_fetchcmd}" ${ban_geoparm} "[ ${uplink_list} ]" "${ban_geourl}" 2>>"${ban_errorlog}" | + "${ban_jsoncmd}" -qe '@[*&&@.status="success"]' | + "${ban_awkcmd}" -v feed="homeIP" '{printf ",{\"%s\": %s}\n",feed,$0}' >"${map_jsn}.home" + if [ -s "${map_jsn}.home" ]; then + "${ban_catcmd}" "${map_jsn}.home" >>"${map_jsn}" + fi + "${ban_rmcmd}" -f "${map_jsn}.home" + fi + if [ -s "${map_jsn}" ]; then + json_init + if json_load_file "${report_jsn}" >/dev/null 2>&1; then + json_select "sets" >/dev/null 2>&1 + json_get_keys table_sets >/dev/null 2>&1 + if [ -n "${table_sets}" ]; then + for item in ${table_sets}; do + [ "${item%%_*}" = "allowlist" ] && continue + json_select "${item}" + json_get_keys set_details + for detail in ${set_details}; do + if [ "${detail}" = "set_elements" ]; then + json_get_values jsnval "${detail}" >/dev/null 2>&1 + jsnval="\"${jsnval// /\", \"}\"" + fi + done + if [ "${jsnval}" != '""' ]; then + ( + quantity="0" + chunk="" + for ip in ${jsnval}; do + chunk="${chunk} ${ip}" + quantity="$((quantity + 1))" + if [ "${quantity}" -eq "100" ]; then + "${ban_fetchcmd}" ${ban_geoparm} "[ ${chunk} ]" "${ban_geourl}" 2>>"${ban_errorlog}" | + "${ban_jsoncmd}" -qe '@[*&&@.status="success"]' | "${ban_awkcmd}" -v feed="${item//_v/.v}" '{printf ",{\"%s\": %s}\n",feed,$0}' >"${map_jsn}.${item}" + chunk="" + quantity="0" + fi + done + if [ "${quantity}" -gt "0" ]; then + "${ban_fetchcmd}" ${ban_geoparm} "[ ${chunk} ]" "${ban_geourl}" 2>>"${ban_errorlog}" | + "${ban_jsoncmd}" -qe '@[*&&@.status="success"]' | "${ban_awkcmd}" -v feed="${item//_v/.v}" '{printf ",{\"%s\": %s}\n",feed,$0}' >>"${map_jsn}.${item}" + fi + ) & + [ "${cnt}" -gt "${ban_cores}" ] && wait -n + cnt="$((cnt + 1))" + fi + json_select ".." + done + wait + + # assemble map data from per-set fragments + # + for item in ${table_sets}; do + if [ -s "${map_jsn}.${item}" ]; then + "${ban_catcmd}" "${map_jsn}.${item}" >>"${map_jsn}" + fi + "${ban_rmcmd}" -f "${map_jsn}.${item}" + done + fi + fi + fi + fi + + # text output preparation + # + if [ "${output}" != "json" ] && [ -s "${report_jsn}" ]; then + json_init + if json_load_file "${report_jsn}" >/dev/null 2>&1; then + json_get_var timestamp "timestamp" >/dev/null 2>&1 + json_get_var autoadd_allow "autoadd_allow" >/dev/null 2>&1 + json_get_var autoadd_block "autoadd_block" >/dev/null 2>&1 + json_get_var sum_synflood "sum_synflood" >/dev/null 2>&1 + json_get_var sum_udpflood "sum_udpflood" >/dev/null 2>&1 + json_get_var sum_icmpflood "sum_icmpflood" >/dev/null 2>&1 + json_get_var sum_ctinvalid "sum_ctinvalid" >/dev/null 2>&1 + json_get_var sum_tcpinvalid "sum_tcpinvalid" >/dev/null 2>&1 + json_get_var sum_bcp38 "sum_bcp38" >/dev/null 2>&1 + json_get_var sum_sets "sum_sets" >/dev/null 2>&1 + json_get_var sum_setinbound "sum_setinbound" >/dev/null 2>&1 + json_get_var sum_setoutbound "sum_setoutbound" >/dev/null 2>&1 + json_get_var sum_cntelements "sum_cntelements" >/dev/null 2>&1 + json_get_var sum_cntinbound "sum_cntinbound" >/dev/null 2>&1 + json_get_var sum_cntoutbound "sum_cntoutbound" >/dev/null 2>&1 + json_get_var sum_setports "sum_setports" >/dev/null 2>&1 + json_get_var sum_setelements "sum_setelements" >/dev/null 2>&1 + { + printf '%s\n%s\n%s\n' ":::" "::: banIP Set Statistics" ":::" + printf '%s\n' " Timestamp: ${timestamp}" + printf '%s\n' " ------------------------------" + printf '%s\n' " blocked syn-flood packets : ${sum_synflood}" + printf '%s\n' " blocked udp-flood packets : ${sum_udpflood}" + printf '%s\n' " blocked icmp-flood packets : ${sum_icmpflood}" + printf '%s\n' " blocked invalid ct packets : ${sum_ctinvalid}" + printf '%s\n' " blocked invalid tcp packets: ${sum_tcpinvalid}" + printf '%s\n' " blocked bcp38 packets : ${sum_bcp38}" + printf '%s\n' " ---" + printf '%s\n' " auto-added IPs to allowlist: ${autoadd_allow}" + printf '%s\n\n' " auto-added IPs to blocklist: ${autoadd_block}" + json_select "sets" >/dev/null 2>&1 + json_get_keys table_sets >/dev/null 2>&1 + table_sets="$(printf '%s\n' ${table_sets} | "${ban_sortcmd}")" + if [ -n "${table_sets}" ]; then + printf '%-25s%-15s%-24s%-24s%-24s%-24s\n' " Set" "| Count " "| Inbound (packets)" "| Outbound (packets)" "| Port/Protocol " "| Elements (max. 50) " + printf '%s\n' " ---------------------+--------------+-----------------------+-----------------------+-----------------------+------------------------" + for item in ${table_sets}; do + printf ' %-21s' "${item//_v/.v}" + json_select "${item}" + json_get_keys set_details + for detail in ${set_details}; do + case "${detail}" in "cnt_elements") - printf "%-15s" "| ${jsnval}" + json_get_var jsnval "${detail}" >/dev/null 2>&1 + printf '%-15s' "| ${jsnval}" ;; - "cnt_input" | "cnt_forwardwan" | "cnt_forwardlan") + "cnt_inbound" | "cnt_outbound") + json_get_var jsnval "${detail}" >/dev/null 2>&1 [ -n "${jsnval}" ] && tmp_val=": ${jsnval}" ;; + "set_elements") + json_get_values jsnval "${detail}" >/dev/null 2>&1 + jsnval="${jsnval// /, }" + printf '%-24s' "| ${jsnval:0:24}" + jsnval="${jsnval:24}" + while [ -n "${jsnval}" ]; do + printf '\n%-25s%-15s%-24s%-24s%-24s%-24s' "" "|" "|" "|" "|" "| ${jsnval:0:24}" + jsnval="${jsnval:24}" + done + ;; *) - printf "%-24s" "| ${jsnval}${tmp_val}" + json_get_var jsnval "${detail}" >/dev/null 2>&1 + printf '%-24s' "| ${jsnval}${tmp_val}" tmp_val="" ;; - esac + esac + done + printf '\n' + json_select ".." done - printf "\n" - json_select ".." - done - printf "%s\n" " ---------------------+--------------+-----------------------+-----------------------+-----------------------+------------------------" - printf "%-25s%-15s%-24s%-24s%s\n" " ${sum_sets}" "| ${sum_setelements}" "| ${sum_setinput} (${sum_cntinput})" "| ${sum_setforwardwan} (${sum_cntforwardwan})" "| ${sum_setforwardlan} (${sum_cntforwardlan})" - fi - } >>"${report_txt}" + printf '%s\n' " ---------------------+--------------+-----------------------+-----------------------+-----------------------+------------------------" + printf '%-25s%-15s%-24s%-24s%-24s%-24s\n' " ${sum_sets}" "| ${sum_cntelements}" "| ${sum_setinbound} (${sum_cntinbound})" "| ${sum_setoutbound} (${sum_cntoutbound})" "| ${sum_setports}" "| ${sum_setelements}" + fi + } >>"${report_txt}" + fi fi fi # output channel (text|json|mail) # case "${output}" in - "text") - [ -s "${report_txt}" ] && "${ban_catcmd}" "${report_txt}" - ;; - "json") - [ -s "${report_jsn}" ] && "${ban_catcmd}" "${report_jsn}" - ;; - "mail") - [ -n "${ban_mailreceiver}" ] && [ -x "${ban_mailcmd}" ] && f_mail - ;; + "text") + [ -s "${report_txt}" ] && "${ban_catcmd}" "${report_txt}" + ;; + "json") + if [ "${ban_nftcount}" = "1" ] && [ "${ban_map}" = "1" ]; then + jsn="$("${ban_catcmd}" ${report_jsn} ${map_jsn} 2>>"${ban_errorlog}")" + [ -n "${jsn}" ] && printf '[%s]]\n' "${jsn}" + else + jsn="$("${ban_catcmd}" ${report_jsn} 2>>"${ban_errorlog}")" + [ -n "${jsn}" ] && printf '[%s]\n' "${jsn}" + fi + ;; + "mail") + [ -n "${ban_mailreceiver}" ] && [ -x "${ban_mailcmd}" ] && f_mail + ;; + "gen") + printf '%s\n' "1" >"${ban_rundir}/banIP.report" + ;; esac : >"${report_txt}" } -# Set search -# f_search() { - local item table_sets ip proto hold cnt result_flag="/var/run/banIP.search" input="${1}" - - if [ -n "${input}" ]; then - ip="$(printf "%s" "${input}" | "${ban_awkcmd}" 'BEGIN{RS="(([1-9][0-9]{0,2}\\.){1}([0-9]{1,3}\\.){2}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?[[:space:]]*$)"}{printf "%s",RT}')" - [ -n "${ip}" ] && proto="v4" - if [ -z "${proto}" ]; then - ip="$(printf "%s" "${input}" | "${ban_awkcmd}" 'BEGIN{RS="(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)([[:space:]].*|$)"}{printf "%s",RT}')" - [ -n "${ip}" ] && proto="v6" - fi - fi + local item table_sets ip proto cnt tmp_result result res input="${1}" + + # prepare result file + # + tmp_result="${ban_rundir}/banIP.search.tmp" + result="${ban_rundir}/banIP.search" + + # validate input + # + case "${input}" in + '' | *[!0-9A-Fa-f:/.]*) + printf '%s\n%s\n%s\n' ":::" "::: no valid search input" ":::" + printf '%s\n%s\n%s\n' ":::" "::: no valid search input" ":::" >"${result}" + return + ;; + esac + + # determine protocol via awk + # + res="$(printf '%s' "${input}" | "${ban_awkcmd}" ' + { + if (match($0,/(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])(\/([0-9]|[12][0-9]|3[012]))?[[:space:]]*$/)) { + printf "v4 %s",substr($0,RSTART,RLENGTH) + } else if (match($0,/(([0-9A-Fa-f]{0,4}:){1,7}[0-9A-Fa-f]{0,4}:?(\/([0-9]|[1-9][0-9]|1[01][0-9]|12[0-8]))?)/)) { + printf "v6 %s",substr($0,RSTART,RLENGTH) + } + }')" + proto="${res%% *}" + ip="${res#* }" + [ "${proto}" != "v4" ] && [ "${proto}" != "v6" ] && proto="" && ip="" + + # get relevant Sets + # if [ -n "${proto}" ]; then - table_sets="$("${ban_nftcmd}" -tj list ruleset 2>/dev/null | "${ban_jsoncmd}" -qe "@.nftables[@.set.table=\"banIP\"&&@.set.type=\"ip${proto}_addr\"].set.name")" + table_sets="$("${ban_nftcmd}" -tj list table inet banIP 2>>"${ban_errorlog}" | + "${ban_jsoncmd}" -qe "@.nftables[@.set.type=\"ip${proto}_addr\"].set.name")" else - printf "%s\n%s\n%s\n" ":::" "::: no valid search input" ":::" + printf '%s\n%s\n%s\n' ":::" "::: no valid search input" ":::" + printf '%s\n%s\n%s\n' ":::" "::: no valid search input" ":::" >"${result}" return fi + + # initial output + # + { + printf '%s\n%s\n%s\n' ":::" "::: banIP Search" ":::" + printf ' %s\n' "Looking for IP '${ip}' on $(date "+%Y-%m-%d %H:%M:%S")" + printf ' %s\n' "---" + } >"${tmp_result}" + + # search for IP in Sets + # cnt="1" for item in ${table_sets}; do - [ -f "${result_flag}" ] && break + case "${item}" in + *[!a-zA-Z0-9_.]*) + continue + ;; + esac ( if "${ban_nftcmd}" get element inet banIP "${item}" "{ ${ip} }" >/dev/null 2>&1; then - printf "%s\n%s\n%s\n" ":::" "::: banIP Search" ":::" - printf " %s\n" "Looking for IP '${ip}' on $(date "+%Y-%m-%d %H:%M:%S")" - printf " %s\n" "---" - printf " %s\n" "IP found in Set '${item}'" - : >"${result_flag}" + printf ' %s\n' "IP found in Set '${item}'" >"${tmp_result}.${item}" fi ) & - hold="$((cnt % ban_cores))" - [ "${hold}" = "0" ] && wait + [ "${cnt}" -gt "${ban_cores}" ] && wait -n cnt="$((cnt + 1))" done wait - if [ -f "${result_flag}" ]; then - rm -f "${result_flag}" - else - printf "%s\n%s\n%s\n" ":::" "::: banIP Search" ":::" - printf " %s\n" "Looking for IP '${ip}' on $(date "+%Y-%m-%d %H:%M:%S")" - printf " %s\n" "---" - printf " %s\n" "IP not found" + + # assemble search results from per-set fragments + # + for item in ${table_sets}; do + if [ -s "${tmp_result}.${item}" ]; then + "${ban_catcmd}" "${tmp_result}.${item}" >>"${tmp_result}" + "${ban_rmcmd}" -f "${tmp_result}.${item}" + fi + done + + # output result + # + if ! "${ban_grepcmd}" -qm1 "found" "${tmp_result}"; then + printf ' %s\n' "IP not found" >>"${tmp_result}" fi + "${ban_mvcmd}" -f "${tmp_result}" "${result}" + "${ban_catcmd}" "${result}" } -# Set survey +# Set content # -f_survey() { - local set_elements input="${1}" +f_content() { + local set_raw set_elements input="${1}" filter="${2}" + + # load config if not already done + # + [ -z "${ban_dev}" ] && f_conf + + # validate input + # + case "${input}" in + "" | *[!a-zA-Z0-9_.]*) + printf '%s\n%s\n%s\n' ":::" "::: no valid Set input" ":::" + return + ;; + esac + + case "${filter}" in + "" | "false") + filter="false" + ;; + "true") + filter="true" + ;; + *) + printf '%s\n%s\n%s\n' ":::" "::: no valid filter input" ":::" + return + ;; + esac - if [ -z "${input}" ]; then - printf "%s\n%s\n%s\n" ":::" "::: no valid survey input" ":::" + # check if Set exists + # + if ! "${ban_nftcmd}" -t list set inet banIP "${input}" >/dev/null 2>&1; then + printf '%s\n%s\n%s\n' ":::" "::: Set '${input}' not found" ":::" return fi - set_elements="$("${ban_nftcmd}" -j list set inet banIP "${input}" 2>/dev/null | "${ban_jsoncmd}" -qe '@.nftables[*].set.elem[*]')" - printf "%s\n%s\n%s\n" ":::" "::: banIP Survey" ":::" - printf " %s\n" "List of elements in the Set '${input}' on $(date "+%Y-%m-%d %H:%M:%S")" - printf " %s\n" "---" - [ -n "${set_elements}" ] && printf "%s\n" "${set_elements}" || printf " %s\n" "empty Set" + + # get Set content + # + set_raw="$("${ban_nftcmd}" -j list set inet banIP "${input}" 2>>"${ban_errorlog}")" + if [ "${ban_nftcount}" = "1" ]; then + if [ "${filter}" = "true" ]; then + set_elements="$(printf '%s' "${set_raw}" | "${ban_jsoncmd}" -qe '@.nftables[*].set.elem[*][@.counter.packets>0].*' | + "${ban_awkcmd}" 'NR%2==1{ip=$0;next}BEGIN{FS="[:,{}\"]+"}{print ip ", packets: "$4 }')" + else + set_elements="$(printf '%s' "${set_raw}" | "${ban_jsoncmd}" -qe '@.nftables[*].set.elem[*].elem["val","counter"]' | + "${ban_awkcmd}" 'NR%2==1{ip=$0;next}BEGIN{FS="[:,{}\"]+"}{print ip ", packets: "$4 }')" + fi + else + set_elements="$(printf '%s' "${set_raw}" | "${ban_jsoncmd}" -qe '@.nftables[*].set.elem[*]')" + fi + + # output result + # + printf '%s\n%s\n%s\n' ":::" "::: banIP Set Content" ":::" + printf ' %s\n' "List elements of the Set '${input}' on $(date "+%Y-%m-%d %H:%M:%S")" + printf ' %s\n' "---" + [ -n "${set_elements}" ] && printf '%s\n' "${set_elements}" || printf ' %s\n' "no elements in Set" } # send status mail @@ -1646,111 +2605,350 @@ f_mail() { # send mail # ban_mailhead="From: ${ban_mailsender}\nTo: ${ban_mailreceiver}\nSubject: ${ban_mailtopic}\nReply-to: ${ban_mailsender}\nMime-Version: 1.0\nContent-Type: text/html;charset=utf-8\nContent-Disposition: inline\n\n" - printf "%b" "${ban_mailhead}${mail_text}" | "${ban_mailcmd}" --timeout=10 ${msmtp_debug} -a "${ban_mailprofile}" "${ban_mailreceiver}" >/dev/null 2>&1 - f_log "info" "send status mail (${?})" + printf '%b' "${ban_mailhead}${mail_text}" | "${ban_mailcmd}" --timeout=10 ${msmtp_debug} -a "${ban_mailprofile}" "${ban_mailreceiver}" >/dev/null 2>&1 - f_log "debug" "f_mail ::: notification: ${ban_mailnotification}, template: ${ban_mailtemplate}, profile: ${ban_mailprofile}, receiver: ${ban_mailreceiver}, rc: ${?}" + f_log "debug" "f_mail ::: notification: ${ban_mailnotification}, template: ${ban_mailtemplate}, profile: ${ban_mailprofile}, receiver: ${ban_mailreceiver}, rc: ${?}" } # log monitor # f_monitor() { - local daemon logread_cmd loglimit_cmd nft_expiry line proto ip log_raw log_count idx prefix cidr rdap_log rdap_rc rdap_idx rdap_info + local nft_expiry ip proto idx base cidr rdap_log rdap_rc rdap_idx rdap_info log_type allow_v4 allow_v6 block_v4 block_v6 + local file cache_ts date_stamp time_now time_elapsed cache_interval rdap_interval rdap_tsfile rdap_lock rdap_jobs + local rdap_ts block_cache block_cache_limit block_cache_cnt - if [ -f "${ban_logreadfile}" ]; then - logread_cmd="${ban_logreadcmd} -qf ${ban_logreadfile} 2>/dev/null | ${ban_grepcmd} -e \"${ban_logterm%%??}\" 2>/dev/null" - loglimit_cmd="${ban_logreadcmd} -qn ${ban_loglimit} ${ban_logreadfile} 2>/dev/null" - else - logread_cmd="${ban_logreadcmd} -fe \"${ban_logterm%%??}\" 2>/dev/null" - loglimit_cmd="${ban_logreadcmd} -l ${ban_loglimit} 2>/dev/null" + # intervals for periodic cache refresh and RDAP queries + # + cache_interval=300 + rdap_interval=2 + rdap_tsfile="${ban_rundir}/banIP_rdap_ts" + printf '%s' "0" >"${rdap_tsfile}" + + # determine log reader type + # + if [ -f "${ban_logreadfile}" ] && [ -x "${ban_logreadcmd}" ] && [ "${ban_logreadcmd##*/}" = "tail" ]; then + log_type="tail" + elif [ -x "${ban_logreadcmd}" ] && [ "${ban_logreadcmd##*/}" = "logread" ]; then + log_type="logread" fi - if [ -x "${ban_logreadcmd}" ] && [ -n "${logread_cmd}" ] && [ -n "${loglimit_cmd}" ] && [ -n "${ban_logterm%%??}" ] && [ "${ban_loglimit}" != "0" ]; then + # start log monitoring + # + if [ -n "${log_type}" ] && [ -n "${ban_logterm}" ] && [ "${ban_loglimit}" != "0" ]; then f_log "info" "start detached banIP log service (${ban_logreadcmd})" - [ -n "${ban_nftexpiry}" ] && nft_expiry="timeout $(printf "%s" "${ban_nftexpiry}" | "${ban_grepcmd}" -oE "([0-9]+[d|h|m|s])+$")" - eval "${logread_cmd}" | - while read -r line; do - proto="" - : >"${ban_rdapfile}" - if [ -z "${daemon}" ]; then - daemon="$(printf "%s" "${line}" | "${ban_awkcmd}" 'BEGIN{RS="dropbear"}{if(!seen[RT]++)printf "%s",RT}')" - [ -z "${daemon}" ] && daemon="sshd" + + # determine nft timeout expression and cache interval + # + if printf '%s' "${ban_nftexpiry}" | grep -qE '^([1-9][0-9]*(ms|s|m|h|d|w))+$'; then + nft_expiry="timeout ${ban_nftexpiry}" + cache_interval="$(printf '%s' "${ban_nftexpiry}" | "${ban_awkcmd}" '{ + s = 0 + str = $0 + while (match(str, /([0-9]+)(ms|s|m|h|d|w)/, a)) { + if (a[2] == "ms") s += a[1] / 1000 + else if (a[2] == "s") s += a[1] + else if (a[2] == "m") s += a[1] * 60 + else if (a[2] == "h") s += a[1] * 3600 + else if (a[2] == "d") s += a[1] * 86400 + else if (a[2] == "w") s += a[1] * 604800 + str = substr(str, RSTART + RLENGTH) + } + interval = int(s / 2) + if (interval < 30) interval = 30 + if (interval > 300) interval = 300 + printf "%d", interval + }')" + fi + + # helper function to extract space-padded bare IPs/CIDRs from nft set listing + # + nft_cache() { + "${ban_nftcmd}" list set inet banIP "${1}" 2>/dev/null | + "${ban_awkcmd}" '{gsub(/[,{}]/, " "); for(i=1;i<=NF;i++) if($i~/^[0-9A-Fa-f].*[.:]/) printf " %s ",$i}' + } + + # retrieve/cache current allowlist/blocklist content + # + allow_v4="$(nft_cache allowlist.v4)" + allow_v6="$(nft_cache allowlist.v6)" + block_v4="$(nft_cache blocklist.v4)" + block_v6="$(nft_cache blocklist.v6)" + + # initial cache timestamp and datestamp + # + read -r cache_ts _ <"/proc/uptime" + cache_ts="${cache_ts%%.*}" + date_stamp="$(date "+%Y-%m-%d %H:%M:%S")" + block_cache="" + block_cache_limit="500" + block_cache_cnt="0" + + # clean up stale RDAP lock/done markers from previous runs + # + "${ban_rmcmd}" -f "${ban_rdapfile}".* + + # log monitoring loop + # awk handles IP extraction, counting and threshold detection internally, + # only IPs reaching ban_logcount are emitted as "BLOCK ip proto" to the shell loop + # + { + case "${log_type}" in + tail) + "${ban_logreadcmd}" -qf "${ban_logreadfile}" 2>/dev/null | + "${ban_grepcmd}" -e "${ban_logterm}" 2>/dev/null + ;; + logread) + "${ban_logreadcmd}" -fe "${ban_logterm}" 2>/dev/null + ;; + esac + } | ban_logterm_map="${ban_logterm_map}" "${ban_awkcmd}" -v threshold="${ban_logcount}" -v limit=5000 ' + function pick_ip(s, mode, m, res) { + res = "" + while (match(s, /([0-9]{1,3}\.){3}[0-9]{1,3}|([A-Fa-f0-9]{0,4}:){2,7}[A-Fa-f0-9]{0,4}/, m)) { + res = m[0] + if (mode == "first") break + s = substr(s, RSTART + RLENGTH) + } + return res + } + function anchor_for(line, k) { for (k = 1; k <= nterm; k++) if (line ~ pat[k]) return anc[k]; return "last" } + BEGIN { + unique = 0 + map = ENVIRON["ban_logterm_map"] + n = split(map, recs, "\036") + nterm = 0 + all_last = 1 + for (i = 1; i <= n; i++) { + if (recs[i] == "") continue + split(recs[i], f, "\037") + nterm++ + anc[nterm] = f[1] + pat[nterm] = f[2] + if (f[1] != "last") all_last = 0 + } + } + { + pos = all_last ? "last" : anchor_for($0) + $0 = gensub(/(([0-9]{1,3}\.){3}[0-9]{1,3}):[0-9]+/, "\\1", "g", $0) + sub(/\]:[0-9]+/, "]", $0) + gsub(/[<>[\]]/, "", $0) + ip = "" + proto = "" + cand = pick_ip($0, pos) + if (cand ~ /\./) { + if (cand !~ /^127\./ && cand !~ /^0\./) { + ip = cand + proto = ".v4" + } + } else if (cand ~ /:/) { + sub(/%.*/, "", cand) + if (cand !~ /^[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$/ && cand !~ /^([A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}$/) { + ip = cand + proto = ".v6" + } + } + if (!ip) { + next + } + cnt[ip]++ + if (cnt[ip] == 1) { + unique++ + if (unique >= limit) { + delete cnt + unique = 0 + print "RESET" + fflush() + } + } + if (cnt[ip] == threshold) { + print "BLOCK " ip " " proto + fflush() + delete cnt[ip] + unique-- + } + }' | while read -r action ip proto; do + + # process only BLOCK/RESET actions emitted by awk + # + case "${action}" in + BLOCK) + f_log "debug" "f_monitor ::: block request for IP '${ip}' (protocol: IP${proto})" + + # periodic monitor cache refresh (only on BLOCK events to reduce /proc/uptime reads) + # + read -r time_now _ <"/proc/uptime" + time_now="${time_now%%.*}" + if [ "$((time_now - cache_ts))" -ge "${cache_interval}" ]; then + block_v4="$(nft_cache blocklist.v4)" + block_v6="$(nft_cache blocklist.v6)" + date_stamp="$(date "+%Y-%m-%d %H:%M:%S")" + cache_ts="${time_now}" + block_cache="" + block_cache_cnt="0" + "${ban_rmcmd}" -f "${ban_rdapfile}".*.done + f_log "debug" "f_monitor ::: refreshed monitor cache at ${date_stamp}" fi - ip="$(printf "%s" "${line}" | "${ban_awkcmd}" 'BEGIN{RS="(([1-9][0-9]{0,2}\\.){1}([0-9]{1,3}\\.){2}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5]))+"}{if(!seen[RT]++)printf "%s ",RT}')" - ip="$(f_trim "${ip}")" - ip="${ip##* }" - [ -n "${ip}" ] && [ "${ip%%.*}" != "127" ] && [ "${ip%%.*}" != "0" ] && proto="v4" - if [ -z "${proto}" ]; then - if [ "${daemon}" = "dropbear" ]; then - ip="$(printf "%s" "${line}" | "${ban_awkcmd}" 'BEGIN{RS="([A-Fa-f0-9]{1,4}::?){3,7}([A-Fa-f0-9]:?)+"}{if(!seen[RT]++)printf "%s ",RT}')" - ip="${ip%:*}" - else - ip="$(printf "%s" "${line}" | "${ban_awkcmd}" 'BEGIN{RS="([A-Fa-f0-9]{1,4}::?){3,7}[A-Fa-f0-9]{1,4}"}{if(!seen[RT]++)printf "%s ",RT}')" + + # fast exact string match against cached Set content + # + case "${proto}" in + .v4) + case "${allow_v4} ${block_v4} ${block_cache}" in + *" ${ip} "*) + f_log "debug" "f_monitor ::: skip IP '${ip}', found in cached IP${proto} Sets" + continue + ;; + esac + ;; + .v6) + case "${allow_v6} ${block_v6} ${block_cache}" in + *" ${ip} "*) + f_log "debug" "f_monitor ::: skip IP '${ip}', found in cached IP${proto} Sets" + continue + ;; + esac + ;; + esac + + # CIDR-aware allowlist lookup (only at block-time, not every IP) + # + if "${ban_nftcmd}" get element inet banIP "allowlist${proto}" { ${ip} } >/dev/null 2>&1; then + block_cache_cnt="$((block_cache_cnt + 1))" + if [ "${block_cache_cnt}" -ge "${block_cache_limit}" ]; then + block_cache="" + block_cache_cnt="1" + f_log "debug" "f_monitor ::: refreshed local monitor cache at ${date_stamp}" fi - ip="$(f_trim "${ip}")" - ip="${ip##* }" - [ -n "${ip%%::*}" ] && proto="v6" + block_cache="${block_cache} ${ip} " + f_log "debug" "f_monitor ::: skip IP '${ip}', found via allowlist CIDR lookup" + continue fi - if [ -n "${proto}" ] && ! "${ban_nftcmd}" get element inet banIP allowlist"${proto}" "{ ${ip} }" >/dev/null 2>&1 && ! "${ban_nftcmd}" get element inet banIP blocklist"${proto}" "{ ${ip} }" >/dev/null 2>&1; then - f_log "info" "suspicious IP '${ip}'" - log_raw="$(eval ${loglimit_cmd})" - log_count="$(printf "%s\n" "${log_raw}" | "${ban_grepcmd}" -c "suspicious IP '${ip}'")" - if [ "${log_count}" -ge "${ban_logcount}" ]; then - if "${ban_nftcmd}" add element inet banIP "blocklist${proto}" { ${ip} ${nft_expiry} } >/dev/null 2>&1; then - f_log "info" "add IP '${ip}' (expiry: ${ban_nftexpiry:-"-"}) to blocklist${proto} set" - fi - if [ "${ban_autoblocksubnet}" = "1" ]; then - rdap_log="$("${ban_fetchcmd}" ${ban_rdapparm} "${ban_rdapfile}" "${ban_rdapurl}${ip}" 2>&1)" - rdap_rc="${?}" - if [ "${rdap_rc}" = "0" ] && [ -s "${ban_rdapfile}" ]; then - [ "${proto}" = "v4" ] && rdap_idx="$("${ban_jsoncmd}" -i "${ban_rdapfile}" -qe '@.cidr0_cidrs[@.v4prefix].*' | "${ban_awkcmd}" '{ORS=" "; print}')" - [ "${proto}" = "v6" ] && rdap_idx="$("${ban_jsoncmd}" -i "${ban_rdapfile}" -qe '@.cidr0_cidrs[@.v6prefix].*' | "${ban_awkcmd}" '{ORS=" "; print}')" - rdap_info="$("${ban_jsoncmd}" -l1 -i "${ban_rdapfile}" -qe '@.country' -qe '@.notices[@.title="Source"].description[1]' | "${ban_awkcmd}" 'BEGIN{RS="";FS="\n"}{printf "%s, %s",$1,$2}')" - [ -z "${rdap_info}" ] && rdap_info="$("${ban_jsoncmd}" -l1 -i "${ban_rdapfile}" -qe '@.notices[0].links[0].value' | "${ban_awkcmd}" 'BEGIN{FS="[/.]"}{printf"%s, %s","n/a",toupper($4)}')" - for idx in ${rdap_idx}; do - if [ -z "${prefix}" ]; then - prefix="${idx}" - continue - else - cidr="${prefix}/${idx}" - if "${ban_nftcmd}" add element inet banIP "blocklist${proto}" { ${cidr} ${nft_expiry} } >/dev/null 2>&1; then - f_log "info" "add IP range '${cidr}' (source: ${rdap_info:-"n/a"} ::: expiry: ${ban_nftexpiry:-"-"}) to blocklist${proto} set" - fi - prefix="" + + # try to add IP to the blocklist Set with appropriate expiry + # + if "${ban_nftcmd}" add element inet banIP "blocklist${proto}" { ${ip} ${nft_expiry} } >/dev/null 2>&1; then + block_cache_cnt="$((block_cache_cnt + 1))" + if [ "${block_cache_cnt}" -ge "${block_cache_limit}" ]; then + block_cache="" + block_cache_cnt="1" + f_log "debug" "f_monitor ::: refreshed local monitor cache at ${date_stamp}" + fi + block_cache="${block_cache} ${ip} " + f_log "info" "add IP '${ip}' (cnt: ${ban_logcount}, expiry: ${ban_nftexpiry:-"0"}) to blocklist${proto} Set" + else + f_log "info" "failed to add IP '${ip}' to blocklist${proto} Set with rc '${?}'" + continue + fi + + # RDAP subnet lookup with rate limiting (background, non-blocking) + # + if [ "${ban_autoblocksubnet}" = "1" ]; then + + # per-IP dedup — skip if already in-flight or completed + # + rdap_lock="${ban_rdapfile}.${ip}.lock" + if [ ! -f "${rdap_lock}" ] && [ ! -f "${ban_rdapfile}.${ip}.done" ]; then + + # global job limit — max. concurrent RDAP subshells (ban_cores), + # to avoid excessive load and potential DoS against RDAP service when multiple IPs are blocked in a short time frame + # + rdap_jobs=0 + for file in "${ban_rdapfile}".*.lock; do + [ -e "${file}" ] || continue + rdap_jobs="$((rdap_jobs + 1))" + done + + # only spawn new RDAP subshell if current number of in-flight RDAP lookups is below ban_cores limit + # + if [ "${rdap_jobs}" -lt "${ban_cores}" ]; then + : >"${rdap_lock}" + ( + # rate limiting via shared timestamp file + # + ( + "${ban_flockcmd}" -x 9 + read -r rdap_ts <"${rdap_tsfile}" 2>/dev/null + rdap_ts="${rdap_ts:-0}" + read -r time_now _ <"/proc/uptime" + time_now="${time_now%%.*}" + time_elapsed=$((time_now - rdap_ts)) + if [ "${time_elapsed}" -lt "${rdap_interval}" ]; then + sleep "$((rdap_interval - time_elapsed))" fi - done - else - f_log "info" "rdap request failed (rc: ${rdap_rc:-"-"}/log: ${rdap_log})" - fi - fi - if [ -z "${ban_nftexpiry}" ] && [ "${ban_autoblocklist}" = "1" ] && ! "${ban_grepcmd}" -q "^${ip}" "${ban_blocklist}"; then - printf "%-45s%s\n" "${ip}" "# added on $(date "+%Y-%m-%d %H:%M:%S")" >>"${ban_blocklist}" - f_log "info" "add IP '${ip}' to local blocklist" + read -r rdap_ts _ <"/proc/uptime" + rdap_ts="${rdap_ts%%.*}" + printf '%s' "${rdap_ts}" >"${rdap_tsfile}" + ) 9>"${rdap_tsfile}.lock" + : >"${ban_rdapfile}.${ip}" + rdap_log="$("${ban_fetchcmd}" ${ban_rdapparm} "${ban_rdapfile}.${ip}" "${ban_rdapurl}${ip}" 2>&1)" + rdap_rc="${?}" + + # process RDAP response if valid JSON with expected content, otherwise log error + # + if [ "${rdap_rc}" = "0" ] && [ -s "${ban_rdapfile}.${ip}" ]; then + [ "${proto}" = ".v4" ] && rdap_idx="$("${ban_jsoncmd}" -i "${ban_rdapfile}.${ip}" -qe '@.cidr0_cidrs[@.v4prefix].*' | "${ban_awkcmd}" '{ORS=" "; print}')" + [ "${proto}" = ".v6" ] && rdap_idx="$("${ban_jsoncmd}" -i "${ban_rdapfile}.${ip}" -qe '@.cidr0_cidrs[@.v6prefix].*' | "${ban_awkcmd}" '{ORS=" "; print}')" + rdap_info="$("${ban_jsoncmd}" -l1 -i "${ban_rdapfile}.${ip}" -qe '@.country' -qe '@.notices[@.title="Source"].description[1]' | "${ban_awkcmd}" 'BEGIN{RS="";FS="\n"}{c=($1!=""?$1:"-"); s=($2!=""?$2:"-"); printf "%s, %s", c, s}')" + [ -z "${rdap_info}" ] || [ "${rdap_info}" = "-, -" ] && rdap_info="$("${ban_jsoncmd}" -l1 -i "${ban_rdapfile}.${ip}" -qe '@.notices[0].links[0].value' | "${ban_awkcmd}" 'BEGIN{FS="[/.]"}{printf"%s, %s","n/a",toupper($4)}')" + + # if RDAP response contains (multiple) valid CIDR info, + # attempt to add entire range to blocklist set with same expiry as individual IP + # + base="" + for idx in ${rdap_idx}; do + if [ -z "${base}" ]; then + base="${idx}" + continue + else + case "${base}" in + "" | "::"* | "127."* | "0."* | "fe80:"*) + base="" + continue + ;; + esac + [ -z "${base}" ] && continue + cidr="${base}/${idx}" + if "${ban_nftcmd}" add element inet banIP "blocklist${proto}" { ${cidr} ${nft_expiry} } >/dev/null 2>&1; then + f_log "info" "add IP range '${cidr}' (source: ${rdap_info:-"n/a"} ::: expiry: ${ban_nftexpiry:-"-"}) to blocklist${proto} set" + fi + base="" + fi + done + else + f_log "info" "rdap request failed (rc: ${rdap_rc:-"-"}/log: ${rdap_log:-"-"}) for IP '${ip}'" + fi + : >"${ban_rdapfile}.${ip}.done" + "${ban_rmcmd}" -f "${ban_rdapfile}.${ip}" "${rdap_lock}" + ) & fi fi fi - done + + # persist to local blocklist file if no expiry + # + if [ -z "${ban_nftexpiry}" ] && [ "${ban_autoblocklist}" = "1" ] && ! "${ban_grepcmd}" -q "^${ip}[[:space:]]" "${ban_blocklist}"; then + printf '%-45s%s\n' "${ip}" "# added on ${date_stamp}" >>"${ban_blocklist}" + f_log "info" "add IP '${ip}' to local blocklist" + fi + ;; + RESET) + f_log "debug" "monitor counter limit reached (5000 unique IPs), awk reset" + ;; + esac + done else + + # no valid log reader configuration, start detached no-op service to keep monitor option enabled + # f_log "info" "start detached no-op banIP service" sleep infinity fi } -# initial sourcing -# -if [ -r "/lib/functions.sh" ] && [ -r "/lib/functions/network.sh" ] && [ -r "/usr/share/libubox/jshn.sh" ]; then - . "/lib/functions.sh" - . "/lib/functions/network.sh" - . "/usr/share/libubox/jshn.sh" -else - f_log "emerg" "system libraries not found" -fi - # reference required system utilities # -ban_awkcmd="$(f_cmd gawk awk)" +ban_awkcmd="$(f_cmd gawk)" ban_catcmd="$(f_cmd cat)" -ban_fw4cmd="$(f_cmd fw4)" ban_grepcmd="$(f_cmd grep)" ban_jsoncmd="$(f_cmd jsonfilter)" ban_logcmd="$(f_cmd logger)" @@ -1758,14 +2956,33 @@ ban_lookupcmd="$(f_cmd nslookup)" ban_mailcmd="$(f_cmd msmtp optional)" ban_nftcmd="$(f_cmd nft)" ban_pgrepcmd="$(f_cmd pgrep)" +ban_xargscmd="$(f_cmd xargs)" +ban_flockcmd="$(f_cmd flock)" ban_sedcmd="$(f_cmd sed)" ban_ubuscmd="$(f_cmd ubus)" ban_zcatcmd="$(f_cmd zcat)" ban_gzipcmd="$(f_cmd gzip)" +ban_sortcmd="$(f_cmd sort)" +ban_wccmd="$(f_cmd wc)" +ban_mvcmd="$(f_cmd mv)" +ban_rmcmd="$(f_cmd rm)" -f_system -if [ "${ban_action}" != "stop" ]; then +# initial sourcing +# +if [ -r "/lib/functions.sh" ] && [ -r "/lib/functions/network.sh" ] && [ -r "/usr/share/libubox/jshn.sh" ]; then + . "/lib/functions.sh" + . "/lib/functions/network.sh" + . "/usr/share/libubox/jshn.sh" +else + f_log "emerg" "system libraries not found" +fi + +# initial system check +# +[ -S "/var/run/ubus/ubus.sock" ] && f_system + +if [ -n "${ban_action}" ] && [ "${ban_action}" != "stop" ]; then [ ! -d "/etc/banip" ] && f_log "err" "no banIP config directory" [ ! -r "/etc/config/banip" ] && f_log "err" "no banIP config" [ "$(uci_get banip global ban_enabled)" = "0" ] && f_log "err" "banIP is disabled" -fi +fi \ No newline at end of file diff --git a/packages/banip/files/banip-service.sh b/packages/banip/files/banip-service.sh index 1f38e07ec..2cab11c56 100755 --- a/packages/banip/files/banip-service.sh +++ b/packages/banip/files/banip-service.sh @@ -1,37 +1,37 @@ #!/bin/sh # banIP main service script - ban incoming and outgoing IPs via named nftables Sets -# Copyright (c) 2018-2024 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2018-2026 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. # (s)hellcheck exceptions # shellcheck disable=all ban_action="${1}" -ban_starttime="$(date "+%s")" +read -r ban_starttime _ <"/proc/uptime" +ban_starttime="${ban_starttime%%.*}" ban_funlib="/usr/lib/banip-functions.sh" -[ -z "${ban_ver}" ] && . "${ban_funlib}" +[ -z "${ban_bver}" ] && . "${ban_funlib}" # load config and set banIP environment # [ "${ban_action}" = "boot" ] && sleep "$(uci_get banip global ban_triggerdelay "20")" f_conf -f_log "info" "start banIP processing (${ban_action}, ${ban_ver:-"n/a"})" +f_log "info" "start banIP processing (${ban_action}, ${ban_bver:-"n/a"})" f_genstatus "processing" f_tmp -f_getfetch +f_getdl f_getif f_getdev -f_getuplink +f_getup f_mkdir "${ban_backupdir}" f_mkfile "${ban_allowlist}" f_mkfile "${ban_blocklist}" +f_rmdir "${ban_errordir}" # firewall/fw4 pre-check # -if [ ! -x "${ban_fw4cmd}" ] || [ ! -x "/etc/init.d/firewall" ]; then - f_log "err" "firewall/fw4 not found" -elif ! /etc/init.d/firewall status >/dev/null 2>&1; then - f_log "info" "firewall/fw4 is not running" +if ! /etc/init.d/firewall status >/dev/null 2>&1; then + f_log "info" "the main firewall is not running" fi # init banIP nftables namespace @@ -40,28 +40,45 @@ if [ "${ban_action}" != "reload" ] || ! "${ban_nftcmd}" list chain inet banIP pr f_nftinit "${ban_tmpfile}".init.nft fi -# handle downloads +# start banIP processing # f_log "info" "start banIP download processes" -if [ "${ban_allowlistonly}" = "1" ]; then - ban_feed="" -else - f_getfeed -fi -[ "${ban_deduplicate}" = "1" ] && printf "\n" >"${ban_tmpfile}.deduplicate" +f_getfeed +[ "${ban_deduplicate}" = "1" ] && printf '\n' >"${ban_tmpfile}.deduplicate" +# handle downloads +# cnt="1" for feed in allowlist ${ban_feed} blocklist; do + # local feeds (sequential processing) # if [ "${feed}" = "allowlist" ] || [ "${feed}" = "blocklist" ]; then for proto in 4MAC 6MAC 4 6; do - [ "${feed}" = "blocklist" ] && wait - f_down "${feed}" "${proto}" + f_down "${feed}" "${proto}" "-" "-" "inout" done continue fi + # skip external feeds in allowlistonly mode + # + if [ "${ban_allowlistonly}" = "1" ]; then + case " ${ban_feedin} " in + *" allowlist "*) ;; + + *) + case " ${ban_feedout} " in + *" allowlist "*) ;; + + *) + f_log "info" "skip feed '${feed}' in allowlistonly mode" + continue + ;; + esac + ;; + esac + fi + # external feeds (parallel processing on multicore hardware) # if ! json_select "${feed}" >/dev/null 2>&1; then @@ -70,56 +87,73 @@ for feed in allowlist ${ban_feed} blocklist; do uci_commit "banip" continue fi - json_objects="url_4 rule_4 url_6 rule_6 flag" + json_objects="url_4 url_6 rule chain flag" for object in ${json_objects}; do - eval json_get_var feed_"${object}" '${object}' >/dev/null 2>&1 + json_get_var "feed_${object}" "${object}" >/dev/null 2>&1 done json_select .. # skip incomplete feeds # - if { { [ -n "${feed_url_4}" ] && [ -z "${feed_rule_4}" ]; } || { [ -z "${feed_url_4}" ] && [ -n "${feed_rule_4}" ]; }; } || - { { [ -n "${feed_url_6}" ] && [ -z "${feed_rule_6}" ]; } || { [ -z "${feed_url_6}" ] && [ -n "${feed_rule_6}" ]; }; } || - { [ -z "${feed_url_4}" ] && [ -z "${feed_rule_4}" ] && [ -z "${feed_url_6}" ] && [ -z "${feed_rule_6}" ]; }; then + if { [ -z "${feed_url_4}" ] && [ -z "${feed_url_6}" ]; } || + { { [ -n "${feed_url_4}" ] || [ -n "${feed_url_6}" ]; } && [ -z "${feed_rule}" ]; }; then f_log "info" "skip incomplete feed '${feed}'" continue fi - # handle IPv4/IPv6 feeds with a single download URL + # handle IPv4 feeds # - if [ "${feed_url_4}" = "${feed_url_6}" ]; then - if [ "${ban_protov4}" = "1" ] && [ -n "${feed_url_4}" ] && [ -n "${feed_rule_4}" ]; then - (f_down "${feed}" "4" "${feed_url_4}" "${feed_rule_4}" "${feed_flag}") & - feed_url_6="local" - wait - fi - if [ "${ban_protov6}" = "1" ] && [ -n "${feed_url_6}" ] && [ -n "${feed_rule_6}" ]; then - (f_down "${feed}" "6" "${feed_url_6}" "${feed_rule_6}" "${feed_flag}") & - hold="$((cnt % ban_cores))" - [ "${hold}" = "0" ] && wait - cnt="$((cnt + 1))" + if [ "${ban_protov4}" = "1" ] && [ -n "${feed_url_4}" ] && [ -n "${feed_rule}" ]; then + feed_ipv="4" + if [ "${feed}" = "country" ] && [ "${ban_countrysplit}" = "1" ]; then + for country in ${ban_country}; do + (f_down "${feed}.${country}" "${feed_ipv}" "${feed_url_4}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") & + [ "${cnt}" -gt "${ban_cores}" ] && wait -n + cnt="$((cnt + 1))" + done + elif [ "${feed}" = "asn" ] && [ "${ban_asnsplit}" = "1" ]; then + for asn in ${ban_asn}; do + (f_down "${feed}.${asn}" "${feed_ipv}" "${feed_url_4}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") & + [ "${cnt}" -gt "${ban_cores}" ] && wait -n + cnt="$((cnt + 1))" + done + else + if [ "${feed_url_4}" = "${feed_url_6}" ]; then + feed_url_6="local" + f_down "${feed}" "${feed_ipv}" "${feed_url_4}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}" + else + (f_down "${feed}" "${feed_ipv}" "${feed_url_4}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") & + [ "${cnt}" -gt "${ban_cores}" ] && wait -n + cnt="$((cnt + 1))" + fi fi - continue fi - # handle IPv4/IPv6 feeds with separate download URLs + # handle IPv6 feeds # - if [ "${ban_protov4}" = "1" ] && [ -n "${feed_url_4}" ] && [ -n "${feed_rule_4}" ]; then - (f_down "${feed}" "4" "${feed_url_4}" "${feed_rule_4}" "${feed_flag}") & - hold="$((cnt % ban_cores))" - [ "${hold}" = "0" ] && wait - cnt="$((cnt + 1))" - fi - if [ "${ban_protov6}" = "1" ] && [ -n "${feed_url_6}" ] && [ -n "${feed_rule_6}" ]; then - (f_down "${feed}" "6" "${feed_url_6}" "${feed_rule_6}" "${feed_flag}") & - hold="$((cnt % ban_cores))" - [ "${hold}" = "0" ] && wait - cnt="$((cnt + 1))" + if [ "${ban_protov6}" = "1" ] && [ -n "${feed_url_6}" ] && [ -n "${feed_rule}" ]; then + feed_ipv="6" + if [ "${feed}" = "country" ] && [ "${ban_countrysplit}" = "1" ]; then + for country in ${ban_country}; do + (f_down "${feed}.${country}" "${feed_ipv}" "${feed_url_6}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") & + [ "${cnt}" -gt "${ban_cores}" ] && wait -n + cnt="$((cnt + 1))" + done + elif [ "${feed}" = "asn" ] && [ "${ban_asnsplit}" = "1" ]; then + for asn in ${ban_asn}; do + (f_down "${feed}.${asn}" "${feed_ipv}" "${feed_url_6}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") & + [ "${cnt}" -gt "${ban_cores}" ] && wait -n + cnt="$((cnt + 1))" + done + else + (f_down "${feed}" "${feed_ipv}" "${feed_url_6}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") & + [ "${cnt}" -gt "${ban_cores}" ] && wait -n + cnt="$((cnt + 1))" + fi fi done wait f_rmset -f_rmdir "${ban_tmpdir}" f_genstatus "active" # start domain lookup @@ -128,20 +162,21 @@ f_log "info" "start banIP domain lookup" cnt="1" for list in allowlist blocklist; do (f_lookup "${list}") & - hold="$((cnt % ban_cores))" - [ "${hold}" = "0" ] && wait + [ "${cnt}" -gt "${ban_cores}" ] && wait -n cnt="$((cnt + 1))" done wait # end processing # +f_log "info" "finish banIP processing" ( sleep 5 if [ "${ban_mailnotification}" = "1" ] && [ -n "${ban_mailreceiver}" ] && [ -x "${ban_mailcmd}" ]; then f_mail fi json_cleanup + f_rmdir "${ban_tmpdir}" rm -rf "${ban_lock}" ) & diff --git a/packages/banip/files/banip.cgi b/packages/banip/files/banip.cgi index 975e3c5b2..4cb8a22ac 100644 --- a/packages/banip/files/banip.cgi +++ b/packages/banip/files/banip.cgi @@ -1,36 +1,61 @@ #!/bin/sh # banIP cgi remote logging script - ban incoming and outgoing IPs via named nftables Sets -# Copyright (c) 2018-2024 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2018-2026 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. # (s)hellcheck exceptions # shellcheck disable=all -# handle post/get requests +# output HTTP response header # -post_string="$(cat)" +cat <|] Check if an element exists in a banIP Set" -extra_command "survey" "[] List all elements of a given banIP Set" -extra_command "lookup" "Lookup the IPs of domain names in the local lists and update them" +extra_command "content" "[] [true|false] Listing of all or only elements with hits of a given banIP Set" +extra_command "actual" "Print nft/monitor actuals" ban_init="/etc/init.d/banip" ban_service="/usr/bin/banip-service.sh" ban_funlib="/usr/lib/banip-functions.sh" -ban_pidfile="/var/run/banip.pid" -ban_lock="/var/run/banip.lock" - -[ "${action}" = "boot" ] && "${ban_init}" running && exit 0 -{ [ "${action}" = "stop" ] || [ "${action}" = "report" ] || [ "${action}" = "search" ] || [ "${action}" = "survey" ] || [ "${action}" = "lookup" ]; } && ! "${ban_init}" running && exit 0 -[ ! -r "${ban_funlib}" ] && { [ "${action}" = "boot" ] || [ "${action}" = "start" ] || [ "${action}" = "restart" ] || [ "${action}" = "reload" ] || [ "${action}" = "stop" ] || [ "${action}" = "report" ] || [ "${action}" = "search" ] || [ "${action}" = "survey" ] || [ "${action}" = "lookup" ] || [ "${action}" = "status" ]; } && exit 1 -[ -d "${ban_lock}" ] && { [ "${action}" = "boot" ] || [ "${action}" = "start" ] || [ "${action}" = "restart" ] || [ "${action}" = "reload" ] || [ "${action}" = "lookup" ] || [ "${action}" = "search" ]; } && exit 1 -[ ! -d "${ban_lock}" ] && { [ "${action}" = "boot" ] || [ "${action}" = "start" ] || [ "${action}" = "restart" ] || [ "${action}" = "reload" ] || [ "${action}" = "lookup" ] || [ "${action}" = "search" ]; } && mkdir -p "${ban_lock}" -{ [ "${action}" = "boot" ] || [ "${action}" = "start" ] || [ "${action}" = "restart" ] || [ "${action}" = "reload" ] || [ "${action}" = "stop" ] || [ "${action}" = "report" ] || [ "${action}" = "search" ] || [ "${action}" = "survey" ] || [ "${action}" = "lookup" ] || [ "${action}" = "status" ]; } && . "${ban_funlib}" -[ ! -d "${ban_lock}" ] && { [ "${action}" = "boot" ] || [ "${action}" = "start" ] || [ "${action}" = "restart" ] || [ "${action}" = "reload" ] || [ "${action}" = "lookup" ] || [ "${action}" = "search" ]; } && exit 1 +ban_pidfile="/var/run/banIP/banIP.pid" +ban_lock="/var/run/banIP/banIP.lock" + +if [ -z "${IPKG_INSTROOT}" ]; then + + # check for running instance and handle boot trigger + # + case "${action}" in + "boot") + "${ban_init}" running && exit 0 + ;; + "stop" | "report" | "content") + "${ban_init}" running || exit 0 + ;; + esac + + # reset pidfile if no/stale process is found, + # otherwise exit with error to prevent multiple instances + # + case "${action}" in + "boot" | "start" | "restart" | "reload" | "search") + if [ -d "${ban_lock}" ]; then + pid="$(cat "${ban_pidfile}" 2>/dev/null)" + if [ -n "${pid}" ] && kill -0 "${pid}" 2>/dev/null; then + exit 1 + else + rm -rf "${ban_lock}" + fi + fi + mkdir -p "${ban_lock}" + ;; + esac + . "${ban_funlib}" +fi boot() { - : >"${ban_pidfile}" rc_procd start_service boot } @@ -42,11 +65,10 @@ start_service() { procd_set_param nice "$(uci_get banip global ban_nicelimit "0")" procd_set_param limits nofile="$(uci_get banip global ban_filelimit "1024")" procd_set_param stdout 0 - procd_set_param stderr 0 + procd_set_param stderr 1 procd_close_instance else f_log "err" "banIP service autostart is disabled" - rm -rf "${ban_lock}" fi } @@ -84,21 +106,12 @@ search() { rm -rf "${ban_lock}" } -survey() { - f_survey "${1}" +content() { + f_content "${1}" "${2:-"false"}" } -lookup() { - local list hold cnt="1" - - for list in allowlist blocklist; do - (f_lookup "${list}") & - hold="$((cnt % ban_cores))" - [ "${hold}" = "0" ] && wait - cnt="$((cnt + 1))" - done - wait - rm -rf "${ban_lock}" +actual() { + f_actual } service_triggers() { @@ -111,7 +124,4 @@ service_triggers() { for iface in ${trigger}; do procd_add_interface_trigger "interface.*.up" "${iface}" "${ban_init}" start done - - PROCD_RELOAD_DELAY="$((2 * 1000))" - procd_add_reload_trigger banip } diff --git a/packages/banip/files/banip.tpl b/packages/banip/files/banip.tpl index 924ffe0b6..13662aaa2 100644 --- a/packages/banip/files/banip.tpl +++ b/packages/banip/files/banip.tpl @@ -1,36 +1,50 @@ # banIP mail template/include - ban incoming and outgoing IPs via named nftables Sets -# Copyright (c) 2018-2024 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2018-2026 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. -# info preparation -# -local banip_info report_info log_info system_info mail_text logread_cmd +local banip_info report_info log_info system_info mail_text -if [ -f "${ban_logreadfile}" ]; then - logread_cmd="${ban_logreadcmd} -qn ${ban_loglimit} ${ban_logreadfile} 2>/dev/null | ${ban_grepcmd} -e \"banIP/\" 2>/dev/null" -elif printf "%s" "${ban_packages}" | "${ban_grepcmd}" -q '"logd'; then - logread_cmd="${ban_logreadcmd} -l ${ban_loglimit} -e "banIP/" 2>/dev/null" +# log info preparation +# +if [ -f "${ban_logreadfile}" ] && [ -x "${ban_logreadcmd}" ] && [ "${ban_logreadcmd##*/}" = "tail" ]; then + log_info="$("${ban_logreadcmd}" -qn "${ban_loglimit}" "${ban_logreadfile}" 2>/dev/null | "${ban_grepcmd}" -e "banIP-" 2>/dev/null)" +elif [ -x "${ban_logreadcmd}" ] && [ "${ban_logreadcmd##*/}" = "logread" ]; then + log_info="$("${ban_logreadcmd}" -l "${ban_loglimit}" -e "banIP-" 2>/dev/null)" fi +# banIP status and report info preparation +# banip_info="$(/etc/init.d/banip status 2>/dev/null)" report_info="$("${ban_catcmd}" "${ban_reportdir}/ban_report.txt" 2>/dev/null)" -log_info="$(${logread_cmd})" system_info="$( strings /etc/banner 2>/dev/null - "${ban_ubuscmd}" call system board | "${ban_awkcmd}" 'BEGIN{FS="[{}\"]"}{if($2=="kernel"||$2=="hostname"||$2=="system"||$2=="model"||$2=="description")printf " + %-12s: %s\n",$2,$4}' + "${ban_ubuscmd}" call system board | + "${ban_awkcmd}" 'BEGIN{FS="[{}\"]"}{if($2=="kernel"||$2=="hostname"||$2=="system"||$2=="model"||$2=="description")printf " + %-12s: %s\n",$2,$4}' )" -# content header -# -mail_text="$(printf "%s\n" "
")"
-
-# content body
-#
-mail_text="$(printf "%s\n" "${mail_text}\n++\n++ System Information ++\n++\n${system_info:-"-"}")"
-mail_text="$(printf "%s\n" "${mail_text}\n\n++\n++ banIP Status ++\n++\n${banip_info:-"-"}")"
-[ -n "${report_info}" ] && mail_text="$(printf "%s\n" "${mail_text}\n\n++\n++ banIP Report ++\n++\n${report_info}")"
-[ -n "${log_info}" ] && mail_text="$(printf "%s\n" "${mail_text}\n\n++\n++ Logfile Information ++\n++\n${log_info}")"
-
-# content footer
+# mail text preparation
 #
-mail_text="$(printf "%s\n" "${mail_text}
")" +mail_text="$( + printf '%s\n' "
"
+	printf '\n%s\n' "++
+++ System Information ++
+++"
+	printf '%s\n' "${system_info:-"-"}"
+	printf '\n%s\n' "++
+++ banIP Status ++
+++"
+	printf '%s\n' "${banip_info:-"-"}"
+	[ -n "${report_info}" ] && {
+		printf '\n%s\n' "++
+++ banIP Report ++
+++"
+		printf '%s\n' "${report_info}"
+	}
+	[ -n "${log_info}" ] && {
+		printf '\n%s\n' "++
+++ Logfile Information ++
+++"
+		printf '%s\n' "${log_info}"
+	}
+	printf '%s\n' "
" +)" diff --git a/packages/ns-api-server/Makefile b/packages/ns-api-server/Makefile index 86cbe291c..f5dd4a0a5 100644 --- a/packages/ns-api-server/Makefile +++ b/packages/ns-api-server/Makefile @@ -33,7 +33,7 @@ define Package/ns-api-server CATEGORY:=NethSecurity TITLE:=NethSecurity REST API server URL:=https://github.com/NethServer/nethsecurity-api - DEPENDS:=$(GO_ARCH_DEPENDS) + DEPENDS:=$(GO_ARCH_DEPENDS) +victoria-metrics endef define Package/ns-api-server/description diff --git a/packages/ns-api-server/README.md b/packages/ns-api-server/README.md index cf0e4fede..ed5440841 100644 --- a/packages/ns-api-server/README.md +++ b/packages/ns-api-server/README.md @@ -4,6 +4,32 @@ NS API server, see [source code](https://github.com/NethServer/nethsecurity-api) The server is configured to listen on `127.0.0.1:8090`. +## Metrics and alerts proxies + +Reverse proxies to the local VictoriaMetrics and vmalert APIs, for the authenticated UI. Live +inside the JWT-protected group: authenticated, rate-limited, `Authorization`/`Cookie` headers +stripped before forwarding. Registered routes accept any HTTP method; unregistered paths 404. +Backend unreachable or slow → `502`. + +| Route | Backend | +|---|---| +| `/api/metrics/query` | VictoriaMetrics `/api/v1/query` | +| `/api/metrics/query_range` | VictoriaMetrics `/api/v1/query_range` | +| `/api/alerts/alerts` | vmalert `/api/v1/alerts` | + +Backend addresses: `VICTORIA_METRICS_URL` / `VMALERT_URL` env vars in `ns-api-server.initd`, +read from `victoria-metrics.main.http_listen_addr` / `vmalert.main.http_listen_addr` (default +`http://127.0.0.1:8428` / `http://127.0.0.1:8082`). Restarts on `victoria-metrics`/`vmalert` +config change (`service_triggers`). + +Example: + +``` +GET /api/metrics/query?query=up +GET /api/metrics/query_range?query=&start=&end=&step= +GET /api/alerts/alerts +``` + ## Rate limiting The server applies a generous global per-client-IP rate limit as a coarse safety net across diff --git a/packages/ns-api-server/files/ns-api-server.initd b/packages/ns-api-server/files/ns-api-server.initd index 54a888ae4..9376dc73a 100644 --- a/packages/ns-api-server/files/ns-api-server.initd +++ b/packages/ns-api-server/files/ns-api-server.initd @@ -39,6 +39,22 @@ start_service() { echo "$SECRET_JWT" > ${WORK_DIR}/secret_jwt fi + # read the config address for proxy fallback + config_load victoria-metrics + local vm_http_listen_addr + config_get vm_http_listen_addr main http_listen_addr "127.0.0.1:8428" + # a host-less "addr" (e.g. ":8428") binds all interfaces; reach it via loopback + case "$vm_http_listen_addr" in + :*) vm_http_listen_addr="127.0.0.1${vm_http_listen_addr}" ;; + esac + + config_load vmalert + local vmalert_http_listen_addr + config_get vmalert_http_listen_addr main http_listen_addr "127.0.0.1:8082" + case "$vmalert_http_listen_addr" in + :*) vmalert_http_listen_addr="127.0.0.1${vmalert_http_listen_addr}" ;; + esac + procd_set_param env GIN_MODE=release \ LISTEN_ADDRESS=127.0.0.1:8090 \ SECRET_JWT="${SECRET_JWT}" \ @@ -47,6 +63,8 @@ start_service() { TOKENS_DIR=${TOKENS_DIR} \ UPLOAD_FILE_PATH=${UPLOAD_FILE_PATH} \ UPLOAD_FILE_MAX_SIZE=${UPLOAD_FILE_MAX_SIZE} \ + VICTORIA_METRICS_URL="http://${vm_http_listen_addr}" \ + VMALERT_URL="http://${vmalert_http_listen_addr}" \ GLOBAL_RATE_LIMIT_AVERAGE=${GLOBAL_RATE_LIMIT_AVERAGE} \ GLOBAL_RATE_LIMIT_BURST=${GLOBAL_RATE_LIMIT_BURST} @@ -57,6 +75,11 @@ start_service() { procd_close_instance } +service_triggers() { + procd_add_reload_trigger victoria-metrics vmalert +} + reload_service() { - procd_send_signal ns-api-server '*' USR1 + stop + start } diff --git a/packages/ns-api-server/files/src/configuration/configuration.go b/packages/ns-api-server/files/src/configuration/configuration.go index 482f575fc..8087f366c 100644 --- a/packages/ns-api-server/files/src/configuration/configuration.go +++ b/packages/ns-api-server/files/src/configuration/configuration.go @@ -31,6 +31,9 @@ type Configuration struct { UploadFilePath string `json:"upload_file_path"` DownloadFilePath string `json:"download_file_path"` + VictoriaMetricsURL string `json:"victoria_metrics_url"` + VMAlertURL string `json:"vmalert_url"` + // Generous global per-IP rate limit applied to every API route as a coarse // safety net; 0 disables it GlobalRateLimitAverage int `json:"global_rate_limit_average"` @@ -97,6 +100,18 @@ func Init() { Config.UploadFileMaxSize = 32 } + if os.Getenv("VICTORIA_METRICS_URL") != "" { + Config.VictoriaMetricsURL = os.Getenv("VICTORIA_METRICS_URL") + } else { + Config.VictoriaMetricsURL = "http://127.0.0.1:8428" + } + + if os.Getenv("VMALERT_URL") != "" { + Config.VMAlertURL = os.Getenv("VMALERT_URL") + } else { + Config.VMAlertURL = "http://127.0.0.1:8082" + } + if v, err := strconv.Atoi(os.Getenv("GLOBAL_RATE_LIMIT_AVERAGE")); err == nil { Config.GlobalRateLimitAverage = v } else { diff --git a/packages/ns-api-server/files/src/configuration/configuration_test.go b/packages/ns-api-server/files/src/configuration/configuration_test.go index 8b74f86b8..adc338680 100644 --- a/packages/ns-api-server/files/src/configuration/configuration_test.go +++ b/packages/ns-api-server/files/src/configuration/configuration_test.go @@ -10,6 +10,60 @@ import ( "testing" ) +func TestInitVictoriaMetricsURLDefault(t *testing.T) { + os.Unsetenv("VICTORIA_METRICS_URL") + os.Setenv("SECRET_JWT", "test-secret") + os.Setenv("SECRETS_DIR", "/tmp/secrets") + os.Setenv("TOKENS_DIR", "/tmp/tokens") + + Init() + + if Config.VictoriaMetricsURL != "http://127.0.0.1:8428" { + t.Fatalf("VictoriaMetricsURL = %q, want %q", Config.VictoriaMetricsURL, "http://127.0.0.1:8428") + } +} + +func TestInitVictoriaMetricsURLFromEnv(t *testing.T) { + os.Setenv("SECRET_JWT", "test-secret") + os.Setenv("SECRETS_DIR", "/tmp/secrets") + os.Setenv("TOKENS_DIR", "/tmp/tokens") + os.Setenv("VICTORIA_METRICS_URL", "http://127.0.0.1:9428") + defer os.Unsetenv("VICTORIA_METRICS_URL") + + Init() + + if Config.VictoriaMetricsURL != "http://127.0.0.1:9428" { + t.Fatalf("VictoriaMetricsURL = %q, want %q", Config.VictoriaMetricsURL, "http://127.0.0.1:9428") + } +} + +func TestInitVMAlertURLDefault(t *testing.T) { + os.Unsetenv("VMALERT_URL") + os.Setenv("SECRET_JWT", "test-secret") + os.Setenv("SECRETS_DIR", "/tmp/secrets") + os.Setenv("TOKENS_DIR", "/tmp/tokens") + + Init() + + if Config.VMAlertURL != "http://127.0.0.1:8082" { + t.Fatalf("VMAlertURL = %q, want %q", Config.VMAlertURL, "http://127.0.0.1:8082") + } +} + +func TestInitVMAlertURLFromEnv(t *testing.T) { + os.Setenv("SECRET_JWT", "test-secret") + os.Setenv("SECRETS_DIR", "/tmp/secrets") + os.Setenv("TOKENS_DIR", "/tmp/tokens") + os.Setenv("VMALERT_URL", "http://127.0.0.1:9082") + defer os.Unsetenv("VMALERT_URL") + + Init() + + if Config.VMAlertURL != "http://127.0.0.1:9082" { + t.Fatalf("VMAlertURL = %q, want %q", Config.VMAlertURL, "http://127.0.0.1:9082") + } +} + func TestInitGlobalRateLimitDefaults(t *testing.T) { os.Unsetenv("GLOBAL_RATE_LIMIT_AVERAGE") os.Unsetenv("GLOBAL_RATE_LIMIT_BURST") diff --git a/packages/ns-api-server/files/src/main.go b/packages/ns-api-server/files/src/main.go index 17f0fd071..a958614e2 100644 --- a/packages/ns-api-server/files/src/main.go +++ b/packages/ns-api-server/files/src/main.go @@ -10,10 +10,11 @@ package main import ( - "github.com/NethServer/nethsecurity-api/sudo" "io" "net/http" + "github.com/NethServer/nethsecurity-api/sudo" + "github.com/fatih/structs" "github.com/gin-contrib/cors" "github.com/gin-contrib/gzip" @@ -117,6 +118,12 @@ func main() { authGroup.POST("/files", methods.UploadFile) authGroup.DELETE("/files/:filename", methods.DeleteFile) + // reverse proxies to VictoriaMetrics/vmalert + victoriaMetricsProxy := methods.NewReverseProxy(configuration.Config.VictoriaMetricsURL) + authGroup.Any("/metrics/query", methods.ProxyTo(victoriaMetricsProxy, "/api/v1/query")) + authGroup.Any("/metrics/query_range", methods.ProxyTo(victoriaMetricsProxy, "/api/v1/query_range")) + authGroup.Any("/alerts/alerts", methods.ProxyTo(methods.NewReverseProxy(configuration.Config.VMAlertURL), "/api/v1/alerts")) + // handle missing endpoint router.NoRoute(func(c *gin.Context) { c.JSON(http.StatusNotFound, structs.Map(response.StatusNotFound{ diff --git a/packages/ns-api-server/files/src/methods/proxy.go b/packages/ns-api-server/files/src/methods/proxy.go new file mode 100644 index 000000000..6f9063933 --- /dev/null +++ b/packages/ns-api-server/files/src/methods/proxy.go @@ -0,0 +1,75 @@ +/* +Copyright (C) 2026 Nethesis S.r.l. +SPDX-License-Identifier: GPL-2.0-only +*/ + +package methods + +import ( + "encoding/json" + "net/http" + "net/http/httputil" + "net/url" + "strings" + "time" + + "github.com/NethServer/nethsecurity-api/logs" + "github.com/NethServer/nethsecurity-api/response" + "github.com/fatih/structs" + "github.com/gin-gonic/gin" +) + +// NewReverseProxy builds a reverse proxy to a local, unauthenticated backend +func NewReverseProxy(rawBaseURL string) *httputil.ReverseProxy { + target, err := url.Parse(rawBaseURL) + if err != nil { + logs.Logs.Println("[CRITICAL][PROXY] invalid backend URL:", rawBaseURL, err.Error()) + } + + proxy := httputil.NewSingleHostReverseProxy(target) + proxy.Transport = &http.Transport{ + ResponseHeaderTimeout: 10 * time.Second, + } + proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) { + logs.Logs.Println("[ERROR][PROXY] backend unreachable:", err.Error()) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadGateway) + json.NewEncoder(w).Encode(structs.Map(response.StatusBadGateway{ + Code: 502, + Message: "bad gateway", + Data: nil, + })) + } + + // The API server owns CORS (see main.go) + // strip any CORS headers the backend sets (e.g. VictoriaMetrics adds Access-Control-Allow-Origin) + // so they don't duplicate and trigger a browser "multiple values" CORS error + proxy.ModifyResponse = func(resp *http.Response) error { + for key := range resp.Header { + if strings.HasPrefix(http.CanonicalHeaderKey(key), "Access-Control-") { + resp.Header.Del(key) + } + } + return nil + } + + return proxy +} + +// ProxyTo returns a handler that forwards the request to proxy at the given +// fixed backendPath, passing the query string through unchanged. It is meant +// to be registered against a single hardcoded path (e.g. authGroup.GET +// ("/metrics/query", ProxyTo(...))) - it does not accept caller-controlled +// path segments. +func ProxyTo(proxy *httputil.ReverseProxy, backendPath string) gin.HandlerFunc { + return func(c *gin.Context) { + c.Request.Header.Del("Authorization") + c.Request.Header.Del("Cookie") + // Let the API server's gzip middleware be the single compression layer. + // Without this the backend (VictoriaMetrics) would compress too, and the + // middleware would compress again, yielding a double-gzipped body. + c.Request.Header.Del("Accept-Encoding") + c.Request.URL.Path = backendPath + proxy.ServeHTTP(c.Writer, c.Request) + } +} diff --git a/packages/ns-api-server/files/src/methods/proxy_test.go b/packages/ns-api-server/files/src/methods/proxy_test.go new file mode 100644 index 000000000..7625f47ca --- /dev/null +++ b/packages/ns-api-server/files/src/methods/proxy_test.go @@ -0,0 +1,283 @@ +/* +Copyright (C) 2026 Nethesis S.r.l. +SPDX-License-Identifier: GPL-2.0-only +*/ + +package methods + +import ( + "bytes" + "compress/gzip" + "io" + "log" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "github.com/NethServer/nethsecurity-api/logs" + gzipmw "github.com/gin-contrib/gzip" + "github.com/gin-gonic/gin" +) + +// proxy.go logs backend errors; logs.Logs is otherwise only set up in main(). +func TestMain(m *testing.M) { + logs.Logs = log.New(os.Stderr, "test ", 0) + os.Exit(m.Run()) +} + +// httputil.ReverseProxy needs a real ResponseWriter (CloseNotifier/Flusher), +// which httptest.NewRecorder doesn't implement. +func startTestServer(t *testing.T, r *gin.Engine) *httptest.Server { + t.Helper() + srv := httptest.NewServer(r) + t.Cleanup(srv.Close) + return srv +} + +func TestProxyToForwardsRequest(t *testing.T) { + gin.SetMode(gin.TestMode) + + var gotPath, gotQuery, gotAuth, gotCookie string + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotQuery = r.URL.RawQuery + gotAuth = r.Header.Get("Authorization") + gotCookie = r.Header.Get("Cookie") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"status":"success"}`)) + })) + defer backend.Close() + + r := gin.New() + r.GET("/x", ProxyTo(NewReverseProxy(backend.URL), "/api/v1/query")) + srv := startTestServer(t, r) + + req, _ := http.NewRequest(http.MethodGet, srv.URL+"/x?query=up", nil) + req.Header.Set("Authorization", "Bearer secret-jwt") + req.Header.Set("Cookie", "session=abc") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if string(body) != `{"status":"success"}` { + t.Fatalf("body = %q, want backend body passed through", string(body)) + } + if resp.Header.Get("Content-Type") != "application/json" { + t.Fatalf("Content-Type = %q, want application/json", resp.Header.Get("Content-Type")) + } + if gotPath != "/api/v1/query" { + t.Fatalf("backend received path = %q, want /api/v1/query", gotPath) + } + if gotQuery != "query=up" { + t.Fatalf("backend received query = %q, want query=up", gotQuery) + } + if gotAuth != "" { + t.Fatalf("backend received Authorization header %q, want it stripped", gotAuth) + } + if gotCookie != "" { + t.Fatalf("backend received Cookie header %q, want it stripped", gotCookie) + } +} + +func TestProxyStripsBackendCorsHeaders(t *testing.T) { + gin.SetMode(gin.TestMode) + + // VictoriaMetrics sets its own CORS headers; the API server owns CORS, so the + // proxy must strip them to avoid duplicate Access-Control-Allow-Origin values. + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"status":"success"}`)) + })) + defer backend.Close() + + r := gin.New() + r.GET("/x", ProxyTo(NewReverseProxy(backend.URL), "/api/v1/query")) + srv := startTestServer(t, r) + + resp, err := http.Get(srv.URL + "/x?query=up") + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + + if got := resp.Header.Get("Access-Control-Allow-Origin"); got != "" { + t.Fatalf("Access-Control-Allow-Origin = %q, want it stripped", got) + } + if got := resp.Header.Get("Access-Control-Allow-Methods"); got != "" { + t.Fatalf("Access-Control-Allow-Methods = %q, want it stripped", got) + } + if resp.Header.Get("Content-Type") != "application/json" { + t.Fatalf("Content-Type = %q, want it preserved", resp.Header.Get("Content-Type")) + } +} + +func TestProxyDoesNotDoubleCompress(t *testing.T) { + gin.SetMode(gin.TestMode) + + const payload = `{"status":"success","data":{"resultType":"matrix","result":[]}}` + + // Emulate VictoriaMetrics: gzip the body when the client accepts gzip. + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + w.Header().Set("Content-Encoding", "gzip") + gz := gzip.NewWriter(w) + gz.Write([]byte(payload)) + gz.Close() + return + } + w.Write([]byte(payload)) + })) + defer backend.Close() + + // Same setup as main.go: the global gzip middleware sits in front of the proxy. + r := gin.New() + r.Use(gzipmw.Gzip(gzipmw.DefaultCompression)) + r.GET("/x", ProxyTo(NewReverseProxy(backend.URL), "/api/v1/query")) + srv := startTestServer(t, r) + + req, _ := http.NewRequest(http.MethodGet, srv.URL+"/x?query=up", nil) + // Explicit Accept-Encoding disables Go's transparent client-side decompression, + // so we inspect exactly what the server sent. + req.Header.Set("Accept-Encoding", "gzip") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + + // If compressed, decompress exactly once and expect the original payload. A + // double-gzipped body would still be gzip after a single decompression pass. + if resp.Header.Get("Content-Encoding") == "gzip" { + gr, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + t.Fatalf("response is not valid gzip: %v", err) + } + decoded, err := io.ReadAll(gr) + if err != nil { + t.Fatalf("failed to read gzip body: %v", err) + } + body = decoded + } + + if string(body) != payload { + t.Fatalf("body = %q, want the original payload (double compression?)", string(body)) + } +} + +func TestProxyToBackendUnreachable(t *testing.T) { + gin.SetMode(gin.TestMode) + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + backendURL := backend.URL + backend.Close() // nothing is listening on this address anymore + + r := gin.New() + r.GET("/x", ProxyTo(NewReverseProxy(backendURL), "/api/v1/query")) + srv := startTestServer(t, r) + + resp, err := http.Get(srv.URL + "/x") + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadGateway { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusBadGateway) + } + if resp.Header.Get("Content-Type") != "application/json" { + t.Fatalf("Content-Type = %q, want application/json", resp.Header.Get("Content-Type")) + } +} + +func TestNewReverseProxyResponseHeaderTimeout(t *testing.T) { + proxy := NewReverseProxy("http://127.0.0.1:8428") + + transport, ok := proxy.Transport.(*http.Transport) + if !ok { + t.Fatalf("Transport = %T, want *http.Transport", proxy.Transport) + } + if transport.ResponseHeaderTimeout != 10*time.Second { + t.Fatalf("ResponseHeaderTimeout = %s, want 10s", transport.ResponseHeaderTimeout) + } +} + +// Mirrors main.go's route table: any method is forwarded on the three registered +// paths, everything else 404s regardless of method. +func TestMinimalRouteTableRejectsEverythingElse(t *testing.T) { + gin.SetMode(gin.TestMode) + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer backend.Close() + + victoriaMetricsProxy := NewReverseProxy(backend.URL) + r := gin.New() + r.Any("/metrics/query", ProxyTo(victoriaMetricsProxy, "/api/v1/query")) + r.Any("/metrics/query_range", ProxyTo(victoriaMetricsProxy, "/api/v1/query_range")) + r.Any("/alerts/alerts", ProxyTo(NewReverseProxy(backend.URL), "/api/v1/alerts")) + srv := startTestServer(t, r) + + allowed := []struct { + method string + path string + }{ + {http.MethodGet, "/metrics/query"}, + {http.MethodPost, "/metrics/query"}, + {http.MethodPut, "/metrics/query"}, + {http.MethodGet, "/metrics/query_range"}, + {http.MethodPost, "/metrics/query_range"}, + {http.MethodGet, "/alerts/alerts"}, + {http.MethodPost, "/alerts/alerts"}, + } + for _, tc := range allowed { + req, _ := http.NewRequest(tc.method, srv.URL+tc.path, nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("%s %s: request failed: %v", tc.method, tc.path, err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("%s %s: status = %d, want %d", tc.method, tc.path, resp.StatusCode, http.StatusOK) + } + } + + rejected := []struct { + method string + path string + }{ + {http.MethodGet, "/metrics/admin/tsdb/delete_series"}, + {http.MethodGet, "/metrics/admin/tsdb/snapshot"}, + {http.MethodPost, "/metrics/admin/tsdb/delete_series"}, + {http.MethodGet, "/alerts/-/reload"}, + {http.MethodGet, "/bogus"}, + {http.MethodPost, "/bogus"}, + } + for _, tc := range rejected { + req, _ := http.NewRequest(tc.method, srv.URL+tc.path, nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("%s %s: request failed: %v", tc.method, tc.path, err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("%s %s: status = %d, want %d", tc.method, tc.path, resp.StatusCode, http.StatusNotFound) + } + } +} diff --git a/packages/ns-api-server/files/src/response/response.go b/packages/ns-api-server/files/src/response/response.go index 7289bd4f4..622434da8 100644 --- a/packages/ns-api-server/files/src/response/response.go +++ b/packages/ns-api-server/files/src/response/response.go @@ -85,3 +85,9 @@ type StatusServiceUnavailable struct { Message string `json:"message" example:"Service unavailable" structs:"message"` Data interface{} `json:"data" structs:"data"` } + +type StatusBadGateway struct { + Code int `json:"code" example:"502" structs:"code"` + Message string `json:"message" example:"Bad gateway" structs:"message"` + Data interface{} `json:"data" structs:"data"` +} \ No newline at end of file diff --git a/packages/ns-api/README.md b/packages/ns-api/README.md index d3a913ee0..653a09e86 100644 --- a/packages/ns-api/README.md +++ b/packages/ns-api/README.md @@ -6094,9 +6094,8 @@ Configure banip settings: - `enabled`: disable or enable banip (true or false). - `ban_logprerouting`: Log suspicious packets in the prerouting chain (true or false). -- `ban_loginput`: Log suspicious packets in the WAN-input chain (true or false). -- `ban_logforwardwan`: Log suspicious packets in the WAN-forward chain (true or false). -- `ban_logforwardlan`: Log suspicious packets in the LAN-forward chain (true or false). +- `ban_loginbound`: Log suspicious packets in inbound traffic (WAN-input chain) (true or false). +- `ban_logoutbound`: Log suspicious packets in outbound traffic (LAN-forward chain) (true or false). - `ban_loglimit`: Enable or disable scanning of logfiles (true or false). - `ban_logcount`: Specify how many times an IP must appear in the log to be considered suspicious (integer). - `ban_logterm`: List of regex entries for logfile parsing (list of strings). @@ -6107,7 +6106,7 @@ Configure banip settings: ```bash -api-cli ns.threatshield edit-settings --data '{"enabled": true, "ban_logprerouting": true, "ban_loginput": true, "ban_logforwardwan": true, "ban_logforwardlan": true, "ban_loglimit": false, "ban_logcount": 5, "ban_logterm": ["regex1", "regex2"], "ban_icmplimit": true, "ban_synlimit": true, "ban_udplimit": true, "ban_nftexpiry": "1d"}' +api-cli ns.threatshield edit-settings --data '{"enabled": true, "ban_logprerouting": true, "ban_loginbound": true, "ban_logoutbound": true, "ban_loglimit": false, "ban_logcount": 5, "ban_logterm": ["regex1", "regex2"], "ban_icmplimit": true, "ban_synlimit": true, "ban_udplimit": true, "ban_nftexpiry": "1d"}' ``` Response example: diff --git a/packages/ns-api/files/ns.controller b/packages/ns-api/files/ns.controller index 8e7a7d3a2..e34ac5d38 100755 --- a/packages/ns-api/files/ns.controller +++ b/packages/ns-api/files/ns.controller @@ -20,11 +20,32 @@ import time import sqlite3 import glob import socket +import urllib.parse +import urllib.request AUTHORIZED_KEYS = "/etc/dropbear/authorized_keys" dns_cache = {} +try: + _vm_addr = EUci().get("victoria-metrics", "main", "http_listen_addr", default="127.0.0.1:8428") +except Exception: + _vm_addr = "127.0.0.1:8428" +VM_URL = f"http://{_vm_addr}" + +def vm_export(metric, start, end): + params = urllib.parse.urlencode({"match[]": metric, "start": start, "end": end}) + series = [] + try: + with urllib.request.urlopen(f"{VM_URL}/api/v1/export?{params}", timeout=10) as resp: + for line in resp.read().decode().splitlines(): + line = line.strip() + if line: + series.append(json.loads(line)) + except Exception: + pass + return series + # Utility functions def reverse_dns(ip): @@ -171,52 +192,32 @@ def remove_ssh_key(ssh_key): return {"result": "success"} def dump_ts_malware(): - # Parse /var/log/messages for banIP blocks - # Line example: - # Aug 20 12:31:58 fw kernel: [1117985.535261] banIP/inp-wan/drop/nethesislvl3v4: IN=eth5 OUT= MAC=e4:3a:6e:15:8d:dd:14:49:bc:17:33:78:08:00 SRC=193.41.206.156 DST=150.252.232.82 LEN=44 TOS=0x00 PREC=0x00 TTL=247 ID=54321 PROTO=TCP SPT=51000 DPT=8728 WINDOW=65535 RES=0x00 SYN URGP=0 MARK=0x3f00" ret = [] - # regexp pattern to extract src and dst - ip_pattern = re.compile(r'SRC=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s+DST=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})') - # regexp pattern to extract banIP//drop/ - tag_pattern = re.compile(r'banIP/([a-zA-Z0-9\-]+)/(drop|reject)[/]*([a-zA-Z0-9]+)*:') - - for line in read_log("banIP/.*/(drop|reject)"): - if line: - try: - time = datetime.strptime(f"{datetime.now().year} {line[:15]}", '%Y %b %d %H:%M:%S').timestamp() - time = datetime.utcfromtimestamp(time).timestamp() - except: - continue - src = dst = category = chain = "" - ips = ip_pattern.search(line) - if ips: - src = ips.group(1) - dst = ips.group(2) - tag = tag_pattern.search(line) - if tag: - chain = tag.group(1) - category = tag.group(3) - - ret.append({"timestamp": int(time), "src": src, "dst": dst, "category": category, "chain": chain}) + end = int(time.time()) + start = end - 1200 # last 20 minutes + for serie in vm_export("banip_blocked_bytes", start, end): + labels = serie.get("metric", {}) + src = labels.get("src", "") + dst = labels.get("dst", "") + category = labels.get("feed", "") + chain = labels.get("chain", "") + for ts in serie.get("timestamps", []): + # VM timestamps are in milliseconds + ret.append({"timestamp": int(ts) // 1000, "src": src, "dst": dst, "category": category, "chain": chain}) return {"data": ret} def dump_ts_attacks(): - # Parse /var/log/messages for banIP blocked IPs(fail2ban-like) - # Line example: - # Aug 20 04:06:30 fw banIP-1.0.0-5[18692]: add IP '47.98.249.3' ret = [] - log = read_log("banIP.*add") - - # regexp pattern to extract suspicious IP - ip_pattern = re.compile(r"add IP '(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'") - for line in log: - if line: - ip = ip_pattern.search(line) - time = datetime.strptime(f"{datetime.now().year} {line[:15]}", '%Y %b %d %H:%M:%S').timestamp() - time = datetime.utcfromtimestamp(time).timestamp() - if ip: - ret.append({"timestamp": int(time), "ip": ip.group(1)}) + end = int(time.time()) + start = end - 1200 # last 20 minutes + for serie in vm_export("banip_banned", start, end): + ip = serie.get("metric", {}).get("ip", "") + if not ip: + continue + for ts in serie.get("timestamps", []): + # VM timestamps are in milliseconds + ret.append({"timestamp": int(ts) // 1000, "ip": ip}) return {"data": ret} def dump_mwan_events(): diff --git a/packages/ns-api/files/ns.dashboard b/packages/ns-api/files/ns.dashboard index 031dd5047..5b15ca332 100644 --- a/packages/ns-api/files/ns.dashboard +++ b/packages/ns-api/files/ns.dashboard @@ -20,6 +20,13 @@ from glob import glob from euci import EUci from nethsec import utils, ovpn +# VictoriaMetrics HTTP API base URL +try: + _vm_addr = EUci().get("victoria-metrics", "main", "http_listen_addr", default="127.0.0.1:8428") +except Exception: + _vm_addr = "127.0.0.1:8428" +VM_URL = f"http://{_vm_addr}" + def get_uptime(): with open('/proc/uptime', 'r') as f: uptime_seconds = float(f.readline().split()[0]) @@ -52,13 +59,6 @@ def get_hardware(): return f.read().lstrip().rstrip() return "Unknown" -def _run(cmd): - try: - proc = subprocess.run(cmd, shell=True, check=True, capture_output=True, text=True) - return proc.stdout.rstrip().lstrip() - except: - return '' - def get_memory(): try: mem_total = None @@ -140,6 +140,25 @@ def count_hosts(): def count_openvpn_rw(): return len(ovpn.list_connected_clients('ns_roadwarrior1').keys()) +def count_threat_shield_ip(): + ret = 0 + + def vm_query(expr): + params = urllib.parse.urlencode({"query": expr}) + with urllib.request.urlopen(f"{VM_URL}/api/v1/query?{params}", timeout=5) as resp: + data = json.loads(resp.read()) + result = data.get("data", {}).get("result", []) + return result[0].get("value", []) if result else [] + + try: + value = vm_query("sum(count_over_time(banip_blocked_bytes[1h]))") + if value: + ret = int(float(value[1])) + except Exception: + pass + + return ret + def check_ts_dns(): u = EUci() adb_status = check_adblock() @@ -278,7 +297,7 @@ def system_info(): def interface_traffic(interface): ret = {"labels": [], "data": []} - vm_url = "http://127.0.0.1:8428/api/v1/query_range" + vm_url = f"{VM_URL}/api/v1/query_range" now = int(time.time()) one_hour_ago = now - 3600 @@ -371,7 +390,7 @@ def counter(service): if service == "hosts": ret = count_hosts() elif service == "threat_shield_ip": - ret = _run("grep -e \"$(date -d '-1 hour' +'%b %e %H:%M')\" -e \"$(date +'%b %e %H:')\" /var/log/messages | grep -E 'banIP.+add IP|banIP.+drop|banIP.+reject' | wc -l") + ret = count_threat_shield_ip() elif service == "openvpn_rw": ret = count_openvpn_rw() diff --git a/packages/ns-api/files/ns.report b/packages/ns-api/files/ns.report index 47dc1c315..2f11c849b 100755 --- a/packages/ns-api/files/ns.report +++ b/packages/ns-api/files/ns.report @@ -19,6 +19,13 @@ import urllib.parse import urllib.request from euci import EUci +# VictoriaMetrics HTTP API base URL +try: + _vm_addr = EUci().get("victoria-metrics", "main", "http_listen_addr", default="127.0.0.1:8428") +except Exception: + _vm_addr = "127.0.0.1:8428" +VM_URL = f"http://{_vm_addr}" + ## Utility functions def format_hourly_data(hourly_data): @@ -333,7 +340,7 @@ def get_victoria_metrics_ping_data(host): ret_latency = {"labels": ["time", "minimum", "maximum", "average"], "data": []} ret_quality = {"labels": ["time", "returned"], "data": []} - vm_url = "http://127.0.0.1:8428/api/v1/query_range" + vm_url = f"{VM_URL}/api/v1/query_range" now = int(time.time()) one_hour_ago = now - 3600 timeout = 5 diff --git a/packages/ns-api/files/ns.telegraf b/packages/ns-api/files/ns.telegraf index 507691465..1eec5d7ae 100755 --- a/packages/ns-api/files/ns.telegraf +++ b/packages/ns-api/files/ns.telegraf @@ -70,8 +70,17 @@ def set_config(config): return {"success": True} -VM_URL = "http://127.0.0.1:8428/api/v1/query_range" -VM_ALERTS_URL = "http://127.0.0.1:8082/api/v1/alerts" +# VictoriaMetrics and vmalert HTTP API base URL +_vm_addr = "127.0.0.1:8428" +_vmalert_addr = "127.0.0.1:8082" +try: + _euci = EUci() + _vm_addr = _euci.get("victoria-metrics", "main", "http_listen_addr", default=_vm_addr) + _vmalert_addr = _euci.get("vmalert", "main", "http_listen_addr", default=_vmalert_addr) +except Exception: + pass +VM_URL = f"http://{_vm_addr}" +VM_ALERTS_URL = f"http://{_vmalert_addr}" def _get_interface_to_zone_map(): @@ -123,7 +132,7 @@ def _vm_query(expr, start, end, step): params = urllib.parse.urlencode( {"query": expr, "start": start, "end": end, "step": step} ) - with urllib.request.urlopen(f"{VM_URL}?{params}", timeout=10) as resp: + with urllib.request.urlopen(f"{VM_URL}/api/v1/query_range?{params}", timeout=10) as resp: data = json.loads(resp.read()) return data.get("data", {}).get("result", []) @@ -141,7 +150,7 @@ def _single_series(results): def list_alerts(): try: - with urllib.request.urlopen(VM_ALERTS_URL, timeout=10) as resp: + with urllib.request.urlopen(f"{VM_ALERTS_URL}/api/v1/alerts", timeout=10) as resp: data = json.loads(resp.read()) except (TimeoutError, urllib.error.URLError): return {"error": "cannot_retrieve_alerts"} diff --git a/packages/ns-api/files/ns.threatshield b/packages/ns-api/files/ns.threatshield index fa061d7d0..97b935a46 100644 --- a/packages/ns-api/files/ns.threatshield +++ b/packages/ns-api/files/ns.threatshield @@ -483,9 +483,8 @@ def list_settings(e_uci): 'data': { 'enabled': e_uci.get('banip', 'global', 'ban_enabled') == '1', 'ban_logprerouting': e_uci.get('banip', 'global', 'ban_logprerouting', default=False) == '1', - 'ban_loginput': e_uci.get('banip', 'global', 'ban_loginput', default=False) == '1', - 'ban_logforwardwan': e_uci.get('banip', 'global', 'ban_logforwardwan', default=False) == '1', - 'ban_logforwardlan': e_uci.get('banip', 'global', 'ban_logforwardlan', default=False) == '1', + 'ban_loginbound': e_uci.get('banip', 'global', 'ban_loginbound', default=False) == '1', + 'ban_logoutbound': e_uci.get('banip', 'global', 'ban_logoutbound', default=False) == '1', 'ban_loglimit': True if int(e_uci.get('banip', 'global', 'ban_loglimit', default=100)) > 0 else False, 'ban_logcount': e_uci.get('banip', 'global', 'ban_logcount', default=1), 'ban_logterm': e_uci.get('banip', 'global', 'ban_logterm', list=True, default=[]), @@ -533,18 +532,14 @@ def edit_settings(e_uci, payload): raise ValidationError('ban_logprerouting', 'required') if not isinstance(payload['ban_logprerouting'], bool): raise ValidationError('ban_logprerouting', 'invalid', payload['ban_logprerouting']) - if 'ban_loginput' not in payload: - raise ValidationError('ban_loginput', 'required') - if not isinstance(payload['ban_loginput'], bool): - raise ValidationError('ban_loginput', 'invalid', payload['ban_loginput']) - if 'ban_logforwardwan' not in payload: - raise ValidationError('ban_logforwardwan', 'required') - if not isinstance(payload['ban_logforwardwan'], bool): - raise ValidationError('ban_logforwardwan', 'invalid', payload['ban_logforwardwan']) - if 'ban_logforwardlan' not in payload: - raise ValidationError('ban_logforwardlan', 'required') - if not isinstance(payload['ban_logforwardlan'], bool): - raise ValidationError('ban_logforwardlan', 'invalid', payload['ban_logforwardlan']) + if 'ban_loginbound' not in payload: + raise ValidationError('ban_loginbound', 'required') + if not isinstance(payload['ban_loginbound'], bool): + raise ValidationError('ban_loginbound', 'invalid', payload['ban_loginbound']) + if 'ban_logoutbound' not in payload: + raise ValidationError('ban_logoutbound', 'required') + if not isinstance(payload['ban_logoutbound'], bool): + raise ValidationError('ban_logoutbound', 'invalid', payload['ban_logoutbound']) if 'ban_icmplimit' not in payload: raise ValidationError('ban_icmplimit', 'required') if not (isinstance(payload['ban_icmplimit'], bool) or isinstance(payload['ban_icmplimit'], int)): @@ -583,9 +578,8 @@ def edit_settings(e_uci, payload): set_default(e_uci, 'ban_protov6', '1') e_uci.set('banip', 'global', 'ban_logprerouting', payload['ban_logprerouting']) - e_uci.set('banip', 'global', 'ban_loginput', payload['ban_loginput']) - e_uci.set('banip', 'global', 'ban_logforwardwan', payload['ban_logforwardwan']) - e_uci.set('banip', 'global', 'ban_logforwardlan', payload['ban_logforwardlan']) + e_uci.set('banip', 'global', 'ban_loginbound', payload['ban_loginbound']) + e_uci.set('banip', 'global', 'ban_logoutbound', payload['ban_logoutbound']) e_uci.set('banip', 'global', 'ban_loglimit', 100 if payload['ban_loglimit'] else 0) if isinstance(payload['ban_icmplimit'], int): @@ -908,7 +902,7 @@ def __parse_set(blocklist): def list_active_blocks(): try: - output_ipv4 = subprocess.run(["nft", "-j", "list", "set", "inet", "banIP", "blocklistv4"], check=True, + output_ipv4 = subprocess.run(["nft", "-j", "list", "set", "inet", "banIP", "blocklist.v4"], check=True, capture_output=True, text=True) data = json.loads(output_ipv4.stdout) except subprocess.CalledProcessError: @@ -917,7 +911,7 @@ def list_active_blocks(): blocks = __parse_set(data) try: - output_ipv6 = subprocess.run(["nft", "-j", "list", "set", "inet", "banIP", "blocklistv6"], check=True, + output_ipv6 = subprocess.run(["nft", "-j", "list", "set", "inet", "banIP", "blocklist.v6"], check=True, capture_output=True, text=True) data = json.loads(output_ipv6.stdout) except subprocess.CalledProcessError: @@ -965,9 +959,9 @@ def remove_active_block(): return validation_error("ip", "invalid", str(ip)) if isinstance(ip, ipaddress.IPv4Address): - blocklist = "blocklistv4" + blocklist = "blocklist.v4" else: - blocklist = "blocklistv6" + blocklist = "blocklist.v6" try: subprocess.run(["nft", "delete", "element", "inet", "banIP", blocklist, "{" + str(ip) + "}"], check=True) @@ -1032,22 +1026,15 @@ def set_geoblocking_configuration(e_uci, payload): # handle the country feed enable/disable feeds = list(e_uci.get('banip', 'global', 'ban_feed', list=True, default=[])) - ban_blockinput = list(e_uci.get('banip', 'global', 'ban_blockinput', list=True, default=[])) - + if payload['enabled']: # add 'country' feed if not present if 'country' not in feeds: feeds.append('country') - if 'country' not in ban_blockinput: - ban_blockinput.append('country') - e_uci.set('banip', 'global', 'ban_blockinput', ban_blockinput) else: # remove 'country' feed if present if 'country' in feeds: feeds.remove('country') - if 'country' in ban_blockinput: - ban_blockinput.remove('country') - e_uci.set('banip', 'global', 'ban_blockinput', ban_blockinput) # convert country codes to lowercase countries = [c.lower() for c in payload['countries']] @@ -1069,9 +1056,8 @@ if cmd == 'list': 'edit-settings': { 'enabled': True, 'ban_logprerouting': True, - 'ban_loginput': True, - 'ban_logforwardwan': True, - 'ban_logforwardlan': True, + 'ban_loginbound': True, + 'ban_logoutbound': True, 'ban_loglimit': True, 'ban_logcount': 3, 'ban_logterm': ['string'], diff --git a/packages/ns-threat_shield/Makefile b/packages/ns-threat_shield/Makefile index f077c9d00..25ebc6c74 100644 --- a/packages/ns-threat_shield/Makefile +++ b/packages/ns-threat_shield/Makefile @@ -58,6 +58,7 @@ define Package/ns-threat_shield/install $(INSTALL_BIN) ./files/configure-banip-wans.py $(1)/usr/libexec/ns-api/pre-commit/ $(INSTALL_DIR) $(1)/etc/uci-defaults $(INSTALL_BIN) ./files/banip-defaults $(1)/etc/uci-defaults/99-nethsec-banip + $(INSTALL_BIN) ./files/banip-extra-defaults $(1)/etc/uci-defaults/96-nethsec-banip-extra $(INSTALL_BIN) ./files/35_ns-threat_shield $(1)/etc/uci-defaults/35_ns-threat_shield gzip -9n $(1)/usr/share/threat_shield/nethesis-dns.sources gzip -9n $(1)/usr/share/threat_shield/community-dns.sources diff --git a/packages/ns-threat_shield/files/adjust-banip.py b/packages/ns-threat_shield/files/adjust-banip.py index d08e92f0c..8dc17825e 100755 --- a/packages/ns-threat_shield/files/adjust-banip.py +++ b/packages/ns-threat_shield/files/adjust-banip.py @@ -16,7 +16,7 @@ # if ban_country property was modified, remove stale backup files to prevent old country selections from being restored if 'banip' in changes and any('ban_country' in op for op in changes['banip']): - for backup_file in ['/tmp/banIP-backup/banIP.countryv4.gz', '/tmp/banIP-backup/banIP.countryv6.gz']: + for backup_file in ['/tmp/banIP-backup/banIP.country.v4.gz', '/tmp/banIP-backup/banIP.country.v6.gz']: try: os.remove(backup_file) except FileNotFoundError: diff --git a/packages/ns-threat_shield/files/banip-defaults b/packages/ns-threat_shield/files/banip-defaults index f8e665414..52af4f6ae 100644 --- a/packages/ns-threat_shield/files/banip-defaults +++ b/packages/ns-threat_shield/files/banip-defaults @@ -1,10 +1,9 @@ -[ "$(uci -q get banip.global.ban_logforwardwan)" != "" ] && exit 0 +[ "$(uci -q get banip.global.ban_loginbound)" != "" ] && exit 0 uci -q batch << EOI -set banip.global.ban_logforwardwan="1" -set banip.global.ban_logforwardlan="1" +set banip.global.ban_loginbound="1" +set banip.global.ban_logoutbound="1" set banip.global.ban_logprerouting="0" -set banip.global.ban_loginput="0" set banip.global.ban_loglimit="100" set banip.global.ban_logcount="3" diff --git a/packages/ns-threat_shield/files/banip-extra-defaults b/packages/ns-threat_shield/files/banip-extra-defaults new file mode 100644 index 000000000..ce030618d --- /dev/null +++ b/packages/ns-threat_shield/files/banip-extra-defaults @@ -0,0 +1,22 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# set banIP default log file to /var/log/messages +case "$(uci -q get banip.global.ban_logreadfile)" in + "" | "/var/log/banip.log") + uci -q set banip.global.ban_logreadfile="/var/log/messages" + ;; +esac + +# enable packet counters on the banIP chains and set elements +if [ -z "$(uci -q get banip.global.ban_nftcount)" ]; then + uci -q set banip.global.ban_nftcount="1" +fi + +[ -n "$(uci -q changes banip)" ] && uci -q commit banip + +exit 0 diff --git a/packages/ns-threat_shield/files/banip.nethesis.feeds b/packages/ns-threat_shield/files/banip.nethesis.feeds index e9f6d4db1..3ec2273a8 100644 --- a/packages/ns-threat_shield/files/banip.nethesis.feeds +++ b/packages/ns-threat_shield/files/banip.nethesis.feeds @@ -1,27 +1,32 @@ { - "yoroimallvl1": { - "url_4": "https://__USER__:__PASSWORD__@bl.nethesis.it/plain/__TYPE__/nethesis-blacklists/yoroi_malware_level1.ipset", - "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}", - "descr": "Yoroi malware - Level 1" - }, - "yoroimallvl2": { - "url_4": "https://__USER__:__PASSWORD__@bl.nethesis.it/plain/__TYPE__/nethesis-blacklists/yoroi_malware_level2.ipset", - "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}", - "descr": "Yoroi malware - Level 2" - }, - "yoroisusplvl1": { - "url_4": "https://__USER__:__PASSWORD__@bl.nethesis.it/plain/__TYPE__/nethesis-blacklists/yoroi_souspicious_level1.ipset", - "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}", - "descr": "Yoroi suspicious - Level 1" - }, - "yoroisusplvl2": { - "url_4": "https://__USER__:__PASSWORD__@bl.nethesis.it/plain/__TYPE__/nethesis-blacklists/yoroi_souspicious_level2.ipset", - "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}", - "descr": "Yoroi suspicious - Level 2" - }, - "nethesislvl3": { - "url_4": "https://__USER__:__PASSWORD__@bl.nethesis.it/plain/__TYPE__/nethesis-blacklists/nethesis_level3.netset", - "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}", - "descr": "Nethesis suspicious - Level 3" - } + "yoroimallvl1": { + "url_4": "https://__USER__:__PASSWORD__@bl.nethesis.it/plain/__TYPE__/nethesis-blacklists/yoroi_malware_level1.ipset", + "rule": "feed 1", + "chain": "in", + "descr": "Yoroi malware - Level 1" + }, + "yoroimallvl2": { + "url_4": "https://__USER__:__PASSWORD__@bl.nethesis.it/plain/__TYPE__/nethesis-blacklists/yoroi_malware_level2.ipset", + "rule": "feed 1", + "chain": "in", + "descr": "Yoroi malware - Level 2" + }, + "yoroisusplvl1": { + "url_4": "https://__USER__:__PASSWORD__@bl.nethesis.it/plain/__TYPE__/nethesis-blacklists/yoroi_souspicious_level1.ipset", + "rule": "feed 1", + "chain": "in", + "descr": "Yoroi suspicious - Level 1" + }, + "yoroisusplvl2": { + "url_4": "https://__USER__:__PASSWORD__@bl.nethesis.it/plain/__TYPE__/nethesis-blacklists/yoroi_souspicious_level2.ipset", + "rule": "feed 1", + "chain": "in", + "descr": "Yoroi suspicious - Level 2" + }, + "nethesislvl3": { + "url_4": "https://__USER__:__PASSWORD__@bl.nethesis.it/plain/__TYPE__/nethesis-blacklists/nethesis_level3.netset", + "rule": "feed 1", + "chain": "in", + "descr": "Nethesis suspicious - Level 3" + } } diff --git a/packages/ns-ui/Makefile b/packages/ns-ui/Makefile index ba1fe7e70..d0a076253 100644 --- a/packages/ns-ui/Makefile +++ b/packages/ns-ui/Makefile @@ -12,7 +12,7 @@ PKG_RELEASE:=1 PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://github.com/NethServer/nethsecurity-ui.git -PKG_SOURCE_VERSION:=$(PKG_VERSION) +PKG_SOURCE_VERSION:=3f2c6d56440c8fe15d0ca89eda3ec6344afcd216 PKG_SOURCE_SUBDIR:=nethsecurity-ui-$(PKG_SOURCE_VERSION) PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) PKG_MIRROR_HASH:=skip diff --git a/packages/telegraf/Makefile b/packages/telegraf/Makefile index c8facc1db..e88d5212f 100644 --- a/packages/telegraf/Makefile +++ b/packages/telegraf/Makefile @@ -47,8 +47,10 @@ GO_PKG_TAGS:= \ inputs.sensors \ inputs.system \ inputs.wireguard \ + inputs.tail \ outputs.influxdb \ outputs.prometheus_client \ + parsers.grok \ parsers.json_v2 include $(INCLUDE_DIR)/package.mk @@ -89,6 +91,7 @@ define Package/telegraf/install $(INSTALL_DATA) ./files/telegraf.conf.d/services.conf $(1)/etc/telegraf.conf.d/services.conf $(INSTALL_DATA) ./files/telegraf.conf.d/mwan.conf $(1)/etc/telegraf.conf.d/mwan.conf $(INSTALL_DATA) ./files/telegraf.conf.d/ha.conf $(1)/etc/telegraf.conf.d/ha.conf + $(INSTALL_DATA) ./files/telegraf.conf.d/banip.conf $(1)/etc/telegraf.conf.d/banip.conf $(INSTALL_DIR) $(1)/etc/uci-defaults $(INSTALL_BIN) ./files/uci-defaults/99-telegraf-migrate-netdata $(1)/etc/uci-defaults/99-telegraf-migrate-netdata $(INSTALL_DIR) $(1)/usr/sbin diff --git a/packages/telegraf/files/telegraf.conf.d/banip.conf b/packages/telegraf/files/telegraf.conf.d/banip.conf new file mode 100644 index 000000000..eec34d159 --- /dev/null +++ b/packages/telegraf/files/telegraf.conf.d/banip.conf @@ -0,0 +1,23 @@ +# banIP monitoring + +# blocked packets (nftables drop/reject) +[[inputs.tail]] + name_override = "banip" + files = ["/var/log/messages"] + from_beginning = false + data_format = "grok" + grok_patterns = ['banIP/%{DATA:chain:tag}/%{WORD:verdict:tag}/%{DATA:feed:tag}: %{GREEDYDATA}SRC=%{IP:src:tag}%{GREEDYDATA}DST=%{IP:dst:tag}%{GREEDYDATA}LEN=%{NUMBER:blocked_bytes:int}'] + + [inputs.tail.tags] + influxdb_db = "os-metrics" + +# dynamically banned IPs (fail2ban-like "add IP") +[[inputs.tail]] + name_override = "banip" + files = ["/var/log/messages"] + from_beginning = false + data_format = "grok" + grok_patterns = ["banIP%{DATA}\\[%{NUMBER:banned:int}\\]: add IP '%{IP:ip:tag}'"] + + [inputs.tail.tags] + influxdb_db = "os-metrics"