snort3: finish up several incomplete capabilities

Reporting
 - Use json alert data for 10x speed improvement in report generation
 - Include both gid and sid, plus packet direction in report output
 - Add by-date incident filtering
 - Add verbose mode which displays actual rules triggered and their source
 - Attempt to look up host names from IPs in verbose mode
 - Clean up display of port number involved in incidents

Rules
 - Complete downloader for subscription rules using oinkcode (only tested
   with snort.org's "free" tier subscription)
 - Auto-detect multiple rules files and include them in lua 'ips.rules'
 - Add '--backup' option to copy out current rules before installing new
 - Add '--persistent' option to 'snort-rules', storing in persistent location

CLI interface
 - Completely rework command line option parsing in all user scripts
 - Allow options and commands to be in any order on command line
 - Add long-form names for all options ('--help' for '-h' and so on)
 - Detect errors properly in options, enhance help pages

Bug fixes
 - Use 'mkdir -p' on all directory creation
 - Use proper tmp directory from 'snort.snort.temp_dir' everywhere

Signed-off-by: Eric Fahlgren <ericfahlgren@gmail.com>

1 files changed, 260 insertions, 137 deletions

diff --git a/net/snort3/files/snort-mgr b/net/snort3/files/snort-mgr
index cc60abf65..625157967 100644
--- a/net/snort3/files/snort-mgr
+++ b/net/snort3/files/snort-mgr
@@ -1,24 +1,29 @@
 #!/bin/sh
-# Copyright (c) 2023 Eric Fahlgren <eric.fahlgren@gmail.com>
+# Copyright (c) 2023-2024 Eric Fahlgren <eric.fahlgren@gmail.com>
 # SPDX-License-Identifier: GPL-2.0
 # shellcheck disable=SC2039,SC2155  # "local" not defined in POSIX sh
 
-PROG="/usr/bin/snort"
+PROG="$(command -v snort)"
 MAIN="/usr/share/snort/main.uc"
-CONF_DIR="/var/snort.d"
+CONF_DIR=$(uci -q get snort.snort.temp_dir || echo "/var/snort.d")
 CONF="${CONF_DIR}/snort_conf.lua"
 
-VERBOSE=
+ACTION="usage"  # Show help by default.
+VERBOSE=false
+QUIET=false
 TESTING=
+TABLE=
 NLINES=0
+DATE_SPEC=
+PATTERN=
 
-[ ! -e "$CONF_DIR" ] && mkdir "$CONF_DIR"
+[ ! -e "$CONF_DIR" ] && mkdir -p "$CONF_DIR"
 [ -e /dev/stdin ] && STDIN=/dev/stdin || STDIN=/proc/self/fd/0
 [ -e /dev/stdout ] && STDOUT=/dev/stdout || STDOUT=/proc/self/fd/1
 [ -t 2 ] && export TTY=1
 
 die() {
-	[ -n "$QUIET" ] || echo "$@" >&2
+	$QUIET || echo "$@" >&2
 	exit 1
 }
 
@@ -47,8 +52,10 @@ nft_rm_table() {
 
 nft_add_table() {
 	if [ "$(uci -q get snort.snort.method)" = "nfq" ]; then
-		print nftables | nft $VERBOSE -f $STDIN
-		[ -n "$VERBOSE" ] && nft list table inet snort
+		local options
+		$VERBOSE && options='-e'
+		print nftables | nft $options -f $STDIN
+		$VERBOSE && nft list table inet snort
 	fi
 }
 
@@ -69,23 +76,30 @@ teardown() {
 	[ -e "$CONF" ] && rm "$CONF"
 }
 
+resetup() {
+	QUIET=true check || die "The generated snort lua configuration contains errors, not restarting.  Run 'snort-mgr check'"
+	teardown
+	setup
+}
+
 update_rules() {
 	/usr/bin/snort-rules $TESTING
 }
 
 print() {
-	# '$1' is file type to generate, one of:
-	#     config, snort or nftables
-	TYPE=$1 utpl -S "$MAIN"
+	# '$1' is optional file type to generate, one of:
+	#     config, snort, nftables or help
+	local table="${1:-$TABLE}"
+	utpl -D TYPE="$table" -D QUIET=$QUIET -S "$MAIN"
 }
 
 check() {
 	local manual=$(uci get snort.snort.manual)
 	[ "$manual" = 1 ] && return 0
 
-	[ -n "$QUIET" ] && OUT=/dev/null || OUT=$STDOUT
+	$QUIET && OUT=/dev/null || OUT=$STDOUT
 	local warn no_rules
-	if [ -n "$VERBOSE" ]; then
+	if $VERBOSE; then
 		warn='--warn-all'
 		no_rules=0
 	else
@@ -94,146 +108,191 @@ check() {
 	fi
 
 	local test_conf="${CONF_DIR}/test_conf.lua"
-	_SNORT_WITHOUT_RULES="$no_rules" print snort > "${test_conf}" || die "Errors during generation of snort config."
+	_SNORT_WITHOUT_RULES="$no_rules" print snort > "${test_conf}" || die "Errors during generation of snort config"
 	if $PROG -T $warn -c "${test_conf}" 2> $OUT ; then
 		rm "${test_conf}"
 	else
-		die "Errors in snort config tests.  Examine ${test_conf} for issues."
+		die "Errors in snort config tests.  Examine ${test_conf} for issues"
 	fi
 
 	if [ "$(uci -q get snort.snort.method)" = "nfq" ]; then
+		local options
 		local test_nft="${CONF_DIR}/test_conf.nft"
-		print nftables > "${test_nft}" || die "Errors during generation of nftables config."
-		if nft $VERBOSE --check -f "${test_nft}" ; then
+		print nftables > "${test_nft}" || die "Errors during generation of nftables config"
+		$VERBOSE && options='-e'
+		if nft $options --check -f "${test_nft}" ; then
 			rm "${test_nft}"
 		else
-			die "Errors in nftables config tests.  Examine ${test_nft} for issues."
+			die "Errors in nftables config tests.  Examine ${test_nft} for issues"
 		fi
 	fi
 
 }
 
+_filter_by_date() {
+	# Grab all the alert_json files in the log directory, scan them
+	# for matching timestamps and return those lines that match.
+	local log_dir="$1"
+
+	local operator date
+	case "$DATE_SPEC" in
+		('')    operator='>' ; date=''                   ;;
+		(-*)    operator='<' ; date="${DATE_SPEC:1}"     ;;
+		(+*)    operator='>' ; date="${DATE_SPEC:1}"     ;;
+		(today) operator='>' ; date=$(date +'%y/%m/%d-') ;;
+		(*)     die "Invalid date specification '${DATE_SPEC}', did you forget the +/- prefix?" ;;
+	esac
+
+	# We need to create a single json array because 'jsonfilter -a' is
+	# severely broken.
+	awk '
+		BEGIN { print "[" }
+		{ print $0"," }
+		END { print "{}]" }
+	' "${log_dir}"/*alert_json.txt \
+	| jsonfilter -e '$[@.timestamp '${operator}' "'"${date}"'"]'
+}
+
 report() {
-	# Reported IPs have source port stripped, but destination port (if any)
-	# retained.
-	#
-	# json notes
-	# from alert_fast:
-	# 08/30-11:39:57.639021 [**] [1:382:11] "PROTOCOL-ICMP PING Windows" [**] [Classification: Misc activity] [Priority: 3] {ICMP} 10.1.1.186 -> 10.1.1.20
-	#
-	# same event in alert_json (single line broken for clarity):
-	# { "timestamp" : "08/30-11:39:57.639021", "pkt_num" : 5366, "proto" : "ICMP", "pkt_gen" : "raw",
-	#   "pkt_len" : 60, "dir" : "C2S", "src_ap" : "10.1.1.186:0", "dst_ap" : "10.1.1.20:0",
-	#   "rule" : "1:382:11", "action" : "allow" }
-	#
-	# Second part of "rule", 382, is "sid" in ruleset, suffixing 11 is "rev".
-	# grep '\bsid:382\b' /etc/snort/rules/snort.rules  (again, single line broken for clarity):
-	# alert icmp $EXTERNAL_NET any -> $HOME_NET any ( msg:"PROTOCOL-ICMP PING Windows";
-	#     itype:8; content:"abcdefghijklmnop",depth 16; metadata:ruleset community;
-	#     classtype:misc-activity; sid:382; rev:11; )
-	#
-	# Not sure where the prefixing 1 comes from.
+	# Reported IPs have random source port stripped, but destination port
+	# (if any) retained.
+
+	local SORT="$(command -v sort)"
+	if [ ! -x "${SORT}" ] || ! "${SORT}" --version 2> /dev/null | grep -q "coreutils"; then
+		die "'snort-mgr report' requires coreutils-sort package"
+	fi
 
 	local logging=$(uci get snort.snort.logging)
 	local log_dir=$(uci get snort.snort.log_dir)
-	local pattern="$1"
 
 	if [ "$logging" = 0 ]; then
-		die "Logging is not enabled in snort config."
+		die "Logging is not enabled in snort config"
 	fi
 		
+	#-- Collect the inputs --
+	local msg src srcP dst dstP dir gid sid
+	local tmp=$(mktemp -t snort.rep.XXXXXX)
+	_filter_by_date "${log_dir}" | while read -r line; do
+		unset -v src dst srcP dstP
+		eval "$(jsonfilter -s "$line" \
+			-e 'msg=$.msg' \
+			-e 'src=$.src_addr' \
+			-e 'dst=$.dst_addr' \
+			-e 'srcP=$.src_port' \
+			-e 'dstP=$.dst_port' \
+			-e 'dir=$.dir' \
+			-e 'gid=$.gid' \
+			-e 'sid=$.sid')"
+
+		# Append the port to the IP, but only if it's meaningful.
+		[ "$dir" = 'C2S' ] && [ -n "$dstP" ] && dst="${dst}(${dstP})"
+		[ "$dir" = 'S2C' ] && [ -n "$srcP" ] && src="${src}(${srcP})"
+
+		echo "$msg#$src#$dst#$dir#$gid#$sid"
+	done | grep -iE "$PATTERN" > "$tmp"
+
+	#-- Generate output --
+	local output
 	[ "$NLINES" = 0 ] && output="cat" || output="head -n $NLINES"
 
-	local msg src dst dir
-	tmp="/tmp/snort.report.$$"
-	for file in "${log_dir}"/*alert_json.txt; do
-		while read -r line; do
-			eval $(jsonfilter -s "$line" -e 'msg=$.msg' -e 'src=$.src_ap' -e 'dst=$.dst_ap' -e 'dir=$.dir')
-			src=$(echo "$src" | sed 's/:.*$//')  # Delete all source ports.
-			dst=$(echo "$dst" | sed 's/:0$//')   # Delete unspecified dest port.
-			echo "$msg#$src#$dst#$dir"
-		done < "$file"
-	done | grep -i "$pattern" > "$tmp"
-
-	echo "Events involving ${pattern:-all IPs}"
-	n_incidents="$(wc -l < $tmp)"
-	lines=$(sort "$tmp" | uniq -c | sort -nr \
-		| awk -F'#' '{printf "%-80s %s %-13s -> %s\n", $1, $4, $2, $3}')
-	echo "$lines" | $output
-	n_lines=$(echo "$lines" | wc -l)
-	[ "$NLINES" -gt 0 ] && [ "$NLINES" -lt "$n_lines" ] && echo "    ... Only showing $NLINES of $n_lines most frequent incidents."
-	printf "%7d total incidents\n" "$n_incidents"
+	local lines=$($SORT "$tmp" | uniq -c | $SORT -nr | $output)
 	rm "$tmp"
+	if [ -z "$lines" ]; then
+		echo -n "There were no incidents "
+		[ -z "$PATTERN" ] && echo "reported." || echo "matching pattern '$PATTERN'."
+		return
+	fi
+
+	local n_total=$(cat "${log_dir}"/*alert_json.txt | wc -l)
+	local n_incidents=$(echo "$lines" | awk '{total += $1} END {print total}')
+	local mlen=$(echo "$lines" | awk -F'#' '{print $1}' | wc -L)
+	local slen=$(echo "$lines" | awk -F'#' '{print $2}' | wc -L)
+
+	echo "Events involving ${PATTERN:-all IPs} - $(date -Is)"
+	printf "%-*s %3s %5s %-3s %-*s %s\n" "$mlen"  "  Count Message" "gid" "sid" "Dir" "$slen" "Source" "Destination"
+	echo "$lines" | awk -F'#' '{printf "%-'"$mlen"'s %3d %5d %s %-'"$slen"'s %s\n", $1, $5, $6, $4, $2, $3}'
+
+	printf "%7d incidents shown of %d logged\n" "$n_incidents" "$n_total"
+
+	#-- Lookup rules and references, if requested. --
+	if $VERBOSE; then
+		local rules_dir="$(uci get snort.snort.config_dir)/rules"
+		local usids="$(echo "$lines" | awk -F'#' '{print $5 "#" $6}' | $SORT -u | $SORT -t'#' -k1n -k2n)"
+		local nsids="$(echo "$usids" | wc -w)"
+
+		echo ''
+		echo "$nsids unique rules triggered:"
+		local rule
+		local i=1
+		for sid in $usids; do
+			eval "$(echo "$sid" | awk -F'#' '{printf "export gid=%s;export sid=%s", $1, $2}')"
+			printf "%3d - gid=%3d sid=%5d " "$i" "$gid" "$sid"
+			rule=$(grep -Hn "\bsid:${sid};" "$rules_dir"/*.rules)
+			if [ "$gid" -ne 1 ] && echo "$rule" | grep -qv "\bgid:${gid};"; then
+				# Many rules have gid implicitly '1', zero any that are not
+				# explicit when expecting non-'1'.
+				rule=""
+			fi
+			if [ -n "$rule" ]; then
+				echo "$rule" | cut -c -120
+			else
+				rule=$($PROG --list-builtin | grep "^${gid}:${sid}\b")
+				if [ -n "$rule" ]; then
+					echo "BUILTIN: ${rule}"
+				fi
+			fi
+			i=$((i + 1))
+		done
+		echo ""
+		echo "Per-rule details may be viewed by specifying the appropriate gid and sid, e.g.:"
+		echo "    https://www.snort.org/rule-docs/$gid-$sid"
+
+		# Look up the names of the IPs shown in report.
+		# Note, on my dev box, nslookup fires rule 1:14777, so you get lots
+		# of incidents if not suppressed.
+		echo ''
+		echo 'Hosts by name:'
+		local IP
+		local peerdns=$(ifstatus wan | jsonfilter -e '$["dns-server"][0]')
+		echo "$lines" | awk -F'#' '{printf "%s\n%s\n", $2, $3}' | sed 's/(.*//' | sort -u \
+		| while read -r IP; do
+			[ -z "$IP" ] && continue
+			n=$(nslookup "$IP" | awk '/name = / {n=$NF} END{print n}')
+			[ -z "$n" ] && [ -n "$peerdns" ] && n=$(nslookup "$IP" "$peerdns" | awk '/name = / {n=$NF} END{print n}')
+			[ -z "$n" ] && n='--unknown host--'
+			printf "  %-39s %s\n" "$IP" "$n"
+		done | $SORT -b -k2
+	fi
 }
 
 status() {
 	echo -n 'snort is ' ; service snort status
-	ps w | grep -E 'PID|snort' | grep -v grep
+	local mem_total mem_free
+	eval "$(ubus call system info | jsonfilter -e 'mem_total=$.memory.total' -e 'mem_free=$.memory.free')"
+	awk -v mem_total="$mem_total" -v mem_free="$mem_free" 'BEGIN {
+		mem_used = mem_total - mem_free;
+		printf "Total system memory=%.3fM  Used=%.3fM (%.1f%%)  Free=%.3fM (%.1f%%)\n",
+			mem_total/1024**2,
+			mem_used/1024**2, 100*mem_used/mem_total,
+			mem_free/1024**2, 100*mem_free/mem_total;
+	}'
+	busybox ps w | grep -E "PID|$PROG " | grep -v grep
+
+	if [ "$(uci -q get snort.snort.method)" = "nfq" ]; then
+		nft list table inet snort
+	fi
 }
 
+#-------------------------------------------------------------------------------
 
-while [ -n "$1" ]; do
-	case "$1" in
-		-q)
-			export QUIET=1
-			shift
-		;;
-		-v)
-			export VERBOSE=-e
-			shift
-		;;
-		-t)
-			TESTING=-t
-			shift
-		;;
-		-n)
-			NLINES="$2"
-			shift
-			shift
-		;;
-		*)
-			break
-		;;
-	esac
-done
+usage() {
+	local msg="$1"
+	[ -n "$msg" ] && printf "ERROR: %s\n\n" "$msg"
 
-case "$1" in
-	setup)
-		setup
-	;;
-	teardown)
-		teardown
-	;;
-	resetup)
-		QUIET=1 check || die "The generated snort lua configuration contains errors, not restarting.  Run 'snort-mgr check'"
-		teardown
-		setup
-	;;
-	update-rules)
-		update_rules
-	;;
-	check)
-		check
-	;;
-	print)
-		print "$2"
-	;;
-	report)
-		report "$2"
-	;;
-	status)
-		status
-	;;
-	*)
-		cat <<USAGE
+	cat <<USAGE
 Usage:
 
-  -n = show only NLINES of output
-  -q = quiet
-  -v = verbose
-  -t = testing mode
-
-  $0 [-v] [-q] setup|teardown|resetup
+  $0 setup|teardown|resetup [-v/--verbose] [-q/--quiet]
 
     Normally only used internally by init scripts to manage the generation
     of configuration files and any needed firewall rules.  None of these
@@ -243,36 +302,52 @@ Usage:
       resetup  = shorthand for teardown and then setup.
 
 
-  $0 [-n lines] report [pattern]
+  $0 report [-v/--verbose] [-n/--n-lines N] [-d/--date-spec D] [-p/--pattern P]
 
     Report on incidents.  Note this is somewhat experimental, so suggested
-    improvements are quite welcome.
+    improvements are quite welcome.  Reported Source and Destination are of
+    the form "ip(port)", with zero and random source ports stripped.
+      -v      = Show the rules that were triggered, after report table.
+      -n N    = Show only the N highest frequency incidents.
+      -d D    = Filter entries by date specification in D.
+      -p P    = Grep pattern to filter incidents, applied to all outputs.
       pattern = A case-insensitive grep pattern used to filter output.
 
-  $0 [-t] update-rules
+    The date specification for '-d' can be either literal 'today'
+    or a snort-formatted date prefixed by '-' or '+', meaning 'before'
+    and 'after', respectively.  Snort date reporting has the format
+    'YY/MM/DD-hh:mm:ss.ssssss', and you can use any prefix as a date.
+    For example,
+      > snort-mgr --date-spec +23/12/20-09 report
+    will process all incidents from from 2023-12-20 at 0900 and later.
+
+
+  $0 update-rules [-t/--testing]
 
-    Download and install the snort ruleset.  Testing mode generates a canned
-    rule that matches IPv4 ping requests.  A typical test scenario might look
-    like:
+    Download and install the snort ruleset.
+      -t      = Generate a test-only ruleset, don't download anything.
+
+    Testing mode generates a canned rule that matches IPv4 ping requests.
+    A typical test scenario might look like:
 
       > snort-mgr -t update-rules
       > /etc/init.d/snort start
       > ping -c4 8.8.8.8
-      > logread -e "TEST ALERT"
+      > snort-mgr report
 
 
-  $0 print config|snort|nftables
+  $0 print config|snort|nftables|help
 
-    Print the rendered file contents.
-      config   = Display contents of /etc/config/snort, but with all values and
-                 descriptions.  Missing values shown with defaults.
-      snort    = The snort configuration file, which is a lua script.
-      nftables = The nftables script used to define the input queues when using
-                 the 'nfq' DAQ.
-      help     = Display config file help.
+    Print the rendered file contents.  Table types are:
+      config   - Display contents of /etc/config/snort, but with all values and
+                 descriptions.  Missing entries rendered with defaults.
+      snort    - The top-level snort configuration lua script, with includes.
+      nftables - The nftables script used to define the input queues when using
+                 the 'nfq' DAQ, with any included content.
+      help     - Display config file help.
 
 
-  $0 [-q] check
+  $0 check [-q/--quiet]
 
     Test the rendered config using snort's check mode without
     applying it to the running system.
@@ -280,8 +355,56 @@ Usage:
 
   $0 status
 
-    Print the nfq counter values and blah blah blah
+    Print the service status, system memory use and if nfq is the current daq,
+    then the nftables with counter values and so on.
 
 USAGE
-	;;
-esac
+	exit 1
+}
+
+while [ -n "$1" ]; do
+	case "$1" in
+		-h|--help)
+			usage
+		;;
+		-q|--quiet)
+			QUIET=true
+		;;
+		-v|--verbose)
+			VERBOSE=true
+		;;
+		-t|--testing)
+			TESTING=-t
+		;;
+		-n|--n-lines)
+			[ -z "$2" ] && usage "'--n-lines' requires a value"
+			NLINES="$2"
+			shift
+		;;
+		-d|--date-spec)
+			[ -z "$2" ] && usage "'--date-spec' requires a value"
+			DATE_SPEC="$2"
+			shift
+		;;
+		-p|--pattern)
+			[ -z "$2" ] && usage "'--pattern' requires a value"
+			PATTERN="$2"
+			shift
+		;;
+		print)
+			[ -z "$2" ] && usage "'print' requires a table type"
+			ACTION="$1"
+			TABLE="$2"
+			shift
+		;;
+		setup|teardown|resetup|update-rules|check|report|status)
+			ACTION="$1"
+		;;
+		*)
+			usage "'$1' is not a valid command or option"
+		;;
+	esac
+	shift
+done
+
+[ -n "$ACTION" ] && eval "$ACTION"