snort3: finish up several incomplete capabilities

Reporting
 - Use json alert data for 10x speed improvement in report generation
 - Include both gid and sid, plus packet direction in report output
 - Add by-date incident filtering
 - Add verbose mode which displays actual rules triggered and their source
 - Attempt to look up host names from IPs in verbose mode
 - Clean up display of port number involved in incidents

Rules
 - Complete downloader for subscription rules using oinkcode (only tested
   with snort.org's "free" tier subscription)
 - Auto-detect multiple rules files and include them in lua 'ips.rules'
 - Add '--backup' option to copy out current rules before installing new
 - Add '--persistent' option to 'snort-rules', storing in persistent location

CLI interface
 - Completely rework command line option parsing in all user scripts
 - Allow options and commands to be in any order on command line
 - Add long-form names for all options ('--help' for '-h' and so on)
 - Detect errors properly in options, enhance help pages

Bug fixes
 - Use 'mkdir -p' on all directory creation
 - Use proper tmp directory from 'snort.snort.temp_dir' everywhere

Signed-off-by: Eric Fahlgren <ericfahlgren@gmail.com>

6 files changed, 448 insertions, 223 deletions

diff --git a/net/snort3/Makefile b/net/snort3/Makefile
index b7658c4d0..6f0218d8e 100644
--- a/net/snort3/Makefile
+++ b/net/snort3/Makefile
@@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk
 
 PKG_NAME:=snort3
 PKG_VERSION:=3.1.78.0
-PKG_RELEASE:=1
+PKG_RELEASE:=2
 
 PKG_SOURCE:=$(PKG_VERSION).tar.gz
 PKG_SOURCE_URL:=https://github.com/snort3/snort3/archive/refs/tags/
diff --git a/net/snort3/files/main.uc b/net/snort3/files/main.uc
index 3a15f73c6..4f2a63ca8 100644
--- a/net/snort3/files/main.uc
+++ b/net/snort3/files/main.uc
@@ -1,6 +1,6 @@
 {%
 //------------------------------------------------------------------------------
-// Copyright (c) 2023 Eric Fahlgren <eric.fahlgren@gmail.com>
+// Copyright (c) 2023-2024 Eric Fahlgren <eric.fahlgren@gmail.com>
 // SPDX-License-Identifier: GPL-2.0
 //
 // The tables defined using 'config_item' are the source of record for the
@@ -9,11 +9,14 @@
 //
 //------------------------------------------------------------------------------
 
+QUIET; // Reference globals passed from CLI, so we get errors when missing.
+TYPE;
+
 import { cursor } from 'uci';
 let uci = cursor();
 
 function wrn(fmt, ...args) {
-	if (getenv("QUIET"))
+	if (QUIET)
 		exit(1);
 
 	let msg = "ERROR: " + sprintf(fmt, ...args);
@@ -25,6 +28,15 @@ function wrn(fmt, ...args) {
 	exit(1);
 }
 
+function rpad(str, fill, len)
+{
+    str = rtrim(str) + ' ';
+    while (length(str) < len) {
+        str += fill;
+    }
+    return str;
+}
+
 //------------------------------------------------------------------------------
 
 function config_item(type, values, def) {
@@ -221,11 +233,11 @@ function dump_config(settings) {
 }
 
 function render_snort() {
-	include("templates/snort.uc", { snort, nfq });
+	include("templates/snort.uc", { snort, nfq, rpad });
 }
 
 function render_nftables() {
-	include("templates/nftables.uc", { snort, nfq });
+	include("templates/nftables.uc", { snort, nfq, rpad });
 }
 
 function render_config() {
@@ -242,7 +254,7 @@ function render_help() {
 
 load_all();
 
-let table_type = getenv("TYPE");
+let table_type = TYPE;  // Supply on cli with '-D TYPE=snort'...
 switch (table_type) {
 	case "snort":
 		render_snort();
diff --git a/net/snort3/files/nftables.uc b/net/snort3/files/nftables.uc
index 516033426..74b1678d6 100644
--- a/net/snort3/files/nftables.uc
+++ b/net/snort3/files/nftables.uc
@@ -1,6 +1,6 @@
 # Do not edit, automatically generated.  See /usr/share/snort/templates.
 {%
-// Copyright (c) 2023 Eric Fahlgren <eric.fahlgren@gmail.com>
+// Copyright (c) 2023-2024 Eric Fahlgren <eric.fahlgren@gmail.com>
 // SPDX-License-Identifier: GPL-2.0
 
 let queues     = `${nfq.queue_start}-${int(nfq.queue_start)+int(nfq.queue_count)-1}`;
@@ -14,9 +14,9 @@ table inet snort {
 		{% if (nfq.include) {
 		  // We use the ucode include here, so that the included file is also
 		  // part of the template and can use values passed in from the config.
-		  printf("\n\t\t#-- The following content included from '%s'\n", nfq.include);
+		  printf("\n\t\t" + rpad(`#-- Include from '${nfq.include}'`, ">", 64) + "\n");
 		  include(nfq.include, { snort, nfq });
-		  printf("\t\t#-- End of included file.\n\n");
+		  printf("\t\t" + rpad("#-- End of included file.", "<", 64) + "\n\n");
 		} %}
 		counter  queue flags bypass to {{ queues }}
 	}
diff --git a/net/snort3/files/snort-mgr b/net/snort3/files/snort-mgr
index cc60abf65..625157967 100644
--- a/net/snort3/files/snort-mgr
+++ b/net/snort3/files/snort-mgr
@@ -1,24 +1,29 @@
 #!/bin/sh
-# Copyright (c) 2023 Eric Fahlgren <eric.fahlgren@gmail.com>
+# Copyright (c) 2023-2024 Eric Fahlgren <eric.fahlgren@gmail.com>
 # SPDX-License-Identifier: GPL-2.0
 # shellcheck disable=SC2039,SC2155  # "local" not defined in POSIX sh
 
-PROG="/usr/bin/snort"
+PROG="$(command -v snort)"
 MAIN="/usr/share/snort/main.uc"
-CONF_DIR="/var/snort.d"
+CONF_DIR=$(uci -q get snort.snort.temp_dir || echo "/var/snort.d")
 CONF="${CONF_DIR}/snort_conf.lua"
 
-VERBOSE=
+ACTION="usage"  # Show help by default.
+VERBOSE=false
+QUIET=false
 TESTING=
+TABLE=
 NLINES=0
+DATE_SPEC=
+PATTERN=
 
-[ ! -e "$CONF_DIR" ] && mkdir "$CONF_DIR"
+[ ! -e "$CONF_DIR" ] && mkdir -p "$CONF_DIR"
 [ -e /dev/stdin ] && STDIN=/dev/stdin || STDIN=/proc/self/fd/0
 [ -e /dev/stdout ] && STDOUT=/dev/stdout || STDOUT=/proc/self/fd/1
 [ -t 2 ] && export TTY=1
 
 die() {
-	[ -n "$QUIET" ] || echo "$@" >&2
+	$QUIET || echo "$@" >&2
 	exit 1
 }
 
@@ -47,8 +52,10 @@ nft_rm_table() {
 
 nft_add_table() {
 	if [ "$(uci -q get snort.snort.method)" = "nfq" ]; then
-		print nftables | nft $VERBOSE -f $STDIN
-		[ -n "$VERBOSE" ] && nft list table inet snort
+		local options
+		$VERBOSE && options='-e'
+		print nftables | nft $options -f $STDIN
+		$VERBOSE && nft list table inet snort
 	fi
 }
 
@@ -69,23 +76,30 @@ teardown() {
 	[ -e "$CONF" ] && rm "$CONF"
 }
 
+resetup() {
+	QUIET=true check || die "The generated snort lua configuration contains errors, not restarting.  Run 'snort-mgr check'"
+	teardown
+	setup
+}
+
 update_rules() {
 	/usr/bin/snort-rules $TESTING
 }
 
 print() {
-	# '$1' is file type to generate, one of:
-	#     config, snort or nftables
-	TYPE=$1 utpl -S "$MAIN"
+	# '$1' is optional file type to generate, one of:
+	#     config, snort, nftables or help
+	local table="${1:-$TABLE}"
+	utpl -D TYPE="$table" -D QUIET=$QUIET -S "$MAIN"
 }
 
 check() {
 	local manual=$(uci get snort.snort.manual)
 	[ "$manual" = 1 ] && return 0
 
-	[ -n "$QUIET" ] && OUT=/dev/null || OUT=$STDOUT
+	$QUIET && OUT=/dev/null || OUT=$STDOUT
 	local warn no_rules
-	if [ -n "$VERBOSE" ]; then
+	if $VERBOSE; then
 		warn='--warn-all'
 		no_rules=0
 	else
@@ -94,146 +108,191 @@ check() {
 	fi
 
 	local test_conf="${CONF_DIR}/test_conf.lua"
-	_SNORT_WITHOUT_RULES="$no_rules" print snort > "${test_conf}" || die "Errors during generation of snort config."
+	_SNORT_WITHOUT_RULES="$no_rules" print snort > "${test_conf}" || die "Errors during generation of snort config"
 	if $PROG -T $warn -c "${test_conf}" 2> $OUT ; then
 		rm "${test_conf}"
 	else
-		die "Errors in snort config tests.  Examine ${test_conf} for issues."
+		die "Errors in snort config tests.  Examine ${test_conf} for issues"
 	fi
 
 	if [ "$(uci -q get snort.snort.method)" = "nfq" ]; then
+		local options
 		local test_nft="${CONF_DIR}/test_conf.nft"
-		print nftables > "${test_nft}" || die "Errors during generation of nftables config."
-		if nft $VERBOSE --check -f "${test_nft}" ; then
+		print nftables > "${test_nft}" || die "Errors during generation of nftables config"
+		$VERBOSE && options='-e'
+		if nft $options --check -f "${test_nft}" ; then
 			rm "${test_nft}"
 		else
-			die "Errors in nftables config tests.  Examine ${test_nft} for issues."
+			die "Errors in nftables config tests.  Examine ${test_nft} for issues"
 		fi
 	fi
 
 }
 
+_filter_by_date() {
+	# Grab all the alert_json files in the log directory, scan them
+	# for matching timestamps and return those lines that match.
+	local log_dir="$1"
+
+	local operator date
+	case "$DATE_SPEC" in
+		('')    operator='>' ; date=''                   ;;
+		(-*)    operator='<' ; date="${DATE_SPEC:1}"     ;;
+		(+*)    operator='>' ; date="${DATE_SPEC:1}"     ;;
+		(today) operator='>' ; date=$(date +'%y/%m/%d-') ;;
+		(*)     die "Invalid date specification '${DATE_SPEC}', did you forget the +/- prefix?" ;;
+	esac
+
+	# We need to create a single json array because 'jsonfilter -a' is
+	# severely broken.
+	awk '
+		BEGIN { print "[" }
+		{ print $0"," }
+		END { print "{}]" }
+	' "${log_dir}"/*alert_json.txt \
+	| jsonfilter -e '$[@.timestamp '${operator}' "'"${date}"'"]'
+}
+
 report() {
-	# Reported IPs have source port stripped, but destination port (if any)
-	# retained.
-	#
-	# json notes
-	# from alert_fast:
-	# 08/30-11:39:57.639021 [**] [1:382:11] "PROTOCOL-ICMP PING Windows" [**] [Classification: Misc activity] [Priority: 3] {ICMP} 10.1.1.186 -> 10.1.1.20
-	#
-	# same event in alert_json (single line broken for clarity):
-	# { "timestamp" : "08/30-11:39:57.639021", "pkt_num" : 5366, "proto" : "ICMP", "pkt_gen" : "raw",
-	#   "pkt_len" : 60, "dir" : "C2S", "src_ap" : "10.1.1.186:0", "dst_ap" : "10.1.1.20:0",
-	#   "rule" : "1:382:11", "action" : "allow" }
-	#
-	# Second part of "rule", 382, is "sid" in ruleset, suffixing 11 is "rev".
-	# grep '\bsid:382\b' /etc/snort/rules/snort.rules  (again, single line broken for clarity):
-	# alert icmp $EXTERNAL_NET any -> $HOME_NET any ( msg:"PROTOCOL-ICMP PING Windows";
-	#     itype:8; content:"abcdefghijklmnop",depth 16; metadata:ruleset community;
-	#     classtype:misc-activity; sid:382; rev:11; )
-	#
-	# Not sure where the prefixing 1 comes from.
+	# Reported IPs have random source port stripped, but destination port
+	# (if any) retained.
+
+	local SORT="$(command -v sort)"
+	if [ ! -x "${SORT}" ] || ! "${SORT}" --version 2> /dev/null | grep -q "coreutils"; then
+		die "'snort-mgr report' requires coreutils-sort package"
+	fi
 
 	local logging=$(uci get snort.snort.logging)
 	local log_dir=$(uci get snort.snort.log_dir)
-	local pattern="$1"
 
 	if [ "$logging" = 0 ]; then
-		die "Logging is not enabled in snort config."
+		die "Logging is not enabled in snort config"
 	fi
 		
+	#-- Collect the inputs --
+	local msg src srcP dst dstP dir gid sid
+	local tmp=$(mktemp -t snort.rep.XXXXXX)
+	_filter_by_date "${log_dir}" | while read -r line; do
+		unset -v src dst srcP dstP
+		eval "$(jsonfilter -s "$line" \
+			-e 'msg=$.msg' \
+			-e 'src=$.src_addr' \
+			-e 'dst=$.dst_addr' \
+			-e 'srcP=$.src_port' \
+			-e 'dstP=$.dst_port' \
+			-e 'dir=$.dir' \
+			-e 'gid=$.gid' \
+			-e 'sid=$.sid')"
+
+		# Append the port to the IP, but only if it's meaningful.
+		[ "$dir" = 'C2S' ] && [ -n "$dstP" ] && dst="${dst}(${dstP})"
+		[ "$dir" = 'S2C' ] && [ -n "$srcP" ] && src="${src}(${srcP})"
+
+		echo "$msg#$src#$dst#$dir#$gid#$sid"
+	done | grep -iE "$PATTERN" > "$tmp"
+
+	#-- Generate output --
+	local output
 	[ "$NLINES" = 0 ] && output="cat" || output="head -n $NLINES"
 
-	local msg src dst dir
-	tmp="/tmp/snort.report.$$"
-	for file in "${log_dir}"/*alert_json.txt; do
-		while read -r line; do
-			eval $(jsonfilter -s "$line" -e 'msg=$.msg' -e 'src=$.src_ap' -e 'dst=$.dst_ap' -e 'dir=$.dir')
-			src=$(echo "$src" | sed 's/:.*$//')  # Delete all source ports.
-			dst=$(echo "$dst" | sed 's/:0$//')   # Delete unspecified dest port.
-			echo "$msg#$src#$dst#$dir"
-		done < "$file"
-	done | grep -i "$pattern" > "$tmp"
-
-	echo "Events involving ${pattern:-all IPs}"
-	n_incidents="$(wc -l < $tmp)"
-	lines=$(sort "$tmp" | uniq -c | sort -nr \
-		| awk -F'#' '{printf "%-80s %s %-13s -> %s\n", $1, $4, $2, $3}')
-	echo "$lines" | $output
-	n_lines=$(echo "$lines" | wc -l)
-	[ "$NLINES" -gt 0 ] && [ "$NLINES" -lt "$n_lines" ] && echo "    ... Only showing $NLINES of $n_lines most frequent incidents."
-	printf "%7d total incidents\n" "$n_incidents"
+	local lines=$($SORT "$tmp" | uniq -c | $SORT -nr | $output)
 	rm "$tmp"
+	if [ -z "$lines" ]; then
+		echo -n "There were no incidents "
+		[ -z "$PATTERN" ] && echo "reported." || echo "matching pattern '$PATTERN'."
+		return
+	fi
+
+	local n_total=$(cat "${log_dir}"/*alert_json.txt | wc -l)
+	local n_incidents=$(echo "$lines" | awk '{total += $1} END {print total}')
+	local mlen=$(echo "$lines" | awk -F'#' '{print $1}' | wc -L)
+	local slen=$(echo "$lines" | awk -F'#' '{print $2}' | wc -L)
+
+	echo "Events involving ${PATTERN:-all IPs} - $(date -Is)"
+	printf "%-*s %3s %5s %-3s %-*s %s\n" "$mlen"  "  Count Message" "gid" "sid" "Dir" "$slen" "Source" "Destination"
+	echo "$lines" | awk -F'#' '{printf "%-'"$mlen"'s %3d %5d %s %-'"$slen"'s %s\n", $1, $5, $6, $4, $2, $3}'
+
+	printf "%7d incidents shown of %d logged\n" "$n_incidents" "$n_total"
+
+	#-- Lookup rules and references, if requested. --
+	if $VERBOSE; then
+		local rules_dir="$(uci get snort.snort.config_dir)/rules"
+		local usids="$(echo "$lines" | awk -F'#' '{print $5 "#" $6}' | $SORT -u | $SORT -t'#' -k1n -k2n)"
+		local nsids="$(echo "$usids" | wc -w)"
+
+		echo ''
+		echo "$nsids unique rules triggered:"
+		local rule
+		local i=1
+		for sid in $usids; do
+			eval "$(echo "$sid" | awk -F'#' '{printf "export gid=%s;export sid=%s", $1, $2}')"
+			printf "%3d - gid=%3d sid=%5d " "$i" "$gid" "$sid"
+			rule=$(grep -Hn "\bsid:${sid};" "$rules_dir"/*.rules)
+			if [ "$gid" -ne 1 ] && echo "$rule" | grep -qv "\bgid:${gid};"; then
+				# Many rules have gid implicitly '1', zero any that are not
+				# explicit when expecting non-'1'.
+				rule=""
+			fi
+			if [ -n "$rule" ]; then
+				echo "$rule" | cut -c -120
+			else
+				rule=$($PROG --list-builtin | grep "^${gid}:${sid}\b")
+				if [ -n "$rule" ]; then
+					echo "BUILTIN: ${rule}"
+				fi
+			fi
+			i=$((i + 1))
+		done
+		echo ""
+		echo "Per-rule details may be viewed by specifying the appropriate gid and sid, e.g.:"
+		echo "    https://www.snort.org/rule-docs/$gid-$sid"
+
+		# Look up the names of the IPs shown in report.
+		# Note, on my dev box, nslookup fires rule 1:14777, so you get lots
+		# of incidents if not suppressed.
+		echo ''
+		echo 'Hosts by name:'
+		local IP
+		local peerdns=$(ifstatus wan | jsonfilter -e '$["dns-server"][0]')
+		echo "$lines" | awk -F'#' '{printf "%s\n%s\n", $2, $3}' | sed 's/(.*//' | sort -u \
+		| while read -r IP; do
+			[ -z "$IP" ] && continue
+			n=$(nslookup "$IP" | awk '/name = / {n=$NF} END{print n}')
+			[ -z "$n" ] && [ -n "$peerdns" ] && n=$(nslookup "$IP" "$peerdns" | awk '/name = / {n=$NF} END{print n}')
+			[ -z "$n" ] && n='--unknown host--'
+			printf "  %-39s %s\n" "$IP" "$n"
+		done | $SORT -b -k2
+	fi
 }
 
 status() {
 	echo -n 'snort is ' ; service snort status
-	ps w | grep -E 'PID|snort' | grep -v grep
+	local mem_total mem_free
+	eval "$(ubus call system info | jsonfilter -e 'mem_total=$.memory.total' -e 'mem_free=$.memory.free')"
+	awk -v mem_total="$mem_total" -v mem_free="$mem_free" 'BEGIN {
+		mem_used = mem_total - mem_free;
+		printf "Total system memory=%.3fM  Used=%.3fM (%.1f%%)  Free=%.3fM (%.1f%%)\n",
+			mem_total/1024**2,
+			mem_used/1024**2, 100*mem_used/mem_total,
+			mem_free/1024**2, 100*mem_free/mem_total;
+	}'
+	busybox ps w | grep -E "PID|$PROG " | grep -v grep
+
+	if [ "$(uci -q get snort.snort.method)" = "nfq" ]; then
+		nft list table inet snort
+	fi
 }
 
+#-------------------------------------------------------------------------------
 
-while [ -n "$1" ]; do
-	case "$1" in
-		-q)
-			export QUIET=1
-			shift
-		;;
-		-v)
-			export VERBOSE=-e
-			shift
-		;;
-		-t)
-			TESTING=-t
-			shift
-		;;
-		-n)
-			NLINES="$2"
-			shift
-			shift
-		;;
-		*)
-			break
-		;;
-	esac
-done
+usage() {
+	local msg="$1"
+	[ -n "$msg" ] && printf "ERROR: %s\n\n" "$msg"
 
-case "$1" in
-	setup)
-		setup
-	;;
-	teardown)
-		teardown
-	;;
-	resetup)
-		QUIET=1 check || die "The generated snort lua configuration contains errors, not restarting.  Run 'snort-mgr check'"
-		teardown
-		setup
-	;;
-	update-rules)
-		update_rules
-	;;
-	check)
-		check
-	;;
-	print)
-		print "$2"
-	;;
-	report)
-		report "$2"
-	;;
-	status)
-		status
-	;;
-	*)
-		cat <<USAGE
+	cat <<USAGE
 Usage:
 
-  -n = show only NLINES of output
-  -q = quiet
-  -v = verbose
-  -t = testing mode
-
-  $0 [-v] [-q] setup|teardown|resetup
+  $0 setup|teardown|resetup [-v/--verbose] [-q/--quiet]
 
     Normally only used internally by init scripts to manage the generation
     of configuration files and any needed firewall rules.  None of these
@@ -243,36 +302,52 @@ Usage:
       resetup  = shorthand for teardown and then setup.
 
 
-  $0 [-n lines] report [pattern]
+  $0 report [-v/--verbose] [-n/--n-lines N] [-d/--date-spec D] [-p/--pattern P]
 
     Report on incidents.  Note this is somewhat experimental, so suggested
-    improvements are quite welcome.
+    improvements are quite welcome.  Reported Source and Destination are of
+    the form "ip(port)", with zero and random source ports stripped.
+      -v      = Show the rules that were triggered, after report table.
+      -n N    = Show only the N highest frequency incidents.
+      -d D    = Filter entries by date specification in D.
+      -p P    = Grep pattern to filter incidents, applied to all outputs.
       pattern = A case-insensitive grep pattern used to filter output.
 
-  $0 [-t] update-rules
+    The date specification for '-d' can be either literal 'today'
+    or a snort-formatted date prefixed by '-' or '+', meaning 'before'
+    and 'after', respectively.  Snort date reporting has the format
+    'YY/MM/DD-hh:mm:ss.ssssss', and you can use any prefix as a date.
+    For example,
+      > snort-mgr --date-spec +23/12/20-09 report
+    will process all incidents from from 2023-12-20 at 0900 and later.
+
+
+  $0 update-rules [-t/--testing]
 
-    Download and install the snort ruleset.  Testing mode generates a canned
-    rule that matches IPv4 ping requests.  A typical test scenario might look
-    like:
+    Download and install the snort ruleset.
+      -t      = Generate a test-only ruleset, don't download anything.
+
+    Testing mode generates a canned rule that matches IPv4 ping requests.
+    A typical test scenario might look like:
 
       > snort-mgr -t update-rules
       > /etc/init.d/snort start
       > ping -c4 8.8.8.8
-      > logread -e "TEST ALERT"
+      > snort-mgr report
 
 
-  $0 print config|snort|nftables
+  $0 print config|snort|nftables|help
 
-    Print the rendered file contents.
-      config   = Display contents of /etc/config/snort, but with all values and
-                 descriptions.  Missing values shown with defaults.
-      snort    = The snort configuration file, which is a lua script.
-      nftables = The nftables script used to define the input queues when using
-                 the 'nfq' DAQ.
-      help     = Display config file help.
+    Print the rendered file contents.  Table types are:
+      config   - Display contents of /etc/config/snort, but with all values and
+                 descriptions.  Missing entries rendered with defaults.
+      snort    - The top-level snort configuration lua script, with includes.
+      nftables - The nftables script used to define the input queues when using
+                 the 'nfq' DAQ, with any included content.
+      help     - Display config file help.
 
 
-  $0 [-q] check
+  $0 check [-q/--quiet]
 
     Test the rendered config using snort's check mode without
     applying it to the running system.
@@ -280,8 +355,56 @@ Usage:
 
   $0 status
 
-    Print the nfq counter values and blah blah blah
+    Print the service status, system memory use and if nfq is the current daq,
+    then the nftables with counter values and so on.
 
 USAGE
-	;;
-esac
+	exit 1
+}
+
+while [ -n "$1" ]; do
+	case "$1" in
+		-h|--help)
+			usage
+		;;
+		-q|--quiet)
+			QUIET=true
+		;;
+		-v|--verbose)
+			VERBOSE=true
+		;;
+		-t|--testing)
+			TESTING=-t
+		;;
+		-n|--n-lines)
+			[ -z "$2" ] && usage "'--n-lines' requires a value"
+			NLINES="$2"
+			shift
+		;;
+		-d|--date-spec)
+			[ -z "$2" ] && usage "'--date-spec' requires a value"
+			DATE_SPEC="$2"
+			shift
+		;;
+		-p|--pattern)
+			[ -z "$2" ] && usage "'--pattern' requires a value"
+			PATTERN="$2"
+			shift
+		;;
+		print)
+			[ -z "$2" ] && usage "'print' requires a table type"
+			ACTION="$1"
+			TABLE="$2"
+			shift
+		;;
+		setup|teardown|resetup|update-rules|check|report|status)
+			ACTION="$1"
+		;;
+		*)
+			usage "'$1' is not a valid command or option"
+		;;
+	esac
+	shift
+done
+
+[ -n "$ACTION" ] && eval "$ACTION"
diff --git a/net/snort3/files/snort-rules b/net/snort3/files/snort-rules
index 24ae7a7f7..9547e9b4f 100644
--- a/net/snort3/files/snort-rules
+++ b/net/snort3/files/snort-rules
@@ -1,12 +1,10 @@
 #!/bin/sh
-# Copyright (c) 2023 Eric Fahlgren <eric.fahlgren@gmail.com>
+# Copyright (c) 2023-2024 Eric Fahlgren <eric.fahlgren@gmail.com>
 # SPDX-License-Identifier: GPL-2.0
-# shellcheck disable=SC2039  # "local" not defined in POSIX sh
+# shellcheck disable=SC2039,SC2155  # "local" not defined in POSIX sh
 
 alias log='logger -s -t "snort-rules[$$]" -p "info"'
 
-[ "$1" = "-t" ] && testing=true || testing=false
-
 download_rules() {
 	# Further information:
 	#    https://www.snort.org/products#rule_subscriptions
@@ -14,32 +12,39 @@ download_rules() {
 	#
 	# Also, what to do about "subscription" vs Talos_LightSPD rules when subbed?
 	# Add a "use_rules" list or option or something?
-	oinkcode=$(uci -q get snort.snort.oinkcode)
-
-
+	local oinkcode=$(uci -q get snort.snort.oinkcode)
 
 	local conf_dir=$(uci -q get snort.snort.config_dir || echo "/etc/snort")
-	local rules_file="$conf_dir/rules/snort.rules"
+	local rules_dir="$conf_dir/rules"
 	local data_dir=$(uci -q get snort.snort.temp_dir || echo "/var/snort.d")
 	local data_tar="$data_dir/rules.tar.gz"
 
+	local new_rules
+	local rules_file
+	local archive_loc
+
 	# Make sure everything exists.
 	[ -d "$data_dir" ] || mkdir -p "$data_dir"
 
-
 	if $testing ; then
 		log "Generating testing rules..."
-		new_rules="$data_dir/testing.rules"
-		rm -f "$new_rules"
-		echo 'alert icmp any any <> any any (msg:"TEST ALERT ICMP v4"; icode:0; itype: 8; sid:10000010; rev:001;)' >> "$new_rules"
-		#echo 'alert icmp any any <> any any (msg:"TEST ALERT ICMP v6"; icode:0; itype:33; sid:10000011; rev:001;)' >> "$new_rules"
-		#echo 'alert icmp any any <> any any (msg:"TEST ALERT ICMP v6"; icode:0; itype:34; sid:10000012; rev:001;)' >> "$new_rules"
+		archive_loc="testing-rules"
+		new_rules="$data_dir/$archive_loc"
+		rm -fr "$new_rules"
+		mkdir -p "$new_rules"
+		rules_file="$new_rules/testing.rules"
+		{
+			echo 'alert icmp any any <> any any (msg:"TEST ALERT ICMP v4"; icode:0; itype: 8; sid:99010;)'
+			echo 'alert icmp any any <> any any (msg:"TEST ALERT ICMP v6"; icode:0; itype:33; sid:99011;)'
+			echo 'alert icmp any any <> any any (msg:"TEST ALERT ICMP v6"; icode:0; itype:34; sid:99012;)'
+		} >> "$rules_file"
 
 	else
 		if [ -z "$oinkcode" ]; then
 			# If you do not have a subscription, then we use the community rules:
 			log "Downloading community rules..."
 			url="https://www.snort.org/downloads/community/snort3-community-rules.tar.gz"
+			archive_loc="snort3-community-rules"
 
 		else
 			# If you have a subscription and its corresponding oinkcode, use this:
@@ -62,31 +67,103 @@ download_rules() {
 
 			log "Downloading subscription rules..."
 			url="https://www.snort.org/rules/snortrules-snapshot-$snortver.tar.gz?oinkcode=$oinkcode"
+			# Non-community tar contains many "*.rules" file, we only care about
+			# the one directory.
+			archive_loc="rules"
 		fi
 
 		wget "$url" -O "$data_tar" 2>&1 | log || exit 1
 
-		# ??? Does non-community tar contain just the one "*.rules" file, too???
-		new_rules=$(tar tzf "$data_tar" | grep '\.rules$')
-		new_rules="$data_dir/$new_rules"
-
 		old_rules="$data_dir/old.rules"
-		if [ -e "$new_rules" ]; then
-			# Before we overwrite with the new download.
-			log "Stashing old rules to $old_rules ..."
-			mv -f "$new_rules" "$old_rules"
+		if $backup; then
+			rm -fr "$old_rules"
+			mkdir -p "$old_rules"
+
+			for rules_file in "$rules_dir"/*; do
+				# Before we overwrite with the new download.
+				log "Stashing '$rules_file' to '$old_rules/'..."
+				mv -f "$rules_file" "$old_rules/"
+			done
 		fi
 
-		log "Unpacking $data_tar ..."
-		tar xzvf "$data_tar" -C "$data_dir" | log || exit 1
-		if [ -e "$old_rules" ] && ! cmp -s "$new_rules" "$old_rules" ; then
-			diff "$new_rules" "$old_rules" 2>&1 | log
-		fi
+		log "Unpacking '$data_tar'..."
+		tar xzvof "$data_tar" "$archive_loc" -C "$data_dir" | log || exit 1
+
+		# Get rid of the non-rule files and aggregator.
+		new_rules="$data_dir/$archive_loc"
+		find "$new_rules" \( -iname 'includes.rules' -o ! -iname '*.rules' -type f \) -exec rm '{}' \;
+
+		# Old unfinished experiment with diffing old and new rules.
+		#for rules_file in "$new_rules"/*; do
+		#blah blah
+		#if [ -e "$old_rules" ] && ! cmp -s "$new_rules" "$old_rules" ; then
+		#	diff "$new_rules" "$old_rules" 2>&1 | log
+		#fi
 	fi
 
-	rm -f "$rules_file"
-	ln -s "$new_rules" "$rules_file"
+
+	mkdir -p "$conf_dir"
+	rm -fr "$rules_dir"
+	if $persist; then
+		mv -f "$new_rules" "$rules_dir"
+	else
+		ln -s "$new_rules" "$rules_dir"
+	fi
 
 	log "Snort rules loaded, restart snort now."
 }
+
+#-------------------------------------------------------------------------------
+
+testing=false
+persist=false
+backup=false
+
+usage() {
+	local msg="$1"
+	[ -n "$msg" ] && printf "ERROR: %s\n\n" "$msg"
+
+	cat <<USAGE
+Usage:
+
+  $0 [-b/--backup] [-t/--testing] [-p/--persist]
+
+  -b = Attempt to copy current rules to '\$temp_dir/old.rules/' before
+       installing new rules.
+
+  -t = Don't download any rules, instead create synthetic testing rules.
+
+  -p = Move the downloaded rules to '\$conf_dir/rules/' (usually '/etc/snort/'),
+       so that they persist across reboots and sysupgrades.  If you do not
+       specify this option, then the rules are stored in '\$temp_dir', and a
+       symbolic link is created to them.
+
+After running 'snort-rules', you should run 'snort-mgr check -v' to verify
+that there are no errors.
+
+USAGE
+	exit 1
+}
+
+while [ -n "$1" ]; do
+	case "$1" in
+		-h|--help)
+			usage
+		;;
+		-b|--backup)
+			backup=true
+		;;
+		-t|--testing)
+			testing=true
+		;;
+		-p|--persist)
+			persist=true
+		;;
+		*)
+			usage "'$1' is not a valid option"
+		;;
+	esac
+	shift
+done
+
 download_rules
diff --git a/net/snort3/files/snort.uc b/net/snort3/files/snort.uc
index 62dae71ca..6e14a0ab7 100644
--- a/net/snort3/files/snort.uc
+++ b/net/snort3/files/snort.uc
@@ -1,11 +1,10 @@
 {%
-// Copyright (c) 2023 Eric Fahlgren <eric.fahlgren@gmail.com>
+// Copyright (c) 2023-2024 Eric Fahlgren <eric.fahlgren@gmail.com>
 // SPDX-License-Identifier: GPL-2.0
 
-// Create some snort-format-specific items.
+import { lsdir } from 'fs';
 
-let home_net = snort.home_net == 'any' ? "'any'" : snort.home_net;
-let external_net = snort.external_net;
+// Create some snort-format-specific items.
 
 let line_mode = snort.mode == "ids" ? "tap"     : "inline";
 let mod_mode  = snort.mode == "ids" ? "passive" : "inline";
@@ -33,8 +32,8 @@ case "nfq":
 -- Do not edit, automatically generated.  See /usr/share/snort/templates.
 
 -- These must be defined before processing snort.lua
-HOME_NET     = [[ {{ home_net }} ]]
-EXTERNAL_NET = [[ {{ external_net }} ]]
+HOME_NET     = [[ {{ snort.home_net }} ]]
+EXTERNAL_NET = [[ {{ snort.external_net }} ]]
 
 include('{{ snort.config_dir }}/snort.lua')
 
@@ -43,26 +42,38 @@ snort  = {
   ['-Q'] = true,
 {% endif %}
   ['--daq'] = '{{ snort.method }}',
---['--daq-dir'] = '/usr/lib/daq/',
 {% if (snort.method == 'nfq'): %}
   ['--max-packet-threads'] = {{ nfq.thread_count }},
 {% endif %}
 }
 
 ips = {
+  -- View all options with "snort --help-module ips"
   mode            = '{{ line_mode }}',
   variables       = default_variables,
+--enable_builtin_rules=true,
 {% if (snort.action != 'default'): %}
   action_override = '{{ snort.action }}',
 {% endif %}
 {% if (getenv("_SNORT_WITHOUT_RULES") == "1"): %}
   -- WARNING: THIS IS A TEST-ONLY CONFIGURATION WITHOUT ANY RULES.
 {% else %}
-  include         = '{{ snort.config_dir }}/' .. RULE_PATH .. '/snort.rules',
+  rules = [[
+{%
+    let rules_dir = snort.config_dir + '/rules';
+    for (let rule in lsdir(rules_dir)) {
+      if (wildcard(rule, '*includes.rules', true)) continue;
+      if (wildcard(rule, '*.rules', true)) {
+        printf(`    include ${rules_dir}/${rule}\n`);
+      }
+    }
+%}
+  ]],
 {% endif -%}
 }
 
 daq = {
+  -- View all options with "snort --help-module daq"
   inputs      = {{ inputs }},
   snaplen     = {{ snort.snaplen }},
   module_dirs = { '/usr/lib/daq/', },
@@ -75,57 +86,57 @@ daq = {
   }
 }
 
-alert_syslog = {
-  level = 'info',
-}
+-- alert_syslog = { level = 'info', }  -- Generate output to syslog.
+alert_syslog = nil -- Disable output to syslog
 
 {% if (int(snort.logging)): %}
 -- Note that this is also the location of the PID file, if you use it.
-output.logdir = '{{ snort.log_dir }}'
+output = {
+  -- View all options with "snort --help-module output"
+  logdir    = '{{ snort.log_dir }}',
+
+  show_year = true,  -- Include year in timestamps.
+  -- See also 'process.utc = true' if you wish to record timestamps
+  -- in UTC.
+}
 
--- alert_full = { file = true, }
+--[[
+alert_full = {
+  -- View all options with "snort --help-config alert_full"
+  file = true,
+}
+--]]
 
 --[[
 alert_fast = {
--- bool alert_fast.file   = false: output to alert_fast.txt instead of stdout
--- bool alert_fast.packet = false: output packet dump with alert
--- int alert_fast.limit   = 0: set maximum size in MB before rollover (0 is unlimited) { 0:maxSZ }
+  -- View all options with "snort --help-config alert_fast"
   file = true,
   packet = false,
 }
 --]]
 
 alert_json = {
--- bool   alert_json.file      = false: output to alert_json.txt instead of stdout
--- int    alert_json.limit     = 0: set maximum size in MB before rollover (0 is unlimited) { 0:maxSZ }
--- string alert_json.separator = , : separate fields with this character sequence
--- multi  alert_json.fields    = 'timestamp pkt_num proto pkt_gen pkt_len dir src_ap dst_ap'
---                               Rule action: selected fields will be output in given order left to right.
---				{ action | class | b64_data | client_bytes | client_pkts | dir
---				| dst_addr | dst_ap | dst_port | eth_dst | eth_len | eth_src
---				| eth_type | flowstart_time | geneve_vni | gid | icmp_code
---				| icmp_id | icmp_seq | icmp_type | iface | ip_id | ip_len
---				| msg | mpls | pkt_gen | pkt_len | pkt_num | priority
---				| proto | rev | rule | seconds | server_bytes | server_pkts
---				| service | sgt | sid | src_addr | src_ap | src_port | target
---				| tcp_ack | tcp_flags | tcp_len | tcp_seq | tcp_win | timestamp
---				| tos | ttl | udp_len | vlan }
-
--- This is a minimal set of fields that simply supports 'snort-mgr report'
--- and minimizes log size:
-  fields = 'dir src_ap dst_ap msg',
-
--- This set also supports the report, but closely matches 'alert_fast' contents.
---fields = 'timestamp pkt_num proto pkt_gen pkt_len dir src_ap dst_ap rule action msg',
-
+  -- View all options with "snort --help-config alert_json"
   file = true,
-}
 
---[[
-unified2 = {
-  limit = 10, -- int unified2.limit = 0: set maximum size in MB before rollover (0 is unlimited) { 0:maxSZ }
+  -- This is a minimal set of fields that simply supports 'snort-mgr report'
+  -- and minimizes log size, but loses a lot of information:
+--fields = 'timestamp dir src_addr src_port dst_addr dst_port gid sid msg',
+
+  -- This is our preferred smallish set, which also supports the report, but
+  -- more closely matches 'alert_fast' contents.
+  fields = [[
+    timestamp
+    pkt_num pkt_gen pkt_len
+    proto
+    dir
+    src_addr src_port
+    dst_addr dst_port
+    gid sid rev
+    action
+    msg
+  ]],
 }
---]]
 
 {% endif -%}
 
@@ -136,12 +147,12 @@ normalizer = {
 }
 
 file_policy = {
-  enable_type = true,
+  enable_type      = true,
   enable_signature = true,
   rules = {
     use = {
-      verdict = 'log',
-      enable_file_type = true,
+      verdict               = 'log',
+      enable_file_type      = true,
       enable_file_signature = true,
     }
   }
@@ -150,7 +161,8 @@ file_policy = {
 -- To use openappid with snort, 'opkg install openappid' and enable in config.
 {% if (int(snort.openappid)): %}
 appid = {
-  log_stats = true,
+  -- View all options with "snort --help-module appid"
+  log_stats        = true,
   app_detector_dir = '/usr/lib/openappid',
   app_stats_period = 60,
 }
@@ -160,7 +172,8 @@ appid = {
 if (snort.include) {
   // We use the ucode include here, so that the included file is also
   // part of the template and can use values passed in from the config.
-  printf("-- The following content from included file '%s'\n", snort.include);
+  printf(rpad(`-- Include from '${snort.include}'`, ">", 80) + "\n");
   include(snort.include, { snort, nfq });
+  printf(rpad("-- End of included file.", "<", 80) + "\n");
 }
 %}