banIP: release 0.1.0

* add automatic blocklist backup & restore, they will be used
  in case of download errors or during startup in backup mode
* add a 'backup mode' to re-use blocklist backups during startup,
  get fresh lists via reload or restart action
* procd interface trigger now supports multiple WAN interfaces
* change URL for abuse.ch/feodo list source in default config
* small fixes
* update readme

Signed-off-by: Dirk Brenken <dev@brenken.org>

4 files changed, 190 insertions, 94 deletions

diff --git a/net/banip/files/README.md b/net/banip/files/README.md
index 982a713c7..1df1f7cdc 100644
--- a/net/banip/files/README.md
+++ b/net/banip/files/README.md
@@ -23,6 +23,8 @@ IP address blocking is commonly used to protect against brute force attacks, pre
 * minimal status & error logging to syslog, enable debug logging to receive more output
 * procd based init system support (start/stop/restart/reload/status)
 * procd network interface trigger support
+* automatic blocklist backup & restore, they will be used in case of download errors or during startup in backup mode
+* 'backup mode' to re-use blocklist backups during startup, get fresh lists via reload or restart action
 * output comprehensive runtime information via LuCI or via 'status' init command
 * strong LuCI support
 * optional: add new banIP sources on your own
@@ -43,6 +45,24 @@ IP address blocking is commonly used to protect against brute force attacks, pre
 * install 'luci-app-banip' (_opkg install luci-app-banip_)
 * the application is located in LuCI under 'Services' menu
 
+## banIP config options
+* usually the pre-configured banIP setup works quite well and no manual overrides are needed
+* the following options apply to the 'global' config section:
+    * ban\_enabled => main switch to enable/disable banIP service (bool/default: '0', disabled)
+    * ban\_automatic => determine the L2/L3 WAN network device automatically (bool/default: '1', enabled)
+    * ban\_fetchutil => name of the used download utility: 'uclient-fetch', 'wget', 'curl', 'aria2c', 'wget-nossl'. 'busybox' (default: 'uclient-fetch')
+    * ban\_iface => space separated list of WAN network interface(s)/device(s) used by banIP (default: automatically set by banIP ('ban_automatic'))
+
+* the following options apply to the 'extra' config section:
+    * ban\_debug => enable/disable banIP debug output (default: '0', disabled)
+    * ban\_nice => set the nice level of the banIP process and all sub-processes (int/default: '0', standard priority)
+    * ban\_triggerdelay => additional trigger delay in seconds before banIP processing begins (int/default: '2')
+    * ban\_backup => create compressed blocklist backups, they will be used in case of download errors or during startup in 'backup mode' (bool/default: '0', disabled)
+    * ban\_backupdir => target directory for adblock backups (default: not set)
+    * ban\_backupboot => do not automatically update blocklists during startup, use their backups instead (bool/default: '0', disabled)
+    * ban\_maxqueue => size of the download queue to handle downloads & IPSet processing in parallel (int/default: '8')
+    * ban\_fetchparm => special config options for the download utility (default: not set)
+
 ## Examples
 **receive banIP runtime information:**
 
@@ -50,11 +70,11 @@ IP address blocking is commonly used to protect against brute force attacks, pre
 /etc/init.d/banip status
 ::: banIP runtime information
   + status     : enabled
-  + version    : 0.0.5
+  + version    : 0.1.0
   + fetch_info : /bin/uclient-fetch (libustream-ssl)
-  + ipset_info : 3 IPSets with overall 29510 IPs/Prefixes
-  + last_run   : 08.11.2018 15:03:50
-  + system     : GL-AR750S, OpenWrt SNAPSHOT r8419-860de2e1aa
+  + ipset_info : 1 IPSets with overall 516 IPs/Prefixes (backup mode)
+  + last_run   : 05.01.2019 14:48:18
+  + system     : TP-LINK RE450, OpenWrt SNAPSHOT r8910+72-25d8aa7d02
 </code></pre>
   
 **cronjob for a regular block list update (/etc/crontabs/root):**
@@ -65,7 +85,7 @@ IP address blocking is commonly used to protect against brute force attacks, pre
   
 
 ## Support
-Please join the banIP discussion in this [forum thread](https://forum.openwrt.org/t/banip-new-project-needs-testers-feedback/16985) or contact me by mail <dev@brenken.org>  
+Please join the banIP discussion in this [forum thread](https://forum.openwrt.org/t/banip-support-thread/16985) or contact me by mail <dev@brenken.org>  
 
 ## Removal
 * stop all banIP related services with _/etc/init.d/banip stop_
diff --git a/net/banip/files/banip.conf b/net/banip/files/banip.conf
index d93088dbc..7de03e8af 100644
--- a/net/banip/files/banip.conf
+++ b/net/banip/files/banip.conf
@@ -9,6 +9,7 @@ config banip 'global'
 
 config banip 'extra'
 	option ban_debug '0'
+	option ban_backup '0'
 	option ban_maxqueue '8'
 
 config source 'whitelist'
@@ -116,7 +117,7 @@ config source 'ransomware'
 	option ban_src_on '0'
 
 config source 'feodo'
-	option ban_src 'https://feodotracker.abuse.ch/blocklist/?download=ipblocklist'
+	option ban_src 'https://feodotracker.abuse.ch/downloads/ipblocklist.txt'
 	option ban_src_desc 'Feodo Tracker by abuse.ch (IPv4)'
 	option ban_src_rset '/^(([0-9]{1,3}\.){3}[0-9]{1,3})([[:space:]]|$)/{print \"add feodo \"\$1}'
 	option ban_src_settype 'ip'
diff --git a/net/banip/files/banip.init b/net/banip/files/banip.init
index 1fe5f01d4..a0b583668 100755
--- a/net/banip/files/banip.init
+++ b/net/banip/files/banip.init
@@ -37,15 +37,20 @@ start_service()
 	fi
 }
 
-stop_service()
+refresh()
 {
-	rc_procd "${ban_script}" stop
-	rc_procd start_service
+	rc_procd start_service refresh
 }
 
-refresh()
+reload_service()
 {
-	rc_procd start_service "refresh"
+	rc_procd start_service reload
+}
+
+stop_service()
+{
+	rc_procd "${ban_script}" stop
+	rc_procd start_service
 }
 
 status()
@@ -71,10 +76,13 @@ status()
 
 service_triggers()
 {
-	local iface="$(uci_get banip global ban_iface)"
+	local ban_iface="$(uci_get banip global ban_iface)"
 	local delay="$(uci_get banip extra ban_triggerdelay)"
 
 	PROCD_RELOAD_DELAY=$((${delay:-2} * 1000))
-	procd_add_interface_trigger "interface.*.up" "${iface:-"wan"}" "${ban_init}" start
+	for iface in ${ban_iface:-"wan"}
+	do
+		procd_add_interface_trigger "interface.*.up" "${iface}" "${ban_init}" start
+	done
 	procd_add_reload_trigger "banip" "firewall"
 }
diff --git a/net/banip/files/banip.sh b/net/banip/files/banip.sh
index 963439441..b12aed6b1 100755
--- a/net/banip/files/banip.sh
+++ b/net/banip/files/banip.sh
@@ -10,12 +10,15 @@
 #
 LC_ALL=C
 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
-ban_ver="0.0.7"
+ban_ver="0.1.0"
 ban_sysver="unknown"
 ban_enabled=0
 ban_automatic="1"
 ban_iface=""
 ban_debug=0
+ban_backup=0
+ban_backupboot=0
+ban_backupdir="/mnt"
 ban_maxqueue=8
 ban_fetchutil="uclient-fetch"
 ban_ip="$(command -v ip)"
@@ -105,6 +108,7 @@ f_envload()
 	then
 		f_jsnup disabled
 		f_ipset destroy
+		f_rmbackup
 		f_rmtemp
 		f_log "info" "banIP is currently disabled, please set ban_enabled to '1' to use this service"
 		exit 0
@@ -232,6 +236,16 @@ f_rmtemp()
 	> "${ban_pidfile}"
 }
 
+# remove backup files
+#
+f_rmbackup()
+{
+	if [ -d "${ban_backupdir}" ]
+	then
+		rm -f "${ban_backupdir}/banIP."*.gz
+	fi
+}
+
 # iptables rules engine
 #
 f_iptrule()
@@ -326,6 +340,31 @@ f_ipset()
 	fi
 
 	case "${mode}" in
+		backup)
+			ban_rc=4
+			if [ -d "${ban_backupdir}" ]
+			then
+				gzip -cf "${tmp_load}" 2>/dev/null > "${ban_backupdir}/banIP.${src_name}.gz"
+				ban_rc=${?}
+			fi
+			f_log "debug" "f_ipset ::: name: ${src_name:-"-"}, mode: ${mode:-"-"}, rc: ${ban_rc}"
+		;;
+		restore)
+			ban_rc=4
+			if [ -d "${ban_backupdir}" ] && [ -f "${ban_backupdir}/banIP.${src_name}.gz" ]
+			then
+				gunzip -cf "${ban_backupdir}/banIP.${src_name}.gz" 2>/dev/null > "${tmp_load}"
+				ban_rc=${?}
+			fi
+			f_log "debug" "f_ipset ::: name: ${src_name:-"-"}, mode: ${mode:-"-"}, rc: ${ban_rc}"
+		;;
+		remove)
+			if [ -d "${ban_backupdir}" ] && [ -f "${ban_backupdir}/banIP.${src_name}.gz" ]
+			then
+				rm -f "${ban_backupdir}/banIP.${src_name}.gz"
+			fi
+			f_log "debug" "f_ipset ::: name: ${src_name:-"-"}, mode: ${mode:-"-"}"
+		;;
 		initial)
 			if [ -z "$("${ban_ipt}" "${timeout}" -nL "${ban_chain}" 2>/dev/null)" ]
 			then
@@ -373,7 +412,6 @@ f_ipset()
 				printf "%s\n" "${cnt}" > "${tmp_cnt}"
 			fi
 			f_iptadd
-
 			end_ts="$(date +%s)"
 			f_log "debug" "f_ipset ::: name: ${src_name:-"-"}, mode: ${mode:-"-"}, settype: ${src_settype:-"-"}, setipv: "${src_setipv}", ruletype: ${src_ruletype:-"-"}, count(sum/ip/cidr): ${cnt:-0}/${cnt_ip:-0}/${cnt_cidr:-0}, time(s): $(( end_ts - start_ts ))"
 		;;
@@ -391,7 +429,6 @@ f_ipset()
 				fi
 				f_iptadd
 			fi
-
 			end_ts="$(date +%s)"
 			f_log "debug" "f_ipset ::: name: ${src_name:-"-"}, mode: ${mode:-"-"}, count: ${cnt:-0}/${cnt_ip:-0}/${cnt_cidr:-0}, time(s): $(( end_ts - start_ts ))"
 		;;
@@ -447,6 +484,7 @@ f_log()
 		then
 			f_jsnup error
 			f_ipset destroy
+			f_rmbackup
 			f_rmtemp
 			logger -p "${class}" -t "banIP-[${ban_ver}]" "Please also check 'https://github.com/openwrt/packages/blob/master/net/banip/files/README.md'"
 			exit 1
@@ -465,7 +503,7 @@ f_main()
 
 	mem_total="$(awk '/^MemTotal/ {print int($2/1000)}' "/proc/meminfo" 2>/dev/null)"
 	mem_free="$(awk '/^MemFree/ {print int($2/1000)}' "/proc/meminfo" 2>/dev/null)"
-	f_log "debug" "f_main  ::: fetch_util: ${ban_fetchinfo:-"-"}, fetch_parm: ${ban_fetchparm:-"-"}, interface(s): ${ban_iface:-"-"}, device(s): ${ban_dev:-"-"}, all_devices: ${ban_dev_all:-"-"}, mem_total: ${mem_total:-0}, mem_free: ${mem_free:-0}, max_queue: ${ban_maxqueue}"
+	f_log "debug" "f_main  ::: fetch_util: ${ban_fetchinfo:-"-"}, fetch_parm: ${ban_fetchparm:-"-"}, interface(s): ${ban_iface:-"-"}, device(s): ${ban_dev:-"-"}, all_devices: ${ban_dev_all:-"-"}, backup: ${ban_backup:-"-"}, backup_boot: ${ban_backupboot:-"-"}, backup_dir: ${ban_backupdir:-"-"}, mem_total: ${mem_total:-0}, mem_free: ${mem_free:-0}, max_queue: ${ban_maxqueue}"
 
 	f_ipset initial
 
@@ -515,6 +553,7 @@ f_main()
 			[ -z "${src_settype}" ] || [ -z "${src_ruletype}" ]
 		then
 			f_ipset flush
+			f_ipset remove
 			continue
 		elif [ "${ban_action}" = "refresh" ]
 		then
@@ -526,89 +565,117 @@ f_main()
 		#
 		(
 			start_ts="$(date +%s)"
-			if [ -f "${src_url}" ]
+			if [ ! -f "${src_url}" ] && [ ${ban_backup} -eq 1 ] && [ ${ban_backupboot} -eq 1 ] && [ "${ban_action}" = "start" ]
 			then
-				src_log="$(cat "${src_url}" 2>/dev/null > "${tmp_load}")"
-				ban_rc=${?}
-
-				case "${src_name}" in
-					whitelist)
-						src_addon="${ban_subnets}"
-					;;
-					whitelist_6)
-						src_addon="${ban_subnets6}"
-					;;
-					blacklist)
-						pid_list="$(printf "%s\n" "${log_content}" | grep -F "Exit before auth" | awk 'match($0,/(\[[0-9]+\])/){ORS=" ";print substr($0,RSTART,RLENGTH)}')"
-						for pid in ${pid_list}
-						do
-							src_addon="${src_addon} $(printf "%s\n" "${log_content}" | grep -F "${pid}" | awk 'match($0,/([0-9]{1,3}\.){3}[0-9]{1,3}/){ORS=" ";print substr($0,RSTART,RLENGTH)}')"
-						done
-					;;
-					blacklist_6)
-						pid_list="$(printf "%s\n" "${log_content}" | grep -F "Exit before auth" | awk 'match($0,/(\[[0-9]+\])/){ORS=" ";print substr($0,RSTART,RLENGTH)}')"
-						for pid in ${pid_list}
-						do
-							src_addon="${src_addon} $(printf "%s\n" "${log_content}" | grep -F "${pid}" | awk 'match($0,/([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}/){ORS=" ";print substr($0,RSTART,RLENGTH)}')"
-						done
-					;;
-				esac
+				f_ipset restore
+			fi
 
-				for ip in ${src_addon}
-				do
-					if [ -z "$(grep -F "${ip}" "${src_url}")" ]
-					then
-						printf '\n%s\n' "${ip}" >> "${tmp_load}"
-						printf '\n%s\n' "${ip}" >> "${src_url}"
-					fi
-				done
-			elif [ -n "${src_cat}" ]
+			if [ ${ban_rc} -ne 0 ] || [ ! -s "${tmp_load}" ]
 			then
-				if [ "${src_cat//[0-9]/}" != "${src_cat}" ]
+				if [ -f "${src_url}" ]
 				then
-					for as in ${src_cat}
+					src_log="$(cat "${src_url}" 2>/dev/null > "${tmp_load}")"
+					ban_rc=${?}
+					case "${src_name}" in
+						whitelist)
+							src_addon="${ban_subnets}"
+						;;
+						whitelist_6)
+							src_addon="${ban_subnets6}"
+						;;
+						blacklist)
+							pid_list="$(printf "%s\n" "${log_content}" | grep -F "Exit before auth" | awk 'match($0,/(\[[0-9]+\])/){ORS=" ";print substr($0,RSTART,RLENGTH)}')"
+							for pid in ${pid_list}
+							do
+								src_addon="${src_addon} $(printf "%s\n" "${log_content}" | grep -F "${pid}" | awk 'match($0,/([0-9]{1,3}\.){3}[0-9]{1,3}/){ORS=" ";print substr($0,RSTART,RLENGTH)}')"
+							done
+						;;
+						blacklist_6)
+							pid_list="$(printf "%s\n" "${log_content}" | grep -F "Exit before auth" | awk 'match($0,/(\[[0-9]+\])/){ORS=" ";print substr($0,RSTART,RLENGTH)}')"
+							for pid in ${pid_list}
+							do
+								src_addon="${src_addon} $(printf "%s\n" "${log_content}" | grep -F "${pid}" | awk 'match($0,/([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}/){ORS=" ";print substr($0,RSTART,RLENGTH)}')"
+							done
+						;;
+					esac
+					for ip in ${src_addon}
 					do
-						src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}AS${as}" 2>&1)"
-						ban_rc=${?}
-						if [ ${ban_rc} -eq 0 ]
+						if [ -z "$(grep -F "${ip}" "${src_url}")" ]
 						then
-							jsonfilter -i "${tmp_raw}" -e '@.data.prefixes.*.prefix' 2>/dev/null >> "${tmp_load}"
-						else
-							break
+							printf '%s\n' "${ip}" >> "${tmp_load}"
+							printf '%s\n' "${ip}" >> "${src_url}"
 						fi
 					done
-				else
-					for co in ${src_cat}
-					do
-						src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}${co}&v4_format=prefix" 2>&1)"
-						ban_rc=${?}
-						if [ ${ban_rc} -eq 0 ]
+				elif [ -n "${src_cat}" ]
+				then
+					if [ "${src_cat//[0-9]/}" != "${src_cat}" ]
+					then
+						for as in ${src_cat}
+						do
+							src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}AS${as}" 2>&1)"
+							ban_rc=${?}
+							if [ ${ban_rc} -eq 0 ]
+							then
+								jsonfilter -i "${tmp_raw}" -e '@.data.prefixes.*.prefix' 2>/dev/null >> "${tmp_load}"
+							else
+								break
+							fi
+						done
+						if [ ${ban_rc} -eq 0 ] && [ ${ban_backup} -eq 1 ]
+						then
+							f_ipset backup
+						elif [ ${ban_backup} -eq 1 ]
 						then
-							if [ "${src_name##*_}" = "6" ]
+							f_ipset restore
+						fi
+					else
+						for co in ${src_cat}
+						do
+							src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}${co}&v4_format=prefix" 2>&1)"
+							ban_rc=${?}
+							if [ ${ban_rc} -eq 0 ]
 							then
-								jsonfilter -i "${tmp_raw}" -e '@.data.resources.ipv6.*' 2>/dev/null >> "${tmp_load}"
+								if [ "${src_name##*_}" = "6" ]
+								then
+									jsonfilter -i "${tmp_raw}" -e '@.data.resources.ipv6.*' 2>/dev/null >> "${tmp_load}"
+								else
+									jsonfilter -i "${tmp_raw}" -e '@.data.resources.ipv4.*' 2>/dev/null >> "${tmp_load}"
+								fi
 							else
-								jsonfilter -i "${tmp_raw}" -e '@.data.resources.ipv4.*' 2>/dev/null >> "${tmp_load}"
+								break
 							fi
-						else
-							break
+						done
+						if [ ${ban_rc} -eq 0 ] && [ ${ban_backup} -eq 1 ]
+						then
+							f_ipset backup
+						elif [ ${ban_backup} -eq 1 ]
+						then
+							f_ipset restore
 						fi
-					done
-				fi
-			else
-				src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}" 2>&1)"
-				ban_rc=${?}
-				if [ ${ban_rc} -eq 0 ]
-				then
-					zcat "${tmp_raw}" 2>/dev/null > "${tmp_load}"
+					fi
+				else
+					src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}" 2>&1)"
 					ban_rc=${?}
-					if [ ${ban_rc} -ne 0 ]
+					if [ ${ban_rc} -eq 0 ]
 					then
-						mv -f "${tmp_raw}" "${tmp_load}"
+						zcat "${tmp_raw}" 2>/dev/null > "${tmp_load}"
 						ban_rc=${?}
+						if [ ${ban_rc} -ne 0 ]
+						then
+							mv -f "${tmp_raw}" "${tmp_load}"
+							ban_rc=${?}
+						fi
+						if [ ${ban_rc} -eq 0 ] && [ ${ban_backup} -eq 1 ]
+						then
+							f_ipset backup
+						fi
+					elif [ ${ban_backup} -eq 1 ]
+					then
+						f_ipset restore
 					fi
 				fi
 			fi
+
 			if [ ${ban_rc} -eq 0 ]
 			then
 				awk "${src_rset}" "${tmp_load}" 2>/dev/null | sort -u > "${tmp_file}"
@@ -655,16 +722,26 @@ f_main()
 #
 f_jsnup()
 {
-	local rundate="$(/bin/date "+%d.%m.%Y %H:%M:%S")" status="${1:-"enabled"}"
+	local rundate="$(/bin/date "+%d.%m.%Y %H:%M:%S")" mode="normal mode" status="${1:-"enabled"}"
 
 	ban_cntinfo="${ban_setcnt} IPSets with overall ${ban_cnt} IPs/Prefixes"
 
+	if [ ${ban_backupboot} -eq 1 ]
+	then
+		mode="backup mode"
+	fi
+
+	> "${ban_rtfile}"
+	json_load_file "${ban_rtfile}" >/dev/null 2>&1
+	json_init
+	json_add_object "data"
 	json_add_string "status" "${status}"
 	json_add_string "version" "${ban_ver}"
 	json_add_string "fetch_info" "${ban_fetchinfo:-"-"}"
-	json_add_string "ipset_info" "${ban_cntinfo:-"-"}"
+	json_add_string "ipset_info" "${ban_cntinfo:-"-"} (${mode})"
 	json_add_string "last_run" "${rundate:-"-"}"
 	json_add_string "system" "${ban_sysver}"
+	json_close_object
 	json_dump > "${ban_rtfile}"
 
 	f_log "debug" "f_jsnup ::: status: ${status}, setcnt: ${ban_setcnt}, cnt: ${ban_cnt}"
@@ -681,17 +758,6 @@ else
 	f_log "err" "system libraries not found"
 fi
 
-# initialize json runtime file
-#
-json_load_file "${ban_rtfile}" >/dev/null 2>&1
-json_select data >/dev/null 2>&1
-if [ ${?} -ne 0 ]
-then
-	> "${ban_rtfile}"
-	json_init
-	json_add_object "data"
-fi
-
 # handle different banIP actions
 #
 f_envload
@@ -699,6 +765,7 @@ case "${ban_action}" in
 	stop)
 		f_jsnup stopped
 		f_ipset destroy
+		f_rmbackup
 		f_rmtemp
 	;;
 	start|restart|reload|refresh)