adblock: 0.91.0

* fix blocklist footer/counter
* optimize/rework blocklist rulesets
* change source url for spam404 and yoyo
* code cosmetics & cleanups

Signed-off-by: Dirk Brenken <openwrt@brenken.org>

4 files changed, 35 insertions, 31 deletions

diff --git a/net/adblock/Makefile b/net/adblock/Makefile
index ca3a183ad..e58ffaf84 100644
--- a/net/adblock/Makefile
+++ b/net/adblock/Makefile
@@ -7,7 +7,7 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=adblock
-PKG_VERSION:=0.90.0
+PKG_VERSION:=0.91.0
 PKG_RELEASE:=1
 PKG_LICENSE:=GPL-3.0+
 PKG_MAINTAINER:=Dirk Brenken <openwrt@brenken.org>
diff --git a/net/adblock/files/adblock-helper.sh b/net/adblock/files/adblock-helper.sh
index d96ce4cca..8c5480b44 100644
--- a/net/adblock/files/adblock-helper.sh
+++ b/net/adblock/files/adblock-helper.sh
@@ -169,24 +169,23 @@ f_envparse()
 
     # set adblock source ruleset definitions
     #
-    rset_start="sed -r 's/[[:space:]]|[\[!#/:;_].*|[0-9\.]*localhost.*//g; s/[\^#/:;_\.\t ]*$//g'"
-    rset_end="tr -cd '[0-9a-z\.\-]\n' | sed -r 's/^[ \.\-].*$|^[a-z0-9]*[ \.\-]*$//g; /^[#/:;_\s]*$/d'"
-    rset_adaway="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
-    rset_blacklist="${rset_start} | ${rset_end}"
-    rset_disconnect="${rset_start} | ${rset_end}"
-    rset_dshield="${rset_start} | ${rset_end}"
-    rset_feodo="${rset_start} | ${rset_end}"
-    rset_malware="${rset_start} | ${rset_end}"
-    rset_malwarelist="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
-    rset_openphish="sed -e 's|^[^/]*//||' -e 's|/.*$||'"
-    rset_palevo="${rset_start} | ${rset_end}"
-    rset_ruadlist="sed -e '/^\|\|/! s/.*//; /\^$/! s/.*//; s/\^$//g; /[\.]/! s/.*//; s/^[\|]\{1,2\}//g' | ${rset_end}"
-    rset_shalla="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}$//g' | ${rset_end}"
-    rset_spam404="${rset_start} | sed 's/^\|\|//g' | ${rset_end}"
-    rset_whocares="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
-    rset_winhelp="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
-    rset_yoyo="${rset_start} | sed 's/,/\n/g' | ${rset_end}"
-    rset_zeus="${rset_start} | ${rset_end}"
+    rset_core="([A-Za-z0-9_-]+\.){1,}[A-Za-z]+"
+    rset_adaway="awk '\$0 ~/^127\.0\.0\.1[ ]+${rset_core}/{print tolower(\$2)}'"
+    rset_blacklist="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'"
+    rset_disconnect="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'"
+    rset_dshield="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'"
+    rset_feodo="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'"
+    rset_malware="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'"
+    rset_malwarelist="awk '\$0 ~/^127\.0\.0\.1[ ]+${rset_core}/{print tolower(\$2)}'"
+    rset_openphish="awk -F '/' '\$3 ~/^${rset_core}/{print tolower(\$3)}'"
+    rset_palevo="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'"
+    rset_ruadlist="awk -F '[|^]' '\$0 ~/^\|\|${rset_core}\^$/{print tolower(\$3)}'"
+    rset_shalla="awk -F '/' '\$1 ~/^${rset_core}/{print tolower(\$1)}'"
+    rset_spam404="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'"
+    rset_whocares="awk '\$0 ~/^127\.0\.0\.1[ ]+${rset_core}/{print tolower(\$2)}'"
+    rset_winhelp="awk '\$0 ~/^0\.0\.0\.0[ ]+${rset_core}/{print tolower(\$2)}'"
+    rset_yoyo="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'"
+    rset_zeus="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'"
 
     # get logical wan update interfaces (with default route) and their device names
     #
diff --git a/net/adblock/files/adblock-update.sh b/net/adblock/files/adblock-update.sh
index 32a48b7d3..9ffd09f79 100755
--- a/net/adblock/files/adblock-update.sh
+++ b/net/adblock/files/adblock-update.sh
@@ -27,7 +27,7 @@ fi
 # get current directory, script- and openwrt version
 #
 adb_scriptdir="${0%/*}"
-adb_scriptver="0.90.0"
+adb_scriptver="0.91.0"
 openwrt_version="$(cat /etc/openwrt_version 2>/dev/null)"
 
 # source in adblock function library
@@ -189,8 +189,8 @@ do
     #
     if [ $((rc)) -eq 0 ] && [ -n "${tmp_domains}" ]
     then
-        eval "$(printf "${src}" | sed 's/\(.*\&ruleset=\)/ruleset=\$/g')"
-        count="$(printf "%s\n" "${tmp_domains}" | tr '[A-Z]' '[a-z]' | eval "${ruleset}" | tee "${adb_tmpfile}" | wc -l)"
+        eval "src_rset=\${rset_${src_name}}"
+        count="$(printf "%s\n" "${tmp_domains}" | eval "${src_rset}" | tee "${adb_tmpfile}" | wc -l)"
         f_log "   source download finished (${count} entries)"
         if [ "${src_name}" = "shalla" ]
         then
@@ -220,10 +220,10 @@ do
     then
         if [ -s "${adb_whitelist}" ]
         then
-            grep -Fvxf "${adb_whitelist}" "${adb_tmpfile}" 2>/dev/null | sort 2>/dev/null | uniq -u 2>/dev/null | eval "${adb_dnsformat}" 2>/dev/null > "${adb_dnsfile}"
+            grep -Fvxf "${adb_whitelist}" "${adb_tmpfile}" 2>/dev/null | sort -u 2>/dev/null | eval "${adb_dnsformat}" 2>/dev/null > "${adb_dnsfile}"
             rc=${?}
         else
-            sort "${adb_tmpfile}" 2>/dev/null | uniq -u 2>/dev/null | eval "${adb_dnsformat}" 2>/dev/null > "${adb_dnsfile}"
+            sort -u "${adb_tmpfile}" 2>/dev/null | eval "${adb_dnsformat}" 2>/dev/null > "${adb_dnsfile}"
             rc=${?}
         fi
 
@@ -242,6 +242,7 @@ do
         then
             printf "%s\n" "#---------------------------------------------" >> "${adb_dnsfile}"
             printf "%s\n" "# last modified: ${url_time}" >> "${adb_dnsfile}"
+            printf "%s\n" "##" >> "${adb_dnsfile}"
             f_log "   domain merging finished"
         else
             f_log "   domain merging failed" "${rc}"
@@ -332,7 +333,7 @@ then
 
         # generate a temporary unique overall list
         #
-        head -qn -2 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | sort -u 2>/dev/null > "${adb_dnsdir}/tmp.overall"
+        head -qn -3 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | sort -u 2>/dev/null > "${adb_dnsdir}/tmp.overall"
 
         # loop through all separate lists, ordered by size (ascending)
         #
@@ -351,7 +352,7 @@ then
 
             # write unique result back to original separate list (with list footer)
             #
-            tail -qn -2 "${adb_dnsdir}/$adb_dnsprefix.${list}" 2>/dev/null >> "${adb_dnsdir}/tmp.${list}"
+            tail -qn 3 "${adb_dnsdir}/$adb_dnsprefix.${list}" 2>/dev/null >> "${adb_dnsdir}/tmp.${list}"
             mv -f "${adb_dnsdir}/tmp.${list}" "${adb_dnsdir}/${adb_dnsprefix}.${list}" >/dev/null 2>&1
         done
         rm -f "${adb_dnsdir}/tmp.overall" >/dev/null 2>&1
@@ -363,12 +364,16 @@ fi
 for list in $(ls -Sr "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null)
 do
     list="${list/*./}"
-    count="$(head -qn -2 "${adb_dnsdir}/${adb_dnsprefix}.${list}" | wc -l)"
+    count="$(head -qn -3 "${adb_dnsdir}/${adb_dnsprefix}.${list}" | wc -l)"
     if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ]
     then
         count=$((count / 2))
     fi
-    printf "%s\n" "# ${0##*/} (${adb_scriptver}) - ${count} ad/abuse domains blocked" >> "${adb_dnsdir}/${adb_dnsprefix}.${list}"
+    if [ "$(tail -qn 1 "${adb_dnsdir}/${adb_dnsprefix}.${list}")" = "##" ]
+    then
+        last_line="# ${0##*/} (${adb_scriptver}) - ${count} ad\/abuse domains blocked"
+        sed -i "s/^##$/${last_line}/" "${adb_dnsdir}/${adb_dnsprefix}.${list}"
+    fi
     adb_count=$((adb_count + count))
 done
 
diff --git a/net/adblock/files/adblock.conf b/net/adblock/files/adblock.conf
index 09d8cfa44..ce81264c7 100644
--- a/net/adblock/files/adblock.conf
+++ b/net/adblock/files/adblock.conf
@@ -3,7 +3,7 @@
 
 config adblock 'global'
 	option adb_enabled '1'
-	option adb_cfgver '0.90'
+	option adb_cfgver '0.91'
 	option adb_blacklist '/etc/adblock/adblock.blacklist'
 	option adb_whitelist '/etc/adblock/adblock.whitelist'
 	option adb_forcedns '1'
@@ -69,7 +69,7 @@ config source 'shalla'
 
 config source 'spam404'
 	option enabled '0'
-	option adb_src 'http://spam404bl.com/spam404scamlist.txt&ruleset=rset_spam404'
+	option adb_src 'https://raw.githubusercontent.com/Dawsey21/Lists/master/main-blacklist.txt&ruleset=rset_spam404'
 	option adb_srcdesc 'generic blocklist for suspicious domains, infrequent updates, approx. 5.000 entries'
 
 config source 'whocares'
@@ -84,7 +84,7 @@ config source 'winhelp'
 
 config source 'yoyo'
 	option enabled '1'
-	option adb_src 'https://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=rset_yoyo'
+	option adb_src 'https://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml&showintro=0&mimetype=plaintext&ruleset=rset_yoyo'
 	option adb_srcdesc 'focus on ad related domains, weekly updates, approx. 2.500 entries'
 
 config source 'zeus'