Improved ASN/IP update scripts and CI integration. (#1474)

* CI will print a warning if ASN/IP addresses changed.

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

16 files changed, 59 insertions, 46 deletions

diff --git a/utils/asn_update.sh b/utils/asn_update.sh
index 290204bf5..a023b2468 100755
--- a/utils/asn_update.sh
+++ b/utils/asn_update.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 function processing_list() {
 	local LIST_MERGED="/tmp/list_m"
 
diff --git a/utils/aws_ip_addresses_download.sh b/utils/aws_ip_addresses_download.sh
index e3b2f1e93..5c62183b8 100755
--- a/utils/aws_ip_addresses_download.sh
+++ b/utils/aws_ip_addresses_download.sh
@@ -12,7 +12,7 @@ echo "(1) Downloading file..."
 http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
 if [ $http_response != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return
+    return 1
 fi
 
 echo "(2) Processing IP addresses..."
@@ -21,3 +21,4 @@ jq -r '.prefixes | .[].ip_prefix' $TMP > $LIST # TODO: ipv6
 rm -f $TMP $LIST
 
 echo "(3) Amazon AWS IPs are available in $DEST"
+return 0
diff --git a/utils/azure_ip_addresses_download.sh b/utils/azure_ip_addresses_download.sh
index 8105fb87f..83ae9bb82 100755
--- a/utils/azure_ip_addresses_download.sh
+++ b/utils/azure_ip_addresses_download.sh
@@ -14,21 +14,21 @@ echo "(1) Downloading file... ${LINK_ORIGIN}"
 http_response=$(curl -s -o ${LINK_TMP} -w "%{http_code}" ${LINK_ORIGIN})
 if [ "${http_response}" != "200" ]; then
     echo "Error $http_response: you probably need to update the link origin url!"
-    return
+    return 1
 fi
 
 ORIGIN="$(grep -E 'ServiceTags_Public_[[:digit:]]+.json' ${LINK_TMP} | grep -o -E 'href=\"[^"]+' | sed 's/href="//' | uniq)"
 rm -f ${LINK_TMP}
 if [ -z "${ORIGIN}" ]; then
     echo "Error ${LINK_ORIGIN} does not contain the url format!"
-    return
+    return 1
 fi
 
 echo "(2) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
 if [ "${http_response}" != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return
+    return 1
 fi
 
 echo "(3) Processing IP addresses..."
@@ -38,6 +38,4 @@ tr -d '\r' < $TMP | grep / | tr -d '"' | tr -d " " | tr -d "," | grep -v : > $LI
 rm -f $TMP $LIST
 
 echo "(4) Microsoft Azure IPs are available in $DEST"
-
-
-
+return 0
diff --git a/utils/bitcoinnodes.sh b/utils/bitcoinnodes.sh
index 033a76713..cfc6b3e1a 100755
--- a/utils/bitcoinnodes.sh
+++ b/utils/bitcoinnodes.sh
@@ -5,4 +5,4 @@
 
 # NOTE: JQ can be found at https://stedolan.github.io/jq/
 
-curl -s -H "Accept: application/json; indent=4" https://bitnodes.earn.com/api/v1/snapshots/latest/ | jq -r '.nodes|keys[] as $k | "\($k)"' | grep -v onion | grep -v ']' | cut -d ':' -f 1
+curl -s -H "Accept: application/json; indent=4" https://bitnodes.io/api/v1/snapshots/latest/ | jq -r '.nodes|keys[] as $k | "\($k)"' | grep -v onion | grep -v ']' | cut -d ':' -f 1
diff --git a/utils/cloudflare_ip_addresses_download.sh b/utils/cloudflare_ip_addresses_download.sh
index 8dd1d3176..32f4f3567 100755
--- a/utils/cloudflare_ip_addresses_download.sh
+++ b/utils/cloudflare_ip_addresses_download.sh
@@ -8,11 +8,11 @@ LIST=/tmp/cloudflare.list
 ORIGIN="https://www.cloudflare.com/ips-v4"
 
 
-echo "(1) Downloading file..."
+echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
 if [ $http_response != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return
+    return 1
 fi
 
 echo "(2) Processing IP addresses..."
@@ -20,6 +20,4 @@ echo "(2) Processing IP addresses..."
 rm -f $LIST
 
 echo "(3) Cloudflare IPs are available in $DEST"
-
-
-
+return 0
diff --git a/utils/ethereum_ip_addresses_download.sh b/utils/ethereum_ip_addresses_download.sh
index b01c92400..3497a943f 100755
--- a/utils/ethereum_ip_addresses_download.sh
+++ b/utils/ethereum_ip_addresses_download.sh
@@ -8,11 +8,11 @@ LIST=/tmp/ethereum.list
 ORIGIN="https://raw.githubusercontent.com/ethereum/go-ethereum/master/params/bootnodes.go"
 
 
-echo "(1) Downloading file..."
+echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
 if [ $http_response != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return
+    return 1
 fi
 
 echo "(2) Processing IP addresses..."
@@ -22,6 +22,4 @@ grep 'enode' $TMP | grep -v '^/' | grep ':' | cut -d '@' -f 2 | cut -d ':' -f 1
 rm -f $TMP $LIST
 
 echo "(3) Ethereum/Mining IPs are available in $DEST"
-
-
-
+return 0
diff --git a/utils/get_routes_by_asn.sh b/utils/get_routes_by_asn.sh
index f9292828b..06e320c16 100755
--- a/utils/get_routes_by_asn.sh
+++ b/utils/get_routes_by_asn.sh
@@ -10,10 +10,14 @@ ORIGIN="https://stat.ripe.net/data/announced-prefixes/data.json?resource=$1"
 
 http_response=$(curl -s -o "${LIST}" -w "%{http_code}" "${ORIGIN}")
 if [ "$http_response" != "200" ]; then
+    echo "Error $http_response: service temporiarally not available?" >&2
+    return 1
+elif [ "$http_response" != "200" ]; then
     echo "Error $http_response: wrong ASN number/format?" >&2
-    return
+    return 1
 fi
 
 jq -r '.data.prefixes[].prefix' $LIST | grep -v ":"
 
 rm -f $LIST
+return 0
diff --git a/utils/google.py b/utils/google.py
index e8b4114c8..e8b4114c8 100644..100755
--- a/utils/google.py
+++ b/utils/google.py
diff --git a/utils/google_cloud_ip_addresses_download.sh b/utils/google_cloud_ip_addresses_download.sh
index b14a33af7..97d399e9f 100755
--- a/utils/google_cloud_ip_addresses_download.sh
+++ b/utils/google_cloud_ip_addresses_download.sh
@@ -8,11 +8,11 @@ LIST=/tmp/google_c.list
 ORIGIN="https://www.gstatic.com/ipranges/cloud.json"
 
 
-echo "(1) Downloading file..."
+echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
 if [ "$http_response" != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return
+    return 1
 fi
 
 echo "(2) Processing IP addresses..."
@@ -21,3 +21,4 @@ jq -r '.prefixes | .[].ipv4Prefix  | select( . != null )' $TMP > $LIST # TODO: i
 rm -f $TMP $LIST
 
 echo "(3) Google Cloud IPs are available in $DEST"
+return 0
diff --git a/utils/google_ip_addresses_download.sh b/utils/google_ip_addresses_download.sh
index 9560ef3c9..efb8ee043 100755
--- a/utils/google_ip_addresses_download.sh
+++ b/utils/google_ip_addresses_download.sh
@@ -1,5 +1,7 @@
 #!/bin/sh
 
+set -e
+
 cd "$(dirname "${0}")" || return
 
 DEST=../src/lib/ndpi_google_match.c.inc
@@ -15,3 +17,4 @@ python3 google.py > $LIST
 #rm -f $TMP $LIST
 
 echo "(3) Google IPs are available in $DEST"
+return 0
diff --git a/utils/icloud_private_relay_ip_addresses_download.sh b/utils/icloud_private_relay_ip_addresses_download.sh
index 149185d83..242dd17b9 100755
--- a/utils/icloud_private_relay_ip_addresses_download.sh
+++ b/utils/icloud_private_relay_ip_addresses_download.sh
@@ -9,11 +9,11 @@ LIST_MERGED=/tmp/icloud.list_m
 ORIGIN="https://mask-api.icloud.com/egress-ip-ranges.csv"
 
 
-echo "(1) Downloading file..."
+echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o "$TMP" -w "%{http_code}" ${ORIGIN})
 if [ "$http_response" != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return
+    return 1
 fi
 
 echo "(2) Processing IP addresses..."
@@ -25,6 +25,4 @@ cut -d ',' -f 1 $TMP | grep -v ':' > $LIST
 rm -f $TMP $LIST $LIST_MERGED
 
 echo "(3) iCloud Private Relay IPs are available in $DEST"
-
-
-
+return 0
diff --git a/utils/microsoft_ip_addresses_download.sh b/utils/microsoft_ip_addresses_download.sh
index 3148de44f..29a277480 100755
--- a/utils/microsoft_ip_addresses_download.sh
+++ b/utils/microsoft_ip_addresses_download.sh
@@ -12,11 +12,11 @@ LIST=/tmp/ms.list
 ORIGIN="https://endpoints.office.com/endpoints/worldwide?clientrequestid=b10c5ed1-bad1-445f-b386-b919946339a7"
 
 
-echo "(1) Downloading file..."
+echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
 if [ $http_response != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return
+    return 1
 fi
 
 echo "(2) Processing IP addresses..."
@@ -45,6 +45,4 @@ jq -r '.[] | select(.serviceArea=="Common") | .ips[]?' < $TMP | grep -v ':' | so
 rm -f $TMP $LIST
 
 echo "(3) Microsoft IPs are available in $DEST_OUTLOOK, $DEST_SKYPE_MSTEAMS, $DEST_ONEDRIVE, $DEST_OFFICE365"
-
-
-
+return 0
diff --git a/utils/tor_ip_addresses_download.sh b/utils/tor_ip_addresses_download.sh
index ed5cdc4c7..8598e1151 100755
--- a/utils/tor_ip_addresses_download.sh
+++ b/utils/tor_ip_addresses_download.sh
@@ -11,11 +11,11 @@ LIST=/tmp/tor.list
 ORIGIN="https://check.torproject.org/torbulkexitlist"
 
 
-echo "(1) Downloading file..."
+echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
 if [ $http_response != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return
+    return 1
 fi
 
 echo "(2) Processing IP addresses..."
@@ -23,6 +23,4 @@ echo "(2) Processing IP addresses..."
 rm -f $LIST
 
 echo "(3) TOR IPs are available in $DEST"
-
-
-
+return 0
diff --git a/utils/update_every_lists.sh b/utils/update_every_lists.sh
index cbc3bf68d..155585b3a 100755
--- a/utils/update_every_lists.sh
+++ b/utils/update_every_lists.sh
@@ -2,15 +2,33 @@
 
 cd "$(dirname "${0}")" || return
 
+RETVAL=0
+
 ./aws_ip_addresses_download.sh
+RETVAL=$(( RETVAL + $? ))
 ./azure_ip_addresses_download.sh
+RETVAL=$(( RETVAL + $? ))
 ./cloudflare_ip_addresses_download.sh
+RETVAL=$(( RETVAL + $? ))
 ./ethereum_ip_addresses_download.sh
+RETVAL=$(( RETVAL + $? ))
 ./microsoft_ip_addresses_download.sh
+RETVAL=$(( RETVAL + $? ))
 ./tor_ip_addresses_download.sh
+RETVAL=$(( RETVAL + $? ))
 ./whatsapp_ip_addresses_download.sh
+RETVAL=$(( RETVAL + $? ))
 ./zoom_ip_addresses_download.sh
+RETVAL=$(( RETVAL + $? ))
 ./google_cloud_ip_addresses_download.sh
+RETVAL=$(( RETVAL + $? ))
 ./google_ip_addresses_download.sh
+RETVAL=$(( RETVAL + $? ))
+./icloud_private_relay_ip_addresses_download.sh
+RETVAL=$(( RETVAL + $? ))
 
 ./asn_update.sh
+RETVAL=$(( RETVAL + $? ))
+
+test ${RETVAL} -ne 0 && printf '%s: %s\n' "${0}" "${RETVAL} scripts failed"
+exit ${RETVAL}
diff --git a/utils/whatsapp_ip_addresses_download.sh b/utils/whatsapp_ip_addresses_download.sh
index ca55ef4d6..1f0e99a30 100755
--- a/utils/whatsapp_ip_addresses_download.sh
+++ b/utils/whatsapp_ip_addresses_download.sh
@@ -9,11 +9,11 @@ LIST=/tmp/wa.list
 ORIGIN="https://scontent.fmxp6-1.fna.fbcdn.net/v/t39.8562-6/218944277_794653217800107_785885630662402277_n.zip?_nc_cat=102&ccb=1-5&_nc_sid=ae5e01&_nc_ohc=CxWH4uR6uPsAX-Yga3M&_nc_ht=scontent.fmxp6-1.fna&oh=00_AT9gC0NiHKwmgoBdNX9jbVbxtciJ8HzeGdOLj35n3kWeUw&oe=6201B6A9"
 
 
-echo "(1) Downloading file..."
+echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
 if [ "$http_response" != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return
+    return 0 # WhatsApp URL broken: set this to 1 when the URL was fixed
 fi
 
 echo "(2) Processing IP addresses..."
@@ -22,6 +22,4 @@ zcat $TMP > $LIST
 rm -f $TMP $LIST
 
 echo "(3) WhatsApp IPs are available in $DEST"
-
-
-
+return 0
diff --git a/utils/zoom_ip_addresses_download.sh b/utils/zoom_ip_addresses_download.sh
index b2f04e7fe..3256b5459 100755
--- a/utils/zoom_ip_addresses_download.sh
+++ b/utils/zoom_ip_addresses_download.sh
@@ -9,11 +9,11 @@ LIST=/tmp/zoom.list
 ORIGIN="https://assets.zoom.us/docs/ipranges/Zoom.txt"
 
 
-echo "(1) Downloading file..."
+echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
 if [ "$http_response" != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return
+    return 1
 fi
 
 echo "(2) Processing IP addresses..."
@@ -21,6 +21,4 @@ echo "(2) Processing IP addresses..."
 rm -f $LIST
 
 echo "(3) ZOOM IPs are available in $DEST"
-
-
-
+return 0