Updated `utils/whatsapp_ip_addresses_download.sh' to scrape the required IP addresses/ranges. (#1524)

* Replaced return statements in `utils/*.sh' with exit's (such scripts should never source'd)
 * Ran `utils/update_every_lists.sh'

Signed-off-by: lns <matzeton@googlemail.com>

14 files changed, 39 insertions, 37 deletions

diff --git a/utils/asn_update.sh b/utils/asn_update.sh
index e79c3da86..5224bbc27 100755
--- a/utils/asn_update.sh
+++ b/utils/asn_update.sh
@@ -22,7 +22,7 @@ function create_list() {
 	rm -f $LIST
 }
 
-cd "$(dirname "${0}")" || return
+cd "$(dirname "${0}")" || exit 1
 
 echo "(1) Downloading Apple routes..."
 DEST="../src/lib/inc_generated/ndpi_asn_apple.c.inc"
diff --git a/utils/aws_ip_addresses_download.sh b/utils/aws_ip_addresses_download.sh
index ffaaddf6a..dd7fc1dde 100755
--- a/utils/aws_ip_addresses_download.sh
+++ b/utils/aws_ip_addresses_download.sh
@@ -12,7 +12,7 @@ echo "(1) Downloading file..."
 http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
 if [ $http_response != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return 1
+    exit 1
 fi
 
 echo "(2) Processing IP addresses..."
@@ -21,4 +21,4 @@ jq -r '.prefixes | .[].ip_prefix' $TMP > $LIST # TODO: ipv6
 rm -f $TMP $LIST
 
 echo "(3) Amazon AWS IPs are available in $DEST"
-return 0
+exit 0
diff --git a/utils/azure_ip_addresses_download.sh b/utils/azure_ip_addresses_download.sh
index 822871b7c..9c3cf9d04 100755
--- a/utils/azure_ip_addresses_download.sh
+++ b/utils/azure_ip_addresses_download.sh
@@ -14,21 +14,21 @@ echo "(1) Downloading file... ${LINK_ORIGIN}"
 http_response=$(curl -s -o ${LINK_TMP} -w "%{http_code}" ${LINK_ORIGIN})
 if [ "${http_response}" != "200" ]; then
     echo "Error $http_response: you probably need to update the link origin url!"
-    return 1
+    exit 1
 fi
 
 ORIGIN="$(grep -E 'ServiceTags_Public_[[:digit:]]+.json' ${LINK_TMP} | grep -o -E 'href=\"[^"]+' | sed 's/href="//' | uniq)"
 rm -f ${LINK_TMP}
 if [ -z "${ORIGIN}" ]; then
     echo "Error ${LINK_ORIGIN} does not contain the url format!"
-    return 1
+    exit 1
 fi
 
 echo "(2) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
 if [ "${http_response}" != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return 1
+    exit 1
 fi
 
 echo "(3) Processing IP addresses..."
@@ -38,4 +38,4 @@ tr -d '\r' < $TMP | grep / | tr -d '"' | tr -d " " | tr -d "," | grep -v : > $LI
 rm -f $TMP $LIST
 
 echo "(4) Microsoft Azure IPs are available in $DEST"
-return 0
+exit 0
diff --git a/utils/cloudflare_ip_addresses_download.sh b/utils/cloudflare_ip_addresses_download.sh
index 9327f7b23..02bdffe81 100755
--- a/utils/cloudflare_ip_addresses_download.sh
+++ b/utils/cloudflare_ip_addresses_download.sh
@@ -12,7 +12,7 @@ echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
 if [ $http_response != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return 1
+    exit 1
 fi
 
 echo "(2) Processing IP addresses..."
@@ -20,4 +20,4 @@ echo "(2) Processing IP addresses..."
 rm -f $LIST
 
 echo "(3) Cloudflare IPs are available in $DEST"
-return 0
+exit 0
diff --git a/utils/ethereum_ip_addresses_download.sh b/utils/ethereum_ip_addresses_download.sh
index 2e2c62ea2..0ea496973 100755
--- a/utils/ethereum_ip_addresses_download.sh
+++ b/utils/ethereum_ip_addresses_download.sh
@@ -12,7 +12,7 @@ echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
 if [ $http_response != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return 1
+    exit 1
 fi
 
 echo "(2) Processing IP addresses..."
@@ -22,4 +22,4 @@ grep 'enode' $TMP | grep -v '^/' | grep ':' | cut -d '@' -f 2 | cut -d ':' -f 1
 rm -f $TMP $LIST
 
 echo "(3) Ethereum/Mining IPs are available in $DEST"
-return 0
+exit 0
diff --git a/utils/get_routes_by_asn.sh b/utils/get_routes_by_asn.sh
index 06e320c16..d34186297 100755
--- a/utils/get_routes_by_asn.sh
+++ b/utils/get_routes_by_asn.sh
@@ -2,7 +2,7 @@
 
 if [ "$#" -ne 1 ]; then
     echo "Usage: $0 AS-Number" >&2
-    return
+    exit 1
 fi
 
 LIST=/tmp/asn.json
@@ -11,13 +11,13 @@ ORIGIN="https://stat.ripe.net/data/announced-prefixes/data.json?resource=$1"
 http_response=$(curl -s -o "${LIST}" -w "%{http_code}" "${ORIGIN}")
 if [ "$http_response" != "200" ]; then
     echo "Error $http_response: service temporiarally not available?" >&2
-    return 1
+    exit 1
 elif [ "$http_response" != "200" ]; then
     echo "Error $http_response: wrong ASN number/format?" >&2
-    return 1
+    exit 1
 fi
 
 jq -r '.data.prefixes[].prefix' $LIST | grep -v ":"
 
 rm -f $LIST
-return 0
+exit 0
diff --git a/utils/google_cloud_ip_addresses_download.sh b/utils/google_cloud_ip_addresses_download.sh
index 2f219cb43..8879a93e2 100755
--- a/utils/google_cloud_ip_addresses_download.sh
+++ b/utils/google_cloud_ip_addresses_download.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-cd "$(dirname "${0}")" || return
+cd "$(dirname "${0}")" || exit 1
 
 DEST=../src/lib/inc_generated/ndpi_google_cloud_match.c.inc
 TMP=/tmp/google_c.json
@@ -12,7 +12,7 @@ echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
 if [ "$http_response" != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return 1
+    exit 1
 fi
 
 echo "(2) Processing IP addresses..."
@@ -21,4 +21,4 @@ jq -r '.prefixes | .[].ipv4Prefix  | select( . != null )' $TMP > $LIST # TODO: i
 rm -f $TMP $LIST
 
 echo "(3) Google Cloud IPs are available in $DEST"
-return 0
+exit 0
diff --git a/utils/google_ip_addresses_download.sh b/utils/google_ip_addresses_download.sh
index 04b89b81f..5fc888b47 100755
--- a/utils/google_ip_addresses_download.sh
+++ b/utils/google_ip_addresses_download.sh
@@ -2,7 +2,7 @@
 
 set -e
 
-cd "$(dirname "${0}")" || return
+cd "$(dirname "${0}")" || exit 1
 
 DEST=../src/lib/inc_generated/ndpi_google_match.c.inc
 LIST=/tmp/google.list
@@ -17,4 +17,4 @@ python3 google.py > $LIST
 #rm -f $TMP $LIST
 
 echo "(3) Google IPs are available in $DEST"
-return 0
+exit 0
diff --git a/utils/icloud_private_relay_ip_addresses_download.sh b/utils/icloud_private_relay_ip_addresses_download.sh
index 9b8de3c4c..0579cc757 100755
--- a/utils/icloud_private_relay_ip_addresses_download.sh
+++ b/utils/icloud_private_relay_ip_addresses_download.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-cd "$(dirname "${0}")" || return
+cd "$(dirname "${0}")" || exit 1
 
 DEST=../src/lib/inc_generated/ndpi_icloud_private_relay_match.c.inc
 TMP=/tmp/icloud.csv
@@ -13,7 +13,7 @@ echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o "$TMP" -w "%{http_code}" ${ORIGIN})
 if [ "$http_response" != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return 1
+    exit 1
 fi
 
 echo "(2) Processing IP addresses..."
@@ -25,4 +25,4 @@ cut -d ',' -f 1 $TMP | grep -v ':' > $LIST
 rm -f $TMP $LIST $LIST_MERGED
 
 echo "(3) iCloud Private Relay IPs are available in $DEST"
-return 0
+exit 0
diff --git a/utils/microsoft_ip_addresses_download.sh b/utils/microsoft_ip_addresses_download.sh
index 641372b08..b09674c4f 100755
--- a/utils/microsoft_ip_addresses_download.sh
+++ b/utils/microsoft_ip_addresses_download.sh
@@ -16,7 +16,7 @@ echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
 if [ $http_response != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return 1
+    exit 1
 fi
 
 echo "(2) Processing IP addresses..."
@@ -45,4 +45,4 @@ jq -r '.[] | select(.serviceArea=="Common") | .ips[]?' < $TMP | grep -v ':' | so
 rm -f $TMP $LIST
 
 echo "(3) Microsoft IPs are available in $DEST_OUTLOOK, $DEST_SKYPE_MSTEAMS, $DEST_ONEDRIVE, $DEST_OFFICE365"
-return 0
+exit 0
diff --git a/utils/tor_ip_addresses_download.sh b/utils/tor_ip_addresses_download.sh
index e0f9fa1ff..22b3a4824 100755
--- a/utils/tor_ip_addresses_download.sh
+++ b/utils/tor_ip_addresses_download.sh
@@ -15,7 +15,7 @@ echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
 if [ $http_response != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return 1
+    exit 1
 fi
 
 echo "(2) Processing IP addresses..."
@@ -23,4 +23,4 @@ echo "(2) Processing IP addresses..."
 rm -f $LIST
 
 echo "(3) TOR IPs are available in $DEST"
-return 0
+exit 0
diff --git a/utils/update_every_lists.sh b/utils/update_every_lists.sh
index 155585b3a..e995b80be 100755
--- a/utils/update_every_lists.sh
+++ b/utils/update_every_lists.sh
@@ -1,6 +1,6 @@
 #/bin/sh
 
-cd "$(dirname "${0}")" || return
+cd "$(dirname "${0}")" || exit 1
 
 RETVAL=0
 
diff --git a/utils/whatsapp_ip_addresses_download.sh b/utils/whatsapp_ip_addresses_download.sh
index e53ac68f6..9810c7644 100755
--- a/utils/whatsapp_ip_addresses_download.sh
+++ b/utils/whatsapp_ip_addresses_download.sh
@@ -5,21 +5,23 @@ cd "$(dirname "${0}")"
 DEST=../src/lib/inc_generated/ndpi_whatsapp_match.c.inc
 TMP=/tmp/wa.zip
 LIST=/tmp/wa.list
-# https://developers.facebook.com/docs/whatsapp/guides/network-requirements/
-ORIGIN="https://scontent.fmxp6-1.fna.fbcdn.net/v/t39.8562-6/218944277_794653217800107_785885630662402277_n.zip?_nc_cat=102&ccb=1-5&_nc_sid=ae5e01&_nc_ohc=CxWH4uR6uPsAX-Yga3M&_nc_ht=scontent.fmxp6-1.fna&oh=00_AT9gC0NiHKwmgoBdNX9jbVbxtciJ8HzeGdOLj35n3kWeUw&oe=6201B6A9"
+IP_LINK_URL='https://developers.facebook.com/docs/whatsapp/guides/network-requirements/'
 
 
-echo "(1) Downloading file... ${ORIGIN}"
+echo "(1) Scraping Facebook WhatsApp IP Adresses and Ranges..."
+ORIGIN="$(curl -s "${IP_LINK_URL}" | sed -ne 's/.*<a href="\([^"]*\)" target="_blank">List of the WhatsApp server IP addresses and ranges (.zip file)<\/a>.*/\1/gp' | sed -e 's/\&amp;/\&/g')"
+
+echo "(2) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
 if [ "$http_response" != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return 0 # WhatsApp URL broken: set this to 1 when the URL was fixed
+    exit 1
 fi
 
-echo "(2) Processing IP addresses..."
+echo "(3) Processing IP addresses..."
 zcat $TMP > $LIST
 ./ipaddr2list.py $LIST NDPI_PROTOCOL_WHATSAPP > $DEST
 rm -f $TMP $LIST
 
-echo "(3) WhatsApp IPs are available in $DEST"
-return 0
+echo "(4) WhatsApp IPs are available in $DEST"
+exit 0
diff --git a/utils/zoom_ip_addresses_download.sh b/utils/zoom_ip_addresses_download.sh
index 9fe1a9445..49e149062 100755
--- a/utils/zoom_ip_addresses_download.sh
+++ b/utils/zoom_ip_addresses_download.sh
@@ -13,7 +13,7 @@ echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
 if [ "$http_response" != "200" ]; then
     echo "Error $http_response: you probably need to update the list url!"
-    return 1
+    exit 1
 fi
 
 echo "(2) Processing IP addresses..."
@@ -21,4 +21,4 @@ echo "(2) Processing IP addresses..."
 rm -f $LIST
 
 echo "(3) ZOOM IPs are available in $DEST"
-return 0
+exit 0