Added Edgecast and Cachefly CDNs. (#1540)

* Improved ASN update script
 * Ran `utils/update_every_lists.sh'
 * `tests/do.sh.in' prints the amount of failed pcap(s)
 * `utils/asn_update.sh' prints the amount of failed download(s)

Signed-off-by: lns <matzeton@googlemail.com>

15 files changed, 66 insertions, 11 deletions

diff --git a/utils/asn_update.sh b/utils/asn_update.sh
index 5224bbc27..81098ab6a 100755
--- a/utils/asn_update.sh
+++ b/utils/asn_update.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 
-set -e
+FAILED_ASN=0
+TOTAL_ASN=0
 
 function processing_list() {
 	local LIST_MERGED="/tmp/list_m"
@@ -15,7 +16,11 @@ function create_list() {
 	LIST=/tmp/list
 
 	for i in "${@:3}"; do
-		./get_routes_by_asn.sh "$i" >> $LIST
+		TOTAL_ASN=$(( TOTAL_ASN + 1 ))
+		if ! ./get_routes_by_asn.sh "$i" >> $LIST; then
+			echo "Could not fetch route for ${i} (${1})"
+			FAILED_ASN=$(( FAILED_ASN + 1 ))
+		fi
 	done
 
 	processing_list "$LIST" "$1" "$2"
@@ -113,3 +118,15 @@ echo "(1) Downloading Citrix routes..." #Citrix or a more generic LogMeIn stuff?
 DEST=../src/lib/inc_generated/ndpi_asn_citrix.c.inc
 create_list NDPI_PROTOCOL_CITRIX $DEST "AS395424" "AS21866" "AS213380" "AS20104" "AS16815"
 echo "(3) Citrix IPs are available in $DEST"
+
+echo "(1) Downloading Edgecast routes..."
+DEST=../src/lib/inc_generated/ndpi_asn_edgecast.c.inc
+create_list NDPI_PROTOCOL_EDGECAST $DEST "AS15133"
+echo "(3) Edgecast IPs are available in $DEST"
+
+if [ ${TOTAL_ASN} -eq ${FAILED_ASN} ]; then
+	printf '%s: %s\n' "${0}" "All download(s) failed, ./get_routes_by_asn.sh broken?"
+	exit 1
+else
+	exit 0
+fi
diff --git a/utils/aws_ip_addresses_download.sh b/utils/aws_ip_addresses_download.sh
index dd7fc1dde..cc923a1a2 100755
--- a/utils/aws_ip_addresses_download.sh
+++ b/utils/aws_ip_addresses_download.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-cd "$(dirname "${0}")"
+cd "$(dirname "${0}")" || exit 1
 
 DEST=../src/lib/inc_generated/ndpi_amazon_aws_match.c.inc
 TMP=/tmp/aws.json
diff --git a/utils/azure_ip_addresses_download.sh b/utils/azure_ip_addresses_download.sh
index 9c3cf9d04..3e20b6119 100755
--- a/utils/azure_ip_addresses_download.sh
+++ b/utils/azure_ip_addresses_download.sh
@@ -1,6 +1,8 @@
 #!/bin/sh
 
-cd "$(dirname "${0}")"
+set -e
+
+cd "$(dirname "${0}")" || exit 1
 
 DEST=../src/lib/inc_generated/ndpi_azure_match.c.inc
 LINK_TMP=/tmp/azure_link.txt
diff --git a/utils/bitcoinnodes.sh b/utils/bitcoinnodes.sh
index cfc6b3e1a..03fbedd14 100755
--- a/utils/bitcoinnodes.sh
+++ b/utils/bitcoinnodes.sh
@@ -3,6 +3,8 @@
 # List all the current bittorrent nodes
 #
 
+set -e
+
 # NOTE: JQ can be found at https://stedolan.github.io/jq/
 
 curl -s -H "Accept: application/json; indent=4" https://bitnodes.io/api/v1/snapshots/latest/ | jq -r '.nodes|keys[] as $k | "\($k)"' | grep -v onion | grep -v ']' | cut -d ':' -f 1
diff --git a/utils/cachefly_ip_addresses_download.sh b/utils/cachefly_ip_addresses_download.sh
new file mode 100755
index 000000000..fa1992e64
--- /dev/null
+++ b/utils/cachefly_ip_addresses_download.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+cd "$(dirname "${0}")" || exit 1
+
+DEST=../src/lib/inc_generated/ndpi_cachefly_match.c.inc
+LIST=/tmp/cachefly.list
+ORIGIN='https://cachefly.cachefly.net/ips/cdn.txt'
+
+
+echo "(1) Downloading file..."
+http_response=$(curl -s -o "${LIST}" -w "%{http_code}" "${ORIGIN}")
+if [ "${http_response}" != "200" ]; then
+    echo "Error ${http_response}: you probably need to update the list url!"
+    exit 1
+fi
+
+echo "(2) Processing IP addresses..."
+./ipaddr2list.py "${LIST}" NDPI_PROTOCOL_CACHEFLY > "${DEST}"
+rm -f "${LIST}"
+
+echo "(3) Cachefly IPs are available in ${DEST}"
+exit 0
diff --git a/utils/cloudflare_ip_addresses_download.sh b/utils/cloudflare_ip_addresses_download.sh
index 02bdffe81..93915b6bc 100755
--- a/utils/cloudflare_ip_addresses_download.sh
+++ b/utils/cloudflare_ip_addresses_download.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-cd "$(dirname "${0}")"
+cd "$(dirname "${0}")" || exit 1
 
 DEST=../src/lib/inc_generated/ndpi_cloudflare_match.c.inc
 LIST=/tmp/cloudflare.list
diff --git a/utils/ethereum_ip_addresses_download.sh b/utils/ethereum_ip_addresses_download.sh
index 0ea496973..c6e88b1ce 100755
--- a/utils/ethereum_ip_addresses_download.sh
+++ b/utils/ethereum_ip_addresses_download.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-cd "$(dirname "${0}")"
+cd "$(dirname "${0}")" || exit 1
 
 DEST=../src/lib/inc_generated/ndpi_ethereum_match.c.inc
 TMP=/tmp/ethereum
diff --git a/utils/get_routes_by_asn.sh b/utils/get_routes_by_asn.sh
index d34186297..ce90ffcd5 100755
--- a/utils/get_routes_by_asn.sh
+++ b/utils/get_routes_by_asn.sh
@@ -1,5 +1,7 @@
 #!/bin/sh
 
+set -e
+
 if [ "$#" -ne 1 ]; then
     echo "Usage: $0 AS-Number" >&2
     exit 1
diff --git a/utils/google_cloud_ip_addresses_download.sh b/utils/google_cloud_ip_addresses_download.sh
index 8879a93e2..2c24c5c36 100755
--- a/utils/google_cloud_ip_addresses_download.sh
+++ b/utils/google_cloud_ip_addresses_download.sh
@@ -1,5 +1,7 @@
 #!/bin/sh
 
+set -e
+
 cd "$(dirname "${0}")" || exit 1
 
 DEST=../src/lib/inc_generated/ndpi_google_cloud_match.c.inc
diff --git a/utils/icloud_private_relay_ip_addresses_download.sh b/utils/icloud_private_relay_ip_addresses_download.sh
index 0579cc757..352f0b649 100755
--- a/utils/icloud_private_relay_ip_addresses_download.sh
+++ b/utils/icloud_private_relay_ip_addresses_download.sh
@@ -1,5 +1,7 @@
 #!/bin/sh
 
+set -e
+
 cd "$(dirname "${0}")" || exit 1
 
 DEST=../src/lib/inc_generated/ndpi_icloud_private_relay_match.c.inc
diff --git a/utils/microsoft_ip_addresses_download.sh b/utils/microsoft_ip_addresses_download.sh
index b09674c4f..3c1759993 100755
--- a/utils/microsoft_ip_addresses_download.sh
+++ b/utils/microsoft_ip_addresses_download.sh
@@ -1,6 +1,8 @@
 #!/bin/sh
 
-cd "$(dirname "${0}")"
+set -e
+
+cd "$(dirname "${0}")" || exit 1
 
 DEST_OUTLOOK=../src/lib/inc_generated/ndpi_ms_outlook_match.c.inc
 DEST_SKYPE_MSTEAMS=../src/lib/inc_generated/ndpi_ms_skype_teams_match.c.inc
diff --git a/utils/tor_ip_addresses_download.sh b/utils/tor_ip_addresses_download.sh
index 22b3a4824..397083800 100755
--- a/utils/tor_ip_addresses_download.sh
+++ b/utils/tor_ip_addresses_download.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-cd "$(dirname "${0}")"
+cd "$(dirname "${0}")" || exit 1
 
 DEST=../src/lib/inc_generated/ndpi_tor_match.c.inc
 LIST=/tmp/tor.list
diff --git a/utils/update_every_lists.sh b/utils/update_every_lists.sh
index e995b80be..514a2999a 100755
--- a/utils/update_every_lists.sh
+++ b/utils/update_every_lists.sh
@@ -8,6 +8,8 @@ RETVAL=0
 RETVAL=$(( RETVAL + $? ))
 ./azure_ip_addresses_download.sh
 RETVAL=$(( RETVAL + $? ))
+./cachefly_ip_addresses_download.sh
+RETVAL=$(( RETVAL + $? ))
 ./cloudflare_ip_addresses_download.sh
 RETVAL=$(( RETVAL + $? ))
 ./ethereum_ip_addresses_download.sh
@@ -30,5 +32,5 @@ RETVAL=$(( RETVAL + $? ))
 ./asn_update.sh
 RETVAL=$(( RETVAL + $? ))
 
-test ${RETVAL} -ne 0 && printf '%s: %s\n' "${0}" "${RETVAL} scripts failed"
+test ${RETVAL} -ne 0 && printf '%s: %s\n' "${0}" "${RETVAL} script(s) failed"
 exit ${RETVAL}
diff --git a/utils/whatsapp_ip_addresses_download.sh b/utils/whatsapp_ip_addresses_download.sh
index 9810c7644..7234906d6 100755
--- a/utils/whatsapp_ip_addresses_download.sh
+++ b/utils/whatsapp_ip_addresses_download.sh
@@ -1,6 +1,8 @@
 #!/bin/sh
 
-cd "$(dirname "${0}")"
+set -e
+
+cd "$(dirname "${0}")" || exit 1
 
 DEST=../src/lib/inc_generated/ndpi_whatsapp_match.c.inc
 TMP=/tmp/wa.zip
diff --git a/utils/zoom_ip_addresses_download.sh b/utils/zoom_ip_addresses_download.sh
index 49e149062..3139c1b54 100755
--- a/utils/zoom_ip_addresses_download.sh
+++ b/utils/zoom_ip_addresses_download.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-cd "$(dirname "${0}")"
+cd "$(dirname "${0}")" || exit 1
 
 DEST=../src/lib/inc_generated/ndpi_zoom_match.c.inc
 LIST=/tmp/zoom.list