Add some scripts to easily update some IPs lists (#1449)

While the lists in a6ff0dd0 and 2f5f445f are somehow provided by the
companies themselves (or by some interested parties), these new lists
are directly extracted from BGP information, via AS prefixes.

*Usually*, these new lists are far more stable than the previous ones.

TODO:
* add some other ASNs (see `src/lib/ndpi_content_match.c.inc`)
* IPv6, as usual :-(

4 files changed, 100 insertions, 1 deletions

diff --git a/utils/asn_update.sh b/utils/asn_update.sh
new file mode 100755
index 000000000..290204bf5
--- /dev/null
+++ b/utils/asn_update.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+function processing_list() {
+	local LIST_MERGED="/tmp/list_m"
+
+	echo "(2) Processing IP addresses..."
+	./mergeipaddrlist.py "$1" > $LIST_MERGED
+	./ipaddr2list.py "$LIST_MERGED" "$2" > "$3"
+	rm -f $LIST_MERGED
+}
+
+function create_list() {
+	LIST=/tmp/list
+
+	for i in "${@:3}"; do
+		./get_routes_by_asn.sh "$i" >> $LIST
+	done
+
+	processing_list "$LIST" "$1" "$2"
+	rm -f $LIST
+}
+
+cd "$(dirname "${0}")" || return
+
+echo "(1) Downloading Apple routes..."
+DEST="../src/lib/ndpi_asn_apple.c.inc"
+create_list NDPI_PROTOCOL_APPLE $DEST "AS714" "AS6185" "AS2709"
+echo "(3) Apple IPs are available in $DEST"
+
+echo "(1) Downloading Facebook routes..."
+DEST=../src/lib/ndpi_asn_facebook.c.inc
+create_list NDPI_PROTOCOL_FACEBOOK $DEST "AS63293" "AS54115" "AS34825" "AS32934"
+echo "(3) Facebook IPs are available in $DEST"
+
+echo "(1) Downloading Netflix routes..."
+DEST=../src/lib/ndpi_asn_netflix.c.inc
+create_list NDPI_PROTOCOL_NETFLIX $DEST "AS55095" "AS40027" "AS394406" "AS2906"
+echo "(3) Netflix IPs are available in $DEST"
+
+echo "(1) Downloading Teamviewer routes..."
+DEST=../src/lib/ndpi_asn_teamviewer.c.inc
+create_list NDPI_PROTOCOL_TEAMVIEWER $DEST "AS43304" "AS212710" "AS208187" "AS208175"
+echo "(3) Teamviewer IPs are available in $DEST"
+
+echo "(1) Downloading Telegram routes..."
+DEST=../src/lib/ndpi_asn_telegram.c.inc
+create_list NDPI_PROTOCOL_TELEGRAM $DEST "AS62041" "AS62014" "AS59930" "AS44907" "AS211157"
+echo "(3) Telegram IPs are available in $DEST"
+
+echo "(1) Downloading Twitter routes..."
+DEST=../src/lib/ndpi_asn_twitter.c.inc
+create_list NDPI_PROTOCOL_TWITTER $DEST "AS63179" "AS54888" "AS35995" "AS13414"
+echo "(3) Twitter IPs are available in $DEST"
+
+echo "(1) Downloading Webex routes..."
+DEST=../src/lib/ndpi_asn_webex.c.inc
+create_list NDPI_PROTOCOL_WEBEX $DEST "AS6577" "AS399937" "AS16472" "AS13445"
+echo "(3) Webex IPs are available in $DEST"
diff --git a/utils/get_routes_by_asn.sh b/utils/get_routes_by_asn.sh
new file mode 100755
index 000000000..f9292828b
--- /dev/null
+++ b/utils/get_routes_by_asn.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 AS-Number" >&2
+    return
+fi
+
+LIST=/tmp/asn.json
+ORIGIN="https://stat.ripe.net/data/announced-prefixes/data.json?resource=$1"
+
+http_response=$(curl -s -o "${LIST}" -w "%{http_code}" "${ORIGIN}")
+if [ "$http_response" != "200" ]; then
+    echo "Error $http_response: wrong ASN number/format?" >&2
+    return
+fi
+
+jq -r '.data.prefixes[].prefix' $LIST | grep -v ":"
+
+rm -f $LIST
diff --git a/utils/mergeipaddrlist.py b/utils/mergeipaddrlist.py
new file mode 100755
index 000000000..36dfba43f
--- /dev/null
+++ b/utils/mergeipaddrlist.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+
+import sys
+import socket
+import struct
+import netaddr
+
+if len (sys.argv) == 3:
+    proto = sys.argv[2]
+
+if len(sys.argv) < 2:
+    print("Usage: mergeipaddrlist.py <file>")
+    sys.exit (1)
+
+ipFile = open(sys.argv[1])
+ipAddresses = list(ipFile.readlines())
+ipAddresses = sorted(ipAddresses)
+cidrs = netaddr.cidr_merge(ipAddresses)
+for cidr in cidrs:
+    print(cidr)
diff --git a/utils/update_every_content_match_lists.sh b/utils/update_every_lists.sh
index 74c8e9ca2..cbc3bf68d 100755
--- a/utils/update_every_content_match_lists.sh
+++ b/utils/update_every_lists.sh
@@ -1,6 +1,6 @@
 #/bin/sh
 
-cd "$(dirname "${0}")"
+cd "$(dirname "${0}")" || return
 
 ./aws_ip_addresses_download.sh
 ./azure_ip_addresses_download.sh
@@ -12,3 +12,5 @@ cd "$(dirname "${0}")"
 ./zoom_ip_addresses_download.sh
 ./google_cloud_ip_addresses_download.sh
 ./google_ip_addresses_download.sh
+
+./asn_update.sh