From 8b062295cc76a60e3905c054ce37bd17669464d1 Mon Sep 17 00:00:00 2001
From: Ivan Nardi <12729895+IvanNardi@users.noreply.github.com>
Date: Fri, 25 Feb 2022 14:26:26 +0100
Subject: Add some scripts to easily update some IPs lists (#1449)

While the lists in a6ff0dd0 and 2f5f445f are somehow provided by the
companies themselves (or by some interested parties), these new lists
are directly extracted from BGP information, via AS prefixes.

*Usually*, these new lists are far more stable than the previous ones.

TODO:
* add some other ASNs (see `src/lib/ndpi_content_match.c.inc`)
* IPv6, as usual :-(
---
 utils/asn_update.sh                       | 58 +++++++++++++++++++++++++++++++
 utils/get_routes_by_asn.sh                | 19 ++++++++++
 utils/mergeipaddrlist.py                  | 20 +++++++++++
 utils/update_every_content_match_lists.sh | 14 --------
 utils/update_every_lists.sh               | 16 +++++++++
 5 files changed, 113 insertions(+), 14 deletions(-)
 create mode 100755 utils/asn_update.sh
 create mode 100755 utils/get_routes_by_asn.sh
 create mode 100755 utils/mergeipaddrlist.py
 delete mode 100755 utils/update_every_content_match_lists.sh
 create mode 100755 utils/update_every_lists.sh

(limited to 'utils')

diff --git a/utils/asn_update.sh b/utils/asn_update.sh
new file mode 100755
index 000000000..290204bf5
--- /dev/null
+++ b/utils/asn_update.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+function processing_list() {
+	local LIST_MERGED="/tmp/list_m"
+
+	echo "(2) Processing IP addresses..."
+	./mergeipaddrlist.py "$1" > $LIST_MERGED
+	./ipaddr2list.py "$LIST_MERGED" "$2" > "$3"
+	rm -f $LIST_MERGED
+}
+
+function create_list() {
+	LIST=/tmp/list
+
+	for i in "${@:3}"; do
+		./get_routes_by_asn.sh "$i" >> $LIST
+	done
+
+	processing_list "$LIST" "$1" "$2"
+	rm -f $LIST
+}
+
+cd "$(dirname "${0}")" || return
+
+echo "(1) Downloading Apple routes..."
+DEST="../src/lib/ndpi_asn_apple.c.inc"
+create_list NDPI_PROTOCOL_APPLE $DEST "AS714" "AS6185" "AS2709"
+echo "(3) Apple IPs are available in $DEST"
+
+echo "(1) Downloading Facebook routes..."
+DEST=../src/lib/ndpi_asn_facebook.c.inc
+create_list NDPI_PROTOCOL_FACEBOOK $DEST "AS63293" "AS54115" "AS34825" "AS32934"
+echo "(3) Facebook IPs are available in $DEST"
+
+echo "(1) Downloading Netflix routes..."
+DEST=../src/lib/ndpi_asn_netflix.c.inc
+create_list NDPI_PROTOCOL_NETFLIX $DEST "AS55095" "AS40027" "AS394406" "AS2906"
+echo "(3) Netflix IPs are available in $DEST"
+
+echo "(1) Downloading Teamviewer routes..."
+DEST=../src/lib/ndpi_asn_teamviewer.c.inc
+create_list NDPI_PROTOCOL_TEAMVIEWER $DEST "AS43304" "AS212710" "AS208187" "AS208175"
+echo "(3) Teamviewer IPs are available in $DEST"
+
+echo "(1) Downloading Telegram routes..."
+DEST=../src/lib/ndpi_asn_telegram.c.inc
+create_list NDPI_PROTOCOL_TELEGRAM $DEST "AS62041" "AS62014" "AS59930" "AS44907" "AS211157"
+echo "(3) Telegram IPs are available in $DEST"
+
+echo "(1) Downloading Twitter routes..."
+DEST=../src/lib/ndpi_asn_twitter.c.inc
+create_list NDPI_PROTOCOL_TWITTER $DEST "AS63179" "AS54888" "AS35995" "AS13414"
+echo "(3) Twitter IPs are available in $DEST"
+
+echo "(1) Downloading Webex routes..."
+DEST=../src/lib/ndpi_asn_webex.c.inc
+create_list NDPI_PROTOCOL_WEBEX $DEST "AS6577" "AS399937" "AS16472" "AS13445"
+echo "(3) Webex IPs are available in $DEST"
diff --git a/utils/get_routes_by_asn.sh b/utils/get_routes_by_asn.sh
new file mode 100755
index 000000000..f9292828b
--- /dev/null
+++ b/utils/get_routes_by_asn.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 AS-Number" >&2
+    return
+fi
+
+LIST=/tmp/asn.json
+ORIGIN="https://stat.ripe.net/data/announced-prefixes/data.json?resource=$1"
+
+http_response=$(curl -s -o "${LIST}" -w "%{http_code}" "${ORIGIN}")
+if [ "$http_response" != "200" ]; then
+    echo "Error $http_response: wrong ASN number/format?" >&2
+    return
+fi
+
+jq -r '.data.prefixes[].prefix' $LIST | grep -v ":"
+
+rm -f $LIST
diff --git a/utils/mergeipaddrlist.py b/utils/mergeipaddrlist.py
new file mode 100755
index 000000000..36dfba43f
--- /dev/null
+++ b/utils/mergeipaddrlist.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+
+import sys
+import socket
+import struct
+import netaddr
+
+if len (sys.argv) == 3:
+    proto = sys.argv[2]
+
+if len(sys.argv) < 2:
+    print("Usage: mergeipaddrlist.py <file>")
+    sys.exit (1)
+
+ipFile = open(sys.argv[1])
+ipAddresses = list(ipFile.readlines())
+ipAddresses = sorted(ipAddresses)
+cidrs = netaddr.cidr_merge(ipAddresses)
+for cidr in cidrs:
+    print(cidr)
diff --git a/utils/update_every_content_match_lists.sh b/utils/update_every_content_match_lists.sh
deleted file mode 100755
index 74c8e9ca2..000000000
--- a/utils/update_every_content_match_lists.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#/bin/sh
-
-cd "$(dirname "${0}")"
-
-./aws_ip_addresses_download.sh
-./azure_ip_addresses_download.sh
-./cloudflare_ip_addresses_download.sh
-./ethereum_ip_addresses_download.sh
-./microsoft_ip_addresses_download.sh
-./tor_ip_addresses_download.sh
-./whatsapp_ip_addresses_download.sh
-./zoom_ip_addresses_download.sh
-./google_cloud_ip_addresses_download.sh
-./google_ip_addresses_download.sh
diff --git a/utils/update_every_lists.sh b/utils/update_every_lists.sh
new file mode 100755
index 000000000..cbc3bf68d
--- /dev/null
+++ b/utils/update_every_lists.sh
@@ -0,0 +1,16 @@
+#/bin/sh
+
+cd "$(dirname "${0}")" || return
+
+./aws_ip_addresses_download.sh
+./azure_ip_addresses_download.sh
+./cloudflare_ip_addresses_download.sh
+./ethereum_ip_addresses_download.sh
+./microsoft_ip_addresses_download.sh
+./tor_ip_addresses_download.sh
+./whatsapp_ip_addresses_download.sh
+./zoom_ip_addresses_download.sh
+./google_cloud_ip_addresses_download.sh
+./google_ip_addresses_download.sh
+
+./asn_update.sh
-- 
cgit v1.2.3