IP lists: aggregate addresses wherever possible (#2152)

See #2150

11 files changed, 95 insertions, 24 deletions

diff --git a/utils/aws_ip_addresses_download.sh b/utils/aws_ip_addresses_download.sh
index 04a6a790d..f413e8c6f 100755
--- a/utils/aws_ip_addresses_download.sh
+++ b/utils/aws_ip_addresses_download.sh
@@ -9,6 +9,8 @@ DEST=../src/lib/inc_generated/ndpi_amazon_aws_match.c.inc
 TMP=/tmp/aws.json
 LIST=/tmp/aws.list
 LIST6=/tmp/aws.list6
+LIST_MERGED=/tmp/aws.list_m
+LIST6_MERGED=/tmp/aws.list6_m
 ORIGIN=https://ip-ranges.amazonaws.com/ip-ranges.json
 
 
@@ -20,12 +22,16 @@ is_file_empty "${TMP}"
 echo "(2) Processing IP addresses..."
 jq -r '.prefixes | .[].ip_prefix' $TMP > $LIST
 is_file_empty "${LIST}"
+./mergeipaddrlist.py $LIST > $LIST_MERGED
+is_file_empty "${LIST_MERGED}"
 jq -r '.ipv6_prefixes | .[].ipv6_prefix' $TMP > $LIST6
 is_file_empty "${LIST6}"
-./ipaddr2list.py $LIST NDPI_PROTOCOL_AMAZON_AWS $LIST6 > $DEST
+./mergeipaddrlist.py $LIST6 > $LIST6_MERGED
+is_file_empty "${LIST6_MERGED}"
+./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_AMAZON_AWS $LIST6_MERGED > $DEST
 is_file_empty "${DEST}"
 
-rm -f $TMP $LIST $LIST6
+rm -f ${TMP} ${LIST} ${LIST6} ${LIST_MERGED} ${LIST_MERGED6}
 
 echo "(3) Amazon AWS IPs are available in $DEST"
 exit 0
diff --git a/utils/azure_ip_addresses_download.sh b/utils/azure_ip_addresses_download.sh
index 4aada03ee..243e11fd3 100755
--- a/utils/azure_ip_addresses_download.sh
+++ b/utils/azure_ip_addresses_download.sh
@@ -10,6 +10,8 @@ LINK_TMP=/tmp/azure_link.txt
 TMP=/tmp/azure.json
 LIST=/tmp/azure.list
 LIST6=/tmp/azure.list6
+LIST_MERGED=/tmp/azure.list_m
+LIST6_MERGED=/tmp/azure.list6_m
 # https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519
 # Azure links have the format https://download.microsoft.com/download/7/1/D/71D86715-5596-4529-9B13-DA13A5DE5B63/ServiceTags_Public_<date>.json
 LINK_ORIGIN="https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519"
@@ -31,12 +33,16 @@ is_file_empty "${TMP}"
 echo "(3) Processing IP addresses..."
 tr -d '\r' < $TMP | grep / | tr -d '"' | tr -d " " | tr -d "," | grep -v : > $LIST
 is_file_empty "${LIST}"
+./mergeipaddrlist.py $LIST > $LIST_MERGED
+is_file_empty "${LIST_MERGED}"
 tr -d '\r' < $TMP | grep / | tr -d '"' | tr -d " " | tr -d "," | grep : > $LIST6
 is_file_empty "${LIST6}"
-./ipaddr2list.py $LIST NDPI_PROTOCOL_MICROSOFT_AZURE $LIST6 > $DEST
+./mergeipaddrlist.py $LIST6 > $LIST6_MERGED
+is_file_empty "${LIST6_MERGED}"
+./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_MICROSOFT_AZURE $LIST6_MERGED > $DEST
 is_file_empty "${DEST}"
 
-rm -f $TMP $LIST $LIST6
+rm -f ${TMP} ${LIST} ${LIST6} ${LIST_MERGED} ${LIST_MERGED6}
 
 echo "(4) Microsoft Azure IPs are available in $DEST"
 exit 0
diff --git a/utils/cachefly_ip_addresses_download.sh b/utils/cachefly_ip_addresses_download.sh
index ade534698..6e27b9ff7 100755
--- a/utils/cachefly_ip_addresses_download.sh
+++ b/utils/cachefly_ip_addresses_download.sh
@@ -7,6 +7,7 @@ cd "$(dirname "${0}")" || exit 1
 
 DEST=../src/lib/inc_generated/ndpi_cachefly_match.c.inc
 LIST=/tmp/cachefly.list
+LIST_MERGED=/tmp/cachefly.list_m
 ORIGIN='https://cachefly.cachefly.net/ips/cdn.txt'
 #TODO: ipv6. Is there any ipv6 list?
 
@@ -14,10 +15,12 @@ echo "(1) Downloading file..."
 http_response=$(curl -s -o "${LIST}" -w "%{http_code}" "${ORIGIN}")
 check_http_response "${http_response}"
 is_file_empty "${LIST}"
+./mergeipaddrlist.py $LIST > $LIST_MERGED
+is_file_empty "${LIST_MERGED}"
 
 echo "(2) Processing IP addresses..."
-./ipaddr2list.py "${LIST}" NDPI_PROTOCOL_CACHEFLY > "${DEST}"
-rm -f "${LIST}"
+./ipaddr2list.py "${LIST_MERGED}" NDPI_PROTOCOL_CACHEFLY > "${DEST}"
+rm -f "${LIST}" "${LIST_MERGED}"
 is_file_empty "${DEST}"
 
 echo "(3) Cachefly IPs are available in ${DEST}"
diff --git a/utils/cloudflare_ip_addresses_download.sh b/utils/cloudflare_ip_addresses_download.sh
index c1aca0165..15a20ba9c 100755
--- a/utils/cloudflare_ip_addresses_download.sh
+++ b/utils/cloudflare_ip_addresses_download.sh
@@ -8,6 +8,8 @@ cd "$(dirname "${0}")" || exit 1
 DEST=../src/lib/inc_generated/ndpi_cloudflare_match.c.inc
 LIST=/tmp/cloudflare.list
 LIST6=/tmp/cloudflare.list6
+LIST_MERGED=/tmp/cloudflare.list_m
+LIST6_MERGED=/tmp/cloudflare.list6_m
 ORIGIN="https://www.cloudflare.com/ips-v4"
 ORIGIN6="https://www.cloudflare.com/ips-v6"
 
@@ -15,14 +17,18 @@ echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
 check_http_response "${http_response}"
 is_file_empty "${LIST}"
+./mergeipaddrlist.py "${LIST}" > "${LIST_MERGED}"
+is_file_empty "${LIST_MERGED}"
 
 http_response=$(curl -s -o $LIST6 -w "%{http_code}" ${ORIGIN6})
 check_http_response "${http_response}"
 is_file_empty "${LIST6}"
+./mergeipaddrlist.py "${LIST6}" > "${LIST6_MERGED}"
+is_file_empty "${LIST6_MERGED}"
 
 echo "(2) Processing IP addresses..."
-./ipaddr2list.py $LIST NDPI_PROTOCOL_CLOUDFLARE $LIST6 > $DEST
-rm -f $LIST
+./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_CLOUDFLARE $LIST6_MERGED > $DEST
+rm -f $LIST $LIST_MERGED $LIST6_MERGED
 is_file_empty "${DEST}"
 
 echo "(3) Cloudflare IPs are available in $DEST"
diff --git a/utils/crawlers_ip_addresses_download.sh b/utils/crawlers_ip_addresses_download.sh
index 88ff4aece..45dbdcd4a 100755
--- a/utils/crawlers_ip_addresses_download.sh
+++ b/utils/crawlers_ip_addresses_download.sh
@@ -13,6 +13,8 @@ TMP_BING=/tmp/bot_bing.json
 TMP_FB=/tmp/bot_fb.list
 LIST=/tmp/bot.list
 LIST6=/tmp/bot.list6
+LIST_MERGED=/tmp/bot.list_m
+LIST6_MERGED=/tmp/bot.list6_m
 #Google Common crawlers
 ORIGIN1="https://developers.google.com/static/search/apis/ipranges/googlebot.json"
 #Google Special-case crawlers
@@ -56,6 +58,8 @@ echo "(2) Processing IP addresses..."
     grep -v route6 $TMP_FB | tr -d 'route:^ '
 } > $LIST
 is_file_empty "${LIST}"
+./mergeipaddrlist.py "${LIST}" > "${LIST_MERGED}"
+is_file_empty "${LIST_MERGED}"
 {
     jq -r '.prefixes | .[].ipv6Prefix  | select( . != null )' $TMP1
     jq -r '.prefixes | .[].ipv6Prefix  | select( . != null )' $TMP2
@@ -64,9 +68,11 @@ is_file_empty "${LIST}"
     grep route6 $TMP_FB | cut -c9- | tr -d ' '
 } > $LIST6
 is_file_empty "${LIST6}"
-./ipaddr2list.py $LIST NDPI_HTTP_CRAWLER_BOT $LIST6 > $DEST
+./mergeipaddrlist.py "${LIST6}" > "${LIST6_MERGED}"
+is_file_empty "${LIST6_MERGED}"
+./ipaddr2list.py $LIST_MERGED NDPI_HTTP_CRAWLER_BOT $LIST6_MERGED > $DEST
 is_file_empty "${DEST}"
-rm -f $TMP1 $TMP2 $TMP3 $TMP_BING $TMP_FB $LIST $LIST6
+rm -f $TMP1 $TMP2 $TMP3 $TMP_BING $TMP_FB $LIST $LIST6 $LIST_MERGED $LIST6_MERGED
 
 echo "(3) Crawlers IPs are available in $DEST"
 exit 0
diff --git a/utils/ethereum_ip_addresses_download.sh b/utils/ethereum_ip_addresses_download.sh
index 95f11d052..df7844a16 100755
--- a/utils/ethereum_ip_addresses_download.sh
+++ b/utils/ethereum_ip_addresses_download.sh
@@ -8,6 +8,7 @@ cd "$(dirname "${0}")" || exit 1
 DEST=../src/lib/inc_generated/ndpi_ethereum_match.c.inc
 TMP=/tmp/ethereum
 LIST=/tmp/ethereum.list
+LIST_MERGED=/tmp/ethereum.list_m
 ORIGIN="https://raw.githubusercontent.com/ethereum/go-ethereum/master/params/bootnodes.go"
 
 
@@ -19,9 +20,11 @@ is_file_empty "${TMP}"
 echo "(2) Processing IP addresses..."
 grep 'enode' $TMP | grep -v '^/' | grep ':' | cut -d '@' -f 2 | cut -d ':' -f 1 > $LIST #no ipv6 in this list
 is_file_empty "${LIST}"
+./mergeipaddrlist.py $LIST > $LIST_MERGED
+is_file_empty "${LIST_MERGED}"
 
-./ipaddr2list.py $LIST NDPI_PROTOCOL_ETHEREUM > $DEST
-rm -f $TMP $LIST
+./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_ETHEREUM > $DEST
+rm -f $TMP $LIST $LIST_MERGED
 is_file_empty "${DEST}"
 
 echo "(3) Ethereum IPs are available in $DEST"
diff --git a/utils/google_cloud_ip_addresses_download.sh b/utils/google_cloud_ip_addresses_download.sh
index d2efb2dcb..41fce68c5 100755
--- a/utils/google_cloud_ip_addresses_download.sh
+++ b/utils/google_cloud_ip_addresses_download.sh
@@ -9,6 +9,8 @@ DEST=../src/lib/inc_generated/ndpi_google_cloud_match.c.inc
 TMP=/tmp/google_c.json
 LIST=/tmp/google_c.list
 LIST6=/tmp/google_c.list6
+LIST_MERGED=/tmp/google_c.list_m
+LIST6_MERGED=/tmp/google_c.list6_m
 ORIGIN="https://www.gstatic.com/ipranges/cloud.json"
 
 
@@ -21,10 +23,16 @@ fi
 
 echo "(2) Processing IP addresses..."
 jq -r '.prefixes | .[].ipv4Prefix  | select( . != null )' $TMP > $LIST
+is_file_empty "${LIST}"
+./mergeipaddrlist.py $LIST > $LIST_MERGED
+is_file_empty "${LIST_MERGED}"
 jq -r '.prefixes | .[].ipv6Prefix  | select( . != null )' $TMP > $LIST6
-./ipaddr2list.py $LIST NDPI_PROTOCOL_GOOGLE_CLOUD $LIST6 > $DEST
+is_file_empty "${LIST6}"
+./mergeipaddrlist.py $LIST6 > $LIST6_MERGED
+is_file_empty "${LIST6_MERGED}"
+./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_GOOGLE_CLOUD $LIST6_MERGED > $DEST
 
-rm -f $TMP $LIST $LIST6
+rm -f $TMP $LIST $LIST6 $LIST_MERGED $LIST6_MERGED
 
 echo "(3) Google Cloud IPs are available in $DEST"
 exit 0
diff --git a/utils/google_ip_addresses_download.sh b/utils/google_ip_addresses_download.sh
index 0b586ee2c..1c3e76948 100755
--- a/utils/google_ip_addresses_download.sh
+++ b/utils/google_ip_addresses_download.sh
@@ -8,6 +8,8 @@ cd "$(dirname "${0}")" || exit 1
 DEST=../src/lib/inc_generated/ndpi_google_match.c.inc
 LIST=/tmp/google.list
 LIST6=/tmp/google.list6
+LIST_MERGED=/tmp/google.list_m
+LIST6_MERGED=/tmp/google.list6_m
 
 echo "(1) Downloading file..."
 #Nothing to do
@@ -16,9 +18,13 @@ echo "(2) Processing IP addresses..."
 #https://cloud.google.com/vpc/docs/configure-private-google-access#ip-addr-defaults
 python3 google.py > $LIST
 is_file_empty "${LIST}"
+./mergeipaddrlist.py $LIST > $LIST_MERGED
+is_file_empty "${LIST_MERGED}"
 python3 google6.py > $LIST6
 is_file_empty "${LIST6}"
-./ipaddr2list.py $LIST NDPI_PROTOCOL_GOOGLE $LIST6 > $DEST
+./mergeipaddrlist.py $LIST6 > $LIST6_MERGED
+is_file_empty "${LIST6_MERGED}"
+./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_GOOGLE $LIST6_MERGED > $DEST
 is_file_empty "${DEST}"
 
 rm -f $TMP $LIST $LIST6
diff --git a/utils/microsoft_ip_addresses_download.sh b/utils/microsoft_ip_addresses_download.sh
index 51ef89b8d..4365732bf 100755
--- a/utils/microsoft_ip_addresses_download.sh
+++ b/utils/microsoft_ip_addresses_download.sh
@@ -12,6 +12,8 @@ DEST_OFFICE365=../src/lib/inc_generated/ndpi_ms_office365_match.c.inc
 TMP=/tmp/ms.json
 LIST=/tmp/ms.list
 LIST6=/tmp/ms.list6
+LIST_MERGED=/tmp/ms.list_m
+LIST6_MERGED=/tmp/ms.list6_m
 # https://docs.microsoft.com/en-us/microsoft-365/enterprise/urls-and-ip-address-ranges?view=o365-worldwide
 ORIGIN="https://endpoints.office.com/endpoints/worldwide?clientrequestid=b10c5ed1-bad1-445f-b386-b919946339a7"
 
@@ -26,37 +28,53 @@ echo "(2) Processing IP addresses..."
 #OUTLOOK
 jq -r '.[] | select(.serviceArea=="Exchange") | .ips[]?' < $TMP | grep -v ':' | sort -u | uniq > $LIST
 is_file_empty "${LIST}"
+./mergeipaddrlist.py $LIST > $LIST_MERGED
+is_file_empty "${LIST_MERGED}"
 jq -r '.[] | select(.serviceArea=="Exchange") | .ips[]?' < $TMP | grep ':' | sort -u | uniq > $LIST6
 is_file_empty "${LIST6}"
-./ipaddr2list.py $LIST NDPI_PROTOCOL_MS_OUTLOOK $LIST6 > $DEST_OUTLOOK
+./mergeipaddrlist.py $LIST6 > $LIST6_MERGED
+is_file_empty "${LIST6_MERGED}"
+./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_MS_OUTLOOK $LIST6_MERGED > $DEST_OUTLOOK
 is_file_empty "${DEST_OUTLOOK}"
 
 #SKYPE/TEAMS
 jq -r '.[] | select(.serviceArea=="Skype") | .ips[]?' < $TMP | grep -v ':' | sort -u | uniq > $LIST
 is_file_empty "${LIST}"
+./mergeipaddrlist.py $LIST > $LIST_MERGED
+is_file_empty "${LIST_MERGED}"
 jq -r '.[] | select(.serviceArea=="Skype") | .ips[]?' < $TMP | grep ':' | sort -u | uniq > $LIST6
 is_file_empty "${LIST6}"
-./ipaddr2list.py $LIST NDPI_PROTOCOL_SKYPE_TEAMS $LIST6 > $DEST_SKYPE_MSTEAMS
+./mergeipaddrlist.py $LIST6 > $LIST6_MERGED
+is_file_empty "${LIST6_MERGED}"
+./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_SKYPE_TEAMS $LIST6_MERGED > $DEST_SKYPE_MSTEAMS
 is_file_empty "${DEST_SKYPE_MSTEAMS}"
 
 #ONEDRIVE
 jq -r '.[] | select(.serviceArea=="SharePoint") | .ips[]?' < $TMP | grep -v ':' | sort -u | uniq > $LIST
 is_file_empty "${LIST}"
+./mergeipaddrlist.py $LIST > $LIST_MERGED
+is_file_empty "${LIST_MERGED}"
 jq -r '.[] | select(.serviceArea=="SharePoint") | .ips[]?' < $TMP | grep ':' | sort -u | uniq > $LIST6
 is_file_empty "${LIST6}"
-./ipaddr2list.py $LIST NDPI_PROTOCOL_MS_ONE_DRIVE $LIST6 > $DEST_ONEDRIVE
+./mergeipaddrlist.py $LIST6 > $LIST6_MERGED
+is_file_empty "${LIST6_MERGED}"
+./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_MS_ONE_DRIVE $LIST6_MERGED > $DEST_ONEDRIVE
 is_file_empty "${DEST_ONEDRIVE}"
 
 #OFFICE
 jq -r '.[] | select(.serviceArea=="Common") | .ips[]?' < $TMP | grep -v ':' | sort -u | uniq > $LIST
 is_file_empty "${LIST}"
+./mergeipaddrlist.py $LIST > $LIST_MERGED
+is_file_empty "${LIST_MERGED}"
 jq -r '.[] | select(.serviceArea=="Common") | .ips[]?' < $TMP | grep ':' | sort -u | uniq > $LIST6
 is_file_empty "${LIST6}"
+./mergeipaddrlist.py $LIST6 > $LIST6_MERGED
+is_file_empty "${LIST6_MERGED}"
 #TODO: NDPI_PROTOCOL_MICROSOFT_365 or NDPI_PROTOCOL_MICROSOFT?
-./ipaddr2list.py $LIST NDPI_PROTOCOL_MICROSOFT_365 $LIST6 > $DEST_OFFICE365
+./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_MICROSOFT_365 $LIST6_MERGED > $DEST_OFFICE365
 is_file_empty "${DEST_OFFICE365}"
 
-rm -f ${TMP} ${LIST} ${LIST6}
+rm -f ${TMP} ${LIST} ${LIST6} ${LIST_MERGED} ${LIST_MERGED6}
 
 echo "(3) Microsoft IPs are available in ${DEST_OUTLOOK}, ${DEST_SKYPE_MSTEAMS}, ${DEST_ONEDRIVE}, ${DEST_OFFICE365}"
 exit 0
diff --git a/utils/whatsapp_ip_addresses_download.sh b/utils/whatsapp_ip_addresses_download.sh
index c3b251e6e..cddb21389 100755
--- a/utils/whatsapp_ip_addresses_download.sh
+++ b/utils/whatsapp_ip_addresses_download.sh
@@ -8,6 +8,7 @@ cd "$(dirname "${0}")" || exit 1
 DEST=../src/lib/inc_generated/ndpi_whatsapp_match.c.inc
 TMP=/tmp/wa.zip
 LIST=/tmp/wa.list
+LIST_MERGED=/tmp/wa.list_m
 IP_LINK_URL='https://developers.facebook.com/docs/whatsapp/guides/network-requirements/'
 
 
@@ -23,8 +24,10 @@ is_file_empty "${TMP}"
 echo "(3) Processing IP addresses..."
 unzip -p /tmp/wa.zip "WhatsApp IPs (IPv4 Only) 2022-07-26 - 2022-07-30.txt" > "${LIST}" #TODO: ipv6
 is_file_empty "${LIST}"
-./ipaddr2list.py "${LIST}" NDPI_PROTOCOL_WHATSAPP > "${DEST}"
-rm -f "${TMP}" "${LIST}"
+./mergeipaddrlist.py $LIST > $LIST_MERGED
+is_file_empty "${LIST_MERGED}"
+./ipaddr2list.py "${LIST_MERGED}" NDPI_PROTOCOL_WHATSAPP > "${DEST}"
+rm -f "${TMP}" "${LIST}" "${LIST_MERGED}"
 is_file_empty "${DEST}"
 
 echo "(4) WhatsApp IPs are available in $DEST"
diff --git a/utils/zoom_ip_addresses_download.sh b/utils/zoom_ip_addresses_download.sh
index 10cd602e2..b527abdad 100755
--- a/utils/zoom_ip_addresses_download.sh
+++ b/utils/zoom_ip_addresses_download.sh
@@ -8,6 +8,8 @@ cd "$(dirname "${0}")" || exit 1
 DEST=../src/lib/inc_generated/ndpi_zoom_match.c.inc
 LIST=/tmp/zoom.list
 LIST6=/tmp/zoom.list6
+LIST_MERGED=/tmp/zoom.list_m
+LIST6_MERGED=/tmp/zoom.list6_m
 # https://support.zoom.us/hc/en-us/articles/201362683-Zoom-network-firewall-or-proxy-server-settings
 # There are few lists in this page, partially overlapping. Pick the generic one
 ORIGIN="https://assets.zoom.us/docs/ipranges/Zoom.txt"
@@ -17,6 +19,8 @@ echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o "${LIST}" -w "%{http_code}" "${ORIGIN}")
 check_http_response "${http_response}"
 is_file_empty "${LIST}"
+./mergeipaddrlist.py $LIST > $LIST_MERGED
+is_file_empty "${LIST_MERGED}"
 
 # IPv6: in that page there is a few IPv6 prefixes but these prefixes are not present in
 # the "main" list!
@@ -24,10 +28,12 @@ is_file_empty "${LIST}"
 echo "2620:123:2000::/40" > $LIST6
 echo "2600:9000:2600::/48" >> $LIST6
 echo "2407:30C0::/32" >> $LIST6
+./mergeipaddrlist.py $LIST6 > $LIST6_MERGED
+is_file_empty "${LIST6_MERGED}" 
 
 echo "(2) Processing IP addresses..."
-./ipaddr2list.py "${LIST}" NDPI_PROTOCOL_ZOOM "${LIST6}" > "${DEST}"
-rm -f "${LIST}" "${LIST6}"
+./ipaddr2list.py "${LIST_MERGED}" NDPI_PROTOCOL_ZOOM "${LIST6_MERGED}" > "${DEST}"
+rm -f "${LIST}" "${LIST6}" "${LIST_MERGED}" "${LIST6_MERGED}"
 is_file_empty "${DEST}"
 
 echo "(3) ZOOM IPs are available in ${DEST}"