Ip address list: aggregate Mullvad and Tor lists too (#2154)

Missing from bdb73db1a
See #2150

3 files changed, 14 insertions, 4 deletions

diff --git a/utils/mergeipaddrlist.py b/utils/mergeipaddrlist.py
index cef9b4660..9fced5600 100755
--- a/utils/mergeipaddrlist.py
+++ b/utils/mergeipaddrlist.py
@@ -14,6 +14,7 @@ if len(sys.argv) < 2:
 
 ipFile = open(sys.argv[1])
 ipAddresses = list(ipFile.readlines())
+ipAddresses = [x.replace("\n","") for x in ipAddresses]
 ipAddresses = sorted(ipAddresses)
 cidrs = netaddr.cidr_merge(ipAddresses)
 
diff --git a/utils/mullvad_ip_addresses_download.sh b/utils/mullvad_ip_addresses_download.sh
index ec7597acc..941f50a27 100755
--- a/utils/mullvad_ip_addresses_download.sh
+++ b/utils/mullvad_ip_addresses_download.sh
@@ -9,6 +9,8 @@ DEST=../src/lib/inc_generated/ndpi_mullvad_match.c.inc
 TMP=/tmp/mullvad.json
 LIST=/tmp/mullvad.list
 LIST6=/tmp/mullvad.list6
+LIST_MERGED=/tmp/mullvad.list_m
+LIST6_MERGED=/tmp/mullvad.list6_m
 ORIGIN=https://api-www.mullvad.net/www/relays/all/
 
 
@@ -20,12 +22,16 @@ is_file_empty "${TMP}"
 echo "(2) Processing IP addresses..."
 jq -r '.[].ipv4_addr_in' $TMP > $LIST
 is_file_empty "${LIST}"
+./mergeipaddrlist.py $LIST > $LIST_MERGED
+is_file_empty "${LIST_MERGED}"
 jq -r '.[].ipv6_addr_in | select( . != null )' $TMP > $LIST6
 is_file_empty "${LIST6}"
-./ipaddr2list.py $LIST NDPI_PROTOCOL_MULLVAD $LIST6 > $DEST
+./mergeipaddrlist.py $LIST6 > $LIST6_MERGED
+is_file_empty "${LIST6_MERGED}"
+./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_MULLVAD $LIST6_MERGED > $DEST
 is_file_empty "${DEST}"
 
-rm -f $TMP $LIST $LIST6
+rm -f $TMP $LIST $LIST6 $LIST_MERGED $LIST6_MERGED
 
 echo "(3) Mullvad IPs are available in $DEST"
 exit 0
diff --git a/utils/tor_ip_addresses_download.sh b/utils/tor_ip_addresses_download.sh
index 602556a4e..cfdba68e2 100755
--- a/utils/tor_ip_addresses_download.sh
+++ b/utils/tor_ip_addresses_download.sh
@@ -7,6 +7,7 @@ cd "$(dirname "${0}")" || exit 1
 
 DEST=../src/lib/inc_generated/ndpi_tor_match.c.inc
 LIST=/tmp/tor.list
+LIST_MERGED=/tmp/tor.list_m
 # There are at least two lists:
 #  * https://torstatus.rueckgr.at/ip_list_all.php/Tor_ip_list_ALL.csv
 #  * https://check.torproject.org/torbulkexitlist
@@ -18,12 +19,14 @@ echo "(1) Downloading file... ${ORIGIN}"
 http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
 check_http_response "${http_response}"
 is_file_empty "${LIST}"
+./mergeipaddrlist.py $LIST > $LIST_MERGED
+is_file_empty "${LIST_MERGED}"
 
 #TODO: TOR relays don't support ipv6 yet
 
 echo "(2) Processing IP addresses..."
-./ipaddr2list.py $LIST NDPI_PROTOCOL_TOR > $DEST
-rm -f "${LIST}"
+./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_TOR > $DEST
+rm -f "${LIST}" "${LIST_MERGED}"
 is_file_empty "${DEST}"
 
 echo "(3) TOR IPs are available in $DEST"