diff options
| -rw-r--r-- | configure.ac | 2 | ||||
| -rwxr-xr-x | utils/asn_update.sh | 7 | ||||
| -rwxr-xr-x | utils/aws_ip_addresses_download.sh | 13 | ||||
| -rwxr-xr-x | utils/azure_ip_addresses_download.sh | 24 | ||||
| -rwxr-xr-x | utils/bitcoinnodes.sh | 8 | ||||
| -rwxr-xr-x | utils/cachefly_ip_addresses_download.sh | 12 | ||||
| -rwxr-xr-x | utils/cloudflare_ip_addresses_download.sh | 12 | ||||
| -rwxr-xr-x | utils/common.sh | 39 | ||||
| -rwxr-xr-x | utils/crawlers_ip_addresses_download.sh | 30 | ||||
| -rwxr-xr-x | utils/ethereum_ip_addresses_download.sh | 13 | ||||
| -rwxr-xr-x | utils/gambling_sites_download.sh | 6 | ||||
| -rwxr-xr-x | utils/google_cloud_ip_addresses_download.sh | 3 | ||||
| -rwxr-xr-x | utils/google_ip_addresses_download.sh | 7 | ||||
| -rwxr-xr-x | utils/hostname2list.py | 6 | ||||
| -rwxr-xr-x | utils/icloud_private_relay_ip_addresses_download.sh | 14 | ||||
| -rwxr-xr-x | utils/ipaddr2list.py | 5 | ||||
| -rwxr-xr-x | utils/mergeipaddrlist.py | 6 | ||||
| -rwxr-xr-x | utils/microsoft_ip_addresses_download.sh | 21 | ||||
| -rwxr-xr-x | utils/tor_ip_addresses_download.sh | 14 | ||||
| -rwxr-xr-x | utils/whatsapp_ip_addresses_download.sh | 20 | ||||
| -rwxr-xr-x | utils/zoom_ip_addresses_download.sh | 20 |
21 files changed, 186 insertions, 96 deletions
diff --git a/configure.ac b/configure.ac index 57f83708c..ff4bae88f 100644 --- a/configure.ac +++ b/configure.ac @@ -87,6 +87,8 @@ LT_LIB_M PKG_PROG_PKG_CONFIG AC_PROG_CC +AC_PROG_CPP_WERROR +AC_C_INLINE SYSTEM=`uname -s` if test $SYSTEM = "Darwin"; then diff --git a/utils/asn_update.sh b/utils/asn_update.sh index 38e8b37c5..d5ad78ee8 100755 --- a/utils/asn_update.sh +++ b/utils/asn_update.sh @@ -1,4 +1,6 @@ -#!/bin/bash +#!/usr/bin/env bash + +set -e FAILED_ASN=0 TOTAL_ASN=0 @@ -28,6 +30,7 @@ function create_list() { } cd "$(dirname "${0}")" || exit 1 +. ./common.sh || exit 1 echo "(1) Downloading Apple routes..." DEST="../src/lib/inc_generated/ndpi_asn_apple.c.inc" @@ -194,7 +197,7 @@ DEST=../src/lib/inc_generated/ndpi_asn_nvidia.c.inc create_list NDPI_PROTOCOL_NVIDIA $DEST "AS60977" "AS50889" "AS20347" "AS11414" echo "(3) Nvidia IPs are available in $DEST" -if [ ${TOTAL_ASN} -eq ${FAILED_ASN} ]; then +if [ ${TOTAL_ASN} -eq 0 -o ${TOTAL_ASN} -eq ${FAILED_ASN} ]; then printf '%s: %s\n' "${0}" "All download(s) failed, ./get_routes_by_asn.sh broken?" exit 1 else diff --git a/utils/aws_ip_addresses_download.sh b/utils/aws_ip_addresses_download.sh index cc923a1a2..3379c96eb 100755 --- a/utils/aws_ip_addresses_download.sh +++ b/utils/aws_ip_addresses_download.sh @@ -1,6 +1,9 @@ -#!/bin/sh +#!/usr/bin/env bash + +set -e cd "$(dirname "${0}")" || exit 1 +. ./common.sh || exit 1 DEST=../src/lib/inc_generated/ndpi_amazon_aws_match.c.inc TMP=/tmp/aws.json @@ -10,15 +13,15 @@ ORIGIN=https://ip-ranges.amazonaws.com/ip-ranges.json echo "(1) Downloading file..." http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN}) -if [ $http_response != "200" ]; then - echo "Error $http_response: you probably need to update the list url!" - exit 1 -fi +check_http_response "${http_response}" +is_file_empty "${TMP}" echo "(2) Processing IP addresses..." jq -r '.prefixes | .[].ip_prefix' $TMP > $LIST # TODO: ipv6 +is_file_empty "${LIST}" ./ipaddr2list.py $LIST NDPI_PROTOCOL_AMAZON_AWS > $DEST rm -f $TMP $LIST +is_file_empty "${DEST}" echo "(3) Amazon AWS IPs are available in $DEST" exit 0 diff --git a/utils/azure_ip_addresses_download.sh b/utils/azure_ip_addresses_download.sh index 3e20b6119..7df5d9cca 100755 --- a/utils/azure_ip_addresses_download.sh +++ b/utils/azure_ip_addresses_download.sh @@ -1,8 +1,9 @@ -#!/bin/sh +#!/usr/bin/env bash set -e cd "$(dirname "${0}")" || exit 1 +. ./common.sh || exit 1 DEST=../src/lib/inc_generated/ndpi_azure_match.c.inc LINK_TMP=/tmp/azure_link.txt @@ -14,30 +15,25 @@ LINK_ORIGIN="https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519 echo "(1) Downloading file... ${LINK_ORIGIN}" http_response=$(curl -s -o ${LINK_TMP} -w "%{http_code}" ${LINK_ORIGIN}) -if [ "${http_response}" != "200" ]; then - echo "Error $http_response: you probably need to update the link origin url!" - exit 1 -fi +check_http_response "${http_response}" +is_file_empty "${LINK_TMP}" -ORIGIN="$(grep -E 'ServiceTags_Public_[[:digit:]]+.json' ${LINK_TMP} | grep -o -E 'href=\"[^"]+' | sed 's/href="//' | uniq)" +ORIGIN="$(grep -E 'ServiceTags_Public_[[:digit:]]+.json' ${LINK_TMP} | grep -o -E 'href="[^"]+' | sed 's/href="//' | uniq)" rm -f ${LINK_TMP} -if [ -z "${ORIGIN}" ]; then - echo "Error ${LINK_ORIGIN} does not contain the url format!" - exit 1 -fi +is_str_empty "${ORIGIN}" "${LINK_ORIGIN} does not contain the url format!" echo "(2) Downloading file... ${ORIGIN}" http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN}) -if [ "${http_response}" != "200" ]; then - echo "Error $http_response: you probably need to update the list url!" - exit 1 -fi +check_http_response "${http_response}" +is_file_empty "${TMP}" echo "(3) Processing IP addresses..." # Note: the last "grep -v :" is used to skip IPv6 addresses tr -d '\r' < $TMP | grep / | tr -d '"' | tr -d " " | tr -d "," | grep -v : > $LIST +is_file_empty "${LIST}" ./ipaddr2list.py $LIST NDPI_PROTOCOL_MICROSOFT_AZURE > $DEST rm -f $TMP $LIST +is_file_empty "${DEST}" echo "(4) Microsoft Azure IPs are available in $DEST" exit 0 diff --git a/utils/bitcoinnodes.sh b/utils/bitcoinnodes.sh index 03fbedd14..19b30b4d9 100755 --- a/utils/bitcoinnodes.sh +++ b/utils/bitcoinnodes.sh @@ -1,10 +1,14 @@ -#!/bin/sh +#!/usr/bin/env bash # # List all the current bittorrent nodes # set -e +cd "$(dirname "${0}")" || exit 1 +. ./common.sh || exit 1 + # NOTE: JQ can be found at https://stedolan.github.io/jq/ -curl -s -H "Accept: application/json; indent=4" https://bitnodes.io/api/v1/snapshots/latest/ | jq -r '.nodes|keys[] as $k | "\($k)"' | grep -v onion | grep -v ']' | cut -d ':' -f 1 +RESULT="$(curl -s -H "Accept: application/json; indent=4" https://bitnodes.io/api/v1/snapshots/latest/ | jq -r '.nodes|keys[] as $k | "\($k)"' | grep -v onion | grep -v ']' | cut -d ':' -f 1)" +is_str_empty "${RESULT}" "String empty, please review this script." diff --git a/utils/cachefly_ip_addresses_download.sh b/utils/cachefly_ip_addresses_download.sh index fa1992e64..9d14e379d 100755 --- a/utils/cachefly_ip_addresses_download.sh +++ b/utils/cachefly_ip_addresses_download.sh @@ -1,6 +1,9 @@ -#!/bin/sh +#!/usr/bin/env bash + +set -e cd "$(dirname "${0}")" || exit 1 +. ./common.sh || exit 1 DEST=../src/lib/inc_generated/ndpi_cachefly_match.c.inc LIST=/tmp/cachefly.list @@ -9,14 +12,13 @@ ORIGIN='https://cachefly.cachefly.net/ips/cdn.txt' echo "(1) Downloading file..." http_response=$(curl -s -o "${LIST}" -w "%{http_code}" "${ORIGIN}") -if [ "${http_response}" != "200" ]; then - echo "Error ${http_response}: you probably need to update the list url!" - exit 1 -fi +check_http_response "${http_response}" +is_file_empty "${LIST}" echo "(2) Processing IP addresses..." ./ipaddr2list.py "${LIST}" NDPI_PROTOCOL_CACHEFLY > "${DEST}" rm -f "${LIST}" +is_file_empty "${DEST}" echo "(3) Cachefly IPs are available in ${DEST}" exit 0 diff --git a/utils/cloudflare_ip_addresses_download.sh b/utils/cloudflare_ip_addresses_download.sh index 93915b6bc..aa34e11cb 100755 --- a/utils/cloudflare_ip_addresses_download.sh +++ b/utils/cloudflare_ip_addresses_download.sh @@ -1,6 +1,9 @@ -#!/bin/sh +#!/usr/bin/env bash + +set -e cd "$(dirname "${0}")" || exit 1 +. ./common.sh || exit 1 DEST=../src/lib/inc_generated/ndpi_cloudflare_match.c.inc LIST=/tmp/cloudflare.list @@ -10,14 +13,13 @@ ORIGIN="https://www.cloudflare.com/ips-v4" echo "(1) Downloading file... ${ORIGIN}" http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN}) -if [ $http_response != "200" ]; then - echo "Error $http_response: you probably need to update the list url!" - exit 1 -fi +check_http_response "${http_response}" +is_file_empty "${LIST}" echo "(2) Processing IP addresses..." ./ipaddr2list.py $LIST NDPI_PROTOCOL_CLOUDFLARE > $DEST rm -f $LIST +is_file_empty "${DEST}" echo "(3) Cloudflare IPs are available in $DEST" exit 0 diff --git a/utils/common.sh b/utils/common.sh new file mode 100755 index 000000000..415d6fe87 --- /dev/null +++ b/utils/common.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env sh + +printf 'Running script: %s\n' "$(basename ${0})" >&2 + +function check_http_response() +{ + http_response="${1}" + + if [ "${http_response}" != "200" ]; then + printf '%s error: %s\n' "${0}" "HTTP Response code ${http_response}; you probably need to update the list url!" >&2 + exit 1 + fi +} + +function is_file_empty() +{ + file="${1}" + + if [ ! -r "${file}" ]; then + printf '%s error: %s\n' "${0}" "file ${file} not found or not readable!" >&2 + exit 1 + fi + + if [ `cat "${file}" | wc -l` -eq 0 ]; then + printf '%s error: %s\n' "${0}" "file ${file} empty!" >&2 + exit 1 + fi +} + +function is_str_empty() +{ + str="${1}" + errmsg="${2}" + + if [ -z "${str}" ]; then + printf '%s error: %s\n' "${0}" "${errmsg}" >&2 + exit 1 + fi +} diff --git a/utils/crawlers_ip_addresses_download.sh b/utils/crawlers_ip_addresses_download.sh index 77e70c61b..50b8934a3 100755 --- a/utils/crawlers_ip_addresses_download.sh +++ b/utils/crawlers_ip_addresses_download.sh @@ -1,8 +1,9 @@ -#!/bin/sh +#!/usr/bin/env bash set -e cd "$(dirname "${0}")" || exit 1 +. ./common.sh || exit 1 DEST=../src/lib/inc_generated/ndpi_crawlers_match.c.inc TMP1=/tmp/bot_google_c1.json @@ -23,34 +24,27 @@ ORIGIN_BING="https://www.bing.com/toolbox/bingbot.json" echo "(1) Downloading file... ${ORIGIN1}" http_response=$(curl -s -o $TMP1 -w "%{http_code}" ${ORIGIN1}) -if [ "$http_response" != "200" ]; then - echo "Error $http_response: you probably need to update the list url!" - exit 1 -fi +check_http_response "${http_response}" +is_file_empty "${TMP1}" echo "(1) Downloading file... ${ORIGIN2}" http_response=$(curl -s -o $TMP2 -w "%{http_code}" ${ORIGIN2}) -if [ "$http_response" != "200" ]; then - echo "Error $http_response: you probably need to update the list url!" - exit 1 -fi +check_http_response "${http_response}" +is_file_empty "${TMP2}" echo "(1) Downloading file... ${ORIGIN3}" http_response=$(curl -s -o $TMP3 -w "%{http_code}" ${ORIGIN3}) -if [ "$http_response" != "200" ]; then - echo "Error $http_response: you probably need to update the list url!" - exit 1 -fi +check_http_response "${http_response}" +is_file_empty "${TMP3}" echo "(1) Downloading file... ${ORIGIN_BING}" http_response=$(curl -s -o $TMP_BING -w "%{http_code}" ${ORIGIN_BING}) -if [ "$http_response" != "200" ]; then - echo "Error $http_response: you probably need to update the list url!" - exit 1 -fi +check_http_response "${http_response}" +is_file_empty "${TMP_BING}" echo "(1) Downloading FB crawlers routes... " whois -h whois.radb.net -- '-i origin AS32934' | grep ^route > $TMP_FB +is_file_empty "${TMP_FB}" echo "(2) Processing IP addresses..." { @@ -60,7 +54,9 @@ echo "(2) Processing IP addresses..." jq -r '.prefixes | .[].ipv4Prefix | select( . != null )' $TMP_BING # TODO: ipv6 grep -v route6 $TMP_FB | tr -d 'route:^ ' # TODO: ipv6 } > $LIST +is_file_empty "${LIST}" ./ipaddr2list.py $LIST NDPI_HTTP_CRAWLER_BOT > $DEST +is_file_empty "${DEST}" rm -f $TMP1 $TMP2 $TMP3 $TMP_BING $TMP_FB $LIST echo "(3) Crawlers IPs are available in $DEST" diff --git a/utils/ethereum_ip_addresses_download.sh b/utils/ethereum_ip_addresses_download.sh index c6e88b1ce..172c656d7 100755 --- a/utils/ethereum_ip_addresses_download.sh +++ b/utils/ethereum_ip_addresses_download.sh @@ -1,6 +1,9 @@ -#!/bin/sh +#!/usr/bin/env bash + +set -e cd "$(dirname "${0}")" || exit 1 +. ./common.sh || exit 1 DEST=../src/lib/inc_generated/ndpi_ethereum_match.c.inc TMP=/tmp/ethereum @@ -10,16 +13,16 @@ ORIGIN="https://raw.githubusercontent.com/ethereum/go-ethereum/master/params/boo echo "(1) Downloading file... ${ORIGIN}" http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN}) -if [ $http_response != "200" ]; then - echo "Error $http_response: you probably need to update the list url!" - exit 1 -fi +check_http_response "${http_response}" +is_file_empty "${TMP}" echo "(2) Processing IP addresses..." grep 'enode' $TMP | grep -v '^/' | grep ':' | cut -d '@' -f 2 | cut -d ':' -f 1 > $LIST +is_file_empty "${LIST}" ./ipaddr2list.py $LIST NDPI_PROTOCOL_MINING > $DEST rm -f $TMP $LIST +is_file_empty "${DEST}" echo "(3) Ethereum/Mining IPs are available in $DEST" exit 0 diff --git a/utils/gambling_sites_download.sh b/utils/gambling_sites_download.sh index 614e7c694..3340cf237 100755 --- a/utils/gambling_sites_download.sh +++ b/utils/gambling_sites_download.sh @@ -1,17 +1,21 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash set -e cd "$(dirname "${0}")" || exit 1 +. ./common.sh || exit 1 DEST=../src/lib/inc_generated/ndpi_gambling_match.c.inc LIST=/tmp/gambling.list printf '(1) %s\n' "Scraping Illegal Gambling Sites (Belgium)" DOMAINS="$(curl -s 'https://www.gamingcommission.be/en/gaming-commission/illegal-games-of-chance/list-of-illegal-gambling-sites' | sed -n 's/^<td[^>]\+>\(.\+\.[a-zA-Z0-9]\+\)\(\|\/.*[^<]*\)<\/td>/\1/gp' || exit 1)" +is_str_empty "${DOMAINS}" "Please check gambling sites URL and sed REGEX." +printf '(2) %s\n' "Processing IP addresses..." echo "${DOMAINS}" >${LIST} ./hostname2list.py "${LIST}" "Gambling" NDPI_PROTOCOL_GAMBLING NDPI_PROTOCOL_CATEGORY_WEB NDPI_PROTOCOL_UNSAFE >${DEST} rm -f "${LIST}" +is_file_empty "${DEST}" exit 0 diff --git a/utils/google_cloud_ip_addresses_download.sh b/utils/google_cloud_ip_addresses_download.sh index 2c24c5c36..02a270e53 100755 --- a/utils/google_cloud_ip_addresses_download.sh +++ b/utils/google_cloud_ip_addresses_download.sh @@ -1,8 +1,9 @@ -#!/bin/sh +#!/usr/bin/env bash set -e cd "$(dirname "${0}")" || exit 1 +. ./common.sh || exit 1 DEST=../src/lib/inc_generated/ndpi_google_cloud_match.c.inc TMP=/tmp/google_c.json diff --git a/utils/google_ip_addresses_download.sh b/utils/google_ip_addresses_download.sh index 5fc888b47..bd2acbd4e 100755 --- a/utils/google_ip_addresses_download.sh +++ b/utils/google_ip_addresses_download.sh @@ -1,8 +1,9 @@ -#!/bin/sh +#!/usr/bin/env bash set -e cd "$(dirname "${0}")" || exit 1 +. ./common.sh || exit 1 DEST=../src/lib/inc_generated/ndpi_google_match.c.inc LIST=/tmp/google.list @@ -13,8 +14,10 @@ echo "(1) Downloading file..." echo "(2) Processing IP addresses..." #https://cloud.google.com/vpc/docs/configure-private-google-access#ip-addr-defaults python3 google.py > $LIST +is_file_empty "${LIST}" ./ipaddr2list.py $LIST NDPI_PROTOCOL_GOOGLE > $DEST -#rm -f $TMP $LIST +rm -f "${TMP}" "${LIST}" +is_file_empty "${DEST}" echo "(3) Google IPs are available in $DEST" exit 0 diff --git a/utils/hostname2list.py b/utils/hostname2list.py index af0a5c349..d466b015d 100755 --- a/utils/hostname2list.py +++ b/utils/hostname2list.py @@ -39,11 +39,13 @@ print("""/* print("static ndpi_protocol_match "+proto.lower()+"_hostname_list[] = {") +lines = 0 with open(sys.argv[1]) as fp: for cnt, line in enumerate(fp): line = line.rstrip() if(line != ""): + lines += 1 x = line.split("/") if(len(x) == 2): @@ -57,3 +59,7 @@ with open(sys.argv[1]) as fp: print(" /* End */") print(" { NULL, NULL, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL }") print("};") + +if lines == 0: + sys.stderr.write(f'File {sys.argv[1]} is empty.\n') + sys.exit(1) diff --git a/utils/icloud_private_relay_ip_addresses_download.sh b/utils/icloud_private_relay_ip_addresses_download.sh index 352f0b649..5954a03a5 100755 --- a/utils/icloud_private_relay_ip_addresses_download.sh +++ b/utils/icloud_private_relay_ip_addresses_download.sh @@ -1,8 +1,9 @@ -#!/bin/sh +#!/usr/bin/env bash set -e cd "$(dirname "${0}")" || exit 1 +. ./common.sh || exit 1 DEST=../src/lib/inc_generated/ndpi_icloud_private_relay_match.c.inc TMP=/tmp/icloud.csv @@ -13,18 +14,19 @@ ORIGIN="https://mask-api.icloud.com/egress-ip-ranges.csv" echo "(1) Downloading file... ${ORIGIN}" http_response=$(curl -s -o "$TMP" -w "%{http_code}" ${ORIGIN}) -if [ "$http_response" != "200" ]; then - echo "Error $http_response: you probably need to update the list url!" - exit 1 -fi +check_http_response "${http_response}" +is_file_empty "${TMP}" echo "(2) Processing IP addresses..." # Note: the "grep -v :" is used to skip IPv6 addresses cut -d ',' -f 1 $TMP | grep -v ':' > $LIST +is_file_empty "${LIST}" ./mergeipaddrlist.py $LIST > $LIST_MERGED +is_file_empty "${LIST_MERGED}" ./ipaddr2list.py $LIST_MERGED NDPI_ANONYMOUS_SUBSCRIBER > $DEST -rm -f $TMP $LIST $LIST_MERGED +is_file_empty "${DEST}" +rm -f "${TMP}" "${LIST}" "${LIST_MERGED}" echo "(3) iCloud Private Relay IPs are available in $DEST" exit 0 diff --git a/utils/ipaddr2list.py b/utils/ipaddr2list.py index 5b2b10bf3..bc7114bd4 100755 --- a/utils/ipaddr2list.py +++ b/utils/ipaddr2list.py @@ -40,11 +40,13 @@ print("""/* print("static ndpi_network "+proto.lower()+"_protocol_list[] = {") +lines = 0 with open(sys.argv[1]) as fp: for cnt, line in enumerate(fp): line = line.rstrip() if(line != ""): + lines += 1 x = line.split("/") if(len(x) == 2): @@ -60,3 +62,6 @@ with open(sys.argv[1]) as fp: print(" /* End */") print(" { 0x0, 0, 0 }") print("};") + +if lines == 0: + sys.stderr.write(f'{sys.argv[0]}: File {sys.argv[1]} is empty.\n') diff --git a/utils/mergeipaddrlist.py b/utils/mergeipaddrlist.py index 36dfba43f..cef9b4660 100755 --- a/utils/mergeipaddrlist.py +++ b/utils/mergeipaddrlist.py @@ -16,5 +16,11 @@ ipFile = open(sys.argv[1]) ipAddresses = list(ipFile.readlines()) ipAddresses = sorted(ipAddresses) cidrs = netaddr.cidr_merge(ipAddresses) + +lines = 0 for cidr in cidrs: + lines += 1 print(cidr) + +if lines == 0: + sys.stderr.write(f'{sys.argv[0]}: file {sys.argv[1]} is empty\n') diff --git a/utils/microsoft_ip_addresses_download.sh b/utils/microsoft_ip_addresses_download.sh index 3c1759993..86cbbe83a 100755 --- a/utils/microsoft_ip_addresses_download.sh +++ b/utils/microsoft_ip_addresses_download.sh @@ -1,8 +1,9 @@ -#!/bin/sh +#!/usr/bin/env bash set -e cd "$(dirname "${0}")" || exit 1 +. ./common.sh || exit 1 DEST_OUTLOOK=../src/lib/inc_generated/ndpi_ms_outlook_match.c.inc DEST_SKYPE_MSTEAMS=../src/lib/inc_generated/ndpi_ms_skype_teams_match.c.inc @@ -16,35 +17,41 @@ ORIGIN="https://endpoints.office.com/endpoints/worldwide?clientrequestid=b10c5ed echo "(1) Downloading file... ${ORIGIN}" http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN}) -if [ $http_response != "200" ]; then - echo "Error $http_response: you probably need to update the list url!" - exit 1 -fi +check_http_response "${http_response}" +is_file_empty "${TMP}" echo "(2) Processing IP addresses..." #OUTLOOK # Note: the "grep -v :" is used to skip IPv6 addresses jq -r '.[] | select(.serviceArea=="Exchange") | .ips[]?' < $TMP | grep -v ':' | sort -u | uniq > $LIST +is_file_empty "${LIST}" ./ipaddr2list.py $LIST NDPI_PROTOCOL_MS_OUTLOOK > $DEST_OUTLOOK +is_file_empty "${DEST_OUTLOOK}" #SKYPE/TEAMS # Note: the "grep -v :" is used to skip IPv6 addresses jq -r '.[] | select(.serviceArea=="Skype") | .ips[]?' < $TMP | grep -v ':' | sort -u | uniq > $LIST +is_file_empty "${LIST}" ./ipaddr2list.py $LIST NDPI_PROTOCOL_SKYPE_TEAMS > $DEST_SKYPE_MSTEAMS +is_file_empty "${DEST_SKYPE_MSTEAMS}" #ONEDRIVE # Note: the "grep -v :" is used to skip IPv6 addresses jq -r '.[] | select(.serviceArea=="SharePoint") | .ips[]?' < $TMP | grep -v ':' | sort -u | uniq > $LIST +is_file_empty "${LIST}" ./ipaddr2list.py $LIST NDPI_PROTOCOL_MS_ONE_DRIVE > $DEST_ONEDRIVE +is_file_empty "${DEST_ONEDRIVE}" #OFFICE # Note: the "grep -v :" is used to skip IPv6 addresses jq -r '.[] | select(.serviceArea=="Common") | .ips[]?' < $TMP | grep -v ':' | sort -u | uniq > $LIST +is_file_empty "${LIST}" #TODO: NDPI_PROTOCOL_MICROSOFT_365 or NDPI_PROTOCOL_MICROSOFT? ./ipaddr2list.py $LIST NDPI_PROTOCOL_MICROSOFT_365 > $DEST_OFFICE365 +is_file_empty "${DEST_OFFICE365}" -rm -f $TMP $LIST +rm -f "${TMP}" "${LIST}" -echo "(3) Microsoft IPs are available in $DEST_OUTLOOK, $DEST_SKYPE_MSTEAMS, $DEST_ONEDRIVE, $DEST_OFFICE365" +echo "(3) Microsoft IPs are available in ${DEST_OUTLOOK}, ${DEST_SKYPE_MSTEAMS}, ${DEST_ONEDRIVE}, ${DEST_OFFICE365}" exit 0 diff --git a/utils/tor_ip_addresses_download.sh b/utils/tor_ip_addresses_download.sh index 397083800..db71ca590 100755 --- a/utils/tor_ip_addresses_download.sh +++ b/utils/tor_ip_addresses_download.sh @@ -1,6 +1,9 @@ -#!/bin/sh +#!/usr/bin/env bash + +set -e cd "$(dirname "${0}")" || exit 1 +. ./common.sh || exit 1 DEST=../src/lib/inc_generated/ndpi_tor_match.c.inc LIST=/tmp/tor.list @@ -13,14 +16,13 @@ ORIGIN="https://check.torproject.org/torbulkexitlist" echo "(1) Downloading file... ${ORIGIN}" http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN}) -if [ $http_response != "200" ]; then - echo "Error $http_response: you probably need to update the list url!" - exit 1 -fi +check_http_response "${http_response}" +is_file_empty "${LIST}" echo "(2) Processing IP addresses..." ./ipaddr2list.py $LIST NDPI_PROTOCOL_TOR > $DEST -rm -f $LIST +rm -f "${LIST}" +is_file_empty "${DEST}" echo "(3) TOR IPs are available in $DEST" exit 0 diff --git a/utils/whatsapp_ip_addresses_download.sh b/utils/whatsapp_ip_addresses_download.sh index f36f4fed3..611de4f10 100755 --- a/utils/whatsapp_ip_addresses_download.sh +++ b/utils/whatsapp_ip_addresses_download.sh @@ -1,8 +1,9 @@ -#!/bin/sh +#!/usr/bin/env bash set -e cd "$(dirname "${0}")" || exit 1 +. ./common.sh || exit 1 DEST=../src/lib/inc_generated/ndpi_whatsapp_match.c.inc TMP=/tmp/wa.zip @@ -12,18 +13,19 @@ IP_LINK_URL='https://developers.facebook.com/docs/whatsapp/guides/network-requir echo "(1) Scraping Facebook WhatsApp IP Adresses and Ranges..." ORIGIN="$(curl -s "${IP_LINK_URL}" | sed -ne 's/.*<a href="\([^"]*\)" target="_blank">WhatsApp server IP addresses and ranges (.zip file)<\/a>.*/\1/gp' | sed -e 's/\&/\&/g')" +is_str_empty "${ORIGIN}" "IP webpage list does not contain any addresses. A REGEX update may be required." echo "(2) Downloading file... ${ORIGIN}" -http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN}) -if [ "$http_response" != "200" ]; then - echo "Error $http_response: you probably need to update the list url!" - exit 1 -fi +http_response=$(curl -s -o "${TMP}" -w "%{http_code}" "${ORIGIN}") +check_http_response "${http_response}" +is_file_empty "${TMP}" echo "(3) Processing IP addresses..." -unzip -p /tmp/wa.zip "WhatsApp IPs (IPv4 Only) 2022-07-26 - 2022-07-30.txt" > $LIST -./ipaddr2list.py $LIST NDPI_PROTOCOL_WHATSAPP > $DEST -rm -f $TMP $LIST +unzip -p /tmp/wa.zip "WhatsApp IPs (IPv4 Only) 2022-07-26 - 2022-07-30.txt" > "${LIST}" +is_file_empty "${LIST}" +./ipaddr2list.py "${LIST}" NDPI_PROTOCOL_WHATSAPP > "${DEST}" +rm -f "${TMP}" "${LIST}" +is_file_empty "${DEST}" echo "(4) WhatsApp IPs are available in $DEST" exit 0 diff --git a/utils/zoom_ip_addresses_download.sh b/utils/zoom_ip_addresses_download.sh index 3139c1b54..e74d1c66a 100755 --- a/utils/zoom_ip_addresses_download.sh +++ b/utils/zoom_ip_addresses_download.sh @@ -1,6 +1,9 @@ -#!/bin/sh +#!/usr/bin/env bash + +set -e cd "$(dirname "${0}")" || exit 1 +. ./common.sh || exit 1 DEST=../src/lib/inc_generated/ndpi_zoom_match.c.inc LIST=/tmp/zoom.list @@ -10,15 +13,14 @@ ORIGIN="https://assets.zoom.us/docs/ipranges/Zoom.txt" echo "(1) Downloading file... ${ORIGIN}" -http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN}) -if [ "$http_response" != "200" ]; then - echo "Error $http_response: you probably need to update the list url!" - exit 1 -fi +http_response=$(curl -s -o "${LIST}" -w "%{http_code}" "${ORIGIN}") +check_http_response "${http_response}" +is_file_empty "${LIST}" echo "(2) Processing IP addresses..." -./ipaddr2list.py $LIST NDPI_PROTOCOL_ZOOM > $DEST -rm -f $LIST +./ipaddr2list.py "${LIST}" NDPI_PROTOCOL_ZOOM > "${DEST}" +rm -f "${LIST}" +is_file_empty "${DEST}" -echo "(3) ZOOM IPs are available in $DEST" +echo "(3) ZOOM IPs are available in ${DEST}" exit 0 |