diff options
| -rwxr-xr-x | utils/azure_ip_addresses_download.sh | 2 | ||||
| -rwxr-xr-x | utils/hostname2list.py | 6 | ||||
| -rwxr-xr-x | utils/ipaddr2list.py | 6 | ||||
| -rwxr-xr-x | utils/whatsapp_ip_addresses_download.sh | 11 |
4 files changed, 22 insertions, 3 deletions
diff --git a/utils/azure_ip_addresses_download.sh b/utils/azure_ip_addresses_download.sh index 3e20b6119..0f7ed08e6 100755 --- a/utils/azure_ip_addresses_download.sh +++ b/utils/azure_ip_addresses_download.sh @@ -19,7 +19,7 @@ if [ "${http_response}" != "200" ]; then exit 1 fi -ORIGIN="$(grep -E 'ServiceTags_Public_[[:digit:]]+.json' ${LINK_TMP} | grep -o -E 'href=\"[^"]+' | sed 's/href="//' | uniq)" +ORIGIN="$(grep -E 'ServiceTags_Public_[[:digit:]]+.json' ${LINK_TMP} | grep -o -E 'href="[^"]+' | sed 's/href="//' | uniq)" rm -f ${LINK_TMP} if [ -z "${ORIGIN}" ]; then echo "Error ${LINK_ORIGIN} does not contain the url format!" diff --git a/utils/hostname2list.py b/utils/hostname2list.py index af0a5c349..e3facfdf0 100755 --- a/utils/hostname2list.py +++ b/utils/hostname2list.py @@ -39,8 +39,10 @@ print("""/* print("static ndpi_protocol_match "+proto.lower()+"_hostname_list[] = {") +lines = 0 with open(sys.argv[1]) as fp: for cnt, line in enumerate(fp): + lines += 1 line = line.rstrip() if(line != ""): @@ -57,3 +59,7 @@ with open(sys.argv[1]) as fp: print(" /* End */") print(" { NULL, NULL, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL }") print("};") + +if lines == 0: + sys.stderr.write(f'File {sys.argv[1]} is empty.\n') + sys.exit(1) diff --git a/utils/ipaddr2list.py b/utils/ipaddr2list.py index 5b2b10bf3..ebf3cf2e6 100755 --- a/utils/ipaddr2list.py +++ b/utils/ipaddr2list.py @@ -40,8 +40,10 @@ print("""/* print("static ndpi_network "+proto.lower()+"_protocol_list[] = {") +lines = 0 with open(sys.argv[1]) as fp: for cnt, line in enumerate(fp): + lines += 1 line = line.rstrip() if(line != ""): @@ -60,3 +62,7 @@ with open(sys.argv[1]) as fp: print(" /* End */") print(" { 0x0, 0, 0 }") print("};") + +if lines == 0: + sys.stderr.write(f'File {sys.argv[1]} is empty.\n') + sys.exit(1) diff --git a/utils/whatsapp_ip_addresses_download.sh b/utils/whatsapp_ip_addresses_download.sh index f36f4fed3..722841015 100755 --- a/utils/whatsapp_ip_addresses_download.sh +++ b/utils/whatsapp_ip_addresses_download.sh @@ -7,16 +7,23 @@ cd "$(dirname "${0}")" || exit 1 DEST=../src/lib/inc_generated/ndpi_whatsapp_match.c.inc TMP=/tmp/wa.zip LIST=/tmp/wa.list +USER_AGENT='Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox' IP_LINK_URL='https://developers.facebook.com/docs/whatsapp/guides/network-requirements/' echo "(1) Scraping Facebook WhatsApp IP Adresses and Ranges..." -ORIGIN="$(curl -s "${IP_LINK_URL}" | sed -ne 's/.*<a href="\([^"]*\)" target="_blank">WhatsApp server IP addresses and ranges (.zip file)<\/a>.*/\1/gp' | sed -e 's/\&/\&/g')" +ORIGIN="$(curl -H "User-Agent: ${USER_AGENT}" -s "${IP_LINK_URL}" | sed -n 's/.*<a href="\(https:\/\/scontent.*\.zip?.*\)" target="_blank">.*/\1/gp')" + +if [ -z "${ORIGIN}" ]; then + echo "Error: IP webpage list does not contain any addresses. A REGEX update may be required." + exit 1 +fi echo "(2) Downloading file... ${ORIGIN}" -http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN}) +http_response=$(curl -H "User-Agent: ${USER_AGENT}" -H "Referer: https://developers.facebook.com/" -s -o $TMP -w "%{http_code}" ${ORIGIN}) if [ "$http_response" != "200" ]; then echo "Error $http_response: you probably need to update the list url!" + echo "Response: $(cat ${TMP})" exit 1 fi |