Add support for Google Cloud (#1447)

Differentiate between Google its own apps/services and Google Cloud.
We already do something similar for Amazon vs AWS and Microsoft vs Azure.

4 files changed, 83 insertions, 0 deletions

diff --git a/utils/google.py b/utils/google.py
new file mode 100644
index 000000000..e8b4114c8
--- /dev/null
+++ b/utils/google.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+import json
+import urllib.request
+import netaddr
+
+GOOG_URL="https://www.gstatic.com/ipranges/goog.json"
+CLOUD_URL="https://www.gstatic.com/ipranges/cloud.json"
+
+def read_url(url):
+   try:
+      s = urllib.request.urlopen(url).read()
+      return json.loads(s)
+   except urllib.error.HTTPError:
+      print("Invalid HTTP response from %s" % url)
+      return {}
+   except json.decoder.JSONDecodeError:
+      print("Could not parse HTTP response from %s" % url)
+      return {}
+
+def main():
+   goog_json=read_url(GOOG_URL)
+   cloud_json=read_url(CLOUD_URL)
+
+   if goog_json and cloud_json:
+#      print("{} published: {}".format(GOOG_URL,goog_json.get('creationTime')))
+#      print("{} published: {}".format(CLOUD_URL,cloud_json.get('creationTime')))
+      goog_cidrs = netaddr.IPSet()
+      for pref in goog_json['prefixes']:
+         if pref.get('ipv4Prefix'):
+            goog_cidrs.add(pref.get('ipv4Prefix'))
+      cloud_cidrs = netaddr.IPSet()
+      for pref in cloud_json['prefixes']:
+         if pref.get('ipv4Prefix'):
+            cloud_cidrs.add(pref.get('ipv4Prefix'))
+#      print("IP ranges for Google APIs and services default domains:")
+      for i in goog_cidrs.difference(cloud_cidrs).iter_cidrs():
+         print(i)
+
+if __name__=='__main__':
+   main()
diff --git a/utils/google_cloud_ip_addresses_download.sh b/utils/google_cloud_ip_addresses_download.sh
new file mode 100755
index 000000000..b14a33af7
--- /dev/null
+++ b/utils/google_cloud_ip_addresses_download.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+cd "$(dirname "${0}")" || return
+
+DEST=../src/lib/ndpi_google_cloud_match.c.inc
+TMP=/tmp/google_c.json
+LIST=/tmp/google_c.list
+ORIGIN="https://www.gstatic.com/ipranges/cloud.json"
+
+
+echo "(1) Downloading file..."
+http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
+if [ "$http_response" != "200" ]; then
+    echo "Error $http_response: you probably need to update the list url!"
+    return
+fi
+
+echo "(2) Processing IP addresses..."
+jq -r '.prefixes | .[].ipv4Prefix  | select( . != null )' $TMP > $LIST # TODO: ipv6
+./ipaddr2list.py $LIST NDPI_PROTOCOL_GOOGLE_CLOUD > $DEST
+rm -f $TMP $LIST
+
+echo "(3) Google Cloud IPs are available in $DEST"
diff --git a/utils/google_ip_addresses_download.sh b/utils/google_ip_addresses_download.sh
new file mode 100755
index 000000000..9560ef3c9
--- /dev/null
+++ b/utils/google_ip_addresses_download.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+cd "$(dirname "${0}")" || return
+
+DEST=../src/lib/ndpi_google_match.c.inc
+LIST=/tmp/google.list
+
+echo "(1) Downloading file..."
+#Nothing to do
+
+echo "(2) Processing IP addresses..."
+#https://cloud.google.com/vpc/docs/configure-private-google-access#ip-addr-defaults
+python3 google.py > $LIST
+./ipaddr2list.py $LIST NDPI_PROTOCOL_GOOGLE > $DEST
+#rm -f $TMP $LIST
+
+echo "(3) Google IPs are available in $DEST"
diff --git a/utils/update_every_content_match_lists.sh b/utils/update_every_content_match_lists.sh
index f300ede07..74c8e9ca2 100755
--- a/utils/update_every_content_match_lists.sh
+++ b/utils/update_every_content_match_lists.sh
@@ -10,3 +10,5 @@ cd "$(dirname "${0}")"
 ./tor_ip_addresses_download.sh
 ./whatsapp_ip_addresses_download.sh
 ./zoom_ip_addresses_download.sh
+./google_cloud_ip_addresses_download.sh
+./google_ip_addresses_download.sh