Added scripts to auto generate hostname/SNI *.inc files. (#1984)

* add illegal gambling sites (Belgium)

Signed-off-by: lns <matzeton@googlemail.com>

3 files changed, 79 insertions, 0 deletions

diff --git a/utils/gambling_sites_download.sh b/utils/gambling_sites_download.sh
new file mode 100755
index 000000000..614e7c694
--- /dev/null
+++ b/utils/gambling_sites_download.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env sh
+
+set -e
+
+cd "$(dirname "${0}")" || exit 1
+
+DEST=../src/lib/inc_generated/ndpi_gambling_match.c.inc
+LIST=/tmp/gambling.list
+
+printf '(1) %s\n' "Scraping Illegal Gambling Sites (Belgium)"
+DOMAINS="$(curl -s 'https://www.gamingcommission.be/en/gaming-commission/illegal-games-of-chance/list-of-illegal-gambling-sites' | sed -n 's/^<td[^>]\+>\(.\+\.[a-zA-Z0-9]\+\)\(\|\/.*[^<]*\)<\/td>/\1/gp' || exit 1)"
+
+echo "${DOMAINS}" >${LIST}
+./hostname2list.py "${LIST}" "Gambling" NDPI_PROTOCOL_GAMBLING NDPI_PROTOCOL_CATEGORY_WEB NDPI_PROTOCOL_UNSAFE >${DEST}
+rm -f "${LIST}"
+
+exit 0
diff --git a/utils/hostname2list.py b/utils/hostname2list.py
new file mode 100755
index 000000000..af0a5c349
--- /dev/null
+++ b/utils/hostname2list.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+import sys
+import socket, struct
+
+# This scripts is used to create "hostname/sni -> protocols" lists.
+if len(sys.argv) < 6:
+    print("Usage: {} <file> <name> <protocol> <category> <breed>".format(sys.argv[0]))
+    sys.exit (1)
+
+name = sys.argv[2]
+proto = sys.argv[3]
+category = sys.argv[4]
+breed = sys.argv[5]
+
+
+print("""/*
+ *
+ * This file is generated automatically and part of nDPI
+ *
+ * nDPI is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nDPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/* ****************************************************** */
+
+""")
+
+print("static ndpi_protocol_match "+proto.lower()+"_hostname_list[] = {")
+
+with open(sys.argv[1]) as fp:
+    for cnt, line in enumerate(fp):
+        line = line.rstrip()
+
+        if(line != ""):
+            x = line.split("/")
+
+            if(len(x) == 2):
+                host = x[0]
+            else:
+                host = line
+
+            if(host != ""):
+                print(' { ' + f'"{host}", "{name}", {proto}, {category}, {breed}, NDPI_PROTOCOL_DEFAULT_LEVEL' + ' },')
+
+print(" /* End */")
+print(" { NULL, NULL, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL }")
+print("};")
diff --git a/utils/update_every_lists.sh b/utils/update_every_lists.sh
index 68882fd98..ab35d6485 100755
--- a/utils/update_every_lists.sh
+++ b/utils/update_every_lists.sh
@@ -34,5 +34,8 @@ RETVAL=$(( RETVAL + $? ))
 ./asn_update.sh
 RETVAL=$(( RETVAL + $? ))
 
+./gambling_sites_download.sh
+RETVAL=$(( RETVAL + $? ))
+
 test ${RETVAL} -ne 0 && printf '%s: %s\n' "${0}" "${RETVAL} script(s) failed"
 exit ${RETVAL}