Make CI faster (#2662)

Right now the CI takes ~30 minutes; the goal is to have it ending in
< 15 min.
The basic trick is to run the longer jobs (no_x86_64 and masan) only
with the recently updated pcaps. The same jobs will run again on schedule
(every night) testing all the traces.

This way the CI will be "green" (hopefully!) earlier while pushing new
commit/PR; full tests are simply delayed.

Details: when `NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS` is set,
`tests/do.sh` checks only the latest 10 pcaps (i.e. the more recent pcap
added/updated) for *every* configuration.

Notes that no_x86_64 and masan jobs run twice: when pushing/merging and
on schedule (every night)

4 files changed, 71 insertions, 16 deletions

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 321deb6b0..766e7d3af 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -130,13 +130,6 @@ jobs:
             msan: ""
             nBPF: ""
             global_context: "--disable-global-context-support"
-          - compiler: "clang" # TODO: some issues with masan/clang/ubuntu-24.04
-            os: ubuntu-22.04
-            gcrypt: ""
-            pcre: "--with-pcre2"
-            maxminddb: "--with-maxminddb"
-            msan: "--with-memory-sanitizer --disable-memory-track-origins"
-            nBPF: ""
           - compiler: "cc"
             os: macOS-13
             gcrypt: ""
@@ -251,8 +244,7 @@ jobs:
         run: |
           NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh
       - name: Test nDPI [UNIT]
-        # Some issues with masan + json-c. Disable the test as workaround
-        if: ${{ !startsWith(matrix.msan, '--with-memory-sanitizer') && !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15') }}
+        if: ${{ !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15') }}
         run: |
           ./tests/do-unit.sh
       - name: Test nDPI [DGA]
diff --git a/.github/workflows/build_masan.yml b/.github/workflows/build_masan.yml
new file mode 100644
index 000000000..b10a5d42d
--- /dev/null
+++ b/.github/workflows/build_masan.yml
@@ -0,0 +1,47 @@
+name: Memory Sanitizer
+on:
+  push:
+    branches:
+      - dev
+  pull_request:
+    branches:
+      - dev
+    types: [opened, synchronize, reopened]
+  release:
+    types: [created]
+  workflow_dispatch:
+  schedule:
+    #At the end of every day
+    - cron: '0 0 * * *'
+
+jobs:
+  memorysanitizer:
+    name: masan (ubuntu-22.04)
+    runs-on: ubuntu-22.04 # TODO: some issues with masan/clang/ubuntu-24.04
+    env:
+      CC: clang
+      CFLAGS: -Wextra -Werror -DNDPI_EXTENDED_SANITY_CHECKS
+      # Idea to have significant faster tests while pushing/merging: test all pcaps only on schedule runs; otherwise tests only the pcaps updated/added recently
+      NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS: ${{ (github.event_name == 'schedule') && '0' || '1' }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install Ubuntu Prerequisites
+        run: |
+          sudo apt-get update
+          sudo apt-get install autoconf automake libtool pkg-config gettext flex bison libjson-c-dev libpcap-dev rrdtool librrd-dev parallel libpcre3-dev libmaxminddb-dev
+      - name: Configure nDPI
+        run: |
+          ./autogen.sh --enable-option-checking=fatal --with-pcre2 --with-maxminddb --with-memory-sanitizer --disable-memory-track-origins
+      - name: Build nDPI
+        run: |
+          make -j $(nproc) all
+          make -j $(nproc) -C example ndpiSimpleIntegration
+          make -j $(nproc) -C rrdtool
+      - name: Tests
+        run: |
+          NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh
+          # Some issues with masan + json-c. Disable the test as workaround
+          # ./tests/do-unit.sh
+          ./tests/do-dga.sh
diff --git a/.github/workflows/build_no_x86_64.yml b/.github/workflows/build_no_x86_64.yml
index af28c95a6..1102f2636 100644
--- a/.github/workflows/build_no_x86_64.yml
+++ b/.github/workflows/build_no_x86_64.yml
@@ -1,4 +1,4 @@
-name: Build No x86_64 archs
+name: No x86_64 archs
 on:
   push:
     branches:
@@ -9,6 +9,9 @@ on:
     types: [opened, synchronize, reopened]
   release:
     types: [created]
+  schedule:
+    # At the end of every day
+    - cron: '0 0 * * *'
 jobs:
 
   test:
@@ -16,6 +19,8 @@ jobs:
     runs-on: ubuntu-latest
     env:
       CFLAGS: -Wextra -Werror -DNDPI_EXTENDED_SANITY_CHECKS
+      # Idea to have significant faster tests while pushing/merging: test all pcaps only on schedule runs; otherwise tests only the pcaps updated/added recently
+      NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS: ${{ (github.event_name == 'schedule') && '0' || '1' }}
     strategy:
       fail-fast: true
       matrix:
@@ -45,7 +50,7 @@ jobs:
             "apt-get -y update &&
             apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev &&
             git config --global --add safe.directory $(realpath .) &&
-            env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages &&
+            env CC=gcc ./autogen.sh --enable-option-checking=fatal --enable-debug-messages &&
             make -j $(nproc) all &&
             make -C example ndpiSimpleIntegration &&
             make -C rrdtool &&
@@ -69,7 +74,7 @@ jobs:
             "apt-get -y update &&
             apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev &&
             git config --global --add safe.directory $(realpath .) &&
-            env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages &&
+            env CC=gcc ./autogen.sh --enable-option-checking=fatal --enable-debug-messages &&
             make -j $(nproc) all &&
             make -C example ndpiSimpleIntegration &&
             make -C rrdtool &&
@@ -93,7 +98,7 @@ jobs:
             "apt-get -y update &&
             apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev &&
             git config --global --add safe.directory $(realpath .) &&
-            env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages &&
+            env CC=gcc ./autogen.sh --enable-option-checking=fatal --enable-debug-messages &&
             make -j $(nproc) all &&
             make -C example ndpiSimpleIntegration &&
             make -C rrdtool &&
diff --git a/tests/do.sh.in b/tests/do.sh.in
index 93822576b..aa68401c0 100755
--- a/tests/do.sh.in
+++ b/tests/do.sh.in
@@ -31,6 +31,11 @@ if [ "${NDPI_SKIP_PARALLEL_BAR}" = "1" ]; then
    SKIP_PARALLEL_BAR=1
 fi
 
+TEST_ONLY_RECENTLY_UPDATED_PCAPS=0
+if [ "${NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS}" = "1" ]; then
+   TEST_ONLY_RECENTLY_UPDATED_PCAPS=1
+fi
+
 #Remember: valgrind and *SAN are incompatible!
 CMD_PREFIX="${CMD_PREFIX}"
 if [ "${NDPI_TESTS_WINE}" = "1" ]; then
@@ -239,10 +244,16 @@ for d in $(find ./cfgs/* -type d -maxdepth 0 2>/dev/null) ; do
 
     cd ./cfgs/"$(basename "$d")" || exit 1
 
-    if [ "$#" -ne 0 ]; then
-        PCAPS=$*
+    if [ $TEST_ONLY_RECENTLY_UPDATED_PCAPS -eq 1 ]; then
+        # Latest 10 pcaps updatetd recently (to be more precise, pcap which triggered a change
+        # in their results lately)
+        PCAPS=$(cd result || exit 1; find * -type f -print0  | xargs -0 ls -t | head -10 | xargs basename -s .out)
     else
-        PCAPS=$(cd pcap || exit 1; /bin/ls -- *.*cap*)
+        if [ "$#" -ne 0 ]; then
+            PCAPS=$*
+        else
+            PCAPS=$(cd pcap || exit 1; /bin/ls -- *.*cap*)
+        fi
     fi
     FAILURES=()
     READER_EXTRA_PARAM=""