CI: add runtime tests for packages

Additional to manual runtime tests this CI addition runs a custom test
script per package. Ideally this lowers the errors of package bumps,
something which is time consuming when done manually for multiple
architectures.

This CI uses the official OpenWrt containers and tries to install and
run compiled packages. The run depends on the content of `test.sh`,
which is an `ash` script. It's called with the *packge name* and
*package version* as arguments. This allows different behaviour if
a single package generates multiple IPK files. The version is usable for
the most trivial runtime check, e.g. `tmux -V | grep "$2"`.

The current approach uses the qus project[1] which contains multiple
QEMU binaries to run various architectures.

[1]: https://github.com/dbhi/qus

Signed-off-by: Paul Spooren <mail@aparcar.org>

3 files changed, 70 insertions, 6 deletions

diff --git a/.github/workflows/Dockerfile b/.github/workflows/Dockerfile
new file mode 100644
index 000000000..aa269b778
--- /dev/null
+++ b/.github/workflows/Dockerfile
@@ -0,0 +1,6 @@
+ARG ARCH=x86-64
+FROM openwrtorg/rootfs:$ARCH
+
+ADD entrypoint.sh /entrypoint.sh
+
+CMD ["/entrypoint.sh"]
diff --git a/.github/workflows/entrypoint.sh b/.github/workflows/entrypoint.sh
new file mode 100755
index 000000000..21e9035d0
--- /dev/null
+++ b/.github/workflows/entrypoint.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+mkdir -p /var/lock/
+
+opkg update
+
+for PKG in /ci/*.ipk; do
+	tar -xzOf "$PKG" ./control.tar.gz | tar xzf - ./control 
+	PKG_NAME=$(sed -ne 's#^Package: \(.*\)$#\1#p' ./control)
+	PKG_VERSION=$(sed -ne 's#^Version: \(.*\)$#\1#p' ./control)
+
+	echo "Testing package $PKG_NAME ($PKG_VERSION)"
+
+	opkg install "$PKG"
+
+	TEST_SCRIPT=$(find /ci/ -name "$PKG_NAME" -type d)/test.sh
+	if [ -f "$TEST_SCRIPT" ]; then
+		echo "Use package specific test.sh"
+		if sh "$TEST_SCRIPT" "$PKG_NAME" "$PKG_VERSION"; then
+			echo "Test successfull"
+		else
+			echo "Test failed"
+			exit 1
+		fi
+	else
+		echo "No test.sh script available"
+	fi
+
+	opkg remove "$PKG_NAME"
+done
diff --git a/.github/workflows/multi-arch-test-build.yml b/.github/workflows/multi-arch-test-build.yml
index b026b10b2..97fa8176b 100644
--- a/.github/workflows/multi-arch-test-build.yml
+++ b/.github/workflows/multi-arch-test-build.yml
@@ -7,23 +7,29 @@ on:
 
 jobs:
   build:
-    name: ${{ matrix.arch }} build
+    name: Test ${{ matrix.arch }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         arch:
-          - aarch64_generic
           - arc_arc700
           - arc_archs
-          - arm_cortex-a15_neon-vfpv4
           - arm_cortex-a9_neon
           - arm_cortex-a9_vfpv3-d16
-          - i386_pentium4
           - mips_24kc
           - powerpc_464fp
           - powerpc_8540
-          - x86_64
+        runtime_test: [false]
+        include:
+          - arch: aarch64_generic
+            runtime_test: true
+          - arch: arm_cortex-a15_neon-vfpv4
+            runtime_test: true
+          - arch: i386_pentium4
+            runtime_test: true
+          - arch: x86_64
+            runtime_test: true
 
     steps:
       - uses: actions/checkout@v2
@@ -50,11 +56,14 @@ jobs:
           ARCH: ${{ matrix.arch }}
           FEEDNAME: packages_ci
 
+      - name: Move created packages to project dir
+        run: cp bin/packages/${{ matrix.arch }}/packages_ci/*.ipk . || true
+
       - name: Store packages
         uses: actions/upload-artifact@v2
         with:
           name: ${{ matrix.arch}}-packages
-          path: bin/packages/${{ matrix.arch }}/packages_ci/*.ipk
+          path: "*.ipk"
 
       - name: Store logs
         uses: actions/upload-artifact@v2
@@ -62,3 +71,22 @@ jobs:
           name: ${{ matrix.arch}}-logs
           path: logs/
 
+      - name: Remove logs
+        run: sudo rm -rf logs/ || true
+
+      - name: Register QEMU
+        if: ${{ matrix.runtime_test }}
+        run: |
+          sudo docker run --rm --privileged aptman/qus -s -- -p
+
+      - name: Build Docker container
+        if: ${{ matrix.runtime_test }}
+        run: |
+          docker build -t test-container --build-arg ARCH .github/workflows/
+        env:
+          ARCH: ${{ matrix.arch }}
+
+      - name: Test via Docker container
+        if: ${{ matrix.runtime_test }}
+        run: |
+          docker run --rm -v $GITHUB_WORKSPACE:/ci test-container