CI: Increase setup timeout to 20min, add timestamps

- Increase qemu-1-setup.sh timeout to 20min since it sometimes
  fails to complete after 15min.

- Timestamp all qemu-1-setup.sh lines to look for hangs.

- Add a 'watchdog' process to print out the top running process every
  30sec to help with debugging.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Tony Hutter <hutter2@llnl.gov>
Closes #17714

.github/workflows/scripts/qemu-1-setup.sh

@@ -6,6 +6,13 @@
 
 set -eu
 
+# We've been seeing this script take over 15min to run.  This may or
+# may not be normal.  Just to get a little more insight, print out
+# a message to stdout with the top running process, and do this every
+# 30 seconds.  We can delete this watchdog later once we get a better
+# handle on what the timeout value should be.
+(while [ 1 ] ; do sleep 30 && echo "[watchdog: $(ps -eo cmd --sort=-pcpu  | head -n 2 | tail -n 1)}')]"; done) &
+
 # install needed packages
 export DEBIAN_FRONTEND="noninteractive"
 sudo apt-get -y update
@@ -65,3 +72,6 @@ sudo zpool create -f -o ashift=12 zpool $SSD1 $SSD2 -O relatime=off \
 for i in /sys/block/s*/queue/scheduler; do
   echo "none" | sudo tee $i
 done
+
+# Kill off our watchdog
+kill $(jobs -p)

.github/workflows/zfs-qemu.yml

@@ -77,8 +77,12 @@ jobs:
         ref: ${{ github.event.pull_request.head.sha }}
 
     - name: Setup QEMU
-      timeout-minutes: 15
+      timeout-minutes: 20
-      run: .github/workflows/scripts/qemu-1-setup.sh
+      run: |
+        # Add a timestamp to each line to debug timeouts
+        while IFS=$'\n' read -r line; do
+            echo "$(date +'%H:%M:%S') $line"
+        done < <(.github/workflows/scripts/qemu-1-setup.sh)
 
     - name: Start build machine
       timeout-minutes: 10