Merge pull request #15281 from keszybz/functional-test-rework

Post-merge tweaks to the functional tests

test/TEST-02-CRYPTSETUP/test.sh

@@ -13,13 +13,12 @@ check_result_qemu() {
     [[ -f $initdir/failed ]] && cp -a $initdir/failed $TESTDIR
     cryptsetup luksOpen ${LOOPDEV}p2 varcrypt <$TESTDIR/keyfile
     mount /dev/mapper/varcrypt $initdir/var
-    cp -a $initdir/var/log/journal $TESTDIR
+    save_journal $initdir/var/log/journal
-    rm -r $initdir/var/log/journal/*
     _umount_dir $initdir/var
     _umount_dir $initdir
     cryptsetup luksClose /dev/mapper/varcrypt
     [[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed
-    ls -l $TESTDIR/journal/*/*.journal
+    echo $JOURNAL_LIST
     test -s $TESTDIR/failed && ret=$(($ret+1))
     return $ret
 }

test/TEST-06-SELINUX/test.sh

@@ -46,7 +46,9 @@ test_create_image() {
         cp systemd_test.if $initdir/systemd-test-module
         dracut_install -o sesearch
         dracut_install runcon
-        dracut_install checkmodule semodule semodule_package m4 make /usr/libexec/selinux/hll/pp load_policy sefcontext_compile
+        dracut_install checkmodule semodule semodule_package m4 make load_policy sefcontext_compile
+        dracut_install -o /usr/libexec/selinux/hll/pp # Fedora/RHEL/...
+        dracut_install -o /usr/lib/selinux/hll/pp     # Debian/Ubuntu/...
     )
 }
 

test/TEST-13-NSPAWN-SMOKE/test.sh

@@ -15,8 +15,8 @@ test_create_image() {
         setup_basic_environment
         mask_supporting_services
 
-        ../create-busybox-container $initdir/nc-container
+        ../create-busybox-container $initdir/testsuite-13.nc-container
-        initdir="$initdir/nc-container" dracut_install nc ip
+        initdir="$initdir/testsuite-13.nc-container" dracut_install nc ip
     )
 }
 

test/TEST-24-UNIT-TESTS/test.sh

@@ -30,8 +30,7 @@ check_result_nspawn() {
             cat $1/testok
         fi
     fi
-    cp -a $1/var/log/journal $TESTDIR
+    save_journal $1/var/log/journal
-    rm -r $1/var/log/journal/*
     _umount_dir $initdir
     [[ -n "$TIMED_OUT" ]] && _ret=$(($_ret+1))
     return $_ret
@@ -55,8 +54,7 @@ check_result_qemu() {
             cat $initdir/testok
         fi
     fi
-    cp -a $initdir/var/log/journal $TESTDIR
+    save_journal $initdir/var/log/journal
-    rm -r $initdir/var/log/journal/*
     _umount_dir $initdir
     [[ -n "$TIMED_OUT" ]] && _ret=$(($_ret+1))
     return $_ret

test/run-integration-tests.sh

@@ -4,44 +4,61 @@ set -e
 BUILD_DIR="$($(dirname "$0")/../tools/find-build-dir.sh)"
 if [ $# -gt 0 ]; then
     args="$@"
-    do_clean=0
 else
     args="setup run clean-again"
-    do_clean=1
 fi
+args_no_clean=$(sed -r 's/(^| )clean($| )/ /g' <<<$args)
+do_clean=$( [ "$args" = "$args_no_clean" ]; echo $? )
 
 ninja -C "$BUILD_DIR"
 
 declare -A results
+declare -A times
 
 COUNT=0
 FAILURES=0
 
 cd "$(dirname "$0")"
 
+# Let's always do the cleaning operation first, because it destroys the image
+# cache.
 if [ $do_clean = 1 ]; then
     for TEST in TEST-??-* ; do
         ( set -x ; make -C "$TEST" "BUILD_DIR=$BUILD_DIR" clean )
     done
 fi
 
+pass_blacklist() {
+    for marker in $BLACKLIST_MARKERS; do
+        if [ -f "$1/$marker" ]; then
+            echo "========== BLACKLISTED: $1 ($marker) =========="
+            return 1
+        fi
+    done
+    return 0
+}
+
 for TEST in TEST-??-* ; do
     COUNT=$(($COUNT+1))
 
+    pass_blacklist $TEST || continue
+    start=$(date +%s)
+
     echo -e "\n--x-- Running $TEST --x--"
     set +e
-    ( set -x ; make -C "$TEST" "BUILD_DIR=$BUILD_DIR" $args )
+    ( set -x ; make -C "$TEST" "BUILD_DIR=$BUILD_DIR" $args_no_clean )
     RESULT=$?
     set -e
     echo "--x-- Result of $TEST: $RESULT --x--"
 
     results["$TEST"]="$RESULT"
+    times["$TEST"]=$(( $(date +%s) - $start ))
 
     [ "$RESULT" -ne "0" ] && FAILURES=$(($FAILURES+1))
 done
 
 if [ $FAILURES -eq 0 -a $do_clean = 1 ]; then
-    for TEST in TEST-??-* ; do
+    for TEST in ${!results[@]}; do
         ( set -x ; make -C "$TEST" "BUILD_DIR=$BUILD_DIR" clean-again )
     done
 fi
@@ -50,11 +67,9 @@ echo ""
 
 for TEST in ${!results[@]}; do
     RESULT="${results[$TEST]}"
-    if [ "$RESULT" -eq "0" ] ; then
+    time="${times[$TEST]}"
-        echo "$TEST: SUCCESS"
+    string=$([ "$RESULT" = "0" ] && echo "SUCCESS" || echo "FAIL")
-    else
+    printf "%-35s %-8s (%3s s)\n" "${TEST}:" "${string}" "$time"
-        echo "$TEST: FAIL"
-    fi
 done | sort
 
 if [ "$FAILURES" -eq 0 ] ; then

test/test-functions

@@ -675,7 +675,7 @@ cleanup_loopdev() {
     fi
 }
 
-trap cleanup_loopdev EXIT
+trap cleanup_loopdev EXIT INT QUIT PIPE
 
 create_empty_image() {
     if [ -z "$IMAGE_NAME" ]; then
@@ -790,16 +790,39 @@ check_asan_reports() {
     return $ret
 }
 
+save_journal() {
+    if [ -n "${ARTIFACT_DIRECTORY}" ]; then
+        dest="${ARTIFACT_DIRECTORY}/${testname}.journal"
+    else
+        dest="$TESTDIR/system.journal"
+    fi
+
+    for j in $1/*; do
+        /usr/lib/systemd/systemd-journal-remote \
+            -o $dest \
+            --getter="journalctl -o export -D $j"
+
+        if [ -n "${TEST_SHOW_JOURNAL}" ]; then
+            echo "---- $j ----"
+            journalctl --no-pager -o short-monotonic --no-hostname --priority=${TEST_SHOW_JOURNAL} -D $j
+        fi
+
+        rm -r $j
+    done
+
+    # we want to print this sometime later, so save this in a variable
+    JOURNAL_LIST="$(ls -l $dest*)"
+}
+
 check_result_nspawn() {
     local ret=1
     local journald_report=""
     local pids=""
     [[ -e $1/testok ]] && ret=0
     [[ -f $1/failed ]] && cp -a $1/failed $TESTDIR
-    cp -a $1/var/log/journal $TESTDIR
+    save_journal $1/var/log/journal
-    rm -r $1/var/log/journal/*
     [[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed
-    ls -l $TESTDIR/journal/*/*.journal
+    echo $JOURNAL_LIST
     test -s $TESTDIR/failed && ret=$(($ret+1))
     [ -n "$TIMED_OUT" ] && ret=$(($ret+1))
     check_asan_reports "$1" || ret=$(($ret+1))
@@ -813,12 +836,11 @@ check_result_qemu() {
     mount_initdir
     [[ -e $initdir/testok ]] && ret=0
     [[ -f $initdir/failed ]] && cp -a $initdir/failed $TESTDIR
-    cp -a $initdir/var/log/journal $TESTDIR
+    save_journal $initdir/var/log/journal
-    rm -r $initdir/var/log/journal/*
     check_asan_reports "$initdir" || ret=$(($ret+1))
     _umount_dir $initdir
     [[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed
-    ls -l $TESTDIR/journal/*/*.journal
+    echo $JOURNAL_LIST
     test -s $TESTDIR/failed && ret=$(($ret+1))
     [ -n "$TIMED_OUT" ] && ret=$(($ret+1))
     return $ret
@@ -919,6 +941,12 @@ install_config_files() {
 
     # set the hostname
     echo systemd-testsuite > $initdir/etc/hostname
+
+    # let's set up just one image with the traditional verbose output
+    if [ ${IMAGE_NAME} != "basic" ]; then
+        mkdir -p $initdir/etc/systemd/system.conf.d
+        echo -e '[Manager]\nStatusUnitFormat=name' >$initdir/etc/systemd/system.conf.d/status.conf
+    fi
 }
 
 install_basic_tools() {

test/units/testsuite-13.sh

@@ -33,7 +33,8 @@ fi
 
 function check_bind_tmp_path {
     # https://github.com/systemd/systemd/issues/4789
-    local _root="/var/lib/machines/bind-tmp-path"
+    local _root="/var/lib/machines/testsuite-13.bind-tmp-path"
+    rm -rf "$_root"
     /usr/lib/systemd/tests/testdata/create-busybox-container "$_root"
     >/tmp/bind
     systemd-nspawn --register=no -D "$_root" --bind=/tmp/bind /bin/sh -c 'test -e /tmp/bind'
@@ -41,7 +42,8 @@ function check_bind_tmp_path {
 
 function check_norbind {
     # https://github.com/systemd/systemd/issues/13170
-    local _root="/var/lib/machines/norbind-path"
+    local _root="/var/lib/machines/testsuite-13.norbind-path"
+    rm -rf "$_root"
     mkdir -p /tmp/binddir/subdir
     echo -n "outer" > /tmp/binddir/subdir/file
     mount -t tmpfs tmpfs /tmp/binddir/subdir
@@ -53,8 +55,9 @@ function check_norbind {
 function check_notification_socket {
     # https://github.com/systemd/systemd/issues/4944
     local _cmd='echo a | $(busybox which nc) -U -u -w 1 /run/systemd/nspawn/notify'
-    systemd-nspawn --register=no -D /nc-container /bin/sh -x -c "$_cmd"
+    # /testsuite-13.nc-container is prepared by test.sh
-    systemd-nspawn --register=no -D /nc-container -U /bin/sh -x -c "$_cmd"
+    systemd-nspawn --register=no -D /testsuite-13.nc-container /bin/sh -x -c "$_cmd"
+    systemd-nspawn --register=no -D /testsuite-13.nc-container -U /bin/sh -x -c "$_cmd"
 }
 
 function run {
@@ -67,7 +70,8 @@ function run {
         return 0
     fi
 
-    local _root="/var/lib/machines/unified-$1-cgns-$2-api-vfs-writable-$3"
+    local _root="/var/lib/machines/testsuite-13.unified-$1-cgns-$2-api-vfs-writable-$3"
+    rm -rf "$_root"
     /usr/lib/systemd/tests/testdata/create-busybox-container "$_root"
     SYSTEMD_NSPAWN_UNIFIED_HIERARCHY="$1" SYSTEMD_NSPAWN_USE_CGNS="$2" SYSTEMD_NSPAWN_API_VFS_WRITABLE="$3" systemd-nspawn --register=no -D "$_root" -b
     SYSTEMD_NSPAWN_UNIFIED_HIERARCHY="$1" SYSTEMD_NSPAWN_USE_CGNS="$2" SYSTEMD_NSPAWN_API_VFS_WRITABLE="$3" systemd-nspawn --register=no -D "$_root" --private-network -b