test: Improve tst.dlclose1.sh test for USDT probes

This test has been failing on aarch64/OL8 ever since its XFAIL was removed
in 8359ab8 "tests: fix usdt/tst.dlclose1.sh".

That patch changed the problematic bufsize setting in the test, but it
could have removed the unnecessary bufsize setting altogether.

The test was then passing gratuitously.  As expected, it failed to find
USDT probes after the dlclose() call, but the reason was that there were
no USDT probes at all simply because they are not yet supported in DTrace
on BPF.

Meanwhile, the test was failing on aarch64/OL8 because no probes were
firing in the pause() call.

Fix the test so that it first confirms there are USDT probes before the
dlclose() call and second that there are no such probes after the call.

Use a different mechanism to synchronize between user program and test
script given the increased synchronization.

Eliminate the unnecessary bufsize setting.

When checking for USDT probes, make sure they are for the expected
process and not for any other test_prov* (e.g. an orphaned process from
an earlier test run).

Set @@xfail since USDT probes are not yet fully implemented.  (This new
version of the test has been checked on legacy DTrace.)

Signed-off-by: Eugene Loh <eugene.loh@oracle.com>
Reviewed-by: Kris Van Hees <kris.van.hees@oracle.com>

test/unittest/usdt/tst.dlclose1.r

@@ -1,4 +1,6 @@
-   ID   PROVIDER            MODULE                          FUNCTION NAME
-
+started pid NNN
+ ID PROVIDER MODULE FUNCTION NAME
+NNN test_provNNN livelib.so go go
+ ID PROVIDER MODULE FUNCTION NAME
 -- @@stderr --
-dtrace: failed to match test_prov*:::: No probe matches description
+dtrace: failed to match test_provNNN:::: No probe matches description

test/unittest/usdt/tst.dlclose1.r.p

@@ -0,0 +1,12 @@
+#!/usr/bin/awk -f
+{
+	# ignore the specific probe ID or process ID
+	# (the script ensures the process ID is consistent)
+	gsub(/[0-9]+/, "NNN");
+
+	# ignore the numbers of spaces for alignment
+	# (they depend on the ID widths)
+	gsub(/ +/, " ");
+
+	print;
+}

test/unittest/usdt/tst.dlclose1.sh

@@ -1,19 +1,34 @@
 #!/bin/bash
 #
 # Oracle Linux DTrace.
-# Copyright (c) 2007, 2021, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2007, 2022, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at
 # http://oss.oracle.com/licenses/upl.
 #
-# This test verifies that USDT providers are removed when its associated
+# This test verifies that USDT providers are removed when the associated
 # load object is closed via dlclose(3dl).
-
-PATH=/usr/bin:/usr/sbin:$PATH
-
-if [ $# != 1 ]; then
-	echo expected one argument: '<'dtrace-path'>'
-	exit 2
-fi
+#
+# The expected sequence of events is:
+#
+#     main.c                                 script
+#     ==============================         =================================
+#
+#     load livelib.so
+#     write "started" to myfile.txt
+#                                            see "started" in myfile.txt
+#                                            check USDT provider test_prov$pid
+#                                            signal USR1
+#     get USR1
+#     run dlclose()
+#     write  "dlclosed" to myfile.txt
+#                                            see "dlclosed" in myfile.txt
+#                                            check USDT provider test_prov$pid
+#                                            kill main.c
+#
+# The first USDT provider check should find test_prov$pid.
+# The second should not.
+#
+# @@xfail: dtv2 (USDT probes)
 
 dtrace=$1
 CC=/usr/bin/gcc
@@ -23,6 +38,10 @@ DIRNAME="$tmpdir/usdt-dlclose1.$$.$RANDOM"
 mkdir -p $DIRNAME
 cd $DIRNAME
 
+#
+# set up Makefile
+#
+
 cat > Makefile <<EOF
 all: main livelib.so deadlib.so
 
@@ -59,6 +78,10 @@ clobber: clean
 	rm -f main livelib.so deadlib.so
 EOF
 
+#
+# set up some small files
+#
+
 cat > prov.d <<EOF
 provider test_prov {
 	probe go();
@@ -67,7 +90,6 @@ EOF
 
 cat > livelib.c <<EOF
 #include "prov.h"
-
 void
 go(void)
 {
@@ -82,57 +104,112 @@ go(void)
 }
 EOF
 
+#
+# set up main.c
+#
 
 cat > main.c <<EOF
 #include <dlfcn.h>
+#include <signal.h>
 #include <unistd.h>
 #include <stdio.h>
 
+void *live;
+
+static void
+interrupt(int sig)
+{
+	/* close livelib.so to remove USDT probes */
+	dlclose(live);
+
+	/* notification */
+	printf("dlclosed\n");
+	fflush(stdout);
+}
+
 int
 main(int argc, char **argv)
 {
-	void *live;
+	struct sigaction act;
 
+	/* open livelib.so to make USDT probes visible */
 	if ((live = dlopen("./livelib.so", RTLD_LAZY | RTLD_LOCAL)) == NULL) {
 		printf("dlopen of livelib.so failed: %s\n", dlerror());
 		return 1;
 	}
 
-	dlclose(live);
+	/* set the handler to listen for SIGUSR1 */
+	act.sa_handler = interrupt;
+	act.sa_flags = 0;
+	if (sigaction(SIGUSR1, &act, NULL)) {
+		printf("set handler failed\n");
+		return 1;
+	}
+
+	/* notification */
+	printf("started\n");
+	fflush(stdout);
 
+	/* wait for SIGUSR1 to execute handler */
+	pause();
+
+	/* wait for SIGKILL to terminate */
 	pause();
 
 	return 0;
 }
 EOF
 
+#
+# make
+#
+
 make > /dev/null
 if [ $? -ne 0 ]; then
 	echo "failed to build" >& 2
 	exit 1
 fi
 
-script() {
-	$dtrace -Zw -x bufsize=64k -c ./main -qs /dev/stdin <<EOF
-	syscall::pause:entry,
-	syscall::rt_sig*:entry
-	/pid == \$target/
-	{
-		system("$dtrace -l -P test_prov*");
-		system("kill %d", \$target);
-		exit(0);
-	}
+#
+# set up a wait mechanism
+#
 
-	tick-1s
-	/i++ == 5/
-	{
-		printf("failed\n");
-		exit(1);
-	}
-EOF
+# args:
+#   1: number of seconds to wait
+#   2: string to watch for
+#   3: file to watch
+function mywait() {
+	for iter in `seq $1`; do
+		sleep 1
+		if grep -q "$2" $3; then
+			iter=0
+			break
+		fi
+	done
+	if [[ $iter -ne 0 ]]; then
+		echo did not see message: $2
+		echo "==================== start of messages"
+		cat $3
+		echo "==================== end of messages"
+		kill -s KILL $pid
+		exit 1
+	fi
 }
 
-script
-status=$?
+#
+# run the process in the background and check USDT probes
+#
+
+./main > myfile.txt &
+pid=$!
+echo started pid $pid
+
+mywait 6 "started"  myfile.txt # wait for process to start
+$dtrace -lP test_prov$pid      # check USDT probes
+kill -s USR1 $pid              # signal process
+
+mywait 6 "dlclosed" myfile.txt # wait for process to dlclose
+$dtrace -lP test_prov$pid      # check USDT probes
+kill -s KILL $pid              # kill process
 
-exit $status
+exit 0