services: better logging in test

.github/workflows/ci.yml

@@ -33,7 +33,7 @@ jobs:
       working-directory: ./src/github.com/kevinburke/rickover
 
     - name: Test rickover
-      run: envdir envs/github go test -p 1 -timeout 10s -v ./...
+      run: envdir envs/github go test -count=1 -p=1 -timeout=10s -v ./...
       working-directory: ./src/github.com/kevinburke/rickover
 
     - name: Run benchmarks

services/process_job.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"os"
 	"strings"
 	"time"
 
@@ -73,10 +74,10 @@ func (d *DownstreamHandler) Handle(ctx context.Context, qj *newmodels.QueuedJob)
 				return err
 			default:
 				go func(err error) {
-					if isTimeout(err) {
+					if reachedRemoteServer(err) {
 						metrics.Increment("dequeue.post_job.timeout")
 					} else {
-						log.Printf("Unknown error making POST request to downstream server: %#v", err)
+						log.Printf("Unknown error making POST request to downstream server: %q (%#v)", err, err)
 						metrics.Increment("dequeue.post_job.error_unknown")
 					}
 				}(err)
@@ -109,12 +110,15 @@ func NewJobProcessor(h Handler) *JobProcessor {
 	}
 }
 
-// isTimeout returns true if the err was caused by a request timeout.
-func isTimeout(err error) bool {
-	//var sysErr *os.SyscallError
-	//if errors.As(err, &sysErr) {
-	//return sysErr.Syscall == "connect"
-	//}
+// reachedRemoteServer returns true if the err may have occurred after the
+// remote server was contacted.
+func reachedRemoteServer(err error) bool {
+	var sysErr *os.SyscallError
+	if errors.As(err, &sysErr) {
+		if sysErr.Syscall == "connect" {
+			return false
+		}
+	}
 	if errors.Is(err, context.DeadlineExceeded) {
 		// We can't really introspect the phase of the HTTP process, just
 		// assume these are all timeouts.
@@ -140,7 +144,7 @@ func (jp *JobProcessor) DoWork(ctx context.Context, qj *newmodels.QueuedJob) err
 	}
 	defer cancel()
 	if err := jp.Handler.Handle(tctx, qj); err != nil {
-		if isTimeout(err) {
+		if reachedRemoteServer(err) {
 			// Special case: if the request made it to the downstream server,
 			// but we timed out waiting for a response, we assume the downstream
 			// server received it and will handle it. we see this most often