Add assert func for verifying status, netmap and netcheck (#1723)

.github/workflows/test-integration-v2-TestPingAllByIPPublicDERP.yaml

@@ -0,0 +1,67 @@
+# DO NOT EDIT, generated with cmd/gh-action-integration-generator/main.go
+# To regenerate, run "go generate" in cmd/gh-action-integration-generator/
+
+name: Integration Test v2 - TestPingAllByIPPublicDERP
+
+on: [pull_request]
+
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  TestPingAllByIPPublicDERP:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - uses: satackey/action-docker-layer-caching@main
+        continue-on-error: true
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v34
+        with:
+          files: |
+            *.nix
+            go.*
+            **/*.go
+            integration_test/
+            config-example.yaml
+
+      - name: Run TestPingAllByIPPublicDERP
+        uses: Wandalen/wretry.action@master
+        if: steps.changed-files.outputs.any_changed == 'true'
+        with:
+          attempt_limit: 5
+          command: |
+            nix develop --command -- docker run \
+              --tty --rm \
+              --volume ~/.cache/hs-integration-go:/go \
+              --name headscale-test-suite \
+              --volume $PWD:$PWD -w $PWD/integration \
+              --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
+              golang:1 \
+                go run gotest.tools/gotestsum@latest -- ./... \
+                  -failfast \
+                  -timeout 120m \
+                  -parallel 1 \
+                  -run "^TestPingAllByIPPublicDERP$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

hscontrol/poll.go

@@ -201,9 +201,15 @@ func (h *Headscale) handlePoll(
 			return
 		}
 
+		// TODO(kradalby): Figure out why patch changes does
+		// not show up in output from `tailscale debug netmap`.
+		// stateUpdate := types.StateUpdate{
+		// 	Type:          types.StatePeerChangedPatch,
+		// 	ChangePatches: []*tailcfg.PeerChange{&change},
+		// }
 		stateUpdate := types.StateUpdate{
-			Type:          types.StatePeerChangedPatch,
-			ChangePatches: []*tailcfg.PeerChange{&change},
+			Type:        types.StatePeerChanged,
+			ChangeNodes: types.Nodes{node},
 		}
 		if stateUpdate.Valid() {
 			ctx := types.NotifyCtx(context.Background(), "poll-nodeupdate-peers-patch", node.Hostname)

hscontrol/types/node_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/juanfont/headscale/hscontrol/util"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 )
@@ -366,3 +367,110 @@ func TestPeerChangeFromMapRequest(t *testing.T) {
 		})
 	}
 }
+
+func TestApplyPeerChange(t *testing.T) {
+	tests := []struct {
+		name       string
+		nodeBefore Node
+		change     *tailcfg.PeerChange
+		want       Node
+	}{
+		{
+			name:       "hostinfo-and-netinfo-not-exists",
+			nodeBefore: Node{},
+			change: &tailcfg.PeerChange{
+				DERPRegion: 1,
+			},
+			want: Node{
+				Hostinfo: &tailcfg.Hostinfo{
+					NetInfo: &tailcfg.NetInfo{
+						PreferredDERP: 1,
+					},
+				},
+			},
+		},
+		{
+			name: "hostinfo-netinfo-not-exists",
+			nodeBefore: Node{
+				Hostinfo: &tailcfg.Hostinfo{
+					Hostname: "test",
+				},
+			},
+			change: &tailcfg.PeerChange{
+				DERPRegion: 3,
+			},
+			want: Node{
+				Hostinfo: &tailcfg.Hostinfo{
+					Hostname: "test",
+					NetInfo: &tailcfg.NetInfo{
+						PreferredDERP: 3,
+					},
+				},
+			},
+		},
+		{
+			name: "hostinfo-netinfo-exists-derp-set",
+			nodeBefore: Node{
+				Hostinfo: &tailcfg.Hostinfo{
+					Hostname: "test",
+					NetInfo: &tailcfg.NetInfo{
+						PreferredDERP: 999,
+					},
+				},
+			},
+			change: &tailcfg.PeerChange{
+				DERPRegion: 2,
+			},
+			want: Node{
+				Hostinfo: &tailcfg.Hostinfo{
+					Hostname: "test",
+					NetInfo: &tailcfg.NetInfo{
+						PreferredDERP: 2,
+					},
+				},
+			},
+		},
+		{
+			name:       "endpoints-not-set",
+			nodeBefore: Node{},
+			change: &tailcfg.PeerChange{
+				Endpoints: []netip.AddrPort{
+					netip.MustParseAddrPort("8.8.8.8:88"),
+				},
+			},
+			want: Node{
+				Endpoints: []netip.AddrPort{
+					netip.MustParseAddrPort("8.8.8.8:88"),
+				},
+			},
+		},
+		{
+			name: "endpoints-set",
+			nodeBefore: Node{
+				Endpoints: []netip.AddrPort{
+					netip.MustParseAddrPort("6.6.6.6:66"),
+				},
+			},
+			change: &tailcfg.PeerChange{
+				Endpoints: []netip.AddrPort{
+					netip.MustParseAddrPort("8.8.8.8:88"),
+				},
+			},
+			want: Node{
+				Endpoints: []netip.AddrPort{
+					netip.MustParseAddrPort("8.8.8.8:88"),
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.nodeBefore.ApplyPeerChange(tt.change)
+
+			if diff := cmp.Diff(tt.want, tt.nodeBefore, util.Comparers...); diff != "" {
+				t.Errorf("Patch unexpected result (-want +got):\n%s", diff)
+			}
+		})
+	}
+}

hscontrol/util/test.go

@@ -15,6 +15,10 @@ var IPComparer = cmp.Comparer(func(x, y netip.Addr) bool {
 	return x.Compare(y) == 0
 })
 
+var AddrPortComparer = cmp.Comparer(func(x, y netip.AddrPort) bool {
+	return x == y
+})
+
 var MkeyComparer = cmp.Comparer(func(x, y key.MachinePublic) bool {
 	return x.String() == y.String()
 })
@@ -28,5 +32,5 @@ var DkeyComparer = cmp.Comparer(func(x, y key.DiscoPublic) bool {
 })
 
 var Comparers []cmp.Option = []cmp.Option{
-	IPComparer, PrefixComparer, MkeyComparer, NkeyComparer, DkeyComparer,
+	IPComparer, PrefixComparer, AddrPortComparer, MkeyComparer, NkeyComparer, DkeyComparer,
 }

integration/auth_oidc_test.go

@@ -83,6 +83,8 @@ func TestOIDCAuthenticationPingAll(t *testing.T) {
 	err = scenario.WaitForTailscaleSync()
 	assertNoErrSync(t, err)
 
+	assertClientsState(t, allClients)
+
 	allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string {
 		return x.String()
 	})
@@ -140,6 +142,8 @@ func TestOIDCExpireNodesBasedOnTokenExpiry(t *testing.T) {
 	err = scenario.WaitForTailscaleSync()
 	assertNoErrSync(t, err)
 
+	assertClientsState(t, allClients)
+
 	allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string {
 		return x.String()
 	})

integration/auth_web_flow_test.go

@@ -53,6 +53,8 @@ func TestAuthWebFlowAuthenticationPingAll(t *testing.T) {
 	err = scenario.WaitForTailscaleSync()
 	assertNoErrSync(t, err)
 
+	assertClientsState(t, allClients)
+
 	allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string {
 		return x.String()
 	})
@@ -90,6 +92,8 @@ func TestAuthWebFlowLogoutAndRelogin(t *testing.T) {
 	err = scenario.WaitForTailscaleSync()
 	assertNoErrSync(t, err)
 
+	assertClientsState(t, allClients)
+
 	allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string {
 		return x.String()
 	})

integration/embedded_derp_test.go

@@ -33,20 +33,23 @@ func TestDERPServerScenario(t *testing.T) {
 	defer scenario.Shutdown()
 
 	spec := map[string]int{
-		"user1": len(MustTestVersions),
+		"user1": 10,
+		// "user1": len(MustTestVersions),
 	}
 
-	headscaleConfig := map[string]string{}
-	headscaleConfig["HEADSCALE_DERP_URLS"] = ""
-	headscaleConfig["HEADSCALE_DERP_SERVER_ENABLED"] = "true"
-	headscaleConfig["HEADSCALE_DERP_SERVER_REGION_ID"] = "999"
-	headscaleConfig["HEADSCALE_DERP_SERVER_REGION_CODE"] = "headscale"
-	headscaleConfig["HEADSCALE_DERP_SERVER_REGION_NAME"] = "Headscale Embedded DERP"
-	headscaleConfig["HEADSCALE_DERP_SERVER_STUN_LISTEN_ADDR"] = "0.0.0.0:3478"
-	headscaleConfig["HEADSCALE_DERP_SERVER_PRIVATE_KEY_PATH"] = "/tmp/derp.key"
-	// Envknob for enabling DERP debug logs
-	headscaleConfig["DERP_DEBUG_LOGS"] = "true"
-	headscaleConfig["DERP_PROBER_DEBUG_LOGS"] = "true"
+	headscaleConfig := map[string]string{
+		"HEADSCALE_DERP_URLS":                    "",
+		"HEADSCALE_DERP_SERVER_ENABLED":          "true",
+		"HEADSCALE_DERP_SERVER_REGION_ID":        "999",
+		"HEADSCALE_DERP_SERVER_REGION_CODE":      "headscale",
+		"HEADSCALE_DERP_SERVER_REGION_NAME":      "Headscale Embedded DERP",
+		"HEADSCALE_DERP_SERVER_STUN_LISTEN_ADDR": "0.0.0.0:3478",
+		"HEADSCALE_DERP_SERVER_PRIVATE_KEY_PATH": "/tmp/derp.key",
+
+		// Envknob for enabling DERP debug logs
+		"DERP_DEBUG_LOGS":        "true",
+		"DERP_PROBER_DEBUG_LOGS": "true",
+	}
 
 	err = scenario.CreateHeadscaleEnv(
 		spec,
@@ -67,6 +70,8 @@ func TestDERPServerScenario(t *testing.T) {
 	err = scenario.WaitForTailscaleSync()
 	assertNoErrSync(t, err)
 
+	assertClientsState(t, allClients)
+
 	allHostnames, err := scenario.ListTailscaleClientsFQDNs()
 	assertNoErrListFQDN(t, err)