Merge remote-tracking branch 'upstream/master' into 8-25-2022_downstr…

…eam_merge

.github/actions/cleanup-action/package.json

@@ -14,7 +14,7 @@
     "JavaScript"
   ],
   "dependencies": {
-    "@actions/core": "^1.2.6",
+    "@actions/core": "^1.9.1",
     "@actions/github": "^3.0.0"
   },
   "devDependencies": {

dist/images/Dockerfile.fedora

@@ -9,13 +9,13 @@
 # are built locally and included in the image (instead of the rpm)
 #
 
-FROM fedora:35
+FROM fedora:36
 
 USER root
 
 ENV PYTHONDONTWRITEBYTECODE yes
 
-ARG ovnver=ovn-22.06.0-2.fc35
+ARG ovnver=ovn-22.06.0-33.fc36
 # Automatically populated when using docker buildx
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
@@ -25,7 +25,7 @@ RUN echo "Running on $BUILDPLATFORM, building for $TARGETPLATFORM"
 # install needed rpms - openvswitch must be 2.10.4 or higher
 RUN INSTALL_PKGS=" \
 	python3-pyyaml bind-utils procps-ng openssl numactl-libs firewalld-filesystem \
-	libpcap hostname kubernetes-client \
+	libpcap hostname kubernetes-client util-linux \
         ovn ovn-central ovn-host python3-openvswitch tcpdump openvswitch-test python3-pyOpenSSL \
 	iptables iproute iputils strace socat koji \
         " && \

dist/images/Dockerfile.fedora.dev

@@ -15,7 +15,7 @@
 # this image in any production environment.
 #
 
-FROM fedora:34 AS ovnbuilder
+FROM fedora:36 AS ovnbuilder
 
 USER root
 
@@ -24,7 +24,7 @@ ENV PYTHONDONTWRITEBYTECODE yes
 # Install tools that are required for building ovs/ovn.
 RUN INSTALL_PKGS=" \
     python3-pyyaml bind-utils procps-ng openssl numactl-libs firewalld-filesystem \
-    libpcap hostname \
+    libpcap hostname util-linux\
     python3-openvswitch python3-pyOpenSSL \
     autoconf automake libtool g++ gcc fedora-packager rpmdevtools \
     unbound unbound-devel groff python3-sphinx graphviz openssl openssl-devel \
@@ -63,7 +63,7 @@ RUN rm rpm/rpmbuild/RPMS/x86_64/*docker*
 RUN git log -n 1
 
 # Build the final image
-FROM fedora:34
+FROM fedora:36
 
 # Install needed dependencies.
 RUN INSTALL_PKGS=" \

docs/metrics.md

@@ -17,6 +17,8 @@ Measurement accuracy can be impacted by other parallel processing that might be
 ## Change log
 This list is to help notify if there are additions, changes or removals to metrics.
 
+- Add ovn_controller_southbound_database_connected metric (https://github.com/ovn-org/ovn-kubernetes/pull/3117).
+- Stopwatch metrics now report in seconds instead of milliseconds.
 - Rename (https://github.com/ovn-org/ovn-kubernetes/pull/3022):
   - `ovs_vswitchd_interface_link_resets` -> `ovs_vswitchd_interface_resets_total`
   - `ovs_vswitchd_interface_rx_dropped` -> `ovs_vswitchd_interface_rx_dropped_total`

go-controller/go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/onsi/gomega v1.14.0
 	github.com/openshift/api v0.0.0-20220525145417-ee5b62754c68
 	github.com/openshift/client-go v0.0.0-20220525160904-9e1acff93e4a
-	github.com/ovn-org/libovsdb v0.6.1-0.20220603151050-98c0bad3cff1
+	github.com/ovn-org/libovsdb v0.6.1-0.20220817084941-500126274290
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.12.1
 	github.com/prometheus/client_model v0.2.0

go-controller/go.sum

@@ -415,8 +415,8 @@ github.com/openshift/build-machinery-go v0.0.0-20211213093930-7e33a7eb4ce3/go.mo
 github.com/openshift/client-go v0.0.0-20220525160904-9e1acff93e4a h1:ylsEgoC8Dlg4A0C1TLH0A4x/TZao7k1YveLwROhRUdk=
 github.com/openshift/client-go v0.0.0-20220525160904-9e1acff93e4a/go.mod h1:eDO5QeVi2IiXmDwB0e2z1DpAznWroZKe978pzZwFBzg=
 github.com/ory/dockertest/v3 v3.8.0/go.mod h1:9zPATATlWQru+ynXP+DytBQrsXV7Tmlx7K86H6fQaDo=
-github.com/ovn-org/libovsdb v0.6.1-0.20220603151050-98c0bad3cff1 h1:ZXVHrW4b7RK/emoqTNL58IWUuK0dD8PVb93fbfh9cMs=
-github.com/ovn-org/libovsdb v0.6.1-0.20220603151050-98c0bad3cff1/go.mod h1:BQPdnSM2QOKxPwxl7wHJDSPP4B/CDKq3+vzgFW3J5gE=
+github.com/ovn-org/libovsdb v0.6.1-0.20220817084941-500126274290 h1:perKNOFGKyBExnh9WzUZnbBd84U9FRIKzUVC4AB14b4=
+github.com/ovn-org/libovsdb v0.6.1-0.20220817084941-500126274290/go.mod h1:BQPdnSM2QOKxPwxl7wHJDSPP4B/CDKq3+vzgFW3J5gE=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

go-controller/pkg/config/config.go

@@ -325,7 +325,8 @@ type MetricsConfig struct {
 	NodeServerCert        string `gcfg:"node-server-cert"`
 	// EnableConfigDuration holds the boolean flag to enable OVN-Kubernetes master to monitor OVN-Kubernetes master
 	// configuration duration and optionally, its application to all nodes
-	EnableConfigDuration bool `gcfg:"enable-config-duration"`
+	EnableConfigDuration  bool `gcfg:"enable-config-duration"`
+	EnableEIPScaleMetrics bool `gcfg:"enable-eip-scale-metrics"`
 }
 
 // OVNKubernetesFeatureConfig holds OVN-Kubernetes feature enhancement config file parameters and command-line overrides
@@ -1000,6 +1001,11 @@ var MetricsFlags = []cli.Flag{
 		Usage:       "Enables monitoring OVN-Kubernetes master and OVN configuration duration",
 		Destination: &cliConfig.Metrics.EnableConfigDuration,
 	},
+	&cli.BoolFlag{
+		Name:        "metrics-enable-eip-scale",
+		Usage:       "Enables metrics related to Egress IP scaling",
+		Destination: &cliConfig.Metrics.EnableEIPScaleMetrics,
+	},
 }
 
 // OvnNBFlags capture OVN northbound database options

go-controller/pkg/config/config_test.go

@@ -157,6 +157,7 @@ enable-pprof=true
 node-server-privkey=/path/to/node-metrics-private.key
 node-server-cert=/path/to/node-metrics.crt
 enable-config-duration=true
+enable-eip-scale-metrics=true
 
 [logging]
 loglevel=5
@@ -574,6 +575,7 @@ var _ = Describe("Config Operations", func() {
 			gomega.Expect(Metrics.NodeServerPrivKey).To(gomega.Equal("/path/to/node-metrics-private.key"))
 			gomega.Expect(Metrics.NodeServerCert).To(gomega.Equal("/path/to/node-metrics.crt"))
 			gomega.Expect(Metrics.EnableConfigDuration).To(gomega.Equal(true))
+			gomega.Expect(Metrics.EnableEIPScaleMetrics).To(gomega.Equal(true))
 
 			gomega.Expect(OvnNorth.Scheme).To(gomega.Equal(OvnDBSchemeSSL))
 			gomega.Expect(OvnNorth.PrivKey).To(gomega.Equal("/path/to/nb-client-private.key"))
@@ -657,6 +659,7 @@ var _ = Describe("Config Operations", func() {
 			gomega.Expect(Metrics.NodeServerPrivKey).To(gomega.Equal("/tls/nodeprivkey"))
 			gomega.Expect(Metrics.NodeServerCert).To(gomega.Equal("/tls/nodecert"))
 			gomega.Expect(Metrics.EnableConfigDuration).To(gomega.Equal(true))
+			gomega.Expect(Metrics.EnableEIPScaleMetrics).To(gomega.Equal(true))
 
 			gomega.Expect(OvnNorth.Scheme).To(gomega.Equal(OvnDBSchemeSSL))
 			gomega.Expect(OvnNorth.PrivKey).To(gomega.Equal("/client/privkey"))

go-controller/pkg/factory/factory.go

@@ -294,7 +294,7 @@ func NewNodeWatchFactory(ovnClientset *util.OVNClientset, nodeName string) (*Wat
 			kapi.NamespaceAll,
 			resyncPeriod,
 			cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
-			noServiceNameSelector())
+			withServiceNameAndNoHeadlessServiceSelector())
 	})
 
 	var err error
@@ -757,27 +757,31 @@ func (wf *WatchFactory) EgressQoSInformer() egressqosinformer.EgressQoSInformer
 	return wf.egressQoSFactory.K8s().V1().EgressQoSes()
 }
 
-// noServiceNameSelector is a LabelSelector added to the watcher for
-// EndpointSlices that excludes EndpointSlices that don't
-// have "kubernetes.io/service-name" label and also those
-// that have "kubernetes.io/service-name" label value set to "".
-func noServiceNameSelector() func(options *metav1.ListOptions) {
-	// if the LabelServiceName label doesn't exist, skip it.
+// withServiceNameAndNoHeadlessServiceSelector returns a LabelSelector (added to the
+// watcher for EndpointSlices) that will only choose EndpointSlices with a non-empty
+// "kubernetes.io/service-name" label and without "service.kubernetes.io/headless"
+// label.
+func withServiceNameAndNoHeadlessServiceSelector() func(options *metav1.ListOptions) {
+	// LabelServiceName must exist
 	svcNameLabel, err := labels.NewRequirement(discovery.LabelServiceName, selection.Exists, nil)
 	if err != nil {
 		// cannot occur
 		panic(err)
 	}
-
+	// LabelServiceName value must be non-empty
 	notEmptySvcName, err := labels.NewRequirement(discovery.LabelServiceName, selection.NotEquals, []string{""})
 	if err != nil {
 		// cannot occur
 		panic(err)
 	}
+	// headless service label must not be there
+	noHeadlessService, err := labels.NewRequirement(kapi.IsHeadlessService, selection.DoesNotExist, nil)
+	if err != nil {
+		// cannot occur
+		panic(err)
+	}
 
-	selector := labels.NewSelector()
-	selector.Add(*svcNameLabel)
-	selector.Add(*notEmptySvcName)
+	selector := labels.NewSelector().Add(*svcNameLabel, *notEmptySvcName, *noHeadlessService)
 
 	return func(options *metav1.ListOptions) {
 		options.LabelSelector = selector.String()

go-controller/pkg/factory/factory_test.go

@@ -110,12 +110,15 @@ func newService(name, namespace string) *v1.Service {
 	}
 }
 
-func newEndpointSlice(name, namespace string) *discovery.EndpointSlice {
+func newEndpointSlice(name, namespace, service string) *discovery.EndpointSlice {
 	return &discovery.EndpointSlice{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
 			UID:       types.UID(name),
 			Namespace: namespace,
+			Labels: map[string]string{
+				discovery.LabelServiceName: service,
+			},
 		},
 	}
 }
@@ -418,7 +421,7 @@ var _ = Describe("Watch Factory Operations", func() {
 		})
 
 		It("is called for each existing endpointSlice", func() {
-			endpointSlices = append(endpointSlices, newEndpointSlice("myEndpointSlice", "default"))
+			endpointSlices = append(endpointSlices, newEndpointSlice("myEndpointSlice", "default", "myService"))
 			testExisting(EndpointSliceType, "", nil)
 		})
 
@@ -509,8 +512,8 @@ var _ = Describe("Watch Factory Operations", func() {
 		})
 
 		It("calls ADD for each existing endpointSlices", func() {
-			endpointSlices = append(endpointSlices, newEndpointSlice("myEndpointSlice", "default"))
-			endpointSlices = append(endpointSlices, newEndpointSlice("myEndpointSlice2", "default"))
+			endpointSlices = append(endpointSlices, newEndpointSlice("myEndpointSlice", "default", "myService"))
+			endpointSlices = append(endpointSlices, newEndpointSlice("myEndpointSlice2", "default", "myService"))
 			testExisting(EndpointSliceType)
 		})
 
@@ -1088,7 +1091,7 @@ var _ = Describe("Watch Factory Operations", func() {
 		err = wf.Start()
 		Expect(err).NotTo(HaveOccurred())
 
-		added := newEndpointSlice("myEndpointSlice", "default")
+		added := newEndpointSlice("myEndpointSlice", "default", "myService")
 		h, c := addHandler(wf, EndpointSliceType, cache.ResourceEventHandlerFuncs{
 			AddFunc: func(obj interface{}) {
 				epSlice := obj.(*discovery.EndpointSlice)

go-controller/pkg/libovsdbops/acl.go

@@ -13,9 +13,9 @@ import (
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
 )
 
-// getACLName returns the ACL name if it has one otherwise returns
+// GetACLName returns the ACL name if it has one otherwise returns
 // an empty string.
-func getACLName(acl *nbdb.ACL) string {
+func GetACLName(acl *nbdb.ACL) string {
 	if acl.Name != nil {
 		return *acl.Name
 	}
@@ -30,8 +30,8 @@ func isEquivalentACL(existing *nbdb.ACL, searched *nbdb.ACL) bool {
 		return true
 	}
 
-	eName := getACLName(existing)
-	sName := getACLName(searched)
+	eName := GetACLName(existing)
+	sName := GetACLName(searched)
 	// TODO if we want to support adding/removing external ids,
 	// we need to compare them differently, perhaps just the common subset
 	if eName != "" && eName == sName && reflect.DeepEqual(existing.ExternalIDs, searched.ExternalIDs) {
@@ -55,7 +55,8 @@ func FindACLsWithPredicate(nbClient libovsdbclient.Client, p aclPredicate) ([]*n
 }
 
 // BuildACL builds an ACL with empty optional properties unset
-func BuildACL(name string, direction nbdb.ACLDirection, priority int, match string, action nbdb.ACLAction, meter string, severity nbdb.ACLSeverity, log bool, externalIds map[string]string, options map[string]string) *nbdb.ACL {
+func BuildACL(name string, direction nbdb.ACLDirection, priority int, match string, action nbdb.ACLAction, meter string,
+	severity nbdb.ACLSeverity, log bool, externalIds map[string]string, options map[string]string) *nbdb.ACL {
 	name = fmt.Sprintf("%.63s", name)
 
 	var realName *string
@@ -86,6 +87,15 @@ func BuildACL(name string, direction nbdb.ACLDirection, priority int, match stri
 	return acl
 }
 
+func SetACLLogging(acl *nbdb.ACL, severity nbdb.ACLSeverity, log bool) {
+	var realSeverity *string
+	if len(severity) != 0 {
+		realSeverity = &severity
+	}
+	acl.Severity = realSeverity
+	acl.Log = log
+}
+
 // CreateOrUpdateACLsOps creates or updates the provided ACLs returning the
 // corresponding ops
 func CreateOrUpdateACLsOps(nbClient libovsdbclient.Client, ops []libovsdb.Operation, acls ...*nbdb.ACL) ([]libovsdb.Operation, error) {

go-controller/pkg/metrics/master.go

@@ -189,6 +189,36 @@ var metricEgressIPCount = prometheus.NewGauge(prometheus.GaugeOpts{
 	Help:      "The number of defined egress IP addresses",
 })
 
+var metricEgressIPAssignLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
+	Namespace: MetricOvnkubeNamespace,
+	Subsystem: MetricOvnkubeSubsystemMaster,
+	Name:      "egress_ips_assign_latency_seconds",
+	Help:      "The latency of egress IP assignment to ovn nb database",
+	Buckets:   prometheus.ExponentialBuckets(.001, 2, 15),
+})
+
+var metricEgressIPUnassignLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
+	Namespace: MetricOvnkubeNamespace,
+	Subsystem: MetricOvnkubeSubsystemMaster,
+	Name:      "egress_ips_unassign_latency_seconds",
+	Help:      "The latency of egress IP unassignment from ovn nb database",
+	Buckets:   prometheus.ExponentialBuckets(.001, 2, 15),
+})
+
+var metricEgressIPNodeUnreacheableCount = prometheus.NewCounter(prometheus.CounterOpts{
+	Namespace: MetricOvnkubeNamespace,
+	Subsystem: MetricOvnkubeSubsystemMaster,
+	Name:      "egress_ips_node_unreachable_total",
+	Help:      "The total number of times assigned egress IP(s) were unreachable"},
+)
+
+var metricEgressIPRebalanceCount = prometheus.NewCounter(prometheus.CounterOpts{
+	Namespace: MetricOvnkubeNamespace,
+	Subsystem: MetricOvnkubeSubsystemMaster,
+	Name:      "egress_ips_rebalance_total",
+	Help:      "The total number of times assigned egress IP(s) needed to be moved to a different node"},
+)
+
 var metricEgressFirewallRuleCount = prometheus.NewGauge(prometheus.GaugeOpts{
 	Namespace: MetricOvnkubeNamespace,
 	Subsystem: MetricOvnkubeSubsystemMaster,
@@ -344,6 +374,12 @@ func RegisterMasterFunctional() {
 	prometheus.MustRegister(metricV4AllocatedHostSubnetCount)
 	prometheus.MustRegister(metricV6AllocatedHostSubnetCount)
 	prometheus.MustRegister(metricEgressIPCount)
+	if config.Metrics.EnableEIPScaleMetrics {
+		prometheus.MustRegister(metricEgressIPAssignLatency)
+		prometheus.MustRegister(metricEgressIPUnassignLatency)
+	}
+	prometheus.MustRegister(metricEgressIPNodeUnreacheableCount)
+	prometheus.MustRegister(metricEgressIPRebalanceCount)
 	prometheus.MustRegister(metricEgressFirewallRuleCount)
 	prometheus.MustRegister(metricEgressFirewallCount)
 	prometheus.MustRegister(metricEgressRoutingViaHost)
@@ -433,6 +469,26 @@ func RecordEgressIPCount(count float64) {
 	metricEgressIPCount.Set(count)
 }
 
+// RecordEgressIPAssign records how long it took EgressIP to configure OVN.
+func RecordEgressIPAssign(duration time.Duration) {
+	metricEgressIPAssignLatency.Observe(duration.Seconds())
+}
+
+// RecordEgressIPUnassign records how long it took EgressIP to unconfigure OVN.
+func RecordEgressIPUnassign(duration time.Duration) {
+	metricEgressIPUnassignLatency.Observe(duration.Seconds())
+}
+
+// RecordEgressIPReachableNode records how many times EgressIP detected an unuseable node.
+func RecordEgressIPUnreachableNode() {
+	metricEgressIPNodeUnreacheableCount.Inc()
+}
+
+// RecordEgressIPRebalance records how many EgressIPs had to move to a different egress node.
+func RecordEgressIPRebalance(count int) {
+	metricEgressIPRebalanceCount.Add(float64(count))
+}
+
 // UpdateEgressFirewallRuleCount records the number of Egress firewall rules.
 func UpdateEgressFirewallRuleCount(count float64) {
 	metricEgressFirewallRuleCount.Add(count)