v1.8 backports 2020-10-27

Documentation/concepts/networking/masquerading.rst

@@ -47,7 +47,7 @@ implementation. It requires Linux kernel 4.19 and can be enabled with
 the ``config.bpfMasquerade=true`` helm option (enabled by default).
 
 The current implementation depends on :ref:`the BPF NodePort feature <kubeproxy-free>`.
-The dependency will be removed in the Cilium v1.9 release.
+The dependency will be removed in the future (`GH-13732 <https://github.com/cilium/cilium/issues/13732>`_).
 
 Masquerading can take place only on those devices which run the eBPF masquerading
 program. This means that a packet sent from a pod to an outside will be masqueraded

Documentation/contributing/development/images.rst

@@ -1,5 +1,5 @@
 .. only:: not (epub or latex or html)
-  
+
     WARNING: You are looking at unreleased Cilium documentation.
     Please use the official rendered version released here:
     https://docs.cilium.io
@@ -77,9 +77,11 @@ update:
 .. image:: ../../images/cilium-quayio-tag-3.png
     :align: center
 
-6. A new pop-up will appear and you can select the branch that contains your
+6. A new pop-up will appear to select your desired branch.
+7. If you're interested in simply bumping the image to have the latest
+   packages, then select the release branch (i.e. v1.7, v1.8). If you already
+   have a branch that contains changes, select the branch that contains the new
    changes.
-7. Select the branch that contains the new changes.
 
 .. image:: ../../images/cilium-quayio-tag-4.png
     :align: center

Documentation/contributing/testing/ci.rst

@@ -1,5 +1,5 @@
 .. only:: not (epub or latex or html)
-  
+
     WARNING: You are looking at unreleased Cilium documentation.
     Please use the official rendered version released here:
     https://docs.cilium.io
@@ -37,38 +37,43 @@ After you don't need to run tests on your branch, please remove the branch from
 
 .. note::
 
-   It is also possible to run specific tests from this suite via ``test-focus`` and ``test-gke``. It takes trailing words as a regex. If you want to run only one ``It`` block, you need to prepend it with a test suite and create a regex, e.g ``test-focus K8sDatapathConfig.*Check connectivity with automatic direct nodes routes``
+   It is also possible to run specific tests from this suite via ``test-only``. The comment can contain 3 arguments: ``--focus`` which specifies which tests should be run, ``--kernel_version`` for supported kernel version (net-next, 49, 419 are possible values right now), ``--k8s_version`` for k8s version. If you want to run only one ``It`` block, you need to prepend it with a test suite and create a regex, e.g ``test-only --focus="K8sDatapathConfig.*Check connectivity with automatic direct nodes routes" --k8s_version=1.18 --kernel_version=net-next`` will run specified test in 1.18 Kubernetes cluster running on net-next nodes. Kubernetes version defaults to 1.17, kernel version defaults to 4.19.
+
+   +------------------------------------------------+-------------------------------------------+
+   | ``test-only --focus="K8s"``                    | Runs all kubernetes tests                 |
+   +------------------------------------------------+-------------------------------------------+
+   | ``test-only --focus="K8sConformance"``         | Runs all k8s conformance tests            |
+   +------------------------------------------------+-------------------------------------------+
+   | ``test-only --focus="K8sChaos"``               | Runs all k8s chaos tests                  |
+   +------------------------------------------------+-------------------------------------------+
+   | ``test-only --focus="K8sDatapathConfig"``      | Runs all k8s datapath configuration tests |
+   +------------------------------------------------+-------------------------------------------+
+   | ``test-only --focus="K8sDemos"``               | Runs all k8s demo tests                   |
+   +------------------------------------------------+-------------------------------------------+
+   | ``test-only --focus="K8sKubeProxyFreeMatrix"`` | Runs all k8s kube-proxy free matrix tests |
+   +------------------------------------------------+-------------------------------------------+
+   | ``test-only --focus="K8sFQDNTest"``            | Runs all k8s fqdn tests                   |
+   +------------------------------------------------+-------------------------------------------+
+   | ``test-only --focus="K8sHealthTest"``          | Runs all k8s health tests                 |
+   +------------------------------------------------+-------------------------------------------+
+   | ``test-only --focus="K8sHubbleTest"``          | Runs all k8s Hubble tests                 |
+   +------------------------------------------------+-------------------------------------------+
+   | ``test-only --focus="K8sIdentity"``            | Runs all k8s identity tests               |
+   +------------------------------------------------+-------------------------------------------+
+   | ``test-only --focus="K8sIstioTest"``           | Runs all k8s Istio tests                  |
+   +------------------------------------------------+-------------------------------------------+
+   | ``test-only --focus="K8sKafkaPolicyTest"``     | Runs all k8s Kafka tests                  |
+   +------------------------------------------------+-------------------------------------------+
+   | ``test-only --focus="K8sPolicyTest"``          | Runs all k8s policy tests                 |
+   +------------------------------------------------+-------------------------------------------+
+   | ``test-only --focus="K8sServicesTest"``        | Runs all k8s services tests               |
+   +------------------------------------------------+-------------------------------------------+
+   | ``test-only --focus="K8sUpdates"``             | Runs k8s update tests                     |
+   +------------------------------------------------+-------------------------------------------+
+
+
+   Running Runtime test suite is still done via ``test-focus`` command.
 
-   +---------------------------------------+-------------------------------------------+
-   | ``test-focus K8s``                    | Runs all kubernetes tests                 |
-   +---------------------------------------+-------------------------------------------+
-   | ``test-focus K8sConformance``         | Runs all k8s conformance tests            |
-   +---------------------------------------+-------------------------------------------+
-   | ``test-focus K8sChaos``               | Runs all k8s chaos tests                  |
-   +---------------------------------------+-------------------------------------------+
-   | ``test-focus K8sDatapathConfig``      | Runs all k8s datapath configuration tests |
-   +---------------------------------------+-------------------------------------------+
-   | ``test-focus K8sDemos``               | Runs all k8s demo tests                   |
-   +---------------------------------------+-------------------------------------------+
-   | ``test-focus K8sKubeProxyFreeMatrix`` | Runs all k8s kube-proxy free matrix tests |
-   +---------------------------------------+-------------------------------------------+
-   | ``test-focus K8sFQDNTest``            | Runs all k8s fqdn tests                   |
-   +---------------------------------------+-------------------------------------------+
-   | ``test-focus K8sHealthTest``          | Runs all k8s health tests                 |
-   +---------------------------------------+-------------------------------------------+
-   | ``test-focus K8sHubbleTest``          | Runs all k8s Hubble tests                 |
-   +---------------------------------------+-------------------------------------------+
-   | ``test-focus K8sIdentity``            | Runs all k8s identity tests               |
-   +---------------------------------------+-------------------------------------------+
-   | ``test-focus K8sIstioTest``           | Runs all k8s Istio tests                  |
-   +---------------------------------------+-------------------------------------------+
-   | ``test-focus K8sKafkaPolicyTest``     | Runs all k8s Kafka tests                  |
-   +---------------------------------------+-------------------------------------------+
-   | ``test-focus K8sPolicyTest``          | Runs all k8s policy tests                 |
-   +---------------------------------------+-------------------------------------------+
-   | ``test-focus K8sServicesTest``        | Runs all k8s services tests               |
-   +---------------------------------------+-------------------------------------------+
-   | ``test-focus K8sUpdates``             | Runs k8s update tests                     |
    +---------------------------------------+-------------------------------------------+
    | ``test-focus Runtime``                | Runs all runtime tests                    |
    +---------------------------------------+-------------------------------------------+

Documentation/gettingstarted/grafana.rst

@@ -145,6 +145,10 @@ Hubble General Processing
 
 Hubble Networking
 -----------------
+.. note::
+
+   The ``port-distribution`` metric is disabled by default.
+   Refer to :ref:`metrics` for more details about the individual metrics.
 
 .. image:: images/grafana_hubble_network.png
 .. image:: images/grafana_hubble_tcp.png

Documentation/gettingstarted/k8s-install-openshift-okd.rst

@@ -31,8 +31,10 @@ OpenShift Requirements
      doesn't simply pickup ``az login`` credentials. It's recommended to
      setup a dedicated service principal and use it
    - with the GCP provider ``openshift-install`` will only work with a service
-     account key, which has to be set using ``GOOGLE_APPLICATION_CREDENTIALS``
-     environment variable (e.g. ``GOOGLE_APPLICATION_CREDENTIALS=service-account.json``)
+     account key, which has to be set using ``GOOGLE_CREDENTIALS``
+     environment variable (e.g. ``GOOGLE_CREDENTIALS=service-account.json``).
+     Follow `Openshift Installer documentation <https://github.com/openshift/installer/blob/master/docs/user/gcp/iam.md>`_
+     to assign required roles to your service account.
 
 Create an OpenShift OKD Cluster
 ===============================
@@ -209,11 +211,12 @@ Please note that ``openshift-install`` doesn't support custom firewall
 rules, so you will need to use one of the following scripts if you are
 using AWS or GCP. Azure does not need additional configuration.
 
-.. note::
+.. warning::
 
-   This has to be done just after ``INFO Waiting up to 40m0s for
-   bootstrapping to complete...`` appears in the logs. It is safe to apply
-   these changes once, OpenShift will not override these.
+   **You need to execute the following command to configure firewall rules just after**
+   ``INFO Waiting up to 40m0s for bootstrapping to complete...`` **appears in the logs,
+   or the installation will fail**. It is safe to apply these changes once, OpenShift will
+   not override these.
 
 .. tabs::
 

Documentation/operations/metrics.rst

@@ -98,7 +98,7 @@ section for the full list of available metrics and their options.
    helm install cilium |CHART_RELEASE| \\
      --namespace kube-system \\
      --set global.hubble.enabled=true \\
-     --set global.hubble.metrics.enabled="{dns,drop,tcp,flow,port-distribution,icmp,http}"
+     --set global.hubble.metrics.enabled="{dns,drop,tcp,flow,icmp,http}"
 
 The port of the Hubble metrics can be configured with the
 ``global.hubble.metrics.port`` Helm value.

Documentation/operations/scalability/identity-relevant-labels.rst

@@ -87,10 +87,23 @@ will be used to evaluate Cilium identities:
 The above configuration would only include the following labels when evaluating
 Cilium identities:
 
-- io.kubernetes.pod.namespace=*
-- k8s-app=*
-- app=*
-- name=*
+- io.kubernetes.pod.namespace*=.*
+- k8s-app*=*
+- app*=*
+- name*=*
+
+Labels with the same prefix as defined in the configuration will also be
+considered. This lists some examples of labels that would also be evaluated for
+Cilium identities:
+
+- k8s-app-team*=*
+- app-production*=*
+- name-defined*=*
+
+When a single "inclusive label" is added to the filter, all labels not defined
+in the default list will be excluded. For example, pods running with the
+security labels ``team=team-1, env=prod`` will have the label ``env=prod``
+ignored as soon Cilium is started with the filter ``k8s:team``.
 
 Excluding Labels
 ----------------

Documentation/operations/system_requirements.rst

@@ -89,6 +89,16 @@ RancherOS_                 >= 1.5.5
           Linux distribution that works well, please let us know by opening a
           GitHub issue or by creating a pull request that updates this guide.
 
+.. note:: Systemd 245 and above (``systemctl --version``) overrides ``rp_filter`` setting
+          of Cilium network interfaces. This introduces connectivity issues (see
+          `GH-10645 <https://github.com/cilium/cilium/issues/10645>`_ for details). To
+          avoid that, configure ``rp_filter`` in systemd using the following commands:
+
+          .. code:: bash
+
+              echo 'net.ipv4.conf.lxc*.rp_filter = 0' > /etc/sysctl.d/99-override_cilium_rp_filter.conf
+              systemctl restart systemd-sysctl
+
 .. _admin_kernel_version:
 
 Linux Kernel

contrib/backporting/cherry-pick

@@ -3,6 +3,14 @@ set -e
 
 source $(dirname $(readlink -ne $BASH_SOURCE))/common.sh
 
+cleanup () {
+  if [ -n "$TMPF" ]; then
+    rm $TMPF
+  fi
+}
+
+trap cleanup EXIT
+
 cherry_pick () {
   CID=$1
   BRANCHES=`git branch -q -r --contains $CID $REM/master 2> /dev/null`
@@ -19,7 +27,6 @@ cherry_pick () {
   git format-patch -1 $FULL_ID --stdout | sed -n '/^$/,$p' >> $TMPF
   echo "Applying: $(git log -1 --oneline $FULL_ID)"
   git am --quiet -3 --signoff $TMPF
-  rm $TMPF
 }
 
 main () {

contrib/backporting/submit-backport

@@ -51,9 +51,9 @@ hub pull-request -b "v$BRANCH" -l kind/backports,backport/$BRANCH -F $SUMMARY
 
 prs=$(grep "contrib/backporting/set-labels.py" $SUMMARY | sed -e 's/^.*for pr in \([0-9 ]\+\);.*$/\1/g')
 echo -e "\nUpdating labels for PRs $prs\n" 2>&1
-echo -n "Set labels for all PRs above? [y/N] "
+echo -n "Set labels for all PRs above? [Y/n] "
 read set_all_labels
-if [ "$set_all_labels" = "y" ]; then
+if [ "$set_all_labels" != "n" ]; then
     for pr in $prs; do
         $DIR/set-labels.py $pr pending $BRANCH;
     done

install/kubernetes/Makefile

@@ -20,7 +20,7 @@ CILIUM_PULLPOLICY_REGEX := '\([pP]ullPolicy:\) .*'
 EXPERIMENTAL_OPTIONS := \
     --set global.hubble.enabled=true \
     --set global.hubble.listenAddress=":4244" \
-    --set global.hubble.metrics.enabled="{dns,drop,tcp,flow,port-distribution,icmp,http}" \
+    --set global.hubble.metrics.enabled="{dns,drop,tcp,flow,icmp,http}" \
     --set global.hubble.relay.enabled=true \
     --set global.hubble.ui.enabled=true
 

install/kubernetes/cilium/values.yaml

@@ -524,13 +524,12 @@ global:
       #   - drop
       #   - tcp
       #   - flow
-      #   - port-distribution
       #   - icmp
       #   - http
       #
       # You can specify the list of metrics from the helm CLI:
       #
-      #   --set metrics.enabled="{dns:query;ignoreAAAA,drop,tcp,flow,port-distribution,icmp,http}"
+      #   --set metrics.enabled="{dns:query;ignoreAAAA,drop,tcp,flow,icmp,http}"
       #
       enabled: ~
       # Specifies the port the metric server listens on (e.g. 9091).

install/kubernetes/experimental-install.yaml

@@ -147,7 +147,6 @@ data:
     drop
     tcp
     flow
-    port-distribution
     icmp
     http
   # An additional address for Hubble server to listen to (e.g. ":4244").

operator/k8s_cep_gc.go

@@ -126,7 +126,7 @@ func enableCiliumEndpointSyncGC(once bool) {
 						ctx,
 						cep.Name,
 						meta_v1.DeleteOptions{PropagationPolicy: &PropagationPolicy})
-					if !k8serrors.IsNotFound(err) {
+					if err != nil && !k8serrors.IsNotFound(err) {
 						scopedLog.WithError(err).Warning("Unable to delete orphaned CEP")
 						return err
 					}

pkg/endpoint/endpoint.go

@@ -121,6 +121,10 @@ type Endpoint struct {
 	// ID of the endpoint, unique in the scope of the node
 	ID uint16
 
+	// createdAt stores the time the endpoint was created. This value is
+	// recalculated on endpoint restore.
+	createdAt time.Time
+
 	// mutex protects write operations to this endpoint structure except
 	// for the logger field which has its own mutex
 	mutex lock.RWMutex
@@ -426,6 +430,7 @@ func createEndpoint(owner regeneration.Owner, proxy EndpointProxy, allocator cac
 	ep := &Endpoint{
 		owner:           owner,
 		ID:              ID,
+		createdAt:       time.Now(),
 		proxy:           proxy,
 		ifName:          ifName,
 		OpLabels:        pkgLabels.NewOpLabels(),
@@ -1061,7 +1066,12 @@ func (e *Endpoint) leaveLocked(proxyWaitGroup *completion.WaitGroup, conf Delete
 	}
 
 	if !conf.NoIdentityRelease && e.SecurityIdentity != nil {
-		identitymanager.Remove(e.SecurityIdentity)
+		// Restored endpoint may be created with a reserved identity of 5
+		// (init), which is not registered in the identity manager and
+		// therefore doesn't need to be removed.
+		if e.SecurityIdentity.ID != identity.ReservedIdentityInit {
+			identitymanager.Remove(e.SecurityIdentity)
+		}
 
 		releaseCtx, cancel := context.WithTimeout(context.Background(), option.Config.KVstoreConnectivityTimeout)
 		defer cancel()
@@ -1635,7 +1645,8 @@ type MetadataResolverCB func(ns, podName string) (pod *slim_corev1.Pod, _ []slim
 // will handle updates (such as pkg/k8s/watchers informers).
 func (e *Endpoint) RunMetadataResolver(resolveMetadata MetadataResolverCB) {
 	done := make(chan struct{})
-	controllerName := fmt.Sprintf("resolve-labels-%s", e.GetK8sNamespaceAndPodName())
+	const controllerPrefix = "resolve-labels"
+	controllerName := fmt.Sprintf("%s-%s", controllerPrefix, e.GetK8sNamespaceAndPodName())
 	go func() {
 		select {
 		case <-done:
@@ -1651,7 +1662,7 @@ func (e *Endpoint) RunMetadataResolver(resolveMetadata MetadataResolverCB) {
 				ns, podName := e.GetK8sNamespace(), e.GetK8sPodName()
 				pod, cp, identityLabels, info, _, err := resolveMetadata(ns, podName)
 				if err != nil {
-					e.Logger(controllerName).WithError(err).Warning("Unable to fetch kubernetes labels")
+					e.Logger(controllerPrefix).WithError(err).Warning("Unable to fetch kubernetes labels")
 					return err
 				}
 				e.SetPod(pod)
@@ -2373,3 +2384,8 @@ func (e *Endpoint) setDefaultPolicyConfig() {
 	e.desiredPolicy.IngressPolicyEnabled = alwaysEnforce
 	e.desiredPolicy.EgressPolicyEnabled = alwaysEnforce
 }
+
+// GetCreatedAt returns the endpoint creation time.
+func (e *Endpoint) GetCreatedAt() time.Time {
+	return e.createdAt
+}

pkg/endpoint/restore.go

@@ -1,4 +1,4 @@
-// Copyright 2018-2019 Authors of Cilium
+// Copyright 2018-2020 Authors of Cilium
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -503,6 +503,7 @@ func (ep *Endpoint) MarshalJSON() ([]byte, error) {
 
 func (ep *Endpoint) fromSerializedEndpoint(r *serializableEndpoint) {
 	ep.ID = r.ID
+	ep.createdAt = time.Now()
 	ep.containerName = r.ContainerName
 	ep.containerID = r.ContainerID
 	ep.dockerNetworkID = r.DockerNetworkID