Add ACNP copy span for multi-cluster (#3363)

Signed-off-by: Yang Ding <dingyang@vmware.com>

ci/jenkins/test-mc.sh

@@ -138,7 +138,7 @@ function cleanup_multicluster_antrea {
 
 function clean_multicluster {
     echo "====== Cleanup Multicluster Antrea Installation in clusters ======"
-    for kubeconfig in ${multicluster_kubeconfigs[@]}
+    for kubeconfig in "${multicluster_kubeconfigs[@]}"
     do
         cleanup_multicluster_ns "antrea-multicluster-test" $kubeconfig
         cleanup_multicluster_ns "antrea-mcs-ns" $kubeconfig
@@ -174,7 +174,7 @@ function wait_for_multicluster_controller_ready {
     sed -i 's/antrea-mcs-ns/kube-system/g' ./multicluster/test/yamls/leader-access-token.yml
     echo "type: Opaque" >>./multicluster/test/yamls/leader-access-token.yml
 
-    for config in ${membercluter_kubeconfigs[@]};
+    for config in "${membercluter_kubeconfigs[@]}";
     do
         kubectl apply -f ./multicluster/build/yamls/antrea-multicluster-member.yml ${config}
         kubectl rollout status deployment/antrea-mc-controller -n kube-system ${config}
@@ -203,7 +203,7 @@ function deliver_antrea_multicluster {
     docker save -o ${WORKDIR}/antrea-ubuntu.tar $DOCKER_REGISTRY/antrea/antrea-ubuntu:latest
 
 
-    for kubeconfig in ${multicluster_kubeconfigs[@]}
+    for kubeconfig in "${multicluster_kubeconfigs[@]}"
     do
        kubectl get nodes -o wide --no-headers=true ${kubeconfig}| awk '{print $6}' | while read IP; do
             rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/antrea-ubuntu.tar jenkins@[${IP}]:${WORKDIR}/antrea-ubuntu.tar
@@ -225,7 +225,7 @@ function deliver_multicluster_controller {
     docker save projects.registry.vmware.com/antrea/antrea-mc-controller:latest -o "${WORKDIR}"/antrea-mcs.tar
     ./multicluster/hack/generate-manifest.sh -l antrea-mcs-ns >./multicluster/test/yamls/manifest.yml
 
-    for kubeconfig in ${multicluster_kubeconfigs[@]}
+    for kubeconfig in "${multicluster_kubeconfigs[@]}"
     do
         kubectl get nodes -o wide --no-headers=true "${kubeconfig}"| awk '{print $6}' | while read IP; do
             rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/antrea-mcs.tar jenkins@[${IP}]:${WORKDIR}/antrea-mcs.tar
@@ -237,7 +237,7 @@ function deliver_multicluster_controller {
     sed -i "s|<LEADER_CLUSTER_IP>|${leader_ip}|" ./multicluster/test/yamls/east-member-cluster.yml
     sed -i "s|<LEADER_CLUSTER_IP>|${leader_ip}|" ./multicluster/test/yamls/west-member-cluster.yml
 
-    for kubeconfig in ${membercluter_kubeconfigs[@]}
+    for kubeconfig in "${membercluter_kubeconfigs[@]}"
     do
        ip=$(kubectl get nodes -o wide --no-headers=true ${EAST_CLUSTER_CONFIG} | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}')
        rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" ./multicluster/test/yamls/test-east-serviceexport.yml jenkins@[${ip}]:${WORKDIR}/serviceexport.yml
@@ -265,7 +265,7 @@ function run_multicluster_e2e {
     docker tag "${DOCKER_REGISTRY}/antrea/agnhost:2.26" "agnhost:2.26"
     docker save agnhost:2.26 -o "${WORKDIR}"/agnhost.tar
 
-    for kubeconfig in ${membercluter_kubeconfigs[@]}
+    for kubeconfig in "${membercluter_kubeconfigs[@]}"
     do
         kubectl get nodes -o wide --no-headers=true "${kubeconfig}"| awk '{print $6}' | while read IP; do
             rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/nginx.tar jenkins@["${IP}"]:"${WORKDIR}"/nginx.tar

docs/multicluster/architecture.md

@@ -26,7 +26,8 @@ The MemberClusterAnnounce CRD declares a member cluster configuration to the lea
 
 The Common Area is an abstraction in the Antrea Multi-cluster implementation provides a storage
 interface for resource export/import that can be read/written by all member and leader clusters
-in the ClusterSet. The Common Area is implemented with a Namespace in the leader cluster
+in the ClusterSet. The Common Area is implemented with a Namespace in the leader cluster for a
+given ClusterSet.
 
 ## Antrea Multi-cluster Controller
 
@@ -88,3 +89,17 @@ The Service Ports definition will be the same as exported Services, the Endpoint
 IPs from all member clusters. The new created Antrea Multi-cluster Service is just like a regular
 Kubernetes Service, so Pods in a member cluster can access the multi-cluster Service as usual without
 any extra setting.
+
+## Antrea Multi-cluster policy enforcement
+
+At this moment, Antrea does not support Pod-level policy enforcement for cross-cluster traffic. Access
+towards Multi-cluster Services can be regulated with Antrea ClusterNetworkPolicy `toService` rules. In
+each member cluster, users can create an Antrea ClusterNetworkPolicy selecting Pods in that cluster, with
+the imported Mutli-cluster Service name and Namespace in an egress `toService` rule, and the Action to
+take for traffic matching this rule. For more information regarding Antrea ClusterNetworkPolicy (ACNP),
+refer to [this document](../antrea-network-policy.md).
+
+Multi-cluster admins can also specify certain ClusterNetworkPolicies to be replicated across the entire
+ClusterSet. The ACNP to be replicated should be created as a ResourceExport in the leader cluster, and
+the resource export/import pipeline will ensure member clusters receive this ACNP spec to be replicated.
+Each member cluster's Multi-cluster Controller will then create an ACNP in their respective clusters.

docs/multicluster/getting-started.md

@@ -412,6 +412,91 @@ ResourceExport into the corresponding ResourceImport until users correct it.
 due to forementioned mismatch issue, Antrea Multi-cluster Controller will also skip converging
 the corresponding Endpoints ResourceExport until users correct it.
 
+## Multi-cluster ClusterNetworkPolicy Replication
+
+Since Antrea v1.6.0, Multi-cluster admins can specify certain ClusterNetworkPolicies to be replicated
+across the entire ClusterSet. This is especially useful for ClusterSet admins who want all clusters in
+the ClusterSet to be applied with a consistent security posture (for example, all Namespaces in all
+clusters can only communicate with Pods in their own namespaces). For more information regarding
+Antrea ClusterNetworkPolicy (ACNP), refer to [this document](../antrea-network-policy.md).
+
+To achieve such ACNP replication across clusters, admins can, in the acting leader cluster of a
+Multi-cluster deployment, create a ResourceExport of kind `AntreaClusterNetworkPolicy` which contains
+the ClusterNetworkPolicy spec they wish to be replicated. The ResourceExport should be created in the
+Namespace which implements the  Common Area of the ClusterSet. In future releases, some additional tooling
+may become available to automate the creation of such ResourceExport and make ACNP replication easier.
+
+```yaml
+apiVersion: multicluster.crd.antrea.io/v1alpha1
+kind: ResourceExport
+metadata:
+  name: strict-namespace-isolation-for-test-clusterset
+  namespace: antrea-mcs-ns          # Namespace that implements Common Area of test-clusterset
+spec:
+  kind: AntreaClusterNetworkPolicy  
+  name: strict-namespace-isolation  # In each importing cluster, an ACNP of name antrea-mc-strict-namespace-isolation will be created with the spec below
+  clusternetworkpolicy:
+    priority: 1
+    tier: securityops
+    appliedTo:
+      - namespaceSelector: {}       # Selects all Namespaces in the member cluster
+    ingress:
+      - action: Pass
+        from:
+        - namespaces:
+            match: Self            # Skip drop rule for traffic from Pods in the same Namespace
+        - podSelector:
+            matchLabels:
+              k8s-app: kube-dns    # Skip drop rule for traffic from the core-dns components
+      - action: Drop
+        from:
+        - namespaceSelector: {}    # Drop from Pods from all other Namespaces
+```
+
+The above sample spec will create an ACNP in each member cluster which implements strict namespace
+isolation for that cluster.
+
+Note that because the Tier that an ACNP refers to must exist before the ACNP is applied, an importing
+cluster may fail to create the ACNP to be replicated, if the Tier in the ResourceExport spec cannot be
+found in that particular cluster. If there are such failures, the ACNP creation status of failed member
+clusters will be reported back to the Common Area as K8s Events, and can be checked by describing the
+ResourceImport of the original ResourceExport:
+
+```text
+kubectl describe resourceimport -A
+---
+Name:         strict-namespace-isolation-antreaclusternetworkpolicy
+Namespace:    antrea-mcs-ns
+API Version:  multicluster.crd.antrea.io/v1alpha1
+Kind:         ResourceImport
+Spec:
+  Clusternetworkpolicy:
+    Applied To:
+      Namespace Selector:
+    Ingress:
+      Action:          Pass
+      Enable Logging:  false
+      From:
+        Namespaces:
+          Match:  Self
+        Pod Selector:
+          Match Labels:
+            k8s-app:   kube-dns
+      Action:          Drop
+      Enable Logging:  false
+      From:
+        Namespace Selector:
+    Priority:  1
+    Tier:      random
+  Kind:        AntreaClusterNetworkPolicy
+  Name:        strict-namespace-isolation
+  ...
+Events:
+  Type    Reason               Age    From                       Message
+  ----    ------               ----   ----                       -------
+  Warning ACNPImportFailed     2m11s  resourceimport-controller  ACNP Tier random does not exist in the importing cluster test-cluster-west
+```
+
 ## Known Issue
 
 We recommend user to reinstall or update Antrea Multi-cluster controllers through `kubectl apply`.

multicluster/Makefile

@@ -1,5 +1,10 @@
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
-CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false"
+# For controller-gen, float value is not allowed by default as it is considered dangerous
+# See https://github.com/kubernetes-sigs/controller-tools/issues/245
+# However the ResourceExport/Import refers to ACNP type definition and the priority field in ACNP spec is type float64.
+# Hence, before any ACNP spec bumps that changes the priorty field to a different type,
+# the allowDangerousTypes flag is needed for CRD manifests to generate correctly.
+CRD_OPTIONS ?= "crd:trivialVersions=true,allowDangerousTypes=true,preserveUnknownFields=false"
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))

multicluster/apis/multicluster/v1alpha1/resourceexport_types.go

@@ -20,6 +20,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"antrea.io/antrea/pkg/apis/crd/v1alpha1"
 	"antrea.io/antrea/pkg/apis/crd/v1alpha2"
 )
 
@@ -63,6 +64,8 @@ type ResourceExportSpec struct {
 	Endpoints *EndpointsExport `json:"endpoints,omitempty"`
 	// If exported resource is ExternalEntity.
 	ExternalEntity *ExternalEntityExport `json:"externalentity,omitempty"`
+	// If exported resource is AntreaClusterNetworkPolicy.
+	ClusterNetworkPolicy *v1alpha1.ClusterNetworkPolicySpec `json:"clusternetworkpolicy,omitempty"`
 	// If exported resource Kind is unknown.
 	Raw RawResourceExport `json:"raw,omitempty"`
 }

multicluster/apis/multicluster/v1alpha1/resourceexport_webhook.go

@@ -21,6 +21,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
+
+	"antrea.io/antrea/multicluster/controllers/multicluster/common"
 )
 
 // log is for logging in this package.
@@ -39,6 +41,25 @@ var _ webhook.Defaulter = &ResourceExport{}
 // Default implements webhook.Defaulter so a webhook will be registered for the type
 func (r *ResourceExport) Default() {
 	resourceexportlog.Info("default", "name", r.Name)
+	if r.Spec.ClusterNetworkPolicy == nil {
+		// Only mutate ResourceExport created for ClusterNetworkPolicy resources
+		return
+	}
+	if len(r.Labels) == 0 {
+		r.Labels = map[string]string{}
+	}
+	if nameLabelVal, exists := r.Labels[common.SourceName]; !exists || nameLabelVal != r.Spec.Name {
+		r.Labels[common.SourceName] = r.Spec.Name
+	}
+	if namespaceLabelVal, exists := r.Labels[common.SourceNamespace]; !exists || namespaceLabelVal != "" {
+		r.Labels[common.SourceNamespace] = ""
+	}
+	if kindLabelVal, exists := r.Labels[common.SourceKind]; !exists || kindLabelVal != common.AntreaClusterNetworkPolicyKind {
+		r.Labels[common.SourceKind] = common.AntreaClusterNetworkPolicyKind
+	}
+	if r.DeletionTimestamp.IsZero() && !common.StringExistsInSlice(r.Finalizers, common.ResourceExportFinalizer) {
+		r.Finalizers = append(r.Finalizers, common.ResourceExportFinalizer)
+	}
 }
 
 //+kubebuilder:webhook:path=/validate-multicluster-crd-antrea-io-v1alpha1-resourceexport,mutating=false,failurePolicy=fail,sideEffects=None,groups=multicluster.crd.antrea.io,resources=resourceexports,verbs=create;update,versions=v1alpha1,name=vresourceexport.kb.io,admissionReviewVersions={v1,v1beta1}

multicluster/apis/multicluster/v1alpha1/resourceimport_types.go

@@ -21,6 +21,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	mcs "sigs.k8s.io/mcs-api/pkg/apis/v1alpha1"
 
+	"antrea.io/antrea/pkg/apis/crd/v1alpha1"
 	"antrea.io/antrea/pkg/apis/crd/v1alpha2"
 )
 
@@ -57,6 +58,8 @@ type ResourceImportSpec struct {
 	Endpoints *EndpointsImport `json:"endpoints,omitempty"`
 	// If imported resource is ExternalEntity.
 	ExternalEntity *ExternalEntityImport `json:"externalentity,omitempty"`
+	// If imported resource is AntreaClusterNetworkPolicy.
+	ClusterNetworkPolicy *v1alpha1.ClusterNetworkPolicySpec `json:"clusternetworkpolicy,omitempty"`
 	// If imported resource is ANP.
 	// TODO:
 	// ANP uses float64 as priority.  Type float64 is discouraged by k8s, and is not supported by controller-gen tools.