Merge pull request #480 from kiwiz/master

Per container ingress network override

README.md

@@ -330,7 +330,7 @@ For containers, that would be the container IPs.
 
 Only containers/services that are connected to Caddy ingress networks are used.
 
-:warning: caddy docker proxy does a best effort to automatically detect what are the ingress networks. But that logic fails on some scenarios: [#207](https://github.com/lucaslorentz/caddy-docker-proxy/issues/207). To have a more resilient solution, you can manually configure Caddy ingress network using CLI option `ingress-networks` or environment variable `CADDY_INGRESS_NETWORKS`.
+:warning: caddy docker proxy does a best effort to automatically detect what are the ingress networks. But that logic fails on some scenarios: [#207](https://github.com/lucaslorentz/caddy-docker-proxy/issues/207). To have a more resilient solution, you can manually configure Caddy ingress network using CLI option `ingress-networks`, environment variable `CADDY_INGRESS_NETWORKS` or label `caddy_ingress_network`.
 
 Usage: `upstreams [http|https] [port]`  
 

generator/containers.go

@@ -14,11 +14,23 @@ func (g *CaddyfileGenerator) getContainerCaddyfile(container *types.Container, l
 	})
 }
 
-func (g *CaddyfileGenerator) getContainerIPAddresses(container *types.Container, logger *zap.Logger, ingress bool) ([]string, error) {
+func (g *CaddyfileGenerator) getContainerIPAddresses(container *types.Container, logger *zap.Logger, onlyIngressIps bool) ([]string, error) {
 	ips := []string{}
 
-	for _, network := range container.NetworkSettings.Networks {
-		if !ingress || g.ingressNetworks[network.NetworkID] {
+	ingressNetworkFromLabel, overrideNetwork := container.Labels[IngressNetworkLabel]
+
+	for networkName, network := range container.NetworkSettings.Networks {
+		include := false
+
+		if !onlyIngressIps  {
+			include = true
+		} else if overrideNetwork {
+			include = networkName == ingressNetworkFromLabel
+		} else {
+			include = g.ingressNetworks[network.NetworkID]
+		}
+
+		if include {
 			ips = append(ips, network.IPAddress)
 		}
 	}

generator/containers_test.go

@@ -138,6 +138,52 @@ func TestContainers_ManualIngressNetworks(t *testing.T) {
 	}, expectedCaddyfile, expectedLogs)
 }
 
+func TestContainers_OverrideIngressNetworks(t *testing.T) {
+	dockerClient := createBasicDockerClientMock()
+	dockerClient.NetworksData = []types.NetworkResource{
+		{
+			ID:   "other-network-id",
+			Name: "other-network-name",
+		},
+		{
+			ID:   "another-network-id",
+			Name: "another-network-name",
+		},
+	}
+	dockerClient.ContainersData = []types.Container{
+		{
+			ID: "CONTAINER-ID",
+			NetworkSettings: &types.SummaryNetworkSettings{
+				Networks: map[string]*network.EndpointSettings{
+					"other-network": {
+						IPAddress: "10.0.0.1",
+						NetworkID: "other-network-id",
+					},
+					"another-network": {
+						IPAddress: "10.0.0.2",
+						NetworkID: "other-network-id",
+					},
+				},
+			},
+			Labels: map[string]string{
+				"caddy_ingress_network":      "another-network",
+				fmtLabel("%s"):               "service.testdomain.com",
+				fmtLabel("%s.reverse_proxy"): "{{upstreams}}",
+			},
+		},
+	}
+
+	const expectedCaddyfile = "service.testdomain.com {\n" +
+		"	reverse_proxy 10.0.0.2\n" +
+		"}\n"
+
+	const expectedLogs = otherIngressNetworksMapLog + swarmIsAvailableLog
+
+	testGeneration(t, dockerClient, func(options *config.Options) {
+		options.IngressNetworks = []string{"other-network-name"}
+	}, expectedCaddyfile, expectedLogs)
+}
+
 func TestContainers_Replicas(t *testing.T) {
 	dockerClient := createBasicDockerClientMock()
 	dockerClient.ContainersData = []types.Container{

generator/generator.go

@@ -21,6 +21,8 @@ import (
 // DefaultLabelPrefix for caddy labels in docker
 const DefaultLabelPrefix = "caddy"
 
+const IngressNetworkLabel = "caddy_ingress_network"
+
 const swarmAvailabilityCacheInterval = 1 * time.Minute
 
 // CaddyfileGenerator generates caddyfile from docker configuration

generator/services.go

@@ -20,24 +20,24 @@ func (g *CaddyfileGenerator) getServiceCaddyfile(service *swarm.Service, logger
 	})
 }
 
-func (g *CaddyfileGenerator) getServiceProxyTargets(service *swarm.Service, logger *zap.Logger, ingress bool) ([]string, error) {
+func (g *CaddyfileGenerator) getServiceProxyTargets(service *swarm.Service, logger *zap.Logger, onlyIngressIps bool) ([]string, error) {
 	if g.options.ProxyServiceTasks {
-		return g.getServiceTasksIps(service, logger, ingress)
+		return g.getServiceTasksIps(service, logger, onlyIngressIps)
 	}
 
-	_, err := g.getServiceVirtualIps(service, logger, ingress)
+	_, err := g.getServiceVirtualIps(service, logger, onlyIngressIps)
 	if err != nil {
 		return nil, err
 	}
 
 	return []string{service.Spec.Name}, nil
 }
 
-func (g *CaddyfileGenerator) getServiceVirtualIps(service *swarm.Service, logger *zap.Logger, ingress bool) ([]string, error) {
+func (g *CaddyfileGenerator) getServiceVirtualIps(service *swarm.Service, logger *zap.Logger, onlyIngressIps bool) ([]string, error) {
 	virtualIps := []string{}
 
 	for _, virtualIP := range service.Endpoint.VirtualIPs {
-		if !ingress || g.ingressNetworks[virtualIP.NetworkID] {
+		if !onlyIngressIps || g.ingressNetworks[virtualIP.NetworkID] {
 			virtualIps = append(virtualIps, virtualIP.Addr)
 		}
 	}
@@ -49,7 +49,7 @@ func (g *CaddyfileGenerator) getServiceVirtualIps(service *swarm.Service, logger
 	return virtualIps, nil
 }
 
-func (g *CaddyfileGenerator) getServiceTasksIps(service *swarm.Service, logger *zap.Logger, ingress bool) ([]string, error) {
+func (g *CaddyfileGenerator) getServiceTasksIps(service *swarm.Service, logger *zap.Logger, onlyIngressIps bool) ([]string, error) {
 	taskListFilter := filters.NewArgs()
 	taskListFilter.Add("service", service.ID)
 	taskListFilter.Add("desired-state", "running")
@@ -66,8 +66,20 @@ func (g *CaddyfileGenerator) getServiceTasksIps(service *swarm.Service, logger *
 		for _, task := range tasks {
 			if task.Status.State == swarm.TaskStateRunning {
 				hasRunningTasks = true
+				ingressNetworkFromLabel, overrideNetwork := service.Spec.Labels[IngressNetworkLabel]
+
 				for _, networkAttachment := range task.NetworksAttachments {
-					if !ingress || g.ingressNetworks[networkAttachment.Network.ID] {
+					include := false
+
+					if !onlyIngressIps {
+						include = true
+					} else if overrideNetwork {
+						include = networkAttachment.Network.Spec.Name == ingressNetworkFromLabel
+					} else {
+						include = g.ingressNetworks[networkAttachment.Network.ID]
+					}
+
+					if include {
 						for _, address := range networkAttachment.Addresses {
 							ipAddress, _, _ := net.ParseCIDR(address)
 							tasksIps = append(tasksIps, ipAddress.String())

generator/services_test.go

@@ -382,6 +382,84 @@ func TestServiceTasks_ManualIngressNetwork(t *testing.T) {
 	}, expectedCaddyfile, expectedLogs)
 }
 
+func TestServiceTasks_OverrideIngressNetwork(t *testing.T) {
+	dockerClient := createBasicDockerClientMock()
+	dockerClient.ServicesData = []swarm.Service{
+		{
+			ID: "SERVICEID",
+			Spec: swarm.ServiceSpec{
+				Annotations: swarm.Annotations{
+					Name: "service",
+					Labels: map[string]string{
+						"caddy_ingress_network":      "another-network",
+						fmtLabel("%s"):               "service.testdomain.com",
+						fmtLabel("%s.reverse_proxy"): "{{upstreams 5000}}",
+					},
+				},
+			},
+			Endpoint: swarm.Endpoint{
+				VirtualIPs: []swarm.EndpointVirtualIP{
+					{
+						NetworkID: caddyNetworkID,
+					},
+				},
+			},
+		},
+	}
+	dockerClient.NetworksData = []types.NetworkResource{
+		{
+			ID:   "other-network-id",
+			Name: "other-network-name",
+		},
+		{
+			ID:   "another-network-id",
+			Name: "another-network-name",
+		},
+	}
+	dockerClient.TasksData = []swarm.Task{
+		{
+			ServiceID: "SERVICEID",
+			NetworksAttachments: []swarm.NetworkAttachment{
+				{
+					Network: swarm.Network{
+						ID: "other-network-id",
+						Spec: swarm.NetworkSpec{
+							Annotations: swarm.Annotations{
+								Name: "other-network",
+							},
+						},
+					},
+					Addresses: []string{"10.0.0.1/24"},
+				},
+				{
+					Network: swarm.Network{
+						ID: "another-network-id",
+						Spec: swarm.NetworkSpec{
+							Annotations: swarm.Annotations{
+								Name: "another-network",
+							},
+						},
+					},
+					Addresses: []string{"10.0.0.2/24"},
+				},
+			},
+			DesiredState: swarm.TaskStateRunning,
+			Status:       swarm.TaskStatus{State: swarm.TaskStateRunning},
+		},
+	}
+
+	const expectedCaddyfile = "service.testdomain.com {\n" +
+		"	reverse_proxy 10.0.0.2:5000\n" +
+		"}\n"
+
+	const expectedLogs = otherIngressNetworksMapLog + swarmIsAvailableLog
+
+	testGeneration(t, dockerClient, func(options *config.Options) {
+		options.ProxyServiceTasks = true
+		options.IngressNetworks = []string{"other-network-name"}
+	}, expectedCaddyfile, expectedLogs)
+}
+
 func TestServiceTasks_Running(t *testing.T) {
 	dockerClient := createBasicDockerClientMock()
 	dockerClient.ServicesData = []swarm.Service{

tests/functions.sh

@@ -2,8 +2,8 @@
 
 function retry {
   local n=0
-  local max=5
-  local delay=20
+  local max=20
+  local delay=5
   while true; do
     ((n=n+1))
     "$@" && break || {

tests/ingress-networks/compose.yaml

@@ -11,6 +11,7 @@ services:
       - controller
       - ingress_0
       - ingress_1
+      - ingress_2
     environment:
       - CADDY_DOCKER_MODE=server
       - CADDY_CONTROLLER_NETWORK=10.200.200.0/24
@@ -54,11 +55,29 @@ services:
         caddy.reverse_proxy: "{{upstreams 80}}"
         caddy.tls: "internal"
 
+  # Proxy to service
+  whoami2:
+    image: containous/whoami
+    networks:
+      - internal
+      - ingress_2
+    deploy:
+      labels:
+        caddy: whoami2.example.com
+        caddy.reverse_proxy: "{{upstreams 80}}"
+        caddy.tls: "internal"
+        caddy_ingress_network: ingress_2
+
 networks:
   ingress_0:
     name: ingress_0
   ingress_1:
     name: ingress_1
+  ingress_2:
+    name: ingress_2
+  internal:
+    name: internal
+    internal: true
   controller:
     driver: overlay
     ipam:

tests/ingress-networks/run.sh

@@ -7,7 +7,8 @@ set -e
 docker stack deploy -c compose.yaml --prune caddy_test
 
 retry curl --show-error -s -k -f --resolve whoami0.example.com:443:127.0.0.1 https://whoami0.example.com &&
-retry curl --show-error -s -k -f --resolve whoami1.example.com:443:127.0.0.1 https://whoami1.example.com || {
+retry curl --show-error -s -k -f --resolve whoami1.example.com:443:127.0.0.1 https://whoami1.example.com &&
+retry curl --show-error -s -k -f --resolve whoami2.example.com:443:127.0.0.1 https://whoami2.example.com || {
     docker service logs caddy_test_caddy_controller
     docker service logs caddy_test_caddy_server
     exit 1

tests/standalone/compose.yaml

@@ -56,6 +56,18 @@ services:
       caddy.reverse_proxy: "{{upstreams 80}}"
       caddy.tls: "internal"
 
+  # Proxy to container
+  whoami4:
+    image: containous/whoami
+    networks:
+      - internal
+      - caddy
+    labels:
+      caddy: whoami4.example.com
+      caddy.reverse_proxy: "{{upstreams 80}}"
+      caddy.tls: "internal"
+      caddy_ingress_network: caddy_test
+
   # Proxy with matches and route
   echo_0:
     image: containous/whoami
@@ -74,4 +86,7 @@ services:
 networks:
   caddy:
     name: caddy_test
-    external: true
+    external: true
+  internal:
+    name: internal
+    internal: true

tests/standalone/run.sh

@@ -10,6 +10,7 @@ retry curl --show-error -s -k -f --resolve whoami0.example.com:443:127.0.0.1 htt
 retry curl --show-error -s -k -f --resolve whoami1.example.com:443:127.0.0.1 https://whoami1.example.com &&
 retry curl --show-error -s -k -f --resolve whoami2.example.com:443:127.0.0.1 https://whoami2.example.com &&
 retry curl --show-error -s -k -f --resolve whoami3.example.com:443:127.0.0.1 https://whoami3.example.com &&
+retry curl --show-error -s -k -f --resolve whoami4.example.com:443:127.0.0.1 https://whoami4.example.com &&
 retry curl --show-error -s -k -f --resolve echo0.example.com:443:127.0.0.1 https://echo0.example.com/sourcepath/something || {
     docker service logs caddy_test_caddy
     exit 1