diff --git a/README.md b/README.md index 1d9e1e6e..90f67234 100644 --- a/README.md +++ b/README.md @@ -330,7 +330,7 @@ For containers, that would be the container IPs. Only containers/services that are connected to Caddy ingress networks are used. -:warning: caddy docker proxy does a best effort to automatically detect what are the ingress networks. But that logic fails on some scenarios: [#207](https://github.com/lucaslorentz/caddy-docker-proxy/issues/207). To have a more resilient solution, you can manually configure Caddy ingress network using CLI option `ingress-networks` or environment variable `CADDY_INGRESS_NETWORKS`. +:warning: caddy docker proxy does a best effort to automatically detect what are the ingress networks. But that logic fails on some scenarios: [#207](https://github.com/lucaslorentz/caddy-docker-proxy/issues/207). To have a more resilient solution, you can manually configure Caddy ingress network using CLI option `ingress-networks`, environment variable `CADDY_INGRESS_NETWORKS` or label `caddy_ingress_network`. Usage: `upstreams [http|https] [port]` diff --git a/generator/containers.go b/generator/containers.go index be23e54a..2b12f3ec 100644 --- a/generator/containers.go +++ b/generator/containers.go @@ -14,11 +14,23 @@ func (g *CaddyfileGenerator) getContainerCaddyfile(container *types.Container, l }) } -func (g *CaddyfileGenerator) getContainerIPAddresses(container *types.Container, logger *zap.Logger, ingress bool) ([]string, error) { +func (g *CaddyfileGenerator) getContainerIPAddresses(container *types.Container, logger *zap.Logger, onlyIngressIps bool) ([]string, error) { ips := []string{} - for _, network := range container.NetworkSettings.Networks { - if !ingress || g.ingressNetworks[network.NetworkID] { + ingressNetworkFromLabel, overrideNetwork := container.Labels[IngressNetworkLabel] + + for networkName, network := range container.NetworkSettings.Networks { + include := false + + if !onlyIngressIps { + include = true + } else if overrideNetwork { + include = networkName == ingressNetworkFromLabel + } else { + include = g.ingressNetworks[network.NetworkID] + } + + if include { ips = append(ips, network.IPAddress) } } diff --git a/generator/containers_test.go b/generator/containers_test.go index 15707d44..4707992c 100644 --- a/generator/containers_test.go +++ b/generator/containers_test.go @@ -138,6 +138,52 @@ func TestContainers_ManualIngressNetworks(t *testing.T) { }, expectedCaddyfile, expectedLogs) } +func TestContainers_OverrideIngressNetworks(t *testing.T) { + dockerClient := createBasicDockerClientMock() + dockerClient.NetworksData = []types.NetworkResource{ + { + ID: "other-network-id", + Name: "other-network-name", + }, + { + ID: "another-network-id", + Name: "another-network-name", + }, + } + dockerClient.ContainersData = []types.Container{ + { + ID: "CONTAINER-ID", + NetworkSettings: &types.SummaryNetworkSettings{ + Networks: map[string]*network.EndpointSettings{ + "other-network": { + IPAddress: "10.0.0.1", + NetworkID: "other-network-id", + }, + "another-network": { + IPAddress: "10.0.0.2", + NetworkID: "other-network-id", + }, + }, + }, + Labels: map[string]string{ + "caddy_ingress_network": "another-network", + fmtLabel("%s"): "service.testdomain.com", + fmtLabel("%s.reverse_proxy"): "{{upstreams}}", + }, + }, + } + + const expectedCaddyfile = "service.testdomain.com {\n" + + " reverse_proxy 10.0.0.2\n" + + "}\n" + + const expectedLogs = otherIngressNetworksMapLog + swarmIsAvailableLog + + testGeneration(t, dockerClient, func(options *config.Options) { + options.IngressNetworks = []string{"other-network-name"} + }, expectedCaddyfile, expectedLogs) +} + func TestContainers_Replicas(t *testing.T) { dockerClient := createBasicDockerClientMock() dockerClient.ContainersData = []types.Container{ diff --git a/generator/generator.go b/generator/generator.go index 6a5f5c23..b7f9f513 100644 --- a/generator/generator.go +++ b/generator/generator.go @@ -21,6 +21,8 @@ import ( // DefaultLabelPrefix for caddy labels in docker const DefaultLabelPrefix = "caddy" +const IngressNetworkLabel = "caddy_ingress_network" + const swarmAvailabilityCacheInterval = 1 * time.Minute // CaddyfileGenerator generates caddyfile from docker configuration diff --git a/generator/services.go b/generator/services.go index 116ce5cb..3cbc6ce6 100644 --- a/generator/services.go +++ b/generator/services.go @@ -20,12 +20,12 @@ func (g *CaddyfileGenerator) getServiceCaddyfile(service *swarm.Service, logger }) } -func (g *CaddyfileGenerator) getServiceProxyTargets(service *swarm.Service, logger *zap.Logger, ingress bool) ([]string, error) { +func (g *CaddyfileGenerator) getServiceProxyTargets(service *swarm.Service, logger *zap.Logger, onlyIngressIps bool) ([]string, error) { if g.options.ProxyServiceTasks { - return g.getServiceTasksIps(service, logger, ingress) + return g.getServiceTasksIps(service, logger, onlyIngressIps) } - _, err := g.getServiceVirtualIps(service, logger, ingress) + _, err := g.getServiceVirtualIps(service, logger, onlyIngressIps) if err != nil { return nil, err } @@ -33,11 +33,11 @@ func (g *CaddyfileGenerator) getServiceProxyTargets(service *swarm.Service, logg return []string{service.Spec.Name}, nil } -func (g *CaddyfileGenerator) getServiceVirtualIps(service *swarm.Service, logger *zap.Logger, ingress bool) ([]string, error) { +func (g *CaddyfileGenerator) getServiceVirtualIps(service *swarm.Service, logger *zap.Logger, onlyIngressIps bool) ([]string, error) { virtualIps := []string{} for _, virtualIP := range service.Endpoint.VirtualIPs { - if !ingress || g.ingressNetworks[virtualIP.NetworkID] { + if !onlyIngressIps || g.ingressNetworks[virtualIP.NetworkID] { virtualIps = append(virtualIps, virtualIP.Addr) } } @@ -49,7 +49,7 @@ func (g *CaddyfileGenerator) getServiceVirtualIps(service *swarm.Service, logger return virtualIps, nil } -func (g *CaddyfileGenerator) getServiceTasksIps(service *swarm.Service, logger *zap.Logger, ingress bool) ([]string, error) { +func (g *CaddyfileGenerator) getServiceTasksIps(service *swarm.Service, logger *zap.Logger, onlyIngressIps bool) ([]string, error) { taskListFilter := filters.NewArgs() taskListFilter.Add("service", service.ID) taskListFilter.Add("desired-state", "running") @@ -66,8 +66,20 @@ func (g *CaddyfileGenerator) getServiceTasksIps(service *swarm.Service, logger * for _, task := range tasks { if task.Status.State == swarm.TaskStateRunning { hasRunningTasks = true + ingressNetworkFromLabel, overrideNetwork := service.Spec.Labels[IngressNetworkLabel] + for _, networkAttachment := range task.NetworksAttachments { - if !ingress || g.ingressNetworks[networkAttachment.Network.ID] { + include := false + + if !onlyIngressIps { + include = true + } else if overrideNetwork { + include = networkAttachment.Network.Spec.Name == ingressNetworkFromLabel + } else { + include = g.ingressNetworks[networkAttachment.Network.ID] + } + + if include { for _, address := range networkAttachment.Addresses { ipAddress, _, _ := net.ParseCIDR(address) tasksIps = append(tasksIps, ipAddress.String()) diff --git a/generator/services_test.go b/generator/services_test.go index 458a004f..06fdec57 100644 --- a/generator/services_test.go +++ b/generator/services_test.go @@ -382,6 +382,84 @@ func TestServiceTasks_ManualIngressNetwork(t *testing.T) { }, expectedCaddyfile, expectedLogs) } +func TestServiceTasks_OverrideIngressNetwork(t *testing.T) { + dockerClient := createBasicDockerClientMock() + dockerClient.ServicesData = []swarm.Service{ + { + ID: "SERVICEID", + Spec: swarm.ServiceSpec{ + Annotations: swarm.Annotations{ + Name: "service", + Labels: map[string]string{ + "caddy_ingress_network": "another-network", + fmtLabel("%s"): "service.testdomain.com", + fmtLabel("%s.reverse_proxy"): "{{upstreams 5000}}", + }, + }, + }, + Endpoint: swarm.Endpoint{ + VirtualIPs: []swarm.EndpointVirtualIP{ + { + NetworkID: caddyNetworkID, + }, + }, + }, + }, + } + dockerClient.NetworksData = []types.NetworkResource{ + { + ID: "other-network-id", + Name: "other-network-name", + }, + { + ID: "another-network-id", + Name: "another-network-name", + }, + } + dockerClient.TasksData = []swarm.Task{ + { + ServiceID: "SERVICEID", + NetworksAttachments: []swarm.NetworkAttachment{ + { + Network: swarm.Network{ + ID: "other-network-id", + Spec: swarm.NetworkSpec{ + Annotations: swarm.Annotations{ + Name: "other-network", + }, + }, + }, + Addresses: []string{"10.0.0.1/24"}, + }, + { + Network: swarm.Network{ + ID: "another-network-id", + Spec: swarm.NetworkSpec{ + Annotations: swarm.Annotations{ + Name: "another-network", + }, + }, + }, + Addresses: []string{"10.0.0.2/24"}, + }, + }, + DesiredState: swarm.TaskStateRunning, + Status: swarm.TaskStatus{State: swarm.TaskStateRunning}, + }, + } + + const expectedCaddyfile = "service.testdomain.com {\n" + + " reverse_proxy 10.0.0.2:5000\n" + + "}\n" + + const expectedLogs = otherIngressNetworksMapLog + swarmIsAvailableLog + + testGeneration(t, dockerClient, func(options *config.Options) { + options.ProxyServiceTasks = true + options.IngressNetworks = []string{"other-network-name"} + }, expectedCaddyfile, expectedLogs) +} + func TestServiceTasks_Running(t *testing.T) { dockerClient := createBasicDockerClientMock() dockerClient.ServicesData = []swarm.Service{ diff --git a/tests/functions.sh b/tests/functions.sh index 436da7fd..bf6e8940 100644 --- a/tests/functions.sh +++ b/tests/functions.sh @@ -2,8 +2,8 @@ function retry { local n=0 - local max=5 - local delay=20 + local max=20 + local delay=5 while true; do ((n=n+1)) "$@" && break || { diff --git a/tests/ingress-networks/compose.yaml b/tests/ingress-networks/compose.yaml index 617a3efb..bf3a1a4c 100644 --- a/tests/ingress-networks/compose.yaml +++ b/tests/ingress-networks/compose.yaml @@ -11,6 +11,7 @@ services: - controller - ingress_0 - ingress_1 + - ingress_2 environment: - CADDY_DOCKER_MODE=server - CADDY_CONTROLLER_NETWORK=10.200.200.0/24 @@ -54,11 +55,29 @@ services: caddy.reverse_proxy: "{{upstreams 80}}" caddy.tls: "internal" + # Proxy to service + whoami2: + image: containous/whoami + networks: + - internal + - ingress_2 + deploy: + labels: + caddy: whoami2.example.com + caddy.reverse_proxy: "{{upstreams 80}}" + caddy.tls: "internal" + caddy_ingress_network: ingress_2 + networks: ingress_0: name: ingress_0 ingress_1: name: ingress_1 + ingress_2: + name: ingress_2 + internal: + name: internal + internal: true controller: driver: overlay ipam: diff --git a/tests/ingress-networks/run.sh b/tests/ingress-networks/run.sh index 65209f54..fc453873 100644 --- a/tests/ingress-networks/run.sh +++ b/tests/ingress-networks/run.sh @@ -7,7 +7,8 @@ set -e docker stack deploy -c compose.yaml --prune caddy_test retry curl --show-error -s -k -f --resolve whoami0.example.com:443:127.0.0.1 https://whoami0.example.com && -retry curl --show-error -s -k -f --resolve whoami1.example.com:443:127.0.0.1 https://whoami1.example.com || { +retry curl --show-error -s -k -f --resolve whoami1.example.com:443:127.0.0.1 https://whoami1.example.com && +retry curl --show-error -s -k -f --resolve whoami2.example.com:443:127.0.0.1 https://whoami2.example.com || { docker service logs caddy_test_caddy_controller docker service logs caddy_test_caddy_server exit 1 diff --git a/tests/standalone/compose.yaml b/tests/standalone/compose.yaml index 6bb3bd0f..a78c0712 100644 --- a/tests/standalone/compose.yaml +++ b/tests/standalone/compose.yaml @@ -56,6 +56,18 @@ services: caddy.reverse_proxy: "{{upstreams 80}}" caddy.tls: "internal" + # Proxy to container + whoami4: + image: containous/whoami + networks: + - internal + - caddy + labels: + caddy: whoami4.example.com + caddy.reverse_proxy: "{{upstreams 80}}" + caddy.tls: "internal" + caddy_ingress_network: caddy_test + # Proxy with matches and route echo_0: image: containous/whoami @@ -74,4 +86,7 @@ services: networks: caddy: name: caddy_test - external: true \ No newline at end of file + external: true + internal: + name: internal + internal: true diff --git a/tests/standalone/run.sh b/tests/standalone/run.sh index 71a8ea84..56c51937 100755 --- a/tests/standalone/run.sh +++ b/tests/standalone/run.sh @@ -10,6 +10,7 @@ retry curl --show-error -s -k -f --resolve whoami0.example.com:443:127.0.0.1 htt retry curl --show-error -s -k -f --resolve whoami1.example.com:443:127.0.0.1 https://whoami1.example.com && retry curl --show-error -s -k -f --resolve whoami2.example.com:443:127.0.0.1 https://whoami2.example.com && retry curl --show-error -s -k -f --resolve whoami3.example.com:443:127.0.0.1 https://whoami3.example.com && +retry curl --show-error -s -k -f --resolve whoami4.example.com:443:127.0.0.1 https://whoami4.example.com && retry curl --show-error -s -k -f --resolve echo0.example.com:443:127.0.0.1 https://echo0.example.com/sourcepath/something || { docker service logs caddy_test_caddy exit 1