Add support to disable convox domain tls cert

docs/installation/production-rack/aws.md

@@ -52,6 +52,7 @@ The following environment variables are required:
 | **build_node_min_count** |     0                  | Minimum number of build nodes to keep running |
 | **cert_duration**        | **2160h**              | Certification renew period                                                                                     | 
 | **cidr**                 | **10.1.0.0/16**        | CIDR range for VPC                                                                                             |
+| **convox_domain_tls_cert_disable** | false        | Disable convox domain(*.convox.cloud) tls certificate generation for services |
 | **fluentd_disable**       | **false**              | Disable fluentd installation in the rack |
 | **gpu_tag_enable**       | **false**              | Enable gpu tagging. Some aws region doesn't support gpu tagging  |
 | **high_availability**    | **true**               | Setting this to "false" will create a cluster with less reduntant resources for cost optimization              |

provider/k8s/k8s.go

@@ -40,35 +40,36 @@ const (
 )
 
 type Provider struct {
-	Atom               atom.Interface
-	BuildkitEnabled    string
-	BuildNodeEnabled   string
-	CertManager        bool
-	CertManagerRoleArn string
-	Cluster            kubernetes.Interface
-	Config             *rest.Config
-	Convox             cv.Interface
-	CertManagerClient  cmclient.Interface
-	DiscoveryClient    discovery.DiscoveryInterface
-	Domain             string
-	DomainInternal     string
-	DynamicClient      dynamic.Interface
-	Engine             Engine
-	Image              string
-	JwtMngr            *jwt.JwtManager
-	Name               string
-	MetricScraper      *MetricScraperClient
-	MetricsClient      metricsclientset.Interface
-	Namespace          string
-	Password           string
-	Provider           string
-	RackName           string
-	Resolver           string
-	RestClient         rest.Interface
-	Router             string
-	Socket             string
-	Storage            string
-	Version            string
+	Atom                       atom.Interface
+	BuildkitEnabled            string
+	BuildNodeEnabled           string
+	CertManager                bool
+	CertManagerRoleArn         string
+	Cluster                    kubernetes.Interface
+	Config                     *rest.Config
+	Convox                     cv.Interface
+	ConvoxDomainTLSCertDisable bool
+	CertManagerClient          cmclient.Interface
+	DiscoveryClient            discovery.DiscoveryInterface
+	Domain                     string
+	DomainInternal             string
+	DynamicClient              dynamic.Interface
+	Engine                     Engine
+	Image                      string
+	JwtMngr                    *jwt.JwtManager
+	Name                       string
+	MetricScraper              *MetricScraperClient
+	MetricsClient              metricsclientset.Interface
+	Namespace                  string
+	Password                   string
+	Provider                   string
+	RackName                   string
+	Resolver                   string
+	RestClient                 rest.Interface
+	Router                     string
+	Socket                     string
+	Storage                    string
+	Version                    string
 
 	ctx       context.Context
 	logger    *logger.Logger
@@ -129,33 +130,34 @@ func FromEnv() (*Provider, error) {
 	rn := common.CoalesceString(os.Getenv("RACK_NAME"), ns.Labels["rack"])
 
 	p := &Provider{
-		Atom:               ac,
-		BuildkitEnabled:    "true",
-		BuildNodeEnabled:   os.Getenv("BUILD_NODE_ENABLED"),
-		CertManager:        os.Getenv("CERT_MANAGER") == "true",
-		CertManagerRoleArn: os.Getenv("CERT_MANAGER_ROLE_ARN"),
-		Cluster:            kc,
-		Config:             rc,
-		Convox:             cc,
-		CertManagerClient:  cm,
-		DiscoveryClient:    kc.Discovery(),
-		Domain:             os.Getenv("DOMAIN"),
-		DomainInternal:     os.Getenv("DOMAIN_INTERNAL"),
-		DynamicClient:      dc,
-		Image:              os.Getenv("IMAGE"),
-		MetricScraper:      ms,
-		MetricsClient:      mc,
-		Name:               ns.Labels["rack"],
-		Namespace:          ns.Name,
-		Password:           os.Getenv("PASSWORD"),
-		Provider:           common.CoalesceString(os.Getenv("PROVIDER"), "k8s"),
-		RackName:           rn,
-		Resolver:           os.Getenv("RESOLVER"),
-		RestClient:         kc.RESTClient(),
-		Router:             os.Getenv("ROUTER"),
-		Socket:             common.CoalesceString(os.Getenv("SOCKET"), "/var/run/docker.sock"),
-		Storage:            common.CoalesceString(os.Getenv("STORAGE"), "/var/storage"),
-		Version:            common.CoalesceString(os.Getenv("VERSION"), "dev"),
+		Atom:                       ac,
+		BuildkitEnabled:            "true",
+		BuildNodeEnabled:           os.Getenv("BUILD_NODE_ENABLED"),
+		CertManager:                os.Getenv("CERT_MANAGER") == "true",
+		CertManagerRoleArn:         os.Getenv("CERT_MANAGER_ROLE_ARN"),
+		Cluster:                    kc,
+		Config:                     rc,
+		Convox:                     cc,
+		ConvoxDomainTLSCertDisable: os.Getenv("CONVOX_DOMAIN_TLS_CERT_DISABLE") == "true",
+		CertManagerClient:          cm,
+		DiscoveryClient:            kc.Discovery(),
+		Domain:                     os.Getenv("DOMAIN"),
+		DomainInternal:             os.Getenv("DOMAIN_INTERNAL"),
+		DynamicClient:              dc,
+		Image:                      os.Getenv("IMAGE"),
+		MetricScraper:              ms,
+		MetricsClient:              mc,
+		Name:                       ns.Labels["rack"],
+		Namespace:                  ns.Name,
+		Password:                   os.Getenv("PASSWORD"),
+		Provider:                   common.CoalesceString(os.Getenv("PROVIDER"), "k8s"),
+		RackName:                   rn,
+		Resolver:                   os.Getenv("RESOLVER"),
+		RestClient:                 kc.RESTClient(),
+		Router:                     os.Getenv("ROUTER"),
+		Socket:                     common.CoalesceString(os.Getenv("SOCKET"), "/var/run/docker.sock"),
+		Storage:                    common.CoalesceString(os.Getenv("STORAGE"), "/var/storage"),
+		Version:                    common.CoalesceString(os.Getenv("VERSION"), "dev"),
 	}
 
 	return p, nil

provider/k8s/release.go

@@ -374,14 +374,15 @@ func (p *Provider) releaseTemplateIngress(a *structs.App, ss manifest.Services,
 		}
 
 		params := map[string]interface{}{
-			"Annotations": ans,
-			"App":         a.Name,
-			"Class":       p.Engine.IngressClass(),
-			"Host":        p.Engine.ServiceHost(a.Name, s),
-			"Idles":       common.DefaultBool(opts.Idle, idles),
-			"Namespace":   p.AppNamespace(a.Name),
-			"Rack":        p.Name,
-			"Service":     s,
+			"Annotations":                ans,
+			"App":                        a.Name,
+			"Class":                      p.Engine.IngressClass(),
+			"ConvoxDomainTLSCertDisable": !p.ConvoxDomainTLSCertDisable,
+			"Host":                       p.Engine.ServiceHost(a.Name, s),
+			"Idles":                      common.DefaultBool(opts.Idle, idles),
+			"Namespace":                  p.AppNamespace(a.Name),
+			"Rack":                       p.Name,
+			"Service":                    s,
 		}
 
 		data, err := p.RenderTemplate("app/ingress", params)
@@ -407,14 +408,15 @@ func (p *Provider) releaseTemplateIngressInternal(a *structs.App, ss manifest.Se
 		s := ss[i]
 
 		params := map[string]interface{}{
-			"Annotations": map[string]string{},
-			"App":         a.Name,
-			"Class":       p.Engine.IngressInternalClass(),
-			"Host":        p.Engine.ServiceHost(a.Name, s),
-			"Idles":       common.DefaultBool(opts.Idle, idles),
-			"Namespace":   p.AppNamespace(a.Name),
-			"Rack":        p.Name,
-			"Service":     s,
+			"Annotations":                map[string]string{},
+			"App":                        a.Name,
+			"Class":                      p.Engine.IngressInternalClass(),
+			"ConvoxDomainTLSCertDisable": !p.ConvoxDomainTLSCertDisable,
+			"Host":                       p.Engine.ServiceHost(a.Name, s),
+			"Idles":                      common.DefaultBool(opts.Idle, idles),
+			"Namespace":                  p.AppNamespace(a.Name),
+			"Rack":                       p.Name,
+			"Service":                    s,
 		}
 
 		data, err := p.RenderTemplate("app/ingress-internal", params)

provider/k8s/template/app/ingress.yml.tmpl

@@ -38,9 +38,11 @@ metadata:
 spec:
   ingressClassName: "{{.Class}}"
   tls:
+  {{ if .ConvoxDomainTLSCertDisable }}
   - hosts:
     - {{ safe .Host }}
     secretName: cert-{{.Service.Name}}
+  {{ end }}
   {{ with .Service.Domains }}
   - hosts:
     {{ range . }}

terraform/api/aws/main.tf

@@ -15,20 +15,21 @@ module "k8s" {
     kubernetes = kubernetes
   }
 
-  buildkit_enabled             = var.buildkit_enabled
-  build_node_enabled           = var.build_node_enabled
-  docker_hub_authentication    = var.docker_hub_authentication
-  domain                       = var.domain
-  domain_internal              = var.domain_internal
-  disable_image_manifest_cache = var.disable_image_manifest_cache
-  image                        = var.image
-  metrics_scraper_host         = var.metrics_scraper_host
-  namespace                    = var.namespace
-  rack                         = var.name
-  rack_name                    = var.rack_name
-  release                      = var.release
-  replicas                     = var.high_availability ? 2 : 1
-  resolver                     = var.resolver
+  buildkit_enabled               = var.buildkit_enabled
+  build_node_enabled             = var.build_node_enabled
+  convox_domain_tls_cert_disable = var.convox_domain_tls_cert_disable
+  docker_hub_authentication      = var.docker_hub_authentication
+  domain                         = var.domain
+  domain_internal                = var.domain_internal
+  disable_image_manifest_cache   = var.disable_image_manifest_cache
+  image                          = var.image
+  metrics_scraper_host           = var.metrics_scraper_host
+  namespace                      = var.namespace
+  rack                           = var.name
+  rack_name                      = var.rack_name
+  release                        = var.release
+  replicas                       = var.high_availability ? 2 : 1
+  resolver                       = var.resolver
 
   annotations = {
     "cert-manager.io/cluster-issuer" = "letsencrypt"

terraform/api/aws/variables.tf

@@ -12,6 +12,11 @@ variable "cert_duration" {
   type    = string
 }
 
+variable "convox_domain_tls_cert_disable" {
+  default = false
+  type    = bool
+}
+
 variable "docker_hub_authentication" {
   type = string
 }

terraform/api/k8s/main.tf

@@ -124,6 +124,11 @@ resource "kubernetes_deployment" "api" {
             value = var.build_node_enabled
           }
 
+          env {
+            name  = "CONVOX_DOMAIN_TLS_CERT_DISABLE"
+            value = var.convox_domain_tls_cert_disable
+          }
+
           env {
             name  = "DOMAIN"
             value = var.domain

terraform/api/k8s/variables.tf

@@ -17,6 +17,11 @@ variable "build_node_enabled" {
   type    = bool
 }
 
+variable "convox_domain_tls_cert_disable" {
+  default = false
+  type    = bool
+}
+
 variable "docker_hub_authentication" {
   default = null
   type    = string

terraform/rack/aws/main.tf

@@ -24,23 +24,24 @@ module "api" {
     kubernetes = kubernetes
   }
 
-  buildkit_enabled             = var.buildkit_enabled
-  build_node_enabled           = var.build_node_enabled
-  docker_hub_authentication    = module.k8s.docker_hub_authentication
-  domain                       = try(module.router.endpoint, "") # terraform destroy sometimes failes to resolve the value
-  domain_internal              = module.router.endpoint_internal
-  disable_image_manifest_cache = var.disable_image_manifest_cache
-  high_availability            = var.high_availability
-  metrics_scraper_host         = module.metrics.metrics_scraper_host
-  image                        = var.image
-  name                         = var.name
-  rack_name                    = var.rack_name
-  namespace                    = module.k8s.namespace
-  oidc_arn                     = var.oidc_arn
-  oidc_sub                     = var.oidc_sub
-  release                      = var.release
-  resolver                     = module.resolver.endpoint
-  router                       = module.router.endpoint
+  buildkit_enabled               = var.buildkit_enabled
+  build_node_enabled             = var.build_node_enabled
+  convox_domain_tls_cert_disable = var.convox_domain_tls_cert_disable
+  docker_hub_authentication      = module.k8s.docker_hub_authentication
+  domain                         = try(module.router.endpoint, "") # terraform destroy sometimes failes to resolve the value
+  domain_internal                = module.router.endpoint_internal
+  disable_image_manifest_cache   = var.disable_image_manifest_cache
+  high_availability              = var.high_availability
+  metrics_scraper_host           = module.metrics.metrics_scraper_host
+  image                          = var.image
+  name                           = var.name
+  rack_name                      = var.rack_name
+  namespace                      = module.k8s.namespace
+  oidc_arn                       = var.oidc_arn
+  oidc_sub                       = var.oidc_sub
+  release                        = var.release
+  resolver                       = module.resolver.endpoint
+  router                         = module.router.endpoint
 }
 
 module "metrics" {

terraform/rack/aws/variables.tf

@@ -11,6 +11,11 @@ variable "cluster" {
   type = string
 }
 
+variable "convox_domain_tls_cert_disable" {
+  default = false
+  type    = bool
+}
+
 variable "docker_hub_username" {
   default = ""
 }

terraform/system/aws/main.tf

@@ -127,29 +127,30 @@ module "rack" {
     null_resource.wait_for_cluster
   ]
 
-  build_node_enabled           = var.build_node_enabled
-  cluster                      = module.cluster.id
-  docker_hub_username          = var.docker_hub_username
-  docker_hub_password          = var.docker_hub_password
-  disable_image_manifest_cache = var.disable_image_manifest_cache
-  eks_addons                   = module.cluster.eks_addons
-  high_availability            = var.high_availability
-  idle_timeout                 = var.idle_timeout
-  internal_router              = var.internal_router
-  image                        = local.image
-  name                         = local.name
-  rack_name                    = local.rack_name
-  oidc_arn                     = module.cluster.oidc_arn
-  oidc_sub                     = module.cluster.oidc_sub
-  proxy_protocol               = var.proxy_protocol
-  release                      = local.release
-  ssl_ciphers                  = var.ssl_ciphers
-  ssl_protocols                = var.ssl_protocols
-  subnets                      = module.cluster.subnets
-  tags                         = local.tag_map
-  telemetry                    = var.telemetry
-  telemetry_map                = local.telemetry_map
-  telemetry_default_map        = local.telemetry_default_map
-  whitelist                    = split(",", var.whitelist)
-  ebs_csi_driver_name          = module.cluster.ebs_csi_driver_name
+  build_node_enabled             = var.build_node_enabled
+  cluster                        = module.cluster.id
+  convox_domain_tls_cert_disable = var.convox_domain_tls_cert_disable
+  docker_hub_username            = var.docker_hub_username
+  docker_hub_password            = var.docker_hub_password
+  disable_image_manifest_cache   = var.disable_image_manifest_cache
+  eks_addons                     = module.cluster.eks_addons
+  high_availability              = var.high_availability
+  idle_timeout                   = var.idle_timeout
+  internal_router                = var.internal_router
+  image                          = local.image
+  name                           = local.name
+  rack_name                      = local.rack_name
+  oidc_arn                       = module.cluster.oidc_arn
+  oidc_sub                       = module.cluster.oidc_sub
+  proxy_protocol                 = var.proxy_protocol
+  release                        = local.release
+  ssl_ciphers                    = var.ssl_ciphers
+  ssl_protocols                  = var.ssl_protocols
+  subnets                        = module.cluster.subnets
+  tags                           = local.tag_map
+  telemetry                      = var.telemetry
+  telemetry_map                  = local.telemetry_map
+  telemetry_default_map          = local.telemetry_default_map
+  whitelist                      = split(",", var.whitelist)
+  ebs_csi_driver_name            = module.cluster.ebs_csi_driver_name
 }

terraform/system/aws/telemetry.tf

@@ -10,6 +10,7 @@ locals {
     build_node_type = var.build_node_type
     cert_duration = var.cert_duration
     cidr = var.cidr
+    convox_domain_tls_cert_disable = var.convox_domain_tls_cert_disable
     coredns_version = var.coredns_version
     disable_image_manifest_cache = var.disable_image_manifest_cache
     docker_hub_password = var.docker_hub_password
@@ -60,6 +61,7 @@ locals {
     build_node_type = ""
     cert_duration = "2160h"
     cidr = "10.1.0.0/16"
+    convox_domain_tls_cert_disable = "false"
     coredns_version = "v1.9.3-eksbuild.7"
     disable_image_manifest_cache = "false"
     docker_hub_password = ""