diff --git a/deploy/build/charts/laf-server/templates/cert-issuer.yaml b/deploy/build/charts/laf-server/templates/cert-issuer.yaml new file mode 100644 index 0000000000..5c26c288b4 --- /dev/null +++ b/deploy/build/charts/laf-server/templates/cert-issuer.yaml @@ -0,0 +1,14 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: laf-issuer +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: admin@sealos.io + privateKeySecretRef: + name: letsencrypt-prod + solvers: + - http01: + ingress: + class: apisix \ No newline at end of file diff --git a/server/src/constants.ts b/server/src/constants.ts index 82b1b00fa0..24ad11bdde 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -100,6 +100,10 @@ export class ServerConfig { return process.env.API_SERVER_URL || 'http://localhost:3000' } + static get certManagerIssuerName() { + return process.env.CERT_MANAGER_ISSUER_NAME || 'laf-issuer' + } + /** default region conf */ static get DEFAULT_REGION_DATABASE_URL() { return process.env.DEFAULT_REGION_DATABASE_URL diff --git a/server/src/gateway/apisix-custom-cert.service.ts b/server/src/gateway/apisix-custom-cert.service.ts new file mode 100644 index 0000000000..c147eddf15 --- /dev/null +++ b/server/src/gateway/apisix-custom-cert.service.ts @@ -0,0 +1,171 @@ +import { Injectable, Logger } from '@nestjs/common' +import { Region, WebsiteHosting } from '@prisma/client' +import { LABEL_KEY_APP_ID, ServerConfig } from 'src/constants' +import { ClusterService } from 'src/region/cluster/cluster.service' +import { GetApplicationNamespaceByAppId } from 'src/utils/getter' + +// This class handles the creation and deletion of website domain certificates +// and ApisixTls resources using Kubernetes Custom Resource Definitions (CRDs). +@Injectable() +export class ApisixCustomCertService { + private readonly logger = new Logger(ApisixCustomCertService.name) + constructor(private readonly clusterService: ClusterService) {} + + // Read a certificate for a given website using cert-manager.io CRD + async readWebsiteDomainCert(region: Region, website: WebsiteHosting) { + try { + // Get the namespace based on the application ID + const namespace = GetApplicationNamespaceByAppId(website.appid) + // Create a Kubernetes API client for the specified region + const api = this.clusterService.makeCustomObjectApi(region) + + // Make a request to read the Certificate resource + const res = await api.getNamespacedCustomObject( + 'cert-manager.io', + 'v1', + namespace, + 'certificates', + website.id, + ) + + return res.body + } catch (err) { + if (err?.response?.body?.reason === 'NotFound') return null + this.logger.error(err) + this.logger.error(err?.response?.body) + throw err + } + } + + // Create a certificate for a given website using cert-manager.io CRD + async createWebsiteDomainCert(region: Region, website: WebsiteHosting) { + // Get the namespace based on the application ID + const namespace = GetApplicationNamespaceByAppId(website.appid) + // Create a Kubernetes API client for the specified region + const api = this.clusterService.makeObjectApi(region) + + // Make a request to create the Certificate resource + const res = await api.create({ + apiVersion: 'cert-manager.io/v1', + kind: 'Certificate', + // Set the metadata for the Certificate resource + metadata: { + name: website.id, + namespace, + labels: { + 'laf.dev/website': website.id, + 'laf.dev/website-domain': website.domain, + [LABEL_KEY_APP_ID]: website.appid, + }, + }, + // Define the specification for the Certificate resource + spec: { + secretName: website.id, + dnsNames: [website.domain], + issuerRef: { + name: ServerConfig.certManagerIssuerName, + kind: 'ClusterIssuer', + }, + }, + }) + return res.body + } + + // Delete a certificate for a given website using cert-manager.io CRD + async deleteWebsiteDomainCert(region: Region, website: WebsiteHosting) { + // Get the namespace based on the application ID + const namespace = GetApplicationNamespaceByAppId(website.appid) + // Create a Kubernetes API client for the specified region + const api = this.clusterService.makeObjectApi(region) + + // Make a request to delete the Certificate resource + const res = await api.delete({ + apiVersion: 'cert-manager.io/v1', + kind: 'Certificate', + metadata: { + name: website.id, + namespace, + }, + }) + return res.body + } + + // Read an ApisixTls resource for a given website using apisix.apache.org CRD + async readWebsiteDomainApisixTls(region: Region, website: WebsiteHosting) { + try { + // Get the namespace based on the application ID + const namespace = GetApplicationNamespaceByAppId(website.appid) + // Create an API object for the specified region + const api = this.clusterService.makeCustomObjectApi(region) + + // Make a request to read the ApisixTls resource + const res = await api.getNamespacedCustomObject( + 'apisix.apache.org', + 'v2', + namespace, + 'apisixtlses', + website.id, + ) + return res.body + } catch (err) { + if (err?.response?.body?.reason === 'NotFound') return null + this.logger.error(err) + this.logger.error(err?.response?.body) + throw err + } + } + + // Create an ApisixTls resource for a given website using apisix.apache.org CRD + async createWebsiteDomainApisixTls(region: Region, website: WebsiteHosting) { + // Get the namespace based on the application ID + const namespace = GetApplicationNamespaceByAppId(website.appid) + // Create an API object for the specified region + const api = this.clusterService.makeObjectApi(region) + + // Make a request to create the ApisixTls resource + const res = await api.create({ + apiVersion: 'apisix.apache.org/v2', + kind: 'ApisixTls', + // Set the metadata for the ApisixTls resource + metadata: { + name: website.id, + namespace, + labels: { + 'laf.dev/website': website.id, + 'laf.dev/website-domain': website.domain, + [LABEL_KEY_APP_ID]: website.appid, + }, + }, + // Define the specification for the ApisixTls resource + spec: { + hosts: [website.domain], + secret: { + name: website.id, + namespace, + }, + }, + }) + return res.body + } + + // Deletes the APISIX TLS configuration for a specific website domain + async deleteWebsiteDomainApisixTls(region: Region, website: WebsiteHosting) { + // Get the application namespace using the website's appid + const namespace = GetApplicationNamespaceByAppId(website.appid) + + // Create an API object for the specified region + const api = this.clusterService.makeObjectApi(region) + + // Send a delete request to remove the APISIX TLS configuration + const res = await api.delete({ + apiVersion: 'apisix.apache.org/v2', + kind: 'ApisixTls', + metadata: { + name: website.id, + namespace, + }, + }) + + return res.body + } +} diff --git a/server/src/gateway/apisix.service.ts b/server/src/gateway/apisix.service.ts index 9d43bade0f..2d0cb705c5 100644 --- a/server/src/gateway/apisix.service.ts +++ b/server/src/gateway/apisix.service.ts @@ -14,6 +14,7 @@ export class ApisixService { const namespace = GetApplicationNamespaceByAppId(appid) const upstreamNode = `${appid}.${namespace}:8000` + // TODO: use appid as route id instead of `app-{appid} const id = `app-${appid}` const data = { name: id, @@ -46,6 +47,7 @@ export class ApisixService { } async deleteAppRoute(region: Region, appid: string) { + // TODO: use appid as route id instead of `app-{appid}` const id = `app-${appid}` const res = await this.deleteRoute(region, id) return res @@ -57,6 +59,7 @@ export class ApisixService { const minioUrl = new URL(region.storageConf.internalEndpoint) const upstreamNode = minioUrl.host + // TODO: use bucket object id as route id instead of bucket name const id = `bucket-${bucketName}` const data = { name: id, @@ -88,6 +91,7 @@ export class ApisixService { } async deleteBucketRoute(region: Region, bucketName: string) { + // TODO: use bucket object id as route id instead of bucket name const id = `bucket-${bucketName}` const res = await this.deleteRoute(region, id) return res @@ -162,6 +166,27 @@ export class ApisixService { } } + async getRoute(region: Region, id: string) { + const conf = region.gatewayConf + const api_url = `${conf.apiUrl}/routes/${id}` + + try { + const res = await this.httpService.axiosRef.get(api_url, { + headers: { + 'X-API-KEY': conf.apiKey, + 'Content-Type': 'application/json', + }, + }) + return res.data + } catch (error) { + if (error?.response?.status === 404) { + return null + } + this.logger.error(error, error.response?.data) + return error + } + } + async deleteRoute(region: Region, id: string) { const conf = region.gatewayConf const api_url = `${conf.apiUrl}/routes/${id}` diff --git a/server/src/gateway/gateway.module.ts b/server/src/gateway/gateway.module.ts index 74d072422d..9bb5d48223 100644 --- a/server/src/gateway/gateway.module.ts +++ b/server/src/gateway/gateway.module.ts @@ -6,6 +6,7 @@ import { BucketDomainService } from './bucket-domain.service' import { WebsiteTaskService } from './website-task.service' import { BucketDomainTaskService } from './bucket-domain-task.service' import { RuntimeDomainTaskService } from './runtime-domain-task.service' +import { ApisixCustomCertService } from './apisix-custom-cert.service' @Module({ imports: [HttpModule], @@ -16,6 +17,7 @@ import { RuntimeDomainTaskService } from './runtime-domain-task.service' WebsiteTaskService, BucketDomainTaskService, RuntimeDomainTaskService, + ApisixCustomCertService, ], exports: [RuntimeDomainService, BucketDomainService], }) diff --git a/server/src/gateway/website-task.service.ts b/server/src/gateway/website-task.service.ts index b75ec239f9..40efadee0e 100644 --- a/server/src/gateway/website-task.service.ts +++ b/server/src/gateway/website-task.service.ts @@ -12,6 +12,7 @@ import { SystemDatabase } from 'src/database/system-database' import { RegionService } from 'src/region/region.service' import * as assert from 'node:assert' import { ApisixService } from './apisix.service' +import { ApisixCustomCertService } from './apisix-custom-cert.service' @Injectable() export class WebsiteTaskService { @@ -22,6 +23,7 @@ export class WebsiteTaskService { constructor( private readonly regionService: RegionService, private readonly apisixService: ApisixService, + private readonly customCertService: ApisixCustomCertService, ) {} @Cron(CronExpression.EVERY_SECOND) @@ -62,22 +64,19 @@ export class WebsiteTaskService { .findOneAndUpdate( { phase: DomainPhase.Creating, - lockedAt: { - $lt: new Date(Date.now() - 1000 * this.lockTimeout), - }, - }, - { - $set: { - lockedAt: new Date(), - }, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, + { $set: { lockedAt: new Date() } }, ) if (!res.value) return this.logger.debug(res.value) // get region by appid - const site = res.value + const site = { + ...res.value, + id: res.value._id.toString(), + } const region = await this.regionService.findByAppId(site.appid) assert(region, 'region not found') @@ -97,7 +96,48 @@ export class WebsiteTaskService { site, bucketDomain.domain, ) - this.logger.debug(`create website route: `, route) + this.logger.log(`create website route: ${route?.node?.key}`) + + // create website custom certificate if custom domain is set + if (site.isCustom) { + // create custom domain certificate + let cert = await this.customCertService.readWebsiteDomainCert( + region, + site, + ) + if (!cert) { + cert = await this.customCertService.createWebsiteDomainCert( + region, + site, + ) + this.logger.log(`create website cert: ${site._id}`) + } + + // return to try to create cert again in next tick if cert is not found + if (!cert) { + this.logger.error(`create website cert failed: ${site._id}`) + return + } + + // config custom domain certificate to apisix + let apisixTls = await this.customCertService.readWebsiteDomainApisixTls( + region, + site, + ) + if (!apisixTls) { + apisixTls = await this.customCertService.createWebsiteDomainApisixTls( + region, + site, + ) + this.logger.log(`create website apisix tls: ${site._id}`) + } + + // return to try to create cert again in next tick if cert is not found + if (!apisixTls) { + this.logger.error(`create website apisix tls failed: ${site._id}`) + return + } + } // update phase to `Created` const updated = await db @@ -117,7 +157,10 @@ export class WebsiteTaskService { if (updated.modifiedCount !== 1) { this.logger.error(`update website hosting phase failed: ${site._id}`) + return } + + this.logger.log(`update website phase to 'Created': ${site._id}`) } /** @@ -133,28 +176,55 @@ export class WebsiteTaskService { .findOneAndUpdate( { phase: DomainPhase.Deleting, - lockedAt: { - $lt: new Date(Date.now() - 1000 * this.lockTimeout), - }, - }, - { - $set: { - lockedAt: new Date(), - }, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, + { $set: { lockedAt: new Date() } }, ) if (!res.value) return // get region by appid - const site = res.value + const site = { + ...res.value, + id: res.value._id.toString(), + } const region = await this.regionService.findByAppId(site.appid) assert(region, 'region not found') - // delete website route - const route = await this.apisixService.deleteWebsiteRoute(region, site) - - this.logger.debug(`delete website route: `, route) + // delete website route if exists + const route = await this.apisixService.getRoute(region, site._id.toString()) + if (route) { + await this.apisixService.deleteWebsiteRoute(region, site) + const res = await this.apisixService.deleteWebsiteRoute(region, site) + this.logger.log(`delete website route: ${res?.key}`) + this.logger.debug('delete website route', res) + } + // delete website custom certificate if custom domain is set + if (site.isCustom) { + // delete custom domain certificate + const cert = await this.customCertService.readWebsiteDomainCert( + region, + site, + ) + if (cert) { + await this.customCertService.deleteWebsiteDomainCert(region, site) + this.logger.log(`delete website cert: ${site._id}`) + // return to wait for cert to be deleted + return + } + + // delete custom domain certificate from apisix + const apisixTls = await this.customCertService.readWebsiteDomainApisixTls( + region, + site, + ) + if (apisixTls) { + await this.customCertService.deleteWebsiteDomainApisixTls(region, site) + this.logger.log(`delete website apisix tls: ${site._id}`) + // return to wait for cert to be deleted + return + } + } // update phase to `Deleted` const updated = await db @@ -174,7 +244,10 @@ export class WebsiteTaskService { if (updated.modifiedCount > 1) { this.logger.error(`update website hosting phase failed: ${site._id}`) + return } + + this.logger.log(`update website phase to 'Deleted': ${site._id}`) } /** diff --git a/server/src/instance/instance.service.ts b/server/src/instance/instance.service.ts index 8930826ce2..11351c7bba 100644 --- a/server/src/instance/instance.service.ts +++ b/server/src/instance/instance.service.ts @@ -310,7 +310,7 @@ export class InstanceService { return res.body } catch (error) { if (error?.response?.body?.reason === 'NotFound') return null - return null + throw error } } @@ -324,7 +324,8 @@ export class InstanceService { const res = await coreV1Api.readNamespacedService(serviceName, namespace) return res.body } catch (error) { - return null + if (error?.response?.body?.reason === 'NotFound') return null + throw error } } } diff --git a/server/src/region/cluster/cluster.service.ts b/server/src/region/cluster/cluster.service.ts index 10a945085c..4653e0c8d4 100644 --- a/server/src/region/cluster/cluster.service.ts +++ b/server/src/region/cluster/cluster.service.ts @@ -52,8 +52,8 @@ export class ClusterService { return res.body } catch (err) { this.logger.error(err) - this.logger.debug(err?.response?.body) - return null + this.logger.error(err?.response?.body) + throw err } } @@ -67,8 +67,8 @@ export class ClusterService { } catch (err) { if (err?.response?.body?.reason === 'NotFound') return null this.logger.error(err) - this.logger.debug(err?.response?.body) - return null + this.logger.error(err?.response?.body) + throw err } } @@ -81,8 +81,8 @@ export class ClusterService { return res } catch (err) { this.logger.error(err) - this.logger.debug(err?.response?.body) - return null + this.logger.error(err?.response?.body) + throw err } }