diff --git a/controller-runtime/build.gradle.kts b/controller-runtime/build.gradle.kts index c0b3bc7..d44a354 100644 --- a/controller-runtime/build.gradle.kts +++ b/controller-runtime/build.gradle.kts @@ -14,6 +14,7 @@ dependencies { implementation(libs.clikt) implementation(libs.spring.crypto) implementation(libs.spotify.completablefutures) + implementation(libs.envoy.controlplane) } application { diff --git a/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/ControllerRuntime.kt b/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/ControllerRuntime.kt index b06e823..ae4e5c8 100644 --- a/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/ControllerRuntime.kt +++ b/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/ControllerRuntime.kt @@ -1,6 +1,9 @@ package app.simplecloud.controller.runtime import app.simplecloud.controller.runtime.database.DatabaseFactory +import app.simplecloud.controller.runtime.droplet.ControllerDropletService +import app.simplecloud.controller.runtime.droplet.DropletRepository +import app.simplecloud.controller.runtime.envoy.ControlPlaneServer import app.simplecloud.controller.runtime.group.GroupRepository import app.simplecloud.controller.runtime.group.GroupService import app.simplecloud.controller.runtime.host.ServerHostRepository @@ -29,11 +32,13 @@ class ControllerRuntime( private val database = DatabaseFactory.createDatabase(controllerStartCommand.databaseUrl) private val authCallCredentials = AuthCallCredentials(controllerStartCommand.authSecret) + private val dropletRepository = DropletRepository() private val groupRepository = GroupRepository(controllerStartCommand.groupPath) private val numericalIdRepository = ServerNumericalIdRepository() private val serverRepository = ServerRepository(database, numericalIdRepository) private val hostRepository = ServerHostRepository() private val pubSubService = PubSubService() + private val controlPlaneServer = ControlPlaneServer(controllerStartCommand, dropletRepository) private val authServer = OAuthServer(controllerStartCommand, database) private val reconciler = Reconciler( groupRepository, @@ -50,6 +55,7 @@ class ControllerRuntime( logger.info("Starting controller") setupDatabase() startAuthServer() + startControlPlaneServer() startPubSubGrpcServer() startGrpcServer() startReconciler() @@ -80,6 +86,11 @@ class ControllerRuntime( } + private fun startControlPlaneServer() { + logger.info("Starting envoy control plane...") + controlPlaneServer.start() + } + private fun setupDatabase() { logger.info("Setting up database...") database.setup() @@ -154,6 +165,7 @@ class ControllerRuntime( ) ) ) + .addService(ControllerDropletService(dropletRepository)) .intercept(AuthSecretInterceptor(controllerStartCommand.grpcHost, controllerStartCommand.authorizationPort)) .build() } diff --git a/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/droplet/ControllerDropletService.kt b/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/droplet/ControllerDropletService.kt new file mode 100644 index 0000000..ebca099 --- /dev/null +++ b/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/droplet/ControllerDropletService.kt @@ -0,0 +1,51 @@ +package app.simplecloud.controller.runtime.droplet + +import app.simplecloud.droplet.api.droplet.Droplet +import build.buf.gen.simplecloud.controller.v1.* +import io.grpc.Status +import io.grpc.StatusException + +class ControllerDropletService(private val dropletRepository: DropletRepository) : + ControllerDropletServiceGrpcKt.ControllerDropletServiceCoroutineImplBase() { + override suspend fun getDroplet(request: GetDropletRequest): GetDropletResponse { + val droplet = dropletRepository.find(request.type, request.id) + ?: throw StatusException(Status.NOT_FOUND.withDescription("This Droplet does not exist")) + return getDropletResponse { this.definition = droplet.toDefinition() } + } + + override suspend fun getAllDroplets(request: GetAllDropletsRequest): GetAllDropletsResponse { + val allDroplets = dropletRepository.getAll() + return getAllDropletsResponse { + definition.addAll(allDroplets.map { it.toDefinition() }) + } + } + + override suspend fun getDropletsByType(request: GetDropletsByTypeRequest): GetDropletsByTypeResponse { + val type = request.type + val typedDroplets = dropletRepository.getAll().filter { it.type == type } + return getDropletsByTypeResponse { + definition.addAll(typedDroplets.map { it.toDefinition() }) + } + } + + override suspend fun registerDroplet(request: RegisterDropletRequest): RegisterDropletResponse { + dropletRepository.find(request.definition.type, request.definition.id) + ?: throw StatusException(Status.NOT_FOUND.withDescription("This Droplet does not exist")) + val droplet = Droplet.fromDefinition(request.definition) + + try { + dropletRepository.save(droplet) + } catch (e: Exception) { + throw StatusException(Status.INTERNAL.withDescription("Error whilst updating Droplet").withCause(e)) + } + return registerDropletResponse { this.definition = droplet.toDefinition() } + } + + override suspend fun unregisterDroplet(request: UnregisterDropletRequest): UnregisterDropletResponse { + val droplet = dropletRepository.find(request.id) + ?: throw StatusException(Status.NOT_FOUND.withDescription("This Droplet does not exist")) + val deleted = dropletRepository.delete(droplet) + if (!deleted) throw StatusException(Status.NOT_FOUND.withDescription("Could not delete this Droplet")) + return unregisterDropletResponse { } + } +} \ No newline at end of file diff --git a/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/droplet/DropletRepository.kt b/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/droplet/DropletRepository.kt new file mode 100644 index 0000000..351c45d --- /dev/null +++ b/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/droplet/DropletRepository.kt @@ -0,0 +1,55 @@ +package app.simplecloud.controller.runtime.droplet + +import app.simplecloud.controller.runtime.Repository +import app.simplecloud.controller.runtime.envoy.DropletCache +import app.simplecloud.droplet.api.droplet.Droplet +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch + +class DropletRepository : Repository { + + private val currentDroplets = mutableListOf() + private val dropletCache = DropletCache(this) + + override suspend fun getAll(): List { + return currentDroplets + } + + override suspend fun find(identifier: String): Droplet? { + return currentDroplets.firstOrNull { it.id == identifier } + } + + fun find(type: String, identifier: String): Droplet? { + return currentDroplets.firstOrNull { it.type == type && it.id == identifier } + } + + override fun save(element: Droplet) { + val updated = managePortRange(element) + val droplet = find(element.type, element.id) + if (droplet != null) { + currentDroplets[currentDroplets.indexOf(droplet)] = updated + return + } + currentDroplets.add(updated) + CoroutineScope(Dispatchers.IO).launch { + dropletCache.update() + } + } + + private fun managePortRange(element: Droplet): Droplet { + if (!currentDroplets.any { it.envoyPort == element.envoyPort }) return element + return managePortRange(element.copy(envoyPort = element.envoyPort + 1)) + } + + override suspend fun delete(element: Droplet): Boolean { + val found = find(element.type, element.id) ?: return false + if (!currentDroplets.remove(found)) return false + dropletCache.update() + return true + } + + fun getAsDropletCache(): DropletCache { + return dropletCache + } +} \ No newline at end of file diff --git a/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/envoy/ControlPlaneServer.kt b/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/envoy/ControlPlaneServer.kt new file mode 100644 index 0000000..94a9d07 --- /dev/null +++ b/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/envoy/ControlPlaneServer.kt @@ -0,0 +1,52 @@ +package app.simplecloud.controller.runtime.envoy + +import app.simplecloud.controller.runtime.droplet.DropletRepository +import app.simplecloud.controller.runtime.launcher.ControllerStartCommand +import app.simplecloud.droplet.api.droplet.Droplet +import io.envoyproxy.controlplane.server.V3DiscoveryServer +import io.grpc.ServerBuilder +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import org.apache.logging.log4j.LogManager + +/** + * @see ADS Documentation + */ +class ControlPlaneServer(private val args: ControllerStartCommand, private val dropletRepository: DropletRepository) { + private val server = V3DiscoveryServer(dropletRepository.getAsDropletCache().getCache()) + private val logger = LogManager.getLogger(ControlPlaneServer::class.java) + + fun start() { + val serverBuilder = ServerBuilder.forPort(args.envoyDiscoveryPort) + register(serverBuilder) + val server = serverBuilder.build() + CoroutineScope(Dispatchers.IO).launch { + try { + server.start() + server.awaitTermination() + } catch (e: Exception) { + logger.warn("Error in envoy control server server", e) + throw e + } + } + registerSelf() + } + + private fun registerSelf() { + dropletRepository.save( + Droplet( + type = "controller", + id = "internal-controller", + host = args.grpcHost, + port = args.grpcPort, + envoyPort = 8080, + ) + ) + } + + private fun register(builder: ServerBuilder<*>) { + logger.info("Registering envoy ADS...") + builder.addService(server.aggregatedDiscoveryServiceImpl) + } +} \ No newline at end of file diff --git a/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/envoy/DropletCache.kt b/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/envoy/DropletCache.kt new file mode 100644 index 0000000..b95e9e8 --- /dev/null +++ b/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/envoy/DropletCache.kt @@ -0,0 +1,176 @@ +package app.simplecloud.controller.runtime.envoy + +import app.simplecloud.controller.runtime.droplet.DropletRepository +import app.simplecloud.droplet.api.droplet.Droplet +import com.google.protobuf.Any +import com.google.protobuf.Duration +import com.google.protobuf.UInt32Value +import io.envoyproxy.controlplane.cache.ConfigWatcher +import io.envoyproxy.controlplane.cache.v3.SimpleCache +import io.envoyproxy.controlplane.cache.v3.Snapshot +import io.envoyproxy.envoy.config.cluster.v3.Cluster +import io.envoyproxy.envoy.config.cluster.v3.Cluster.EdsClusterConfig +import io.envoyproxy.envoy.config.core.v3.* +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment +import io.envoyproxy.envoy.config.endpoint.v3.Endpoint +import io.envoyproxy.envoy.config.endpoint.v3.LbEndpoint +import io.envoyproxy.envoy.config.endpoint.v3.LocalityLbEndpoints +import io.envoyproxy.envoy.config.listener.v3.Filter +import io.envoyproxy.envoy.config.listener.v3.FilterChain +import io.envoyproxy.envoy.config.listener.v3.Listener +import io.envoyproxy.envoy.config.route.v3.* +import io.envoyproxy.envoy.extensions.filters.http.connect_grpc_bridge.v3.FilterConfig +import io.envoyproxy.envoy.extensions.filters.http.grpc_web.v3.GrpcWeb +import io.envoyproxy.envoy.extensions.filters.http.router.v3.Router +import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager +import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpFilter +import io.envoyproxy.envoy.extensions.upstreams.http.v3.HttpProtocolOptions +import org.apache.logging.log4j.LogManager +import java.util.* + +/** + * This class handles the remapping of the [DropletRepository] to a [SimpleCache] of [Snapshot]s, which are used by the envoy ADS service. + */ +class DropletCache(private val dropletRepository: DropletRepository) { + private val cache = SimpleCache(SimpleCloudNodeGroup()) + private val logger = LogManager.getLogger(DropletCache::class.java) + + //Create a new Snapshot by the droplet repository's data + suspend fun update() { + logger.info("Detected new droplets in DropletRepository, adding to ADS...") + val clusters = mutableListOf() + val listeners = mutableListOf() + val clas = mutableListOf() + + dropletRepository.getAll().forEach { + clusters.add(createCluster(it)) + listeners.add(createListener(it)) + clas.add(createCLA(it)) + } + + cache.setSnapshot( + SimpleCloudNodeGroup.GROUP, + Snapshot.create( + clusters, + clas, + listeners, + listOf(), // We don't need routes + listOf(), // We don't need secrets + UUID.randomUUID() + .toString() //This can be anything, used internally for versioning. THIS HAS TO BE DIFFERENT FOR EVERY SNAPSHOT + ) + ) + } + + //Creates endpoints users can connect with later + private fun createListener(it: Droplet): Listener { + return Listener.newBuilder().setName("${it.type}-${it.id}").setAddress( + Address.newBuilder().setSocketAddress( + SocketAddress.newBuilder().setProtocol(SocketAddress.Protocol.TCP).setAddress("0.0.0.0") + .setPortValue(it.envoyPort) + ) + ).setDefaultFilterChain(createListenerFilterChain("${it.type}-${it.id}")).build() + + } + + //Creates load assignments for new droplets (I don't yet know if they need to be called every time?) + private fun createCLA(it: Droplet): ClusterLoadAssignment { + return ClusterLoadAssignment.newBuilder().setClusterName("${it.type}-${it.id}") + .addEndpoints( + LocalityLbEndpoints.newBuilder().addLbEndpoints( + LbEndpoint.newBuilder().setEndpoint( + Endpoint.newBuilder() + .setAddress( + Address.newBuilder().setSocketAddress( + SocketAddress.newBuilder().setPortValue(it.port).setAddress(it.host) + .setProtocol(SocketAddress.Protocol.TCP) + ) + ) + ) + ) + ) + .build() + } + + //Creates clusters listening to droplets + private fun createCluster(it: Droplet): Cluster { + return Cluster.newBuilder().setName("${it.type}-${it.id}") + .setConnectTimeout(Duration.newBuilder().setSeconds(5)) + .setType(Cluster.DiscoveryType.EDS) + .setEdsClusterConfig( + EdsClusterConfig.newBuilder() + .setEdsConfig(ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) + ) + .setLbPolicy(Cluster.LbPolicy.ROUND_ROBIN) + .setLoadAssignment( + ClusterLoadAssignment.newBuilder().setClusterName("${it.type}-${it.id}") + .addEndpoints( + LocalityLbEndpoints.newBuilder() + .addLbEndpoints( + LbEndpoint.newBuilder().setEndpoint( + Endpoint.newBuilder() + .setAddress( + Address.newBuilder().setSocketAddress( + SocketAddress.newBuilder().setPortValue(it.port).setAddress(it.host) + ) + ) + ) + ) + ) + ).putTypedExtensionProtocolOptions( + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions", Any.pack( + HttpProtocolOptions.newBuilder().setExplicitHttpConfig( + HttpProtocolOptions.ExplicitHttpConfig.newBuilder().setHttp2ProtocolOptions( + Http2ProtocolOptions.newBuilder().setMaxConcurrentStreams( + UInt32Value.of(100) + ) + ) + ).build() + ) + ) + .build() + + } + + //Creates a filter chain that remaps http to grpc + private fun createListenerFilterChain(cluster: String): FilterChain.Builder { + return FilterChain.newBuilder() + .addFilters( + Filter.newBuilder().setName("envoy.filters.network.http_connection_manager") + .setTypedConfig( + Any.pack( + HttpConnectionManager.newBuilder().setStatPrefix("ingress_http") + .setCodecType(HttpConnectionManager.CodecType.AUTO) + .setRouteConfig( + RouteConfiguration.newBuilder().setName("local_route") + .addVirtualHosts( + VirtualHost.newBuilder().setName("local_service").addDomains("*") + .addRoutes( + Route.newBuilder().setRoute( + RouteAction.newBuilder().setCluster(cluster) + .setTimeout(Duration.newBuilder().setSeconds(0).setNanos(0)) + ).setMatch(RouteMatch.newBuilder().setPrefix("/")) + ) + ) + ).addHttpFilters( + HttpFilter.newBuilder().setName("envoy.filters.http.connect_grpc_bridge") + .setTypedConfig(Any.pack(FilterConfig.getDefaultInstance())) + ).addHttpFilters( + HttpFilter.newBuilder().setName("envoy.filters.http.grpc_web") + .setTypedConfig(Any.pack(GrpcWeb.getDefaultInstance())) + ).addHttpFilters( + HttpFilter.newBuilder().setName("envoy.filters.http.router") + .setTypedConfig(Any.pack(Router.getDefaultInstance())) + ) + + .build() + ) + ) + ) + } + + fun getCache(): ConfigWatcher { + return cache + } + +} \ No newline at end of file diff --git a/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/envoy/SimpleCloudNodeGroup.kt b/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/envoy/SimpleCloudNodeGroup.kt new file mode 100644 index 0000000..b2d02fc --- /dev/null +++ b/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/envoy/SimpleCloudNodeGroup.kt @@ -0,0 +1,22 @@ +package app.simplecloud.controller.runtime.envoy + +import io.envoyproxy.controlplane.cache.NodeGroup +import io.envoyproxy.envoy.config.core.v3.Node + +/** + * SimpleCloud only uses one envoy node. That's why we can just + * have one node in the nodegroup. + */ +class SimpleCloudNodeGroup : NodeGroup { + + companion object { + const val GROUP = "simplecloud" + } + + override fun hash(node: Node?): String { + if (node == null) { + throw IllegalArgumentException("Null node") + } + return GROUP + } +} \ No newline at end of file diff --git a/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/launcher/ControllerStartCommand.kt b/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/launcher/ControllerStartCommand.kt index dccc336..b972c8d 100644 --- a/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/launcher/ControllerStartCommand.kt +++ b/controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/launcher/ControllerStartCommand.kt @@ -45,6 +45,11 @@ class ControllerStartCommand( envvar = "AUTHORIZATION_PORT" ).int().default(5818) + val envoyDiscoveryPort: Int by option( + help = "Authorization port (default: 5814)", + envvar = "ENVOY_DISCOVERY_PORT" + ).int().default(5814) + private val authSecretPath: Path by option( help = "Path to auth secret file (default: .auth.secret)", envvar = "AUTH_SECRET_PATH" diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5ce5e2d..a0b906c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -3,7 +3,7 @@ kotlin = "2.0.20" kotlin-coroutines = "1.9.0" shadow = "8.3.3" log4j = "2.20.0" -droplet-api = "0.0.1-dev.27d7043" +droplet-api = "0.0.1-dev.7e049d6" simplecloud-pubsub = "1.0.5" simplecloud-metrics = "1.0.0" jooq = "3.19.3" @@ -13,6 +13,7 @@ clikt = "5.0.1" sonatype-central-portal-publisher = "1.2.3" spotify-completablefutures = "0.3.6" spring-crypto = "6.3.4" +envoy = "1.0.46" [libraries] kotlin-jvm = { module = "org.jetbrains.kotlin:kotlin-stdlib-jdk8", version.ref = "kotlin" } @@ -42,8 +43,8 @@ clikt = { module = "com.github.ajalt.clikt:clikt", version.ref = "clikt" } spotify-completablefutures = { module = "com.spotify:completable-futures", version.ref = "spotify-completablefutures" } -spring-crypto = { module = "org.springframework.security:spring-security-crypto", version.ref = "spring-crypto"} - +spring-crypto = { module = "org.springframework.security:spring-security-crypto", version.ref = "spring-crypto" } +envoy-controlplane = { module = "io.envoyproxy.controlplane:server", version.ref = "envoy" } [bundles] diff --git a/test-envoy/docker-compose.yml b/test-envoy/docker-compose.yml new file mode 100644 index 0000000..585ac11 --- /dev/null +++ b/test-envoy/docker-compose.yml @@ -0,0 +1,6 @@ +services: + envoy: + network_mode: "host" + image: envoyproxy/envoy:v1.31.4 + volumes: + - ./envoy-bootstrap.yaml:/etc/envoy/envoy.yaml \ No newline at end of file diff --git a/test-envoy/envoy-bootstrap.yaml b/test-envoy/envoy-bootstrap.yaml new file mode 100644 index 0000000..3bd962a --- /dev/null +++ b/test-envoy/envoy-bootstrap.yaml @@ -0,0 +1,45 @@ +node: + cluster: simplecloud + id: simplecloud + +dynamic_resources: + ads_config: + api_type: GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: ads_cluster + cds_config: + ads: {} + lds_config: + ads: {} + +static_resources: + clusters: + - name: ads_cluster + lb_policy: ROUND_ROBIN + type: STRICT_DNS + load_assignment: + cluster_name: ads_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 5814 + # It is recommended to configure either HTTP/2 or TCP keepalives in order to detect + # connection issues, and allow Envoy to reconnect. TCP keepalive is less expensive, but + # may be inadequate if there is a TCP proxy between Envoy and the management server. + # HTTP/2 keepalive is slightly more expensive, but may detect issues through more types + # of intermediate proxies. + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + upstream_connection_options: + tcp_keepalive: {} \ No newline at end of file