diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/AzkarraContext.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/AzkarraContext.java index c6f0790f..38a58f52 100644 --- a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/AzkarraContext.java +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/AzkarraContext.java @@ -96,7 +96,7 @@ public interface AzkarraContext extends ConfigurableComponentFactory, ComponentR * @throws AlreadyExistsException if a {@link StreamsExecutionEnvironment} * is already registered for the given name. */ - AzkarraContext addExecutionEnvironment(final StreamsExecutionEnvironment environment) + AzkarraContext addExecutionEnvironment(final StreamsExecutionEnvironment environment) throws AlreadyExistsException; /** @@ -168,7 +168,16 @@ ApplicationId addTopology(final String type, * * @return a set of {@link TopologyDescriptor}. */ - Set topologyProviders(); + Set getTopologyDescriptors(); + + /** + * Gets all topologies registered into this {@link AzkarraContext} which are available for the given environment. + * + * @param environmentName the {@link StreamsExecutionEnvironment} + * @return a set of {@link TopologyDescriptor}. + */ + Set getTopologyDescriptors(final String environmentName); + /** * Gets all topologies registered into this {@link AzkarraContext} which are available for the given environment. @@ -176,37 +185,30 @@ ApplicationId addTopology(final String type, * @param env the {@link StreamsExecutionEnvironment} * @return a set of {@link TopologyDescriptor}. */ - Set topologyProviders(final StreamsExecutionEnvironment env); + Set getTopologyDescriptors(final StreamsExecutionEnvironment env); /** * Gets all {@link StreamsExecutionEnvironment} registered to this context. * * @return a list of {@link StreamsExecutionEnvironment} instance. */ - List environments(); + List> getAllEnvironments(); /** - * Gets the {@link StreamsExecutionEnvironment} for the specified name or create a new one. + * Gets the {@link StreamsExecutionEnvironment} for the specified name. * * @param envName the environment name. * @return a {@link StreamsExecutionEnvironment} instance with the specified name. */ - StreamsExecutionEnvironment getEnvironmentForNameOrCreate(final String envName); + StreamsExecutionEnvironment getEnvironmentForName(final String envName); /** * Gets the default {@link StreamsExecutionEnvironment}. + * This method can return {@code null} while the context is not started. * * @return a {@link StreamsExecutionEnvironment} instance. */ - StreamsExecutionEnvironment defaultExecutionEnvironment(); - - /** - * Gets the topology for the specified class name or alias. - * - * @param type the topology type. - * @return the {@link TopologyDescriptor}. - */ - TopologyDescriptor getTopology(final String type); + StreamsExecutionEnvironment getDefaultEnvironment(); /** * Starts this {@link AzkarraContext} instance. diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/AzkarraContextListener.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/AzkarraContextListener.java index 998173a5..51356112 100644 --- a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/AzkarraContextListener.java +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/AzkarraContextListener.java @@ -18,9 +18,27 @@ */ package io.streamthoughts.azkarra.api; -public interface AzkarraContextListener { +import io.streamthoughts.azkarra.api.components.Ordered; - void onContextStart(final AzkarraContext context); +/** + * @see AzkarraContext + */ +public interface AzkarraContextListener extends Ordered { + + + /** + * {@inheritDoc} + */ + @Override + default int order() { + return Ordered.LOWEST_ORDER - 100; + } + + default void onContextStart(final AzkarraContext context) { + + } + + default void onContextStop(final AzkarraContext context) { - void onContextStop(final AzkarraContext context); + } } diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/AzkarraStreamsService.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/AzkarraStreamsService.java index ff08ee17..870f019a 100644 --- a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/AzkarraStreamsService.java +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/AzkarraStreamsService.java @@ -20,7 +20,6 @@ import io.streamthoughts.azkarra.api.components.Qualifier; import io.streamthoughts.azkarra.api.config.Conf; -import io.streamthoughts.azkarra.api.errors.InvalidStreamsStateException; import io.streamthoughts.azkarra.api.errors.NoSuchComponentException; import io.streamthoughts.azkarra.api.errors.NotFoundException; import io.streamthoughts.azkarra.api.model.Environment; @@ -31,10 +30,6 @@ import io.streamthoughts.azkarra.api.model.TopologyAndAliases; import io.streamthoughts.azkarra.api.monad.Tuple; import io.streamthoughts.azkarra.api.providers.TopologyDescriptor; -import io.streamthoughts.azkarra.api.query.Queried; -import io.streamthoughts.azkarra.api.query.QueryParams; -import io.streamthoughts.azkarra.api.query.internal.Query; -import io.streamthoughts.azkarra.api.query.result.QueryResult; import io.streamthoughts.azkarra.api.streams.ApplicationId; import io.streamthoughts.azkarra.api.streams.KafkaStreamsContainer; import io.streamthoughts.azkarra.api.streams.ServerMetadata; @@ -201,13 +196,21 @@ TopologyDescriptor getTopologyByAliasAndQualifiers(final String alias, */ Set getAllEnvironments(); + /** + * Gets all supported environment types. + * + * @return the set of the environment types. + */ + Set getSupportedEnvironmentTypes(); + /** * Adds a new environment to this application. * * @param name the environment name. + * @param type the environment type. * @param conf the environment configuration. */ - void addNewEnvironment(final String name, final Conf conf); + void addNewEnvironment(final String name, final String type, final Conf conf); /** * Gets all local and remote streams instances for the specified streams application. diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/StreamsExecutionEnvironment.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/StreamsExecutionEnvironment.java index 62ea8da9..08509124 100644 --- a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/StreamsExecutionEnvironment.java +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/StreamsExecutionEnvironment.java @@ -23,20 +23,24 @@ import io.streamthoughts.azkarra.api.streams.ApplicationId; import io.streamthoughts.azkarra.api.streams.ApplicationIdBuilder; import io.streamthoughts.azkarra.api.streams.KafkaStreamsContainer; -import io.streamthoughts.azkarra.api.streams.KafkaStreamsFactory; -import io.streamthoughts.azkarra.api.streams.TopologyProvider; -import io.streamthoughts.azkarra.api.streams.errors.StreamThreadExceptionHandler; import org.apache.kafka.streams.KafkaStreams; -import org.apache.kafka.streams.processor.StateRestoreListener; import java.time.Duration; import java.util.Collection; +import java.util.Set; import java.util.function.Supplier; /** * A StreamsExecutionEnvironment manages the lifecycle of {@link org.apache.kafka.streams.Topology} instances. */ -public interface StreamsExecutionEnvironment { +public interface StreamsExecutionEnvironment> { + + /** + * Gets the type of this {@link StreamsExecutionEnvironment}. + * + * @return the string type. + */ + String type(); /** * Gets the name of this {@link StreamsExecutionEnvironment}. @@ -46,80 +50,47 @@ public interface StreamsExecutionEnvironment { String name(); /** - * Gets the state f this {@link StreamsExecutionEnvironment}. + * Gets the state of this {@link StreamsExecutionEnvironment}. * @return the {@link State}. */ State state(); /** - * Adds a {@link KafkaStreams.StateListener} instance that will set to all {@link KafkaStreams} instance created - * in this {@link StreamsExecutionEnvironment}. - * - * @see KafkaStreams#setStateListener(KafkaStreams.StateListener). - * - * @param listener the {@link KafkaStreams.StateListener} instance. + * Check whether this {@link StreamsExecutionEnvironment} is marked as default. * - * @throws IllegalStateException if this {@link StreamsExecutionEnvironment} instance is started. - * - * @return this {@link StreamsExecutionEnvironment} instance. + * @return {@code true} if this {@link StreamsExecutionEnvironment} is marked as default. */ - StreamsExecutionEnvironment addStateListener(final KafkaStreams.StateListener listener); + boolean isDefault(); /** - * Adds a {@link StateRestoreListener} instance that will set to all {@link KafkaStreams} instance created - * in this {@link StreamsExecutionEnvironment}. - * - * @see KafkaStreams#setGlobalStateRestoreListener(StateRestoreListener) . - * - * @param listener the {@link StateRestoreListener} instance. - * - * @throws IllegalStateException if this {@link StreamsExecutionEnvironment} instance is started. + * Creates a new {@link StreamsTopologyExecution} to be applied on this {@link StreamsExecutionEnvironment}. * - * @return this {@link StreamsExecutionEnvironment} instance. - */ - StreamsExecutionEnvironment addGlobalStateListener(final StateRestoreListener listener); - - /** - * Adds a streams interceptor that will set to all {@link KafkaStreams} instance created - * in this {@link StreamsExecutionEnvironment}. - * The interceptors will be executed in the order in which they were added. - * - * @param interceptor the {@link {@link StreamsLifecycleInterceptor}}. - * @return this {@link StreamsExecutionEnvironment} instance. + * @param meta the {@link StreamsTopologyMeta} to executed. + * @param executed the execution options. + * @return the new {@link StreamsTopologyExecution} instance. */ - StreamsExecutionEnvironment addStreamsLifecycleInterceptor(final Supplier interceptor); + StreamsTopologyExecution newTopologyExecution(final StreamsTopologyMeta meta, final Executed executed); /** - * Sets the {@link StreamThreadExceptionHandler} invoked when a StreamThread abruptly terminates - * due to an uncaught exception. - * - * @param handler the {@link StreamThreadExceptionHandler}. - * @return this {@link StreamsExecutionEnvironment} instance. + * Returns all containers for active Kafka Streams applications. * - * @see KafkaStreams#setUncaughtExceptionHandler(Thread.UncaughtExceptionHandler) - */ - StreamsExecutionEnvironment setStreamThreadExceptionHandler(final Supplier handler); - - /** - * Gets the {@link StreamThreadExceptionHandler}. - * - * @return the {@link Supplier}, otherwise {@code null} if no handler is set. + * @return a collection of {@link KafkaStreamsContainer} applications. */ - Supplier getStreamThreadExceptionHandler(); + Collection applications(); /** - * Returns all {@link KafkaStreams} started applications. + * Returns all ids for active Kafka Streams applications. * * @return a collection of {@link KafkaStreamsContainer} applications. */ - Collection applications(); + Set applicationIds(); /** * Sets this environment configuration. * * @return this {@link StreamsExecutionEnvironment} instance. */ - StreamsExecutionEnvironment setConfiguration(final Conf configuration); + T setConfiguration(final Conf configuration); /** * Gets this environment configuration. @@ -135,7 +106,7 @@ public interface StreamsExecutionEnvironment { * * @return this {@link StreamsExecutionEnvironment} instance. */ - StreamsExecutionEnvironment setRocksDBConfig(final RocksDBConfig settings); + T setRocksDBConfig(final RocksDBConfig settings); /** * Sets the {@link ApplicationIdBuilder} that should be used for building streams {@code application.id}. @@ -143,7 +114,7 @@ public interface StreamsExecutionEnvironment { * @param supplier the {@link ApplicationIdBuilder} instance supplier. * @return this {@link StreamsExecutionEnvironment} instance. */ - StreamsExecutionEnvironment setApplicationIdBuilder(final Supplier supplier); + T setApplicationIdBuilder(final Supplier supplier); /** * Gets the {@link ApplicationIdBuilder}. @@ -158,37 +129,7 @@ public interface StreamsExecutionEnvironment { * @param settings the {@link Conf} instance. * @return this {@link StreamsExecutionEnvironment} instance. */ - StreamsExecutionEnvironment addFallbackConfiguration(final Conf settings); - - /** - * Sets the {@link KafkaStreamsFactory} that will be used to provide - * the {@link KafkaStreams} to configure and start. - * - * @param kafkaStreamsFactory the {@link KafkaStreamsFactory} instance. - * @return this {@link StreamsExecutionEnvironment} instance. - */ - StreamsExecutionEnvironment setKafkaStreamsFactory(final Supplier kafkaStreamsFactory); - - /** - * Add a new {@link TopologyProvider} instance to this {@link StreamsExecutionEnvironment} to be started. - * - * @param provider the {@link TopologyProvider} supplier. - * - * @return this {@link ApplicationId} instance if the environment is already started, - * otherwise {@code null}. - */ - ApplicationId addTopology(final Supplier provider); - - /** - * Add a new {@link TopologyProvider} instance to this {@link StreamsExecutionEnvironment} to be started. - * - * @param provider the {@link TopologyProvider} supplier. - * @param executed the {@link Executed} instance. - * - * @return this {@link ApplicationId} instance if the environment is already started, - * otherwise {@code null}. - */ - ApplicationId addTopology(final Supplier provider, final Executed executed); + T addFallbackConfiguration(final Conf settings); /** * Starts this {@link StreamsExecutionEnvironment} instance. diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/StreamsExecutionEnvironmentFactory.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/StreamsExecutionEnvironmentFactory.java new file mode 100644 index 00000000..8731a151 --- /dev/null +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/StreamsExecutionEnvironmentFactory.java @@ -0,0 +1,53 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamthoughts.azkarra.api; + +import io.streamthoughts.azkarra.api.config.Conf; + +public interface StreamsExecutionEnvironmentFactory> { + + /** + * Creates a new StreamsExecutionEnvironment from the specified name and an empty configuration. + * + * @param name the environment name. + * @return a new {@link StreamsExecutionEnvironment} of type {@link T}. + */ + default T create(final String name) { + return create(name, Conf.empty()); + } + + /** + * Creates a new StreamsExecutionEnvironment from the specified name and configuration. + * + * @param name the environment name. + * @param conf the environment configuration. + * @return a new {@link StreamsExecutionEnvironment} of type {@link T}. + */ + T create(final String name, final Conf conf); + + /** + * Returns the string type associated with the {@link StreamsExecutionEnvironment} that can + * be created from this factory. + * + * @return the type of the {@link StreamsExecutionEnvironment}. + */ + String type(); + +} diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/StreamsLifecycleContext.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/StreamsLifecycleContext.java index 08ce6b5b..816a0c33 100644 --- a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/StreamsLifecycleContext.java +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/StreamsLifecycleContext.java @@ -39,10 +39,10 @@ default String applicationId() { } /** - * @see KafkaStreamsContainer#topologyDescription() + * @see KafkaStreamsContainer#getTopology() */ default TopologyDescription topologyDescription() { - return container().topologyDescription(); + return container().getTopology().describe(); } /** diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/StreamsTopologyExecution.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/StreamsTopologyExecution.java new file mode 100644 index 00000000..6f27c0f3 --- /dev/null +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/StreamsTopologyExecution.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamthoughts.azkarra.api; + +import io.streamthoughts.azkarra.api.streams.ApplicationId; + +import java.util.concurrent.Callable; + +/** + * TopologyExecution provide a way for topology to be executed in an specific environment. + * + * @see StreamsExecutionEnvironment + */ +public interface StreamsTopologyExecution extends Callable { + + /** + * Starts the streams-topology encapsulated by this object. + * + * @return the {@link ApplicationId}. + */ + ApplicationId start(); + + @Override + default ApplicationId call() { + return start(); + } +} diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/StreamsTopologyMeta.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/StreamsTopologyMeta.java new file mode 100644 index 00000000..ac53cd5d --- /dev/null +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/StreamsTopologyMeta.java @@ -0,0 +1,124 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamthoughts.azkarra.api; + +import io.streamthoughts.azkarra.api.config.Conf; +import io.streamthoughts.azkarra.api.streams.TopologyProvider; +import io.streamthoughts.azkarra.api.util.Version; + +import java.util.Objects; + +public class StreamsTopologyMeta { + + private final String name; + private final Version version; + private final String description; + private final Class type; + private final ClassLoader classLoader; + private final Conf conf; + + + /** + * Creates a new {@link StreamsTopologyMeta} instance. + * + * @param name the name of the topology. + * @param version the version of the topology + * @param description the description of the topology. + * @param type the topology {@link Class}. + * @param classLoader the topology {@link ClassLoader}. + * @param conf the default {@link Conf} for the topology. + */ + public StreamsTopologyMeta(final String name, + final Version version, + final String description, + final Class type, + final ClassLoader classLoader, + final Conf conf) { + this.name = name; + this.version = version; + this.description = description; + this.type = type; + this.classLoader = classLoader; + this.conf = conf; + } + + public String name() { + return name; + } + + public Version version() { + return version; + } + + public Class type() { + return type; + } + + public String description() { + return description; + } + + public Conf configuration() { + return conf; + } + + public ClassLoader classLoader() { + return classLoader; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof StreamsTopologyMeta)) return false; + StreamsTopologyMeta that = (StreamsTopologyMeta) o; + return Objects.equals(name, that.name) && + Objects.equals(version, that.version) && + Objects.equals(description, that.description) && + Objects.equals(type, that.type) && + Objects.equals(classLoader, that.classLoader) && + Objects.equals(conf, that.conf); + } + + /** + * {@inheritDoc} + */ + @Override + public int hashCode() { + return Objects.hash(name, version, description, type, classLoader, conf); + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + return "StreamTopologyMeta{" + + "name='" + name + '\'' + + ", version=" + version + + ", description='" + description + '\'' + + ", type=" + type + + ", classLoader=" + classLoader + + ", conf=" + conf + + '}'; + } +} diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/components/SimpleComponentDescriptor.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/components/SimpleComponentDescriptor.java index e028dd4e..20151f37 100644 --- a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/components/SimpleComponentDescriptor.java +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/components/SimpleComponentDescriptor.java @@ -35,7 +35,7 @@ */ public class SimpleComponentDescriptor implements ComponentDescriptor { - private String name; + private final String name; private final Version version; @@ -45,7 +45,7 @@ public class SimpleComponentDescriptor implements ComponentDescriptor { private final Set aliases; - private ClassLoader classLoader; + private final ClassLoader classLoader; private ComponentMetadata metadata; diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/components/condition/ComponentConditionalContext.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/components/condition/ComponentConditionalContext.java index b573a09b..a1e67c29 100644 --- a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/components/condition/ComponentConditionalContext.java +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/components/condition/ComponentConditionalContext.java @@ -30,7 +30,7 @@ * @see io.streamthoughts.azkarra.api.components.ConditionalDescriptorRegistry */ @FunctionalInterface -public interface ComponentConditionalContext{ +public interface ComponentConditionalContext>{ /** * Verify if this component is enabled for the given context. @@ -38,7 +38,7 @@ public interface ComponentConditionalContext{ * @param factory the {@link ComponentFactory}; cannot be {@code null}. * @param descriptor the {@link ComponentDescriptor}; cannot be {@code null}. * - * @return {@code true} if the compoenent is enable. + * @return {@code true} if the component is enable. */ boolean isEnable(final ComponentFactory factory, final T descriptor); diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/Conf.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/Conf.java index 2b61ae80..a58ab183 100644 --- a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/Conf.java +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/Conf.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.Optional; import java.util.Properties; +import java.util.Set; /** * Class which can be used for configuring components. @@ -131,10 +132,12 @@ static Conf of(final Conf...configurations) { Conf config = empty(); for (Conf conf : configurations) { if (conf != null) { - config = config.withFallback(conf); + Conf copy = Conf.of(conf.getConfAsMap()); + config = config.withFallback(copy); } } - return Conf.of(config.getConfAsMap()); // This will have the effect to flatten the configuration. + // This will have the effect to flatten the configuration. + return Conf.of(config.getConfAsMap()); } /** @@ -147,10 +150,25 @@ static Conf empty() { } /** - * Gets a required parameter a a string. + * Gets a required parameter value. * * @param path the parameter path. - * @return the parameter value as a string. + * + * @return the object value. + */ + Object getValue(final String path); + + /** + * Returns a {@link Set} view of the keys contained in this conf. + * @return a set view of the keys contained in this conf + */ + Set keySet(); + + /** + * Gets a required parameter as a string. + * + * @param path the parameter path. + * @return the parameter value as a string. * * @throws MissingConfException if no parameter can be found for the specified path. */ diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/ConfBuilder.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/ConfBuilder.java index 3430ffdb..da62f945 100644 --- a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/ConfBuilder.java +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/ConfBuilder.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; /** * Helper class which can be used for building new {@link Conf} instance. @@ -51,6 +52,22 @@ public ConfBuilder with(final String key, Object value) { return this; } + /** + * {@inheritDoc} + */ + @Override + public Object getValue(final String path) { + return build().getValue(path); + } + + /** + * {@inheritDoc} + */ + @Override + public Set keySet() { + return build().keySet(); + } + /** * {@inheritDoc} */ diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/ConfEntry.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/ConfEntry.java new file mode 100644 index 00000000..940f9161 --- /dev/null +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/ConfEntry.java @@ -0,0 +1,180 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamthoughts.azkarra.api.config; + +import java.util.Collection; +import java.util.List; +import java.util.Objects; + +public class ConfEntry { + + private final Property property; + + public static ConfEntry of(final String key, final Object value) { + return new ConfEntry(key, value); + } + + /** + * Creates a new {@link ConfEntry} instance. + * @param key the object key. + * @param value the object value. + */ + private ConfEntry(final String key, final Object value) { + this.property = new Property(key, value); + } + + /** + * Gets the object key. + * + * @return the string key. + */ + public String key() { + return property.key(); + } + + /** + * Gets the object value. + * + * @return the object value. + */ + public Object value() { + return property.get(); + } + + /** + * Get the {@link ConfEntry} as {@link Property}. + * @return a new {@link Property} instance. + */ + public Property asProperty() { + return new Property(key(), value()); + } + + + /** + * Gets value as a string. + */ + public String asString() { + return property.getString(key()); + } + + /** + * Gets valuer as a long. + */ + public long asLong() { + return property.getLong(key()); + } + + /** + * Gets value as an integer. + * + * @return an integer value. + */ + public int asInt() { + return property.getInt(key()); + } + + /** + * Gets value as an boolean. + * + * @return the value as as boolean. + */ + public boolean asBoolean() { + return property.getBoolean(key()); + } + + /** + * Gets value as a double. + * + * @return the value as a double. + */ + public double asDouble() { + return property.getDouble(key()); + } + + /** + * Gets value as a list. + * + * @return a string list value. + */ + public List asStringList() { + return property.getStringList(key()); + } + + /** + * Gets value as a {@link Conf}. + * + * @return a new {@link Conf} instance. + */ + public Conf asSubConf() { + return property.getSubConf(key()); + } + + /** + * Gets value as a list of {@link Conf}. + * + * @return a new list of {@link Conf} instances. + */ + public List asSubConfList() { + return property.getSubConfList(key()); + } + + /** + * Gets value as a list of instances of type {@link T}. + * + * @param type the class of the . + * @param the expected type. + * + * @return a new {@link Collection} of {@link T}. + */ + public Collection getClasses(final Class type) { + return property.getClasses(key(), type); + } + + /** + * Gets value as an instances of type {@link T}. + * + * @param type the class of the . + * @param the expected type. + * + * @return a new {@link Collection} of {@link T}. + */ + public T getClass( final Class type) { + return property.getClass(key(), type); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof ConfEntry)) return false; + ConfEntry that = (ConfEntry) o; + return Objects.equals(property, that.property); + } + + /** + * {@inheritDoc} + */ + @Override + public int hashCode() { + return Objects.hash(property); + } +} diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/DelegatingConf.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/DelegatingConf.java index d7aa6f77..aca4c814 100644 --- a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/DelegatingConf.java +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/DelegatingConf.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; /** * Delegates to a {@link Conf}. @@ -39,6 +40,22 @@ public DelegatingConf(final Conf originals) { this.originals = originals; } + /** + * {@inheritDoc} + */ + @Override + public Object getValue(String path) { + return originals.getValue(path); + } + + /** + * {@inheritDoc} + */ + @Override + public Set keySet() { + return originals.keySet(); + } + /** * {@inheritDoc} */ diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/EmptyConf.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/EmptyConf.java index fe97061f..d94f4bdd 100644 --- a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/EmptyConf.java +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/EmptyConf.java @@ -25,9 +25,26 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; public class EmptyConf implements Conf { + /** + * {@inheritDoc} + */ + @Override + public Object getValue(String path) { + throw new MissingConfException(path); + } + + /** + * {@inheritDoc} + */ + @Override + public Set keySet() { + return Collections.emptySet(); + } + /** * {@inheritDoc} */ diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/MapConf.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/MapConf.java index 04dbed37..94a09585 100644 --- a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/MapConf.java +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/MapConf.java @@ -23,9 +23,11 @@ import io.streamthoughts.azkarra.api.monad.Tuple; import io.streamthoughts.azkarra.api.util.TypeConverter; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -84,11 +86,29 @@ static MapConf singletonConf(final String key, final Object value) { protected MapConf(final Map parameters, final Conf fallback, final boolean explode) { - Objects.requireNonNull(parameters, "parameters Conf cannot be null"); + Objects.requireNonNull(parameters, "parameters cannot be null"); this.parameters = explode ? explode(parameters).unwrap() : parameters; this.fallback = fallback; } + /** + * {@inheritDoc} + */ + @Override + public Set keySet() { + final HashSet keySet = new HashSet<>(parameters.keySet()); + keySet.addAll(fallback.keySet()); + return keySet; + } + + /** + * {@inheritDoc} + */ + @Override + public Object getValue(final String path) { + return findForPathOrThrow(path, Conf::getValue); + } + /** * {@inheritDoc} */ @@ -179,7 +199,16 @@ public List getStringList(final String path) { */ @Override public Conf getSubConf(final String path) { - Conf conf = (Conf) findForPathOrThrow(path, Conf::getSubConf); + Object o = findForPathOrThrow(path, Conf::getSubConf); + Conf conf; + if(o instanceof Map) + conf = Conf.of((Map)o) ; + else if (o instanceof Conf) + conf = (Conf)o; + else { + throw new InvalidConfException( + "Type mismatch for path '" + path + "': " + o.getClass().getSimpleName() + "<> [Map, Conf]"); + } if (fallback != null && fallback.hasPath(path)) { conf = conf.withFallback(fallback.getSubConf(path)); } @@ -192,7 +221,19 @@ public Conf getSubConf(final String path) { @Override @SuppressWarnings("unchecked") public List getSubConfList(final String path) { - return (List) findForPathOrThrow(path, Conf::getSubConfList); + List ol = (List) findForPathOrThrow(path, Conf::getSubConfList); + List subConfList = new ArrayList<>(ol.size()); + for (Object o : ol) { + if(o instanceof Map) + subConfList.add(Conf.of((Map)o)); + else if (o instanceof Conf) + subConfList.add((Conf)o); + else { + throw new InvalidConfException( + "Type mismatch for path '" + path + "': " + o.getClass().getSimpleName() + "<> [Map, Conf]"); + } + } + return subConfList; } /** diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/Property.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/Property.java index 1e4ff345..04b53e30 100644 --- a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/Property.java +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/config/Property.java @@ -26,7 +26,7 @@ */ public class Property extends MapConf implements Conf { - private String key; + private final String key; /** * Creates a new {@link Property} instance. * @@ -41,4 +41,8 @@ public class Property extends MapConf implements Conf { public Object get() { return unwrap().get(key); } + + public String key() { + return key; + } } diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/model/Environment.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/model/Environment.java index 6a2f9d3b..e237e345 100644 --- a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/model/Environment.java +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/model/Environment.java @@ -31,6 +31,7 @@ public class Environment { private final String name; private final State state; + private final String type; private final Map config; private final Set applications; private final boolean isDefault; @@ -39,17 +40,20 @@ public class Environment { * Creates a new {@link Environment} instance. * * @param name the environment name. + * @param type the environment type. * @param state the environment state. * @param config the environment configuration. * @param applications the list of active streams applications. * @param isDefault is the default environment. */ public Environment(final String name, + final String type, final State state, final Map config, final Set applications, final boolean isDefault) { this.name = name; + this.type = type; this.state = state; this.config = config; this.applications = new TreeSet<>(applications); @@ -61,6 +65,11 @@ public String name() { return name; } + @JsonProperty("type") + public String type() { + return type; + } + @JsonProperty("state") public State state() { return state; @@ -90,6 +99,7 @@ public boolean equals(Object o) { Environment that = (Environment) o; return isDefault == that.isDefault && Objects.equals(name, that.name) && + Objects.equals(type, that.type) && Objects.equals(config, that.config) && Objects.equals(applications, that.applications); } @@ -109,6 +119,7 @@ public int hashCode() { public String toString() { return "Environment{" + "name='" + name + '\'' + + "name='" + type + '\'' + ", config=" + config + ", applications=" + applications + ", isDefault=" + isDefault + diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/streams/KafkaStreamsContainer.java b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/streams/KafkaStreamsContainer.java index aba28a54..0c6e836f 100644 --- a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/streams/KafkaStreamsContainer.java +++ b/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/streams/KafkaStreamsContainer.java @@ -21,20 +21,23 @@ import io.streamthoughts.azkarra.api.config.Conf; import io.streamthoughts.azkarra.api.events.EventStream; import io.streamthoughts.azkarra.api.events.reactive.EventStreamPublisher; +import io.streamthoughts.azkarra.api.model.Metric; +import io.streamthoughts.azkarra.api.model.MetricGroup; +import io.streamthoughts.azkarra.api.model.StreamsTopologyGraph; import io.streamthoughts.azkarra.api.model.TimestampedValue; +import io.streamthoughts.azkarra.api.monad.Tuple; import io.streamthoughts.azkarra.api.query.LocalStoreAccessor; import io.streamthoughts.azkarra.api.streams.consumer.ConsumerGroupOffsets; import io.streamthoughts.azkarra.api.streams.store.LocalStorePartitionLags; import io.streamthoughts.azkarra.api.streams.topology.TopologyMetadata; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.common.Metric; -import org.apache.kafka.common.MetricName; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.KeyQueryMetadata; import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.TopologyDescription; import org.apache.kafka.streams.processor.ThreadMetadata; import org.apache.kafka.streams.state.HostInfo; @@ -49,18 +52,10 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.Executor; -import java.util.concurrent.Future; +import java.util.function.Predicate; public interface KafkaStreamsContainer { - /** - * Asynchronously start the underlying {@link KafkaStreams} instance. - * - * @param executor the {@link Executor} instance to be used for starting the streams. - */ - Future start(final Executor executor); - /** * Restarts this container. */ @@ -141,20 +136,32 @@ default void close(final Duration timeout) { TopologyMetadata topologyMetadata(); /** - * Gets the {@link TopologyDescription} for this {@link KafkaStreams} instance. + * Gets the {@link StreamsTopologyGraph} for this {@link KafkaStreams} instance. * * @return a new {@link TopologyDescription} instance. */ - TopologyDescription topologyDescription(); + StreamsTopologyGraph topologyGraph(); /** - * Gets all the current {@link Metric}s for this {@link KafkaStreams} instance. + * Gets all {@link Metric}s for this {@link KafkaStreams} instance. * * @see KafkaStreams#metrics() * - * @return a map of {@link Metric}. + * @return a {@code Set} of {@link Metric}. */ - Map metrics(); + default Set metrics() { + return metrics(KafkaMetricFilter.all()); + } + + /** + * Gets all {@link Metric}s for this {@link KafkaStreams} instance matching the specified {@link KafkaMetricFilter}. + * + * @see KafkaStreams#metrics() + * + * @param filter the {@link KafkaMetricFilter} to be used. + * @return a {@code Set} of {@link Metric}. + */ + Set metrics(final KafkaMetricFilter filter); /** * Gets the offsets for the topic/partitions assigned to this {@link KafkaStreams} instance. @@ -181,9 +188,25 @@ default void close(final Duration timeout) { /** * Returns the wrapped {@link KafkaStreams} instance. * + * This method can throw an {@link UnsupportedOperationException} if the {@link KafkaStreamsContainer} + * implementation doesn't manage the {@link KafkaStreams} locally. + * * @return the {@link KafkaStreams}. */ - KafkaStreams kafkaStreams(); + default KafkaStreams getKafkaStreams() { + throw new UnsupportedOperationException(); + } + /** + * Returns the wrapped {@link Topology} instance. + * + * This method can throw an {@link UnsupportedOperationException} if the {@link KafkaStreamsContainer} + * implementation doesn't manage the {@link KafkaStreams} locally. + * + * @return the {@link Topology} instance. + */ + default Topology getTopology() { + throw new UnsupportedOperationException(); + } /** * Checks whether the given {@link HostInfo} is the same as this container. @@ -316,4 +339,26 @@ default boolean accept(final State newState) { */ void onChange(final StateChangeEvent event); } + + /** + * A {@code KafkaMetricFilter} can be used to only get specific metrics. + */ + interface KafkaMetricFilter extends Predicate> { + + static KafkaMetricFilter of(final Predicate> predicate) { + return predicate::test; + } + + static KafkaMetricFilter all() { + return candidate -> true; + } + + static KafkaMetricFilter filterByGroup(final String groupName) { + return candidate -> candidate.left().equals(groupName); + } + + static KafkaMetricFilter filterByGroupAndMetricName(final String groupName, final String metricName) { + return candidate -> candidate.left().equals(groupName) && candidate.right().name().equals(metricName); + } + } } diff --git a/azkarra-examples/src/main/java/io/streamthoughts/examples/azkarra/noannotation/StreamsApplication.java b/azkarra-examples/src/main/java/io/streamthoughts/examples/azkarra/noannotation/StreamsApplication.java index dc9171c2..95a125c1 100644 --- a/azkarra-examples/src/main/java/io/streamthoughts/examples/azkarra/noannotation/StreamsApplication.java +++ b/azkarra-examples/src/main/java/io/streamthoughts/examples/azkarra/noannotation/StreamsApplication.java @@ -60,7 +60,8 @@ public static void main(final String[] args) { .setConfiguration(AzkarraConf.create()) .setBannerMode(Banner.Mode.OFF) .setContext(context) - .enableHttpServer(true, serverConfig) + .setHttpServerEnable(true) + .setHttpServerConf(serverConfig) .setAutoStart(true) // mandatory for auto-starting ConfigurableWordCountTopology. .setEnableComponentScan(false) .run(args); diff --git a/azkarra-examples/src/main/java/io/streamthoughts/examples/azkarra/security/BasicAuthenticationExample.java b/azkarra-examples/src/main/java/io/streamthoughts/examples/azkarra/security/BasicAuthenticationExample.java index acbc0781..cc283a1a 100644 --- a/azkarra-examples/src/main/java/io/streamthoughts/examples/azkarra/security/BasicAuthenticationExample.java +++ b/azkarra-examples/src/main/java/io/streamthoughts/examples/azkarra/security/BasicAuthenticationExample.java @@ -69,7 +69,8 @@ public static void main(final String[] args) { new AzkarraApplication() .setConfiguration(AzkarraConf.create("application")) .addSource(BasicWordCountTopology.class) - .enableHttpServer(true, serverConfig) + .setHttpServerEnable(true) + .setHttpServerConf(serverConfig) .run(args); } } diff --git a/azkarra-examples/src/main/java/io/streamthoughts/examples/azkarra/security/SSLAuthenticationExample.java b/azkarra-examples/src/main/java/io/streamthoughts/examples/azkarra/security/SSLAuthenticationExample.java index 1e591a93..348ee0d5 100644 --- a/azkarra-examples/src/main/java/io/streamthoughts/examples/azkarra/security/SSLAuthenticationExample.java +++ b/azkarra-examples/src/main/java/io/streamthoughts/examples/azkarra/security/SSLAuthenticationExample.java @@ -59,7 +59,8 @@ public static void main(final String[] args) { new AzkarraApplication() .setConfiguration(AzkarraConf.create("application")) .addSource(BasicWordCountTopology.class) - .enableHttpServer(true, securedServer) + .setHttpServerEnable(true) + .setHttpServerConf(securedServer) .run(args); } } diff --git a/azkarra-metrics/pom.xml b/azkarra-metrics/pom.xml index 2e5d60fb..ec8fb335 100644 --- a/azkarra-metrics/pom.xml +++ b/azkarra-metrics/pom.xml @@ -52,8 +52,9 @@ io.streamthoughts - azkarra-api + azkarra-streams ${project.parent.version} + provided diff --git a/azkarra-metrics/src/main/java/io/streamthoughts/azkarra/metrics/AzkarraMetricsConfigEntryLoader.java b/azkarra-metrics/src/main/java/io/streamthoughts/azkarra/metrics/AzkarraMetricsConfigEntryLoader.java new file mode 100644 index 00000000..92ffdcd9 --- /dev/null +++ b/azkarra-metrics/src/main/java/io/streamthoughts/azkarra/metrics/AzkarraMetricsConfigEntryLoader.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamthoughts.azkarra.metrics; + +import io.streamthoughts.azkarra.api.config.Conf; +import io.streamthoughts.azkarra.api.config.ConfEntry; +import io.streamthoughts.azkarra.streams.AbstractConfigEntryLoader; +import io.streamthoughts.azkarra.streams.AzkarraApplication; + +public class AzkarraMetricsConfigEntryLoader extends AbstractConfigEntryLoader { + + public static final String CONFIG_ENTRY_KEY = "metrics"; + + /** + * Creates a new {@link AzkarraMetricsConfigEntryLoader} instance. + */ + public AzkarraMetricsConfigEntryLoader() { + super(CONFIG_ENTRY_KEY); + } + + /** + * {@inheritDoc} + */ + @Override + public void load(final ConfEntry configEntryObject, + final AzkarraApplication application) { + // Injects "metrics." prefixed properties into Azkarra Context. + application.getContext().addConfiguration(Conf.of(CONFIG_ENTRY_KEY, configEntryObject.asSubConf())); + } +} diff --git a/azkarra-metrics/src/main/java/io/streamthoughts/azkarra/metrics/interceptor/MeterKafkaStreamsInterceptor.java b/azkarra-metrics/src/main/java/io/streamthoughts/azkarra/metrics/interceptor/MeterKafkaStreamsInterceptor.java index 9286c1df..d917d5d1 100644 --- a/azkarra-metrics/src/main/java/io/streamthoughts/azkarra/metrics/interceptor/MeterKafkaStreamsInterceptor.java +++ b/azkarra-metrics/src/main/java/io/streamthoughts/azkarra/metrics/interceptor/MeterKafkaStreamsInterceptor.java @@ -82,7 +82,7 @@ public boolean accept(final State newState) { @Override public void onChange(final StateChangeEvent event) { final List tags = List.of(Tag.of(TAG_APPLICATION_ID, context.applicationId())); - final var kafkaStreams = streamsContainer.kafkaStreams(); + final var kafkaStreams = streamsContainer.getKafkaStreams(); MeterRegistry registry = getComponent(MeterRegistry.class, Qualifiers.byPrimary()); metrics = new KafkaStreamsMetrics(kafkaStreams, tags); metrics.bindTo(registry); diff --git a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/AbstractTopologyStreamsExecution.java b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/AbstractTopologyStreamsExecution.java new file mode 100644 index 00000000..cc291d4f --- /dev/null +++ b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/AbstractTopologyStreamsExecution.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamthoughts.azkarra.runtime; + +import io.streamthoughts.azkarra.api.Executed; +import io.streamthoughts.azkarra.api.StreamsTopologyMeta; +import io.streamthoughts.azkarra.api.StreamsExecutionEnvironment; +import io.streamthoughts.azkarra.api.StreamsTopologyExecution; +import io.streamthoughts.azkarra.runtime.streams.topology.InternalExecuted; + +import java.util.Objects; + +public abstract class AbstractTopologyStreamsExecution> + implements StreamsTopologyExecution { + + protected E environment; + protected StreamsTopologyMeta meta; + protected InternalExecuted executed; + + /** + * Creates a new {@link AbstractTopologyStreamsExecution} instance. + * + * @param environment the execution environment. + * @param meta the {@link StreamsTopologyMeta} meta-information to be executed. + * @param executed the execution options. + */ + protected AbstractTopologyStreamsExecution(final E environment, + final StreamsTopologyMeta meta, + final Executed executed) { + this.environment = Objects.requireNonNull(environment, "environment cannot be null"); + this.meta = Objects.requireNonNull(meta, "meta cannot be null"); + this.executed = new InternalExecuted(executed); + } +} diff --git a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/StreamsExecutionEnvironmentAbstractFactory.java b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/StreamsExecutionEnvironmentAbstractFactory.java new file mode 100644 index 00000000..80e7f2d0 --- /dev/null +++ b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/StreamsExecutionEnvironmentAbstractFactory.java @@ -0,0 +1,71 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamthoughts.azkarra.runtime; + +import io.streamthoughts.azkarra.api.StreamsExecutionEnvironment; +import io.streamthoughts.azkarra.api.StreamsExecutionEnvironmentFactory; +import io.streamthoughts.azkarra.api.config.Conf; +import io.streamthoughts.azkarra.api.errors.InvalidStreamsEnvironmentException; + +import java.util.Collection; +import java.util.Map; +import java.util.stream.Collectors; + +public class StreamsExecutionEnvironmentAbstractFactory { + + private final Map> factoryByTypes; + + /** + * Creates a new {@link StreamsExecutionEnvironmentAbstractFactory} instance. + * + * @param factories all the {@link StreamsExecutionEnvironmentFactory}. + */ + @SuppressWarnings("unchecked") + public StreamsExecutionEnvironmentAbstractFactory(final Collection factories) { + this.factoryByTypes = factories + .stream() + .collect(Collectors.toMap(StreamsExecutionEnvironmentFactory::type, it -> it)); + } + + /** + * Creates a new {@link StreamsExecutionEnvironment} for the specified type using the given name and configuration. + * + * @param type the environment type. + * @param name the environment name. + * @param conf the environment configuration. + * @return a new {@link StreamsExecutionEnvironment}. + */ + public StreamsExecutionEnvironment create(final String type, final String name, final Conf conf) { + final StreamsExecutionEnvironmentFactory factory = factoryByTypes.get(type); + if (factory == null) { + throw new InvalidStreamsEnvironmentException("Cannot find factory for environment type " + type); + } + return factory.create(name, conf); + } + + /** + * Adds a new {@link StreamsExecutionEnvironmentFactory} to this one. + * + * @param factory the {@link StreamsExecutionEnvironmentFactory} to be added. + */ + public void addFactory(final StreamsExecutionEnvironmentFactory factory) { + this.factoryByTypes.put(factory.type(), factory); + } +} diff --git a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/components/RestrictedComponentFactory.java b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/components/RestrictedComponentFactory.java new file mode 100644 index 00000000..d71be3cb --- /dev/null +++ b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/components/RestrictedComponentFactory.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamthoughts.azkarra.runtime.components; + +import io.streamthoughts.azkarra.api.components.ComponentFactory; +import io.streamthoughts.azkarra.api.components.GettableComponent; +import io.streamthoughts.azkarra.api.components.Qualifier; +import io.streamthoughts.azkarra.api.components.Restriction; +import io.streamthoughts.azkarra.api.components.qualifier.Qualifiers; +import io.streamthoughts.azkarra.api.config.Conf; +import io.streamthoughts.azkarra.runtime.components.condition.ConfigConditionalContext; + +import java.util.Optional; + +public class RestrictedComponentFactory { + + private final ComponentFactory factory; + + public RestrictedComponentFactory(final ComponentFactory factory) { + this.factory = factory; + } + + /** + * Finds a component for the given type that is available for the given config and restriction. + * + * @param componentType the {@link Class } of the component. + * @param componentConfig the {@link Conf} object to be used for resolved available components. + * @param restriction the {@link Restriction}. + * @param the component type. + * @return an optional {@link GettableComponent}. + */ + public Optional> findComponentByRestriction(final Class componentType, + final Conf componentConfig, + final Restriction restriction) { + + final Qualifier qualifier = Qualifiers.byRestriction(restriction); + if (factory.containsComponent(componentType, qualifier)) { + GettableComponent provider = factory.getComponentProvider(componentType, qualifier); + if (provider.isEnable(new ConfigConditionalContext<>(componentConfig))) + return Optional.of(provider); + } + return Optional.empty(); + } +} diff --git a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/context/DefaultAzkarraContext.java b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/context/DefaultAzkarraContext.java index 5c14498d..8461aa37 100644 --- a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/context/DefaultAzkarraContext.java +++ b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/context/DefaultAzkarraContext.java @@ -25,16 +25,15 @@ import io.streamthoughts.azkarra.api.Executed; import io.streamthoughts.azkarra.api.State; import io.streamthoughts.azkarra.api.StreamsExecutionEnvironment; -import io.streamthoughts.azkarra.api.StreamsLifecycleInterceptor; +import io.streamthoughts.azkarra.api.StreamsExecutionEnvironmentFactory; +import io.streamthoughts.azkarra.api.StreamsTopologyMeta; import io.streamthoughts.azkarra.api.annotations.VisibleForTesting; import io.streamthoughts.azkarra.api.components.ComponentDescriptor; import io.streamthoughts.azkarra.api.components.ComponentDescriptorModifier; import io.streamthoughts.azkarra.api.components.ComponentFactory; import io.streamthoughts.azkarra.api.components.ComponentRegistry; import io.streamthoughts.azkarra.api.components.ContextAwareComponentFactory; -import io.streamthoughts.azkarra.api.components.GettableComponent; import io.streamthoughts.azkarra.api.components.Ordered; -import io.streamthoughts.azkarra.api.components.Qualifier; import io.streamthoughts.azkarra.api.components.Restriction; import io.streamthoughts.azkarra.api.components.qualifier.Qualifiers; import io.streamthoughts.azkarra.api.config.Conf; @@ -44,23 +43,15 @@ import io.streamthoughts.azkarra.api.errors.InvalidStreamsEnvironmentException; import io.streamthoughts.azkarra.api.providers.TopologyDescriptor; import io.streamthoughts.azkarra.api.streams.ApplicationId; -import io.streamthoughts.azkarra.api.streams.ApplicationIdBuilder; -import io.streamthoughts.azkarra.api.streams.KafkaStreamsFactory; import io.streamthoughts.azkarra.api.streams.TopologyProvider; import io.streamthoughts.azkarra.api.streams.errors.StreamThreadExceptionHandler; +import io.streamthoughts.azkarra.runtime.StreamsExecutionEnvironmentAbstractFactory; import io.streamthoughts.azkarra.runtime.components.ClassComponentAliasesGenerator; import io.streamthoughts.azkarra.runtime.components.DefaultComponentDescriptorFactory; import io.streamthoughts.azkarra.runtime.components.DefaultComponentFactory; import io.streamthoughts.azkarra.runtime.components.condition.ConfigConditionalContext; -import io.streamthoughts.azkarra.runtime.context.internal.ClassLoaderAwareKafkaStreamsFactory; -import io.streamthoughts.azkarra.runtime.context.internal.ContextAwareApplicationIdBuilderSupplier; -import io.streamthoughts.azkarra.runtime.context.internal.ContextAwareKafkaStreamsFactorySupplier; -import io.streamthoughts.azkarra.runtime.context.internal.ContextAwareLifecycleInterceptorSupplier; -import io.streamthoughts.azkarra.runtime.context.internal.ContextAwareThreadExceptionHandlerSupplier; -import io.streamthoughts.azkarra.runtime.context.internal.ContextAwareTopologySupplier; -import io.streamthoughts.azkarra.runtime.env.DefaultStreamsExecutionEnvironment; +import io.streamthoughts.azkarra.runtime.env.LocalStreamsExecutionEnvironmentFactory; import io.streamthoughts.azkarra.runtime.interceptors.AutoCreateTopicsInterceptor; -import io.streamthoughts.azkarra.runtime.interceptors.ClassloadingIsolationInterceptor; import io.streamthoughts.azkarra.runtime.interceptors.KafkaBrokerReadyInterceptor; import io.streamthoughts.azkarra.runtime.interceptors.MonitoringStreamsInterceptor; import io.streamthoughts.azkarra.runtime.interceptors.WaitForSourceTopicsInterceptor; @@ -73,6 +64,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -110,7 +102,7 @@ public class DefaultAzkarraContext implements AzkarraContext { * * @return a new {@link AzkarraContext} instance. */ - public static AzkarraContext create() { + public static DefaultAzkarraContext create() { return create(Conf.empty()); } @@ -120,7 +112,7 @@ public static AzkarraContext create() { * * @return a new {@link AzkarraContext} instance. */ - public static AzkarraContext create(final ComponentFactory factory) { + public static DefaultAzkarraContext create(final ComponentFactory factory) { return new DefaultAzkarraContext(Conf.empty(), factory); } @@ -130,7 +122,7 @@ public static AzkarraContext create(final ComponentFactory factory) { * * @return a new {@link AzkarraContext} instance. */ - public static AzkarraContext create(final Conf configuration) { + public static DefaultAzkarraContext create(final Conf configuration) { // Set all default implementations DefaultComponentFactory factory = new DefaultComponentFactory(new DefaultComponentDescriptorFactory()); factory.setComponentAliasesGenerator(new ClassComponentAliasesGenerator()); @@ -139,9 +131,9 @@ public static AzkarraContext create(final Conf configuration) { private boolean registerShutdownHook; - private StreamsExecutionEnvironment defaultEnvironment; + private StreamsExecutionEnvironment defaultEnvironment; - private final Map environments; + private final Map> environments; private final ComponentFactory componentFactory; @@ -149,8 +141,6 @@ public static AzkarraContext create(final Conf configuration) { private State state; - private final RestrictedComponentSupplierFactory componentSupplierFactory; - private Conf contextConfig; // contains the list of topologies to register when starting. @@ -168,7 +158,7 @@ private DefaultAzkarraContext(final Conf configuration, final ComponentFactory c this.listeners = new ArrayList<>(); this.contextConfig = configuration; this.componentFactory = new ContextAwareComponentFactory(this, componentFactory); - this.componentSupplierFactory = new RestrictedComponentSupplierFactory(this); + setState(State.CREATED); initialize(); } @@ -206,6 +196,15 @@ private void initialize() { LocalAzkarraStreamsService::new, withConditions(onMissingComponent(List.of(AzkarraStreamsService.class))) ); + registerSingleton( + StreamsExecutionEnvironmentFactory.class, + LocalStreamsExecutionEnvironmentFactory::new, + withConditions(onMissingComponent(List.of(StreamsExecutionEnvironmentFactory.class))) + ); + registerComponent( + StreamsExecutionEnvironmentAbstractFactory.class, + () -> new StreamsExecutionEnvironmentAbstractFactory(getAllComponents(StreamsExecutionEnvironmentFactory.class)) + ); } /** @@ -266,7 +265,7 @@ public AzkarraContext addConfiguration(final Conf configuration) { * {@inheritDoc} */ @Override - public AzkarraContext addExecutionEnvironment(final StreamsExecutionEnvironment env) + public AzkarraContext addExecutionEnvironment(final StreamsExecutionEnvironment env) throws AlreadyExistsException { Objects.requireNonNull(env, "env cannot be null."); LOG.debug("Adding new streams environment for name '{}'", env.name()); @@ -284,11 +283,7 @@ public AzkarraContext addExecutionEnvironment(final StreamsExecutionEnvironment if (env instanceof AzkarraContextAware) ((AzkarraContextAware)env).setAzkarraContext(this); - if (env.name().equals(DEFAULT_ENV_NAME)) - defaultEnvironment = env; - if (state == State.STARTED) { - initializeEnvironment(env); env.start(); } } else { @@ -388,7 +383,7 @@ private ApplicationId mayAddTopologyToEnvironment(final TopologyRegistration reg } final var streamConfig = registration.executed.config(); - final var conditionalContext = new ConfigConditionalContext( + final var conditionalContext = new ConfigConditionalContext<>( streamConfig .withFallback(environment.getConfiguration()) .withFallback(getConfiguration()) @@ -399,8 +394,16 @@ private ApplicationId mayAddTopologyToEnvironment(final TopologyRegistration reg componentFactory.findDescriptorByAlias(registration.type, conditionalContext, Qualifiers.byVersion(registration.version)); if (opt.isPresent()) { - final TopologyDescriptor descriptor = new TopologyDescriptor<>(opt.get()); - return addTopologyToEnvironment(descriptor, environment, registration.executed); + final TopologyDescriptor descriptor = new TopologyDescriptor<>(opt.get()); + final StreamsTopologyMeta meta = new StreamsTopologyMeta( + descriptor.name(), + descriptor.version(), + descriptor.description(), + descriptor.type(), + descriptor.classLoader(), + descriptor.configuration() + ); + return environment.newTopologyExecution(meta, registration.executed).start(); } else { final String loggedVersion = registration.version != null ? registration.version : "latest"; @@ -412,68 +415,11 @@ private ApplicationId mayAddTopologyToEnvironment(final TopologyRegistration reg } } - @VisibleForTesting - ApplicationId addTopologyToEnvironment(final TopologyDescriptor descriptor, - final StreamsExecutionEnvironment env, - final InternalExecuted executed) { - - // Gets user-defined streams name or fallback on descriptor cannot be null). - final var streamName = executed.nameOrElseGet(descriptor.name()); - - // Gets user-defined description or fallback on descriptor (can be null). - final var description = executed.descriptionOrElseGet(descriptor.description()); - - // Gets user-defined configuration and fallback on inherited configurations. - final var streamsConfig = Conf.of( - executed.config(), // (1) Executed - env.getConfiguration(), // (2) Environment - getConfiguration(), // (3) Context - descriptor.configuration() // (4) Descriptor (i.e: default) - ); - - final var interceptors = executed.interceptors(); - // Register StreamsLifeCycleInterceptor for class-loading isolation (must always be registered first) - interceptors.add(() -> new ClassloadingIsolationInterceptor(descriptor.classLoader())); - - // Get and register all StreamsLifeCycleInterceptors component for any scopes: Application, Env, Streams - interceptors.addAll(componentSupplierFactory.findLifecycleInterceptors( - streamsConfig, - Restriction.application(), - Restriction.env(env.name()), - Restriction.streams(streamName)) - ); - - // Get and register KafkaStreamsFactory for one the scopes: Application, Env, Streams - final var factory = executed.factory() - .or(() -> componentSupplierFactory.findKafkaStreamsFactory(streamsConfig, Restriction.streams(streamName))) - .or(() -> componentSupplierFactory.findKafkaStreamsFactory(streamsConfig, Restriction.env(env.name()))) - .or(() -> componentSupplierFactory.findKafkaStreamsFactory(streamsConfig, Restriction.application())) - .orElse(() -> KafkaStreamsFactory.DEFAULT); - - LOG.info( - "Registered new topology to environment '{}' for type='{}', version='{}', name='{}'.", - env.name(), - descriptor.className() , - descriptor.version(), - streamName - ); - - final var completedExecuted = Executed.as(streamName) - .withConfig(streamsConfig) - .withDescription(Optional.ofNullable(description).orElse("")) - .withInterceptors(interceptors) - .withKafkaStreamsFactory( - () -> new ClassLoaderAwareKafkaStreamsFactory(factory.get(), descriptor.classLoader()) - ); - - return env.addTopology(new ContextAwareTopologySupplier(this, descriptor), completedExecuted); - } - /** * {@inheritDoc} */ @Override - public Set topologyProviders() { + public Set getTopologyDescriptors() { Collection> descriptors = componentFactory.findAllDescriptorsByClass(TopologyProvider.class); return descriptors.stream().map(TopologyDescriptor::new).collect(Collectors.toSet()); @@ -483,57 +429,53 @@ public Set topologyProviders() { * {@inheritDoc} */ @Override - public Set topologyProviders(final StreamsExecutionEnvironment env) { - final Conf componentConfig = env.getConfiguration() - .withFallback(getConfiguration()); - ConfigConditionalContext conditionalContext = new ConfigConditionalContext(componentConfig); - Collection> descriptors = - componentFactory.findAllDescriptorsByClass(TopologyProvider.class, conditionalContext); - return descriptors.stream().map(TopologyDescriptor::new).collect(Collectors.toSet()); + public Set getTopologyDescriptors(final String environmentName) { + return getTopologyDescriptors(getEnvironmentForName(environmentName)); } /** * {@inheritDoc} */ @Override - public List environments() { - initializeDefaultEnvironment(); - return List.copyOf(environments.values()); + public Set getTopologyDescriptors(final StreamsExecutionEnvironment env) { + final Conf componentConfig = env.getConfiguration() + .withFallback(getConfiguration()); + var conditionalContext = new ConfigConditionalContext<>(componentConfig); + Collection> descriptors = + componentFactory.findAllDescriptorsByClass( + TopologyProvider.class, + conditionalContext, Qualifiers.byAnyRestrictions( + Restriction.application(), + Restriction.env(env.name()) + ) + ); + return descriptors.stream().map(TopologyDescriptor::new).collect(Collectors.toSet()); } /** * {@inheritDoc} */ @Override - public StreamsExecutionEnvironment getEnvironmentForNameOrCreate(final String envName) { - Objects.requireNonNull(envName, "envName cannot be null"); - StreamsExecutionEnvironment environment = environments.get(envName); - if (environment == null) { - environment = DefaultStreamsExecutionEnvironment.create(envName); - addExecutionEnvironment(environment); - } - return environment; + public List> getAllEnvironments() { + return List.copyOf(environments.values()); } /** * {@inheritDoc} */ @Override - public StreamsExecutionEnvironment defaultExecutionEnvironment() { - initializeDefaultEnvironment(); - return defaultEnvironment; + public StreamsExecutionEnvironment getEnvironmentForName(final String name) { + Objects.requireNonNull(name, "name cannot be null"); + checkIfEnvironmentExists(name, "No environment can be found for name " + name); + return environments.get(name); } /** * {@inheritDoc} */ @Override - public TopologyDescriptor getTopology(final String type) { - Optional> descriptor = componentFactory.findDescriptorByAlias(type); - if (descriptor.isPresent()) { - return new TopologyDescriptor<>(descriptor.get()); - } - throw new AzkarraException("Cannot find topology for given alias or class '" + type + "'"); + public StreamsExecutionEnvironment getDefaultEnvironment() { + return defaultEnvironment; } /** @@ -546,30 +488,26 @@ public void start() { "The context is either already started or already stopped, cannot re-start"); } LOG.info("Starting AzkarraContext"); - preStart(); componentFactory.init(getConfiguration()); + registerShutdownHook(); try { - listeners.forEach(listeners -> listeners.onContextStart(this)); - registerShutdownHook(); + listeners.addAll(getAllComponents(AzkarraContextListener.class)); + Collections.sort(listeners); + listeners.forEach(listener -> { + LOG.debug("Executing context listener {}#onContextStart", listener.getClass()); + listener.onContextStart(this); + }); + preStart(); // Initialize and start all streams environments. - for (StreamsExecutionEnvironment env : environments()) { - initializeEnvironment(env); - LOG.info("Starting streams environment: {}", env.name()); + for (StreamsExecutionEnvironment env : getAllEnvironments()) { + LOG.info("Starting streams execution environment: {}", env.name()); env.start(); } setState(State.STARTED); } catch (Exception e) { LOG.error("Unexpected error happens while starting AzkarraContext", e); stop(); // stop properly to close potentially open resources. - } - } - - @VisibleForTesting - void preStart() { - initializeDefaultEnvironment(); - if (!topologyRegistrations.isEmpty()) { - LOG.info("Adding all registered topologies to declared streams environments"); - topologyRegistrations.forEach(this::mayAddTopologyToEnvironment); + throw e; } } @@ -579,17 +517,21 @@ void preStart() { @Override public void stop(boolean cleanUp) { LOG.info("Stopping AzkarraContext"); + Collections.sort(listeners); listeners.forEach(listener -> { try { + LOG.debug("Executing context listener {}#onContextStop", listener.getClass()); listener.onContextStop(this); } catch (Exception e) { - LOG.error("Unexpected error happens while invoking listener '{}#onContextStop' : ", + LOG.error( + "Unexpected error happens while invoking listener '{}#onContextStop' : ", listener.getClass().getName(), - e); + e + ); } }); if (state == State.STARTED) { - environments().forEach(env -> env.stop(cleanUp)); + getAllEnvironments().forEach(env -> env.stop(cleanUp)); } try { componentFactory.close(); @@ -599,21 +541,13 @@ public void stop(boolean cleanUp) { LOG.info("AzkarraContext stopped completely"); } - private void initializeEnvironment(final StreamsExecutionEnvironment env) { - - final Conf componentResolutionConfig = env.getConfiguration().withFallback(getConfiguration()); - - // Inject environment or application scoped ApplicationIdBuilder - Optional.ofNullable(env.getApplicationIdBuilder()) - .or(() -> componentSupplierFactory.findApplicationIdBuilder(componentResolutionConfig, Restriction.env(env.name()))) - .or(() -> componentSupplierFactory.findApplicationIdBuilder(componentResolutionConfig, Restriction.application())) - .ifPresent(env::setApplicationIdBuilder); - - // Inject environment, global or default StreamsThreadExceptionHandler - Optional.ofNullable(env.getStreamThreadExceptionHandler()) - .or(() -> componentSupplierFactory.findStreamThreadExceptionHandler(componentResolutionConfig, Restriction.env(env.name()))) - .or(() -> componentSupplierFactory.findStreamThreadExceptionHandler(componentResolutionConfig, Restriction.application())) - .ifPresent(env::setStreamThreadExceptionHandler); + @VisibleForTesting + void preStart() { + initDefaultStreamsExecutionEnvironment(); + if (!topologyRegistrations.isEmpty()) { + LOG.info("Adding all registered topologies to declared streams environments"); + topologyRegistrations.forEach(this::mayAddTopologyToEnvironment); + } } private void checkIfEnvironmentExists(final String name, final String errorMessage) { @@ -622,9 +556,26 @@ private void checkIfEnvironmentExists(final String name, final String errorMessa } } - private void initializeDefaultEnvironment() { - if (defaultEnvironment == null) { - defaultEnvironment = DefaultStreamsExecutionEnvironment.create(DEFAULT_ENV_NAME); + private void initDefaultStreamsExecutionEnvironment() { + final List> defaults = environments + .values() + .stream() + .filter(env -> env.isDefault() || env.name().equals(DEFAULT_ENV_NAME)) + .collect(Collectors.toList()); + + if (defaults.size() > 1) { + final String strDefault = defaults + .stream() + .map(StreamsExecutionEnvironment::name) + .collect(Collectors.joining(",", "[", "]")); + throw new AzkarraContextException("Too many default environments " + strDefault); + } + + if (defaults.size() == 1) { + defaultEnvironment = defaults.get(0); + } else { + LOG.warn("No default environment can be found, initializing a new one with name {}", DEFAULT_ENV_NAME); + defaultEnvironment = getComponent(StreamsExecutionEnvironmentFactory.class).create(DEFAULT_ENV_NAME); addExecutionEnvironment(defaultEnvironment); } } @@ -715,9 +666,9 @@ public boolean equals(Object o) { if (!(o instanceof TopologyRegistration)) return false; TopologyRegistration that = (TopologyRegistration) o; return Objects.equals(type, that.type) && - Objects.equals(version, that.version) && - Objects.equals(environment, that.environment) && - Objects.equals(executed, that.executed); + Objects.equals(version, that.version) && + Objects.equals(environment, that.environment) && + Objects.equals(executed, that.executed); } /** @@ -728,66 +679,4 @@ public int hashCode() { return Objects.hash(type, version, environment, executed); } } - - @VisibleForTesting - static class RestrictedComponentSupplierFactory { - - private final ComponentFactory factory; - private final AzkarraContext context; - - RestrictedComponentSupplierFactory(final AzkarraContext context) { - this.context = context; - this.factory = context.getComponentFactory(); - } - - List> findLifecycleInterceptors(final Conf componentConfig, - final Restriction... restrictions) { - final Qualifier qualifier = Qualifiers.byAnyRestrictions(restrictions); - return factory.getAllComponentProviders(StreamsLifecycleInterceptor.class, qualifier) - .stream() - .filter(ConfigConditionalContext.of(componentConfig)) - .map(provider -> new ContextAwareLifecycleInterceptorSupplier(context, provider)) - .collect(Collectors.toList()); - } - - Optional> findApplicationIdBuilder(final Conf componentConfig, - final Restriction restriction) { - return findComponentByRestriction(ApplicationIdBuilder.class, componentConfig, restriction) - .map(gettable -> new ContextAwareApplicationIdBuilderSupplier(context, gettable)); - } - - Optional> findKafkaStreamsFactory(final Conf componentConfig, - final Restriction restriction) { - return findComponentByRestriction(KafkaStreamsFactory.class, componentConfig, restriction) - .map(gettable -> new ContextAwareKafkaStreamsFactorySupplier(context, gettable)); - } - - Optional> findStreamThreadExceptionHandler(final Conf conf, - final Restriction restriction) { - return findComponentByRestriction(StreamThreadExceptionHandler.class, conf, restriction) - .map(gettable -> new ContextAwareThreadExceptionHandlerSupplier(context, gettable)); - } - - /** - * Finds a component for the given type that is available for the given config and restriction. - * - * @param componentType the {@link Class } of the component. - * @param componentConfig the {@link Conf} object to be used for resolved available components. - * @param restriction the {@link Restriction}. - * @param the component type. - * @return an optional {@link GettableComponent}. - */ - private Optional> findComponentByRestriction(final Class componentType, - final Conf componentConfig, - final Restriction restriction) { - - final Qualifier qualifier = Qualifiers.byRestriction(restriction); - if (factory.containsComponent(componentType, qualifier)) { - GettableComponent provider = factory.getComponentProvider(componentType, qualifier); - if (provider.isEnable(new ConfigConditionalContext<>(componentConfig))) - return Optional.of(provider); - } - return Optional.empty(); - } - } } diff --git a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/context/internal/ContextAwareTopologySupplier.java b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/context/internal/ContextAwareTopologySupplier.java index d1c846ad..1d421aa7 100644 --- a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/context/internal/ContextAwareTopologySupplier.java +++ b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/context/internal/ContextAwareTopologySupplier.java @@ -20,8 +20,8 @@ import io.streamthoughts.azkarra.api.AzkarraContext; import io.streamthoughts.azkarra.api.AzkarraContextAware; +import io.streamthoughts.azkarra.api.StreamsTopologyMeta; import io.streamthoughts.azkarra.api.StreamsExecutionEnvironmentAware; -import io.streamthoughts.azkarra.api.components.ComponentDescriptor; import io.streamthoughts.azkarra.api.components.ComponentFactory; import io.streamthoughts.azkarra.api.components.qualifier.Qualifiers; import io.streamthoughts.azkarra.api.config.Conf; @@ -29,7 +29,6 @@ import io.streamthoughts.azkarra.api.config.ConfigurableSupplier; import io.streamthoughts.azkarra.api.events.EventStream; import io.streamthoughts.azkarra.api.events.EventStreamProvider; -import io.streamthoughts.azkarra.api.providers.TopologyDescriptor; import io.streamthoughts.azkarra.api.streams.TopologyProvider; import io.streamthoughts.azkarra.api.util.ClassUtils; import org.apache.kafka.streams.Topology; @@ -39,18 +38,18 @@ public class ContextAwareTopologySupplier extends ConfigurableSupplier { - private final ComponentDescriptor descriptor; + private final StreamsTopologyMeta meta; private final AzkarraContext context; /** * Creates a new {@link ContextAwareTopologySupplier} instance. * * @param context the {@link AzkarraContext} instance. - * @param descriptor the {@link TopologyDescriptor} instance. + * @param meta the {@link StreamsTopologyMeta} instance. */ public ContextAwareTopologySupplier(final AzkarraContext context, - final ComponentDescriptor descriptor) { - this.descriptor = descriptor; + final StreamsTopologyMeta meta) { + this.meta = meta; this.context = context; } @@ -62,9 +61,9 @@ public TopologyProvider get(final Conf configs) { final ComponentFactory factory = context.getComponentFactory(); final TopologyProvider provider = factory.getComponent( - descriptor.type(), + meta.type(), configs, - Qualifiers.byVersion(descriptor.version()) + Qualifiers.byVersion(meta.version()) ); if (provider instanceof AzkarraContextAware) { @@ -73,8 +72,8 @@ public TopologyProvider get(final Conf configs) { // The components returned from the registry may be already configured. // Thus, here we need to wrap the component into a non-configurable one so that the configure method - // will be not invoke a second time by the StreamsExecutionEnvironment. - return new ClassLoaderAwareTopologyProvider(provider, descriptor.classLoader()); + // will be not invoked a second time by the StreamsExecutionEnvironment. + return new ClassLoaderAwareTopologyProvider(provider, meta.classLoader()); } /** diff --git a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/env/LocalStreamsExecution.java b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/env/LocalStreamsExecution.java new file mode 100644 index 00000000..c003a905 --- /dev/null +++ b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/env/LocalStreamsExecution.java @@ -0,0 +1,143 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamthoughts.azkarra.runtime.env; + +import io.streamthoughts.azkarra.api.AzkarraContext; +import io.streamthoughts.azkarra.api.Executed; +import io.streamthoughts.azkarra.api.StreamsTopologyMeta; +import io.streamthoughts.azkarra.api.StreamsLifecycleInterceptor; +import io.streamthoughts.azkarra.api.components.Qualifier; +import io.streamthoughts.azkarra.api.components.Restriction; +import io.streamthoughts.azkarra.api.components.qualifier.Qualifiers; +import io.streamthoughts.azkarra.api.config.Conf; +import io.streamthoughts.azkarra.api.streams.ApplicationId; +import io.streamthoughts.azkarra.api.streams.KafkaStreamsFactory; +import io.streamthoughts.azkarra.runtime.AbstractTopologyStreamsExecution; +import io.streamthoughts.azkarra.runtime.components.RestrictedComponentFactory; +import io.streamthoughts.azkarra.runtime.components.condition.ConfigConditionalContext; +import io.streamthoughts.azkarra.runtime.context.internal.ClassLoaderAwareKafkaStreamsFactory; +import io.streamthoughts.azkarra.runtime.context.internal.ContextAwareKafkaStreamsFactorySupplier; +import io.streamthoughts.azkarra.runtime.context.internal.ContextAwareLifecycleInterceptorSupplier; +import io.streamthoughts.azkarra.runtime.context.internal.ContextAwareTopologySupplier; +import io.streamthoughts.azkarra.runtime.interceptors.ClassloadingIsolationInterceptor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Optional; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +public class LocalStreamsExecution extends AbstractTopologyStreamsExecution { + + private static final Logger LOG = LoggerFactory.getLogger(LocalStreamsExecution.class); + private final AzkarraContext context; + + /** + * Creates a new {@link LocalStreamsExecution} instance. + * + * @param meta the {@link StreamsTopologyMeta} object. + * @param executed the {@link Executed} object. + * @param context the {@link AzkarraContext} object. + * @param environment the {@link LocalStreamsExecutionEnvironment} object. + */ + LocalStreamsExecution(final StreamsTopologyMeta meta, + final Executed executed, + final AzkarraContext context, + final LocalStreamsExecutionEnvironment environment) { + super(environment, meta, executed); + this.context = context; + } + + /** + * {@inheritDoc} + */ + @Override + public ApplicationId start() { + // Gets user-defined streams name or fallback on descriptor cannot be null). + final var streamName = executed.nameOrElseGet(meta.name()); + + // Gets user-defined description or fallback on descriptor (can be null). + final var description = executed.descriptionOrElseGet(meta.description()); + + // Gets user-defined configuration and fallback on inherited configurations. + final var streamsConfig = Conf.of( + executed.config(), // (1) Executed + environment.getConfiguration(), // (2) Environment + context.getConfiguration(), // (3) Context + meta.configuration() // (4) Streams (i.e: default) + ); + + final var interceptors = executed.interceptors(); + // Register StreamsLifeCycleInterceptor for class-loading isolation (must always be registered first) + interceptors.add(() -> new ClassloadingIsolationInterceptor(meta.classLoader())); + + // Get and register all StreamsLifeCycleInterceptors component for any scopes: Application, Env, Streams + interceptors.addAll(findLifecycleInterceptors( + streamsConfig, + Restriction.application(), + Restriction.env(environment.name()), + Restriction.streams(streamName)) + ); + + // Get and register KafkaStreamsFactory for one the scopes: Application, Env, Streams + final var factory = executed.factory() + .or(() -> findKafkaStreamsFactory(streamsConfig, Restriction.streams(streamName))) + .or(() -> findKafkaStreamsFactory(streamsConfig, Restriction.env(environment.name()))) + .or(() -> findKafkaStreamsFactory(streamsConfig, Restriction.application())) + .orElse(() -> KafkaStreamsFactory.DEFAULT); + + LOG.info( + "Registered new topology to environment '{}' for type='{}', version='{}', name='{}'.", + environment.name(), + meta.type().getName(), + meta.version(), + streamName + ); + + final var completedExecuted = Executed.as(streamName) + .withConfig(streamsConfig) + .withDescription(Optional.ofNullable(description).orElse("")) + .withInterceptors(interceptors) + .withKafkaStreamsFactory( + () -> new ClassLoaderAwareKafkaStreamsFactory(factory.get(), meta.classLoader()) + ); + + return environment.addTopology(new ContextAwareTopologySupplier(context, meta), completedExecuted); + } + + private List> findLifecycleInterceptors(final Conf componentConfig, + final Restriction... restrictions) { + final Qualifier qualifier = Qualifiers.byAnyRestrictions(restrictions); + return context.getComponentFactory() + .getAllComponentProviders(StreamsLifecycleInterceptor.class, qualifier) + .stream() + .filter(ConfigConditionalContext.of(componentConfig)) + .map(provider -> new ContextAwareLifecycleInterceptorSupplier(context, provider)) + .collect(Collectors.toList()); + } + + private Optional> findKafkaStreamsFactory(final Conf componentConfig, + final Restriction restriction) { + return new RestrictedComponentFactory(context.getComponentFactory()) + .findComponentByRestriction(KafkaStreamsFactory.class, componentConfig, restriction) + .map(gettable -> new ContextAwareKafkaStreamsFactorySupplier(context, gettable)); + } +} diff --git a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/env/DefaultStreamsExecutionEnvironment.java b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/env/LocalStreamsExecutionEnvironment.java similarity index 74% rename from azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/env/DefaultStreamsExecutionEnvironment.java rename to azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/env/LocalStreamsExecutionEnvironment.java index adcf7aa7..b3b6f00f 100644 --- a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/env/DefaultStreamsExecutionEnvironment.java +++ b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/env/LocalStreamsExecutionEnvironment.java @@ -24,6 +24,8 @@ import io.streamthoughts.azkarra.api.State; import io.streamthoughts.azkarra.api.StreamsExecutionEnvironment; import io.streamthoughts.azkarra.api.StreamsLifecycleInterceptor; +import io.streamthoughts.azkarra.api.StreamsTopologyExecution; +import io.streamthoughts.azkarra.api.StreamsTopologyMeta; import io.streamthoughts.azkarra.api.annotations.VisibleForTesting; import io.streamthoughts.azkarra.api.config.Conf; import io.streamthoughts.azkarra.api.config.RocksDBConfig; @@ -33,7 +35,7 @@ import io.streamthoughts.azkarra.api.events.EventStreamProvider; import io.streamthoughts.azkarra.api.streams.ApplicationId; import io.streamthoughts.azkarra.api.streams.ApplicationIdBuilder; -import io.streamthoughts.azkarra.api.streams.DefaultKafkaStreamsContainer; +import io.streamthoughts.azkarra.runtime.streams.LocalKafkaStreamsContainer; import io.streamthoughts.azkarra.api.streams.KafkaStreamsContainer; import io.streamthoughts.azkarra.api.streams.KafkaStreamsFactory; import io.streamthoughts.azkarra.api.streams.TopologyProvider; @@ -59,6 +61,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; @@ -68,17 +71,20 @@ /** * The default {@link StreamsExecutionEnvironment} implementation. */ -public class DefaultStreamsExecutionEnvironment implements StreamsExecutionEnvironment, AzkarraContextAware { +public class LocalStreamsExecutionEnvironment + implements StreamsExecutionEnvironment, AzkarraContextAware { - private static final Logger LOG = LoggerFactory.getLogger(DefaultStreamsExecutionEnvironment.class); + public final static String TYPE = "local"; + + private static final Logger LOG = LoggerFactory.getLogger(LocalStreamsExecutionEnvironment.class); /** * Static helper that can be used to creates a new {@link StreamsExecutionEnvironment} instance * using the empty configuration and a generated unique name. * - * @return a new {@link StreamsExecutionEnvironment} instance. + * @return a new {@link LocalStreamsExecutionEnvironment} instance. */ - public static StreamsExecutionEnvironment create() { + public static LocalStreamsExecutionEnvironment create() { return create(Conf.empty()); } @@ -88,9 +94,9 @@ public static StreamsExecutionEnvironment create() { * * @param envName the name to be used for identifying this environment. * - * @return a new {@link StreamsExecutionEnvironment} instance. + * @return a new {@link LocalStreamsExecutionEnvironment} instance. */ - public static StreamsExecutionEnvironment create(final String envName) { + public static LocalStreamsExecutionEnvironment create(final String envName) { return create(Conf.empty(), envName); } @@ -100,9 +106,9 @@ public static StreamsExecutionEnvironment create(final String envName) { * * @param settings the {@link Conf} instance. * - * @return a new {@link StreamsExecutionEnvironment} instance. + * @return a new {@link LocalStreamsExecutionEnvironment} instance. */ - public static StreamsExecutionEnvironment create(final Conf settings) { + public static LocalStreamsExecutionEnvironment create(final Conf settings) { return create(settings, EnvironmentNameGenerator.generate()); } @@ -113,13 +119,13 @@ public static StreamsExecutionEnvironment create(final Conf settings) { * @param settings the {@link Conf} instance. * @param envName the name to be used for identifying this environment. * - * @return a new {@link StreamsExecutionEnvironment} instance. + * @return a new {@link LocalStreamsExecutionEnvironment} instance. */ - public static StreamsExecutionEnvironment create(final Conf settings, final String envName) { - return new DefaultStreamsExecutionEnvironment(settings, envName); + public static LocalStreamsExecutionEnvironment create(final Conf settings, final String envName) { + return new LocalStreamsExecutionEnvironment(settings, envName); } - private static ThreadPerStreamsExecutor STREAMS_EXECUTOR = new ThreadPerStreamsExecutor(); + private static final ThreadPerStreamsExecutor STREAMS_EXECUTOR = new ThreadPerStreamsExecutor(); /** * An internal name used to identify this environment. @@ -133,9 +139,9 @@ public static StreamsExecutionEnvironment create(final Conf settings, final Stri private Conf configuration; - private List stateListeners = new LinkedList<>(); + private final List stateListeners = new LinkedList<>(); - private List restoreListeners = new LinkedList<>(); + private final List restoreListeners = new LinkedList<>(); /** * The list of topologies to initialize when the environment is started. @@ -157,22 +163,24 @@ public static StreamsExecutionEnvironment create(final Conf settings, final Stri private Supplier applicationIdBuilderSupplier; + private boolean isDefault; + /** - * Creates a new {@link DefaultStreamsExecutionEnvironment} instance. + * Creates a new {@link LocalStreamsExecutionEnvironment} instance. * * @param configuration the default {@link Conf} instance. */ - public DefaultStreamsExecutionEnvironment(final Conf configuration) { + public LocalStreamsExecutionEnvironment(final Conf configuration) { this(configuration, EnvironmentNameGenerator.generate()); } /** - * Creates a new {@link DefaultStreamsExecutionEnvironment} instance. + * Creates a new {@link LocalStreamsExecutionEnvironment} instance. * * @param envName the environment name to be used. */ - private DefaultStreamsExecutionEnvironment(final Conf config, - final String envName) { + private LocalStreamsExecutionEnvironment(final Conf config, + final String envName) { Objects.requireNonNull(config, "config cannot be null"); Objects.requireNonNull(envName, "envName cannot be null"); this.configuration = config; @@ -185,6 +193,14 @@ private DefaultStreamsExecutionEnvironment(final Conf config, setState(State.CREATED); } + /** + * {@inheritDoc} + */ + @Override + public String type() { + return TYPE; + } + /** * {@inheritDoc} */ @@ -205,46 +221,92 @@ public State state() { * {@inheritDoc} */ @Override - public StreamsExecutionEnvironment addStateListener(final KafkaStreams.StateListener listener) { + public boolean isDefault() { + return isDefault; + } + + public boolean isDefault(final boolean isDefault) { + return this.isDefault = isDefault; + } + + /** + * {@inheritDoc} + */ + @Override + public StreamsTopologyExecution newTopologyExecution(final StreamsTopologyMeta meta, final Executed executed) { + return new LocalStreamsExecution(meta, executed, context, this); + } + + /** + * Adds a {@link KafkaStreams.StateListener} instance that will set to all {@link KafkaStreams} instance created + * in this {@link StreamsExecutionEnvironment}. + * + * @see KafkaStreams#setStateListener(KafkaStreams.StateListener). + * + * @param listener the {@link KafkaStreams.StateListener} instance. + * + * @throws IllegalStateException if this {@link StreamsExecutionEnvironment} instance is started. + * + * @return this {@link StreamsExecutionEnvironment} instance. + */ + public LocalStreamsExecutionEnvironment addStateListener(final KafkaStreams.StateListener listener) { Objects.requireNonNull(listener, "Cannot add empty listener"); stateListeners.add(listener); return this; } /** - * {@inheritDoc} + * Adds a {@link StateRestoreListener} instance that will set to all {@link KafkaStreams} instance created + * in this {@link StreamsExecutionEnvironment}. + * + * @see KafkaStreams#setGlobalStateRestoreListener(StateRestoreListener) . + * + * @param listener the {@link StateRestoreListener} instance. + * + * @throws IllegalStateException if this {@link StreamsExecutionEnvironment} instance is started. + * + * @return this {@link StreamsExecutionEnvironment} instance. */ - @Override - public StreamsExecutionEnvironment addGlobalStateListener(final StateRestoreListener listener) { + public LocalStreamsExecutionEnvironment addGlobalStateListener(final StateRestoreListener listener) { Objects.requireNonNull(listener, "Cannot add empty listener"); restoreListeners.add(listener); return this; } /** - * {@inheritDoc} + * Adds a streams interceptor that will set to all {@link KafkaStreams} instance created + * in this {@link StreamsExecutionEnvironment}. + * The interceptors will be executed in the order in which they were added. + * + * @param interceptor the {@link {@link StreamsLifecycleInterceptor}}. + * @return this {@link StreamsExecutionEnvironment} instance. */ - @Override - public StreamsExecutionEnvironment addStreamsLifecycleInterceptor( + public LocalStreamsExecutionEnvironment addStreamsLifecycleInterceptor( final Supplier interceptor) { this.interceptors.add(interceptor); return this; } /** - * {@inheritDoc} + * Sets the {@link StreamThreadExceptionHandler} invoked when a StreamThread abruptly terminates + * due to an uncaught exception. + * + * @param handler the {@link StreamThreadExceptionHandler}. + * @return this {@link StreamsExecutionEnvironment} instance. + * + * @see KafkaStreams#setUncaughtExceptionHandler(Thread.UncaughtExceptionHandler) */ - @Override - public StreamsExecutionEnvironment setStreamThreadExceptionHandler( + public LocalStreamsExecutionEnvironment setStreamThreadExceptionHandler( final Supplier handler) { streamThreadExceptionHandler = Objects.requireNonNull(handler, "handle cannot be null"); return this; } /** - * {@inheritDoc} + * Gets the {@link StreamThreadExceptionHandler}. + * + * @return the {@link Supplier}, otherwise {@code null} if no handler is set. */ - @Override public Supplier getStreamThreadExceptionHandler() { return streamThreadExceptionHandler; } @@ -259,9 +321,19 @@ public Collection applications() { /** * {@inheritDoc} + * @return */ @Override - public StreamsExecutionEnvironment setConfiguration(final Conf configuration) { + public Set applicationIds() { + return activeStreams.keySet().stream().map(ApplicationId::toString).collect(Collectors.toSet()); + } + + + /** + * {@inheritDoc} + */ + @Override + public LocalStreamsExecutionEnvironment setConfiguration(final Conf configuration) { this.configuration = configuration; return this; } @@ -278,7 +350,7 @@ public Conf getConfiguration() { * {@inheritDoc} */ @Override - public StreamsExecutionEnvironment setRocksDBConfig(final RocksDBConfig rocksDBConfig) { + public LocalStreamsExecutionEnvironment setRocksDBConfig(final RocksDBConfig rocksDBConfig) { Objects.requireNonNull(rocksDBConfig, "rocksDBConfig cannot be null"); configuration = configuration.withFallback(Conf.of("streams", rocksDBConfig.conf())); return this; @@ -288,7 +360,7 @@ public StreamsExecutionEnvironment setRocksDBConfig(final RocksDBConfig rocksDBC * {@inheritDoc} */ @Override - public StreamsExecutionEnvironment setApplicationIdBuilder(final Supplier supplier) { + public LocalStreamsExecutionEnvironment setApplicationIdBuilder(final Supplier supplier) { Objects.requireNonNull(supplier, "builder cannot be null"); applicationIdBuilderSupplier = supplier; return this; @@ -303,17 +375,26 @@ public Supplier getApplicationIdBuilder() { } /** - * {@inheritDoc} + * Add a new {@link TopologyProvider} instance to this {@link StreamsExecutionEnvironment} to be started. + * + * @param provider the {@link TopologyProvider} supplier. + * + * @return this {@link ApplicationId} instance if the environment is already started, + * otherwise {@code null}. */ - @Override public ApplicationId addTopology(final Supplier provider) { return addTopology(provider, new InternalExecuted()); } /** - * {@inheritDoc} + * Add a new {@link TopologyProvider} instance to this {@link StreamsExecutionEnvironment} to be started. + * + * @param provider the {@link TopologyProvider} supplier. + * @param executed the {@link Executed} instance. + * + * @return this {@link ApplicationId} instance if the environment is already started, + * otherwise {@code null}. */ - @Override public ApplicationId addTopology(final Supplier provider, final Executed executed) { final TopologyDefinitionHolder internalProvider = new TopologyDefinitionHolder(provider, executed); topologies.add(internalProvider); @@ -353,7 +434,7 @@ private ApplicationId start(final TopologyDefinitionHolder topologyHolder) { var threadExceptionHandler = supply(streamThreadExceptionHandler, topologyConfig); - var kafkaStreamsContainer = DefaultKafkaStreamsContainer.newBuilder() + var kafkaStreamsContainer = LocalKafkaStreamsContainer.newBuilder() .withStateListeners(stateListeners) .withRestoreListeners(restoreListeners) .withStreamThreadExceptionHandlers(List.of(threadExceptionHandler)) @@ -457,17 +538,20 @@ private void checkIsStarted() { * {@inheritDoc} */ @Override - public StreamsExecutionEnvironment addFallbackConfiguration(final Conf fallback) { + public LocalStreamsExecutionEnvironment addFallbackConfiguration(final Conf fallback) { configuration = configuration.withFallback(fallback); return this; } /** - * {@inheritDoc} + * Sets the {@link KafkaStreamsFactory} that will be used to provide + * the {@link KafkaStreams} to configure and start. + * + * @param factory the {@link KafkaStreamsFactory} instance. + * @return this {@link StreamsExecutionEnvironment} instance. */ - @Override - public StreamsExecutionEnvironment setKafkaStreamsFactory(final Supplier kafkaStreamsFactory) { - this.kafkaStreamsFactory = kafkaStreamsFactory; + public LocalStreamsExecutionEnvironment setKafkaStreamsFactory(final Supplier factory) { + this.kafkaStreamsFactory = factory; return this; } @@ -559,7 +643,7 @@ KafkaStreamsFactory getKafkaStreamsFactory() { Conf getTopologyConfig() { var ctxConfig = context != null ? context.getConfiguration() : Conf.empty(); - var envConfig = DefaultStreamsExecutionEnvironment.this.getConfiguration(); + var envConfig = LocalStreamsExecutionEnvironment.this.getConfiguration(); // Merged all configurations return Conf.of(executed.config(), envConfig, envConfig, ctxConfig); } @@ -588,7 +672,7 @@ private static class InternalTopologyDefinition implements TopologyDefinition { private final String description; private final TopologyProvider provider; - private Topology topology; + private final Topology topology; InternalTopologyDefinition(final String name, final String description, diff --git a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/env/LocalStreamsExecutionEnvironmentFactory.java b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/env/LocalStreamsExecutionEnvironmentFactory.java new file mode 100644 index 00000000..c230ea99 --- /dev/null +++ b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/env/LocalStreamsExecutionEnvironmentFactory.java @@ -0,0 +1,101 @@ +/* + * Copyright 2019-2020 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamthoughts.azkarra.runtime.env; + +import io.streamthoughts.azkarra.api.AzkarraContext; +import io.streamthoughts.azkarra.api.AzkarraContextAware; +import io.streamthoughts.azkarra.api.StreamsExecutionEnvironment; +import io.streamthoughts.azkarra.api.StreamsExecutionEnvironmentFactory; +import io.streamthoughts.azkarra.api.components.Restriction; +import io.streamthoughts.azkarra.api.config.Conf; +import io.streamthoughts.azkarra.api.streams.ApplicationIdBuilder; +import io.streamthoughts.azkarra.api.streams.errors.StreamThreadExceptionHandler; +import io.streamthoughts.azkarra.runtime.components.RestrictedComponentFactory; +import io.streamthoughts.azkarra.runtime.context.internal.ContextAwareApplicationIdBuilderSupplier; +import io.streamthoughts.azkarra.runtime.context.internal.ContextAwareThreadExceptionHandlerSupplier; + +import java.util.Optional; +import java.util.function.Supplier; + +/** + * The default {@link StreamsExecutionEnvironment} implementation. + */ +public class LocalStreamsExecutionEnvironmentFactory + implements StreamsExecutionEnvironmentFactory, AzkarraContextAware { + + private AzkarraContext context; + + /** + * {@inheritDoc} + */ + @Override + public LocalStreamsExecutionEnvironment create(final String name, final Conf conf) { + return initialize(LocalStreamsExecutionEnvironment.create(conf, name)); + } + + /** + * {@inheritDoc} + */ + @Override + public String type() { + return LocalStreamsExecutionEnvironment.TYPE; + } + + /** + * {@inheritDoc} + */ + @Override + public void setAzkarraContext(final AzkarraContext context) { + this.context = context; + } + + private LocalStreamsExecutionEnvironment initialize(final LocalStreamsExecutionEnvironment env) { + + final Conf componentResolutionConfig = env.getConfiguration().withFallback(context.getConfiguration()); + + // Inject environment or application scoped ApplicationIdBuilder + Optional.ofNullable(env.getApplicationIdBuilder()) + .or(() -> findApplicationIdBuilder(componentResolutionConfig, Restriction.env(env.name()))) + .or(() -> findApplicationIdBuilder(componentResolutionConfig, Restriction.application())) + .ifPresent(env::setApplicationIdBuilder); + + // Inject environment, global or default StreamsThreadExceptionHandler + Optional.ofNullable(env.getStreamThreadExceptionHandler()) + .or(() -> findStreamThreadExceptionHandler(componentResolutionConfig, Restriction.env(env.name()))) + .or(() -> findStreamThreadExceptionHandler(componentResolutionConfig, Restriction.application())) + .ifPresent(env::setStreamThreadExceptionHandler); + + return env; + } + + public Optional> findApplicationIdBuilder(final Conf componentConfig, + final Restriction restriction) { + return new RestrictedComponentFactory(context.getComponentFactory()) + .findComponentByRestriction(ApplicationIdBuilder.class, componentConfig, restriction) + .map(gettable -> new ContextAwareApplicationIdBuilderSupplier(context, gettable)); + } + + public Optional> findStreamThreadExceptionHandler( + final Conf conf, + final Restriction restriction) { + return new RestrictedComponentFactory(context.getComponentFactory()) + .findComponentByRestriction(StreamThreadExceptionHandler.class, conf, restriction) + .map(gettable -> new ContextAwareThreadExceptionHandlerSupplier(context, gettable)); + } +} \ No newline at end of file diff --git a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/env/internal/EnvironmentAwareComponentSupplier.java b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/env/internal/EnvironmentAwareComponentSupplier.java index c310b9dd..ccc95de2 100644 --- a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/env/internal/EnvironmentAwareComponentSupplier.java +++ b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/env/internal/EnvironmentAwareComponentSupplier.java @@ -41,7 +41,7 @@ public EnvironmentAwareComponentSupplier(final Supplier componentSupplier) { this.componentSupplier = componentSupplier; } - public T get(final StreamsExecutionEnvironment environment, final Conf componentConf) { + public T get(final StreamsExecutionEnvironment environment, final Conf componentConf) { mayConfigure(componentSupplier, componentConf); T component = componentSupplier.get(); mayConfigure(component, componentConf); @@ -50,7 +50,7 @@ public T get(final StreamsExecutionEnvironment environment, final Conf component } private void maySetStreamsExecutionEnvironmentAware(final Object component, - final StreamsExecutionEnvironment environment) { + final StreamsExecutionEnvironment environment) { if (StreamsExecutionEnvironmentAware.class.isAssignableFrom(component.getClass())) { ((StreamsExecutionEnvironmentAware)component).setExecutionEnvironment(environment); } diff --git a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/interceptors/AutoCreateTopicsInterceptor.java b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/interceptors/AutoCreateTopicsInterceptor.java index 6ab28f70..2e87ddd9 100644 --- a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/interceptors/AutoCreateTopicsInterceptor.java +++ b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/interceptors/AutoCreateTopicsInterceptor.java @@ -96,9 +96,9 @@ public class AutoCreateTopicsInterceptor /** * The list of topics actually created for the topology. */ - private Set createdTopics = new HashSet<>(); + private final Set createdTopics = new HashSet<>(); - private AtomicBoolean topicListed = new AtomicBoolean(false); + private final AtomicBoolean topicListed = new AtomicBoolean(false); private final AdminClient adminClient; diff --git a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/service/AbstractAzkarraStreamsService.java b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/service/AbstractAzkarraStreamsService.java index a710d72d..f6e5a2cd 100644 --- a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/service/AbstractAzkarraStreamsService.java +++ b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/service/AbstractAzkarraStreamsService.java @@ -31,6 +31,7 @@ import io.streamthoughts.azkarra.api.providers.TopologyDescriptor; import io.streamthoughts.azkarra.api.streams.TopologyProvider; import io.streamthoughts.azkarra.api.util.Version; +import io.streamthoughts.azkarra.api.StreamsExecutionEnvironmentFactory; import java.util.Collection; import java.util.HashMap; @@ -45,17 +46,19 @@ public abstract class AbstractAzkarraStreamsService implements AzkarraStreamsSer protected AzkarraContext context; /** - * {@inheritDoc} + * Creates a new {@link AbstractAzkarraStreamsService} instance. + * */ - @Override - public Set getTopologyProviders() { - return context.topologyProviders(); - } + protected AbstractAzkarraStreamsService() { } /** * {@inheritDoc} */ @Override + public Set getTopologyProviders() { + return context.getTopologyDescriptors(); + } + public List getAllTopologies() { Map> topologies = new HashMap<>(); var factory = context.getComponentFactory(); @@ -113,6 +116,7 @@ public TopologyDescriptor getTopologyByAliasAndQualifiers(final String alias, return new TopologyDescriptor(descriptor); } + /** * {@inheritDoc} */ @@ -132,6 +136,17 @@ public List getTopologyVersionsByAlias(final String alias) { .collect(Collectors.toList()); } + /** + * {@inheritDoc} + */ + @Override + public Set getSupportedEnvironmentTypes() { + return context.getAllComponents(StreamsExecutionEnvironmentFactory.class) + .stream() + .map(StreamsExecutionEnvironmentFactory::type) + .collect(Collectors.toSet()); + } + /** * {@inheritDoc} */ diff --git a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/service/LocalAzkarraStreamsService.java b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/service/LocalAzkarraStreamsService.java index 4cddb0ca..53e86cf5 100644 --- a/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/service/LocalAzkarraStreamsService.java +++ b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/service/LocalAzkarraStreamsService.java @@ -21,7 +21,9 @@ import io.streamthoughts.azkarra.api.AzkarraStreamsService; import io.streamthoughts.azkarra.api.Executed; import io.streamthoughts.azkarra.api.StreamsExecutionEnvironment; +import io.streamthoughts.azkarra.api.StreamsExecutionEnvironmentFactory; import io.streamthoughts.azkarra.api.config.Conf; +import io.streamthoughts.azkarra.api.errors.InvalidStreamsEnvironmentException; import io.streamthoughts.azkarra.api.errors.NotFoundException; import io.streamthoughts.azkarra.api.model.Environment; import io.streamthoughts.azkarra.api.model.Metric; @@ -33,16 +35,10 @@ import io.streamthoughts.azkarra.api.streams.KafkaStreamsContainer; import io.streamthoughts.azkarra.api.streams.ServerMetadata; import io.streamthoughts.azkarra.api.streams.consumer.ConsumerGroupOffsets; -import io.streamthoughts.azkarra.runtime.env.DefaultStreamsExecutionEnvironment; -import org.apache.kafka.common.MetricName; import java.time.Duration; import java.util.Collection; -import java.util.HashMap; import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -59,9 +55,9 @@ public class LocalAzkarraStreamsService extends AbstractAzkarraStreamsService { */ @Override public Collection getAllStreams() { - return containers() + return context.getAllEnvironments() .stream() - .map(KafkaStreamsContainer::applicationId) + .flatMap(environment -> environment.applicationIds().stream()) .collect(Collectors.toList()); } @@ -78,8 +74,10 @@ public Conf getStreamsConfigById(final String applicationId) { */ @Override public KafkaStreamsContainer getStreamsById(final String applicationId) { - final Optional container = containers() + final Optional container = context + .getAllEnvironments() .stream() + .flatMap(environment -> environment.applications().stream()) .filter(o -> o.applicationId().equals(applicationId)) .findFirst(); @@ -106,8 +104,7 @@ public StreamsStatus getStreamsStatusById(final String applicationId) { */ @Override public StreamsTopologyGraph getStreamsTopologyById(final String applicationId) { - KafkaStreamsContainer streams = getStreamsById(applicationId); - return StreamsTopologyGraph.build(streams.topologyDescription()); + return getStreamsById(applicationId).topologyGraph(); } /** @@ -126,7 +123,7 @@ public ApplicationId startStreamsTopology(final String topologyType, */ @Override public Set getStreamsMetricsById(final String applicationId) { - return getStreamsMetricsById(applicationId, m -> true); + return getStreamsById(applicationId).metrics(); } /** @@ -135,32 +132,7 @@ public Set getStreamsMetricsById(final String applicationId) { @Override public Set getStreamsMetricsById(final String applicationId, final Predicate> filter) { - KafkaStreamsContainer container = getStreamsById(applicationId); - - Map metrics = container.metrics(); - - Map> m = new HashMap<>(metrics.size()); - for (Map.Entry elem : metrics.entrySet()) { - final MetricName metricName = elem.getKey(); - final org.apache.kafka.common.Metric metricValue = elem.getValue(); - - final Metric metric = new Metric( - metricName.name(), - metricName.group(), - metricName.description(), - metricName.tags(), - metricValue.metricValue() - ); - final boolean filtered = filter.test(Tuple.of(metricName.group(), metric)); - if (filtered) { - m.computeIfAbsent(metricName.group(), k -> new LinkedList<>()).add(metric); - } - } - - return m.entrySet() - .stream() - .map(e -> new MetricGroup(e.getKey(), e.getValue())) - .collect(Collectors.toSet()); + return getStreamsById(applicationId).metrics(KafkaStreamsContainer.KafkaMetricFilter.of(filter)); } /** @@ -185,25 +157,33 @@ public Conf getContextConfig() { */ @Override public Set getAllEnvironments() { - StreamsExecutionEnvironment defaultEnv = context.defaultExecutionEnvironment(); - return context.environments() - .stream() - .map( env -> new Environment( - env.name(), - env.state(), - env.getConfiguration().getConfAsMap(), - env.applications().stream().map(KafkaStreamsContainer::applicationId).collect(Collectors.toSet()), - env.name().equals(defaultEnv.name()) - ) - ).collect(Collectors.toSet()); + return context.getAllEnvironments() + .stream() + .map( env -> new Environment( + env.name(), + env.type(), + env.state(), + env.getConfiguration().getConfAsMap(), + env.applicationIds(), + env.isDefault() + ) + ).collect(Collectors.toSet()); } /** * {@inheritDoc} */ @Override - public void addNewEnvironment(final String name, final Conf conf) { - this.context.addExecutionEnvironment(DefaultStreamsExecutionEnvironment.create(conf, name)); + public void addNewEnvironment(final String name, final String type, final Conf conf) { + var factories = context.getAllComponents(StreamsExecutionEnvironmentFactory.class); + var opt = factories + .stream() + .filter(factory -> factory.type().equals(type)) + .findAny(); + if (opt.isEmpty()) { + throw new InvalidStreamsEnvironmentException("Cannot find factory for environment type " + type); + } + context.addExecutionEnvironment(opt.get().create(name, conf)); } /** @@ -241,15 +221,15 @@ public void restartStreams(final String applicationId) { @Override public void deleteStreams(final String applicationId) { - StreamsExecutionEnvironment env = null; - Iterator it = context.environments().iterator(); + StreamsExecutionEnvironment env = null; + Iterator> it = context.getAllEnvironments().iterator(); while (it.hasNext() && env == null) { - StreamsExecutionEnvironment e = it.next(); + StreamsExecutionEnvironment e = it.next(); boolean exists = e.applications() - .stream() - .map(KafkaStreamsContainer::applicationId) - .collect(Collectors.toList()) - .contains(applicationId); + .stream() + .map(KafkaStreamsContainer::applicationId) + .collect(Collectors.toList()) + .contains(applicationId); if (exists) { env = e; } @@ -261,10 +241,4 @@ public void deleteStreams(final String applicationId) { "Can't find streams environment running an application with id'" + applicationId+ "'."); } } - - private Collection containers() { - return context.environments().stream() - .flatMap(environment -> environment.applications().stream()) - .collect(Collectors.toList()); - } } diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/streams/DefaultKafkaStreamsContainer.java b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/streams/LocalKafkaStreamsContainer.java similarity index 90% rename from azkarra-api/src/main/java/io/streamthoughts/azkarra/api/streams/DefaultKafkaStreamsContainer.java rename to azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/streams/LocalKafkaStreamsContainer.java index 6283bc44..4007504f 100644 --- a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/streams/DefaultKafkaStreamsContainer.java +++ b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/streams/LocalKafkaStreamsContainer.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.streamthoughts.azkarra.api.streams; +package io.streamthoughts.azkarra.runtime.streams; import io.streamthoughts.azkarra.api.StreamsLifecycleChain; import io.streamthoughts.azkarra.api.StreamsLifecycleContext; @@ -26,9 +26,19 @@ import io.streamthoughts.azkarra.api.events.EventStream; import io.streamthoughts.azkarra.api.events.reactive.AsyncMulticastEventStreamPublisher; import io.streamthoughts.azkarra.api.events.reactive.EventStreamPublisher; +import io.streamthoughts.azkarra.api.model.MetricGroup; +import io.streamthoughts.azkarra.api.model.StreamsTopologyGraph; import io.streamthoughts.azkarra.api.model.TimestampedValue; import io.streamthoughts.azkarra.api.monad.Try; +import io.streamthoughts.azkarra.api.monad.Tuple; import io.streamthoughts.azkarra.api.query.LocalStoreAccessor; +import io.streamthoughts.azkarra.api.streams.KafkaStreamsContainer; +import io.streamthoughts.azkarra.api.streams.KafkaStreamsFactory; +import io.streamthoughts.azkarra.api.streams.ServerHostInfo; +import io.streamthoughts.azkarra.api.streams.ServerMetadata; +import io.streamthoughts.azkarra.api.streams.State; +import io.streamthoughts.azkarra.api.streams.StateChangeEvent; +import io.streamthoughts.azkarra.api.streams.TopicPartitions; import io.streamthoughts.azkarra.api.streams.consumer.ConsumerClientOffsets; import io.streamthoughts.azkarra.api.streams.consumer.ConsumerGroupOffsets; import io.streamthoughts.azkarra.api.streams.consumer.ConsumerLogOffsets; @@ -58,7 +68,7 @@ import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.KeyQueryMetadata; import org.apache.kafka.streams.StreamsConfig; -import org.apache.kafka.streams.TopologyDescription; +import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.errors.StreamsException; import org.apache.kafka.streams.processor.ThreadMetadata; import org.apache.kafka.streams.state.HostInfo; @@ -98,9 +108,9 @@ import static org.apache.kafka.clients.consumer.ConsumerConfig.CLIENT_ID_CONFIG; import static org.apache.kafka.streams.StoreQueryParameters.fromNameAndType; -public class DefaultKafkaStreamsContainer implements KafkaStreamsContainer { +public class LocalKafkaStreamsContainer implements KafkaStreamsContainer { - private static final Logger LOG = LoggerFactory.getLogger(DefaultKafkaStreamsContainer.class); + private static final Logger LOG = LoggerFactory.getLogger(LocalKafkaStreamsContainer.class); private final KafkaStreamsFactory streamsFactory; @@ -160,22 +170,22 @@ public boolean isValidTransition(final ContainerState newState) { } /** - * @return a new {@link KafkaStreamsContainerBuilder} instance. + * @return a new {@link LocalKafkaStreamsContainerBuilder} instance. */ - public static KafkaStreamsContainerBuilder newBuilder() { - return new KafkaStreamsContainerBuilder(); + public static LocalKafkaStreamsContainerBuilder newBuilder() { + return new LocalKafkaStreamsContainerBuilder(); } /** - * Creates a new {@link DefaultKafkaStreamsContainer} instance. + * Creates a new {@link LocalKafkaStreamsContainer} instance. * * @param topologyDefinition the {@link TopologyDefinition} instance. * @param streamsFactory the {@link KafkaStreamsFactory} instance. */ - DefaultKafkaStreamsContainer(final Conf streamsConfig, - final TopologyDefinition topologyDefinition, - final KafkaStreamsFactory streamsFactory, - final List interceptors) { + LocalKafkaStreamsContainer(final Conf streamsConfig, + final TopologyDefinition topologyDefinition, + final KafkaStreamsFactory streamsFactory, + final List interceptors) { Objects.requireNonNull(topologyDefinition, "topologyDefinition cannot be null"); Objects.requireNonNull(streamsFactory, "streamsFactory cannot be null"); this.streamsConfig = Objects.requireNonNull(streamsConfig, "streamConfigs cannot be null"); @@ -192,10 +202,11 @@ public static KafkaStreamsContainerBuilder newBuilder() { } /** - * {@inheritDoc} + *Asynchronously start the underlying {@link KafkaStreams} instance. + * + * @param executor the {@link Executor} instance to be used for starting the streams. */ - @Override - public synchronized Future start(final Executor executor) { + public Future start(final Executor executor) { LOG.info("Starting KafkaStreams container for name='{}', version='{}', id='{}'.", topologyDefinition.getName(), topologyDefinition.getVersion(), @@ -374,17 +385,41 @@ public TopologyMetadata topologyMetadata() { /** * {@inheritDoc} */ - @Override - public TopologyDescription topologyDescription() { - return topologyDefinition.getTopology().describe(); + public StreamsTopologyGraph topologyGraph() { + return StreamsTopologyGraph.build(topologyDefinition.getTopology().describe()); } /** * {@inheritDoc} */ @Override - public Map metrics() { - return initialized() ? kafkaStreams.metrics() : Collections.emptyMap(); + public Set metrics(final KafkaMetricFilter filter) { + final Map kafkaMetrics = initialized() + ? kafkaStreams.metrics() : + Collections.emptyMap(); + + Map> m = new HashMap<>(kafkaMetrics.size()); + for (Map.Entry elem : kafkaMetrics.entrySet()) { + final MetricName metricName = elem.getKey(); + final org.apache.kafka.common.Metric metricValue = elem.getValue(); + + final io.streamthoughts.azkarra.api.model.Metric metric = new io.streamthoughts.azkarra.api.model.Metric( + metricName.name(), + metricName.group(), + metricName.description(), + metricName.tags(), + metricValue.metricValue() + ); + final boolean filtered = filter.test(Tuple.of(metricName.group(), metric)); + if (filtered) { + m.computeIfAbsent(metricName.group(), k -> new LinkedList<>()).add(metric); + } + } + + return m.entrySet() + .stream() + .map(e -> new MetricGroup(e.getKey(), e.getValue())) + .collect(Collectors.toSet()); } /** @@ -587,12 +622,12 @@ private StreamsLifecycleContext newStreamsLifecycleContext() { return new StreamsLifecycleContext() { @Override public void setState(final State state) { - DefaultKafkaStreamsContainer.this.setState(state); + LocalKafkaStreamsContainer.this.setState(state); } @Override public KafkaStreamsContainer container() { - return DefaultKafkaStreamsContainer.this; + return LocalKafkaStreamsContainer.this; } }; } @@ -889,15 +924,22 @@ private static Map getConfigsForKeys(final Map c } /** - * Returns the wrapper {@link KafkaStreams} instance. - * - * @return the {@link KafkaStreams}. + * {@inheritDoc} */ - public KafkaStreams kafkaStreams() { + @Override + public KafkaStreams getKafkaStreams() { validateInitialized(); return kafkaStreams; } + /** + * {@inheritDoc} + */ + @Override + public Topology getTopology() { + return topologyDefinition.getTopology(); + } + /** * {@inheritDoc} */ diff --git a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/streams/KafkaStreamsContainerBuilder.java b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/streams/LocalKafkaStreamsContainerBuilder.java similarity index 86% rename from azkarra-api/src/main/java/io/streamthoughts/azkarra/api/streams/KafkaStreamsContainerBuilder.java rename to azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/streams/LocalKafkaStreamsContainerBuilder.java index f803dea3..2f25f952 100644 --- a/azkarra-api/src/main/java/io/streamthoughts/azkarra/api/streams/KafkaStreamsContainerBuilder.java +++ b/azkarra-runtime/src/main/java/io/streamthoughts/azkarra/runtime/streams/LocalKafkaStreamsContainerBuilder.java @@ -16,10 +16,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.streamthoughts.azkarra.api.streams; +package io.streamthoughts.azkarra.runtime.streams; import io.streamthoughts.azkarra.api.StreamsLifecycleInterceptor; import io.streamthoughts.azkarra.api.config.Conf; +import io.streamthoughts.azkarra.api.streams.KafkaStreamsContainer; +import io.streamthoughts.azkarra.api.streams.KafkaStreamsFactory; +import io.streamthoughts.azkarra.api.streams.State; +import io.streamthoughts.azkarra.api.streams.StateChangeEvent; import io.streamthoughts.azkarra.api.streams.consumer.MonitorOffsetsConsumerInterceptor; import io.streamthoughts.azkarra.api.streams.errors.DelegatingUncaughtExceptionHandler; import io.streamthoughts.azkarra.api.streams.errors.StreamThreadExceptionHandler; @@ -44,7 +48,7 @@ /** * Default builder class for creating and configuring a new wrapped {@link KafkaStreams} instance. */ -public class KafkaStreamsContainerBuilder { +public class LocalKafkaStreamsContainerBuilder { private final static List CONFIG_DECORATORS = List.of( new RocksDBConfigDecorator(), @@ -59,38 +63,38 @@ public class KafkaStreamsContainerBuilder { private List exceptionHandlers = Collections.emptyList(); private List interceptors = Collections.emptyList(); - public KafkaStreamsContainerBuilder withStreamsConfig(final Conf streamsConfig) { + public LocalKafkaStreamsContainerBuilder withStreamsConfig(final Conf streamsConfig) { this.streamsConfig = streamsConfig; return this; } - public KafkaStreamsContainerBuilder withInterceptors(final List interceptors) { + public LocalKafkaStreamsContainerBuilder withInterceptors(final List interceptors) { this.interceptors = interceptors; return this; } - public KafkaStreamsContainerBuilder withKafkaStreamsFactory(final KafkaStreamsFactory kafkaStreamsFactory) { + public LocalKafkaStreamsContainerBuilder withKafkaStreamsFactory(final KafkaStreamsFactory kafkaStreamsFactory) { this.kafkaStreamsFactory = kafkaStreamsFactory; return this; } - public KafkaStreamsContainerBuilder withTopologyDefinition(final TopologyDefinition topologyDefinition) { + public LocalKafkaStreamsContainerBuilder withTopologyDefinition(final TopologyDefinition topologyDefinition) { this.topologyDefinition = topologyDefinition; return this; } - public KafkaStreamsContainerBuilder withRestoreListeners(final List listeners) { + public LocalKafkaStreamsContainerBuilder withRestoreListeners(final List listeners) { this.restoreListeners = listeners; return this; } - public KafkaStreamsContainerBuilder withStreamThreadExceptionHandlers( + public LocalKafkaStreamsContainerBuilder withStreamThreadExceptionHandlers( final List handlers) { this.exceptionHandlers = handlers; return this; } - public KafkaStreamsContainerBuilder withStateListeners(final List listeners) { + public LocalKafkaStreamsContainerBuilder withStateListeners(final List listeners) { this.stateListeners = listeners; return this; } @@ -100,14 +104,14 @@ public KafkaStreamsContainerBuilder withStateListeners(final List executedArgumentCaptor = ArgumentCaptor.forClass(Executed.class); + private final ArgumentCaptor executedArgumentCaptor = ArgumentCaptor.forClass(Executed.class); @BeforeEach public void setUp() { // Create default context with empty configuration. - context = (DefaultAzkarraContext) DefaultAzkarraContext.create(); - } - - @Test - public void shouldRegisteredDefaultEnvironment() { - List environments = context.environments(); - assertEquals(1, environments.size()); - assertEquals(DefaultAzkarraContext.DEFAULT_ENV_NAME, environments.get(0).name()); + context = DefaultAzkarraContext.create(); } @Test public void shouldRegisterTopologyToGivenEnvironmentWhenStart() { - StreamsExecutionEnvironment env = spy(DefaultStreamsExecutionEnvironment.create("env")); + LocalStreamsExecutionEnvironment env = spy(LocalStreamsExecutionEnvironment.create("env")); context.addExecutionEnvironment(env); context.addTopology(TestTopologyProvider.class, "env", Executed.as("test")); context.preStart(); @@ -152,67 +139,6 @@ public void shouldAutomaticallyRegisterConditionalWaitForSourceTopicsInterceptor ); } - @Test - public void shouldProperlyMergedAllConfigsWhenAddingTopology() { - //Setup - var mkEnv = mock(StreamsExecutionEnvironment.class); - when(mkEnv.name()).thenReturn("test"); - when(mkEnv.getConfiguration()).thenReturn(Conf.of("prop.env", "env.value")); - - var mkDescriptor = mock(TopologyDescriptor.class); - when(mkDescriptor.name()).thenReturn("test"); - when(mkDescriptor.configuration()).thenReturn(Conf.of("prop.descriptor", "desc.value")); - Executed executed = Executed.as("test-app").withConfig(Conf.of("prop.executed", "exec.value")); - context.setConfiguration(Conf.of("prop.context", "value")); - - //Execute - context.addTopologyToEnvironment(mkDescriptor, mkEnv, new InternalExecuted(executed)); - - // Assert - verify(mkEnv, times(1)).addTopology( - any(ContextAwareTopologySupplier.class), - executedArgumentCaptor.capture() - ); - - var captured = new InternalExecuted(executedArgumentCaptor.getValue()); - assertEquals("test-app", captured.name()); - assertTrue(captured.config().hasPath("prop.env"), "Missing prop.env"); - assertTrue(captured.config().hasPath("prop.descriptor"), "Missing prop.descriptor"); - assertTrue(captured.config().hasPath("prop.executed"), "Missing prop.executed"); - assertTrue(captured.config().hasPath("prop.context"), "Missing prop.context"); - } - - @Test - public void shouldProperlyConfigureInterceptorsWhenAddingTopology() { - var mkEnv = mock(StreamsExecutionEnvironment.class); - when(mkEnv.name()).thenReturn("test"); - - var mkDescriptor = mock(TopologyDescriptor.class); - when(mkDescriptor.name()).thenReturn("test"); - when(mkDescriptor.configuration()).thenReturn(Conf.empty()); - when(mkDescriptor.classLoader()).thenReturn(this.getClass().getClassLoader()); - - context.setConfiguration(Conf.of( - AUTO_CREATE_TOPICS_ENABLE_CONFIG, true, - MONITORING_STREAMS_INTERCEPTOR_ENABLE_CONFIG, true, - WAIT_FOR_TOPICS_ENABLE_CONFIG, true - )); - - //Execute - context.addTopologyToEnvironment(mkDescriptor, mkEnv, new InternalExecuted(Executed.as("test-app"))); - // Assert - verify(mkEnv, times(1)).addTopology( - any(ContextAwareTopologySupplier.class), - executedArgumentCaptor.capture() - ); - - var captured = new InternalExecuted(executedArgumentCaptor.getValue()); - captured.interceptors().forEach(it -> Configurable.mayConfigure(it, Conf.empty())); - assertEquals(4, captured.interceptors().size()); - assertEquals("ClassloadingIsolationInterceptor", captured.interceptors().get(0).get().name()); // assert first - assertEquals("WaitForSourceTopicsInterceptor", captured.interceptors().get(3).get().name()); // assert last - } - public static class TestTopologyProvider implements TopologyProvider { @Override diff --git a/azkarra-runtime/src/test/java/io/streamthoughts/azkarra/runtime/env/DefaultStreamsExecutionEnvironmentTest.java b/azkarra-runtime/src/test/java/io/streamthoughts/azkarra/runtime/env/LocalStreamsExecutionEnvironmentTest.java similarity index 90% rename from azkarra-runtime/src/test/java/io/streamthoughts/azkarra/runtime/env/DefaultStreamsExecutionEnvironmentTest.java rename to azkarra-runtime/src/test/java/io/streamthoughts/azkarra/runtime/env/LocalStreamsExecutionEnvironmentTest.java index a14048d2..2915d82f 100644 --- a/azkarra-runtime/src/test/java/io/streamthoughts/azkarra/runtime/env/DefaultStreamsExecutionEnvironmentTest.java +++ b/azkarra-runtime/src/test/java/io/streamthoughts/azkarra/runtime/env/LocalStreamsExecutionEnvironmentTest.java @@ -22,7 +22,6 @@ import io.streamthoughts.azkarra.api.config.Conf; import io.streamthoughts.azkarra.api.config.Configurable; import io.streamthoughts.azkarra.api.streams.TopologyProvider; -import io.streamthoughts.azkarra.api.streams.topology.TopologyDefinition; import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.TopologyDescription; import org.junit.jupiter.api.Test; @@ -30,9 +29,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -public class DefaultStreamsExecutionEnvironmentTest { +public class LocalStreamsExecutionEnvironmentTest { - private static Topology MOCK = Mockito.mock(Topology.class); + private static final Topology MOCK = Mockito.mock(Topology.class); static { Mockito.when(MOCK.describe()).thenReturn(Mockito.mock(TopologyDescription.class)); @@ -41,8 +40,7 @@ public class DefaultStreamsExecutionEnvironmentTest { @Test public void shouldFallbackToEnvCongWhenInitializingTopologies() { - DefaultStreamsExecutionEnvironment environment = (DefaultStreamsExecutionEnvironment) - DefaultStreamsExecutionEnvironment.create(); + LocalStreamsExecutionEnvironment environment = LocalStreamsExecutionEnvironment.create(); var envConf = Conf.of( "streams.prop", "value", diff --git a/azkarra-runtime/src/test/java/io/streamthoughts/azkarra/runtime/env/LocalStreamsExecutionTest.java b/azkarra-runtime/src/test/java/io/streamthoughts/azkarra/runtime/env/LocalStreamsExecutionTest.java new file mode 100644 index 00000000..5d655edb --- /dev/null +++ b/azkarra-runtime/src/test/java/io/streamthoughts/azkarra/runtime/env/LocalStreamsExecutionTest.java @@ -0,0 +1,140 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamthoughts.azkarra.runtime.env; + +import io.streamthoughts.azkarra.api.Executed; +import io.streamthoughts.azkarra.api.StreamsTopologyMeta; +import io.streamthoughts.azkarra.api.config.Conf; +import io.streamthoughts.azkarra.api.config.Configurable; +import io.streamthoughts.azkarra.api.providers.TopologyDescriptor; +import io.streamthoughts.azkarra.runtime.context.DefaultAzkarraContext; +import io.streamthoughts.azkarra.runtime.context.internal.ContextAwareTopologySupplier; +import io.streamthoughts.azkarra.runtime.streams.topology.InternalExecuted; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import static io.streamthoughts.azkarra.runtime.interceptors.AutoCreateTopicsInterceptorConfig.AUTO_CREATE_TOPICS_ENABLE_CONFIG; +import static io.streamthoughts.azkarra.runtime.interceptors.MonitoringStreamsInterceptorConfig.MONITORING_STREAMS_INTERCEPTOR_ENABLE_CONFIG; +import static io.streamthoughts.azkarra.runtime.interceptors.WaitForSourceTopicsInterceptorConfig.WAIT_FOR_TOPICS_ENABLE_CONFIG; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class LocalStreamsExecutionTest { + + private final DefaultAzkarraContext context = DefaultAzkarraContext.create(Conf.empty()); + + private final ArgumentCaptor executedArgumentCaptor = ArgumentCaptor.forClass(Executed.class); + + @Test + public void should_properly_merged_all_configs_when_adding_topology() { + //Setup + var mkEnv = mock(LocalStreamsExecutionEnvironment.class); + when(mkEnv.name()).thenReturn("test"); + when(mkEnv.getConfiguration()).thenReturn(Conf.of("prop.env", "env.value")); + + var mkDescriptor = mock(TopologyDescriptor.class); + when(mkDescriptor.name()).thenReturn("test"); + when(mkDescriptor.configuration()).thenReturn(Conf.of("prop.descriptor", "desc.value")); + when(mkDescriptor.type()).thenReturn(TopologyDescriptor.class); + Executed executed = Executed.as("test-app").withConfig(Conf.of("prop.executed", "exec.value")); + context.setConfiguration(Conf.of("prop.context", "value")); + + final LocalStreamsExecution execution = new LocalStreamsExecution( + new StreamsTopologyMeta( + mkDescriptor.name(), + mkDescriptor.version(), + mkDescriptor.description(), + mkDescriptor.type(), + mkDescriptor.classLoader(), + mkDescriptor.configuration() + ), + executed, + context, + mkEnv + ); + + execution.start(); + + // Assert + verify(mkEnv, times(1)).addTopology( + any(ContextAwareTopologySupplier.class), + executedArgumentCaptor.capture() + ); + + var captured = new InternalExecuted(executedArgumentCaptor.getValue()); + assertEquals("test-app", captured.name()); + assertTrue(captured.config().hasPath("prop.env"), "Missing prop.env"); + assertTrue(captured.config().hasPath("prop.descriptor"), "Missing prop.descriptor"); + assertTrue(captured.config().hasPath("prop.executed"), "Missing prop.executed"); + assertTrue(captured.config().hasPath("prop.context"), "Missing prop.context"); + } + + @Test + public void should_properly_configure_interceptors_when_adding_topology() { + var mkEnv = mock(LocalStreamsExecutionEnvironment.class); + when(mkEnv.name()).thenReturn("test"); + + var mkDescriptor = mock(TopologyDescriptor.class); + when(mkDescriptor.name()).thenReturn("test"); + when(mkDescriptor.configuration()).thenReturn(Conf.empty()); + when(mkDescriptor.type()).thenReturn(TopologyDescriptor.class); + when(mkDescriptor.classLoader()).thenReturn(this.getClass().getClassLoader()); + + context.setConfiguration(Conf.of( + AUTO_CREATE_TOPICS_ENABLE_CONFIG, true, + MONITORING_STREAMS_INTERCEPTOR_ENABLE_CONFIG, true, + WAIT_FOR_TOPICS_ENABLE_CONFIG, true + )); + + //Execute + final LocalStreamsExecution execution = new LocalStreamsExecution( + new StreamsTopologyMeta( + mkDescriptor.name(), + mkDescriptor.version(), + mkDescriptor.description(), + mkDescriptor.type(), + mkDescriptor.classLoader(), + mkDescriptor.configuration() + ), + Executed.as("test"), + context, + mkEnv + ); + execution.start(); + + // Assert + verify(mkEnv, times(1)).addTopology( + any(ContextAwareTopologySupplier.class), + executedArgumentCaptor.capture() + ); + + var captured = new InternalExecuted(executedArgumentCaptor.getValue()); + captured.interceptors().forEach(it -> Configurable.mayConfigure(it, Conf.empty())); + assertEquals(4, captured.interceptors().size()); + assertEquals("ClassloadingIsolationInterceptor", captured.interceptors().get(0).get().name()); // assert first + assertEquals("WaitForSourceTopicsInterceptor", captured.interceptors().get(3).get().name()); // assert last + } + +} \ No newline at end of file diff --git a/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/ServerConfig.java b/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/ServerConfig.java index cf2fdcc9..115ce330 100644 --- a/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/ServerConfig.java +++ b/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/ServerConfig.java @@ -21,6 +21,8 @@ import io.streamthoughts.azkarra.api.config.Conf; import io.streamthoughts.azkarra.http.security.SecurityConfig; +import java.util.Map; + /** * The server configuration. */ @@ -32,15 +34,21 @@ public class ServerConfig extends SecurityConfig { public static final String HTTP_SERVER_LISTENER_CONFIG = "listener"; private static final String HTTP_SERVER_LISTENER_DEFAULT = "localhost"; - public static final String HTTP_SERVER_UI_ENABLE_CONFIG = "enable.ui"; + public static final String HTTP_SERVER_WEB_UI_ENABLE_CONFIG = "webui.enable"; public static final String HTTP_SERVER_REST_EXTENSIONS_ENABLE = "rest.extensions.enable"; - - public static ServerConfigBuilder newBuilder() { return new ServerConfigBuilder(); } + public static ServerConfigBuilder newBuilder(final Conf configs) { + return new ServerConfigBuilder(configs); + } + + public static ServerConfigBuilder newBuilder(final Map configs) { + return new ServerConfigBuilder(configs); + } + public static ServerConfig of(final Conf conf) { if (conf instanceof ServerConfig) return (ServerConfig)conf; @@ -66,7 +74,7 @@ public String getListener() { public boolean isUIEnable() { // by default Web UI should always be enable. - return getOptionalBoolean(HTTP_SERVER_UI_ENABLE_CONFIG).orElse(true); + return getOptionalBoolean(HTTP_SERVER_WEB_UI_ENABLE_CONFIG).orElse(true); } public boolean isRestExtensionEnable() { diff --git a/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/ServerConfigBuilder.java b/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/ServerConfigBuilder.java index da78c7a9..7eca559f 100644 --- a/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/ServerConfigBuilder.java +++ b/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/ServerConfigBuilder.java @@ -38,12 +38,25 @@ public class ServerConfigBuilder { private final Map configs; - /** * Creates a new {@link ServerConfigBuilder}. */ ServerConfigBuilder() { - configs = new HashMap<>(); + this(new HashMap<>()); + } + + /** + * Creates a new {@link ServerConfigBuilder}. + */ + ServerConfigBuilder(final Conf configs) { + this(configs.getConfAsMap()); + } + + /** + * Creates a new {@link ServerConfigBuilder}. + */ + ServerConfigBuilder(final Map configs) { + this.configs = configs; } /** diff --git a/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/handler/EnvironmentPostHandler.java b/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/handler/EnvironmentPostHandler.java index b71ab32e..3f64b7cf 100644 --- a/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/handler/EnvironmentPostHandler.java +++ b/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/handler/EnvironmentPostHandler.java @@ -30,7 +30,6 @@ import static io.streamthoughts.azkarra.api.util.Utils.isNullOrEmpty; - public class EnvironmentPostHandler extends AbstractStreamHttpHandler { /** @@ -47,22 +46,25 @@ public EnvironmentPostHandler(final AzkarraStreamsService service) { */ @Override public void handleRequest(final HttpServerExchange exchange) { - final EnvironmentPayload env = ExchangeHelper.readJsonRequest(exchange, EnvironmentPayload.class); - if (isNullOrEmpty(env.name)) { + final EnvironmentRequestBody request = ExchangeHelper.readJsonRequest(exchange, EnvironmentRequestBody.class); + if (isNullOrEmpty(request.name)) { throw new BadRequestException("Invalid JSON field, 'name' cannot be null."); } - service.addNewEnvironment(env.name, Conf.of(env.config)); + service.addNewEnvironment(request.name, request.type, Conf.of(request.config)); } - public static final class EnvironmentPayload { + public static final class EnvironmentRequestBody { public final String name; + public final String type; public final Map config; @JsonCreator - public EnvironmentPayload(@JsonProperty("name") final String name, - @JsonProperty("config") final Map config) { + public EnvironmentRequestBody(@JsonProperty("name") final String name, + @JsonProperty("type") final String type, + @JsonProperty("config") final Map config) { this.name = name; + this.type = type; this.config = config; } } diff --git a/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/handler/EnvironmentTypesGetListHandler.java b/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/handler/EnvironmentTypesGetListHandler.java new file mode 100644 index 00000000..116b7146 --- /dev/null +++ b/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/handler/EnvironmentTypesGetListHandler.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019-2020 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamthoughts.azkarra.http.handler; + +import io.streamthoughts.azkarra.api.AzkarraStreamsService; +import io.streamthoughts.azkarra.http.ExchangeHelper; +import io.undertow.server.HttpServerExchange; + +import java.util.Set; + +public class EnvironmentTypesGetListHandler extends AbstractStreamHttpHandler { + + /** + * Creates a new {@link EnvironmentTypesGetListHandler} instance. + * + * @param service the {@link AzkarraStreamsService} instance. + */ + public EnvironmentTypesGetListHandler(final AzkarraStreamsService service) { + super(service); + } + + /** + * {@inheritDoc} + */ + @Override + public void handleRequest(final HttpServerExchange exchange) { + final Set environments = service.getSupportedEnvironmentTypes(); + ExchangeHelper.sendJsonResponse(exchange, environments); + } +} diff --git a/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/routes/ApiEnvironmentRoutes.java b/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/routes/ApiEnvironmentRoutes.java index 5806a7fe..d0b6639b 100644 --- a/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/routes/ApiEnvironmentRoutes.java +++ b/azkarra-server/src/main/java/io/streamthoughts/azkarra/http/routes/ApiEnvironmentRoutes.java @@ -22,6 +22,7 @@ import io.streamthoughts.azkarra.http.APIVersions; import io.streamthoughts.azkarra.http.handler.EnvironmentGetListHandler; import io.streamthoughts.azkarra.http.handler.EnvironmentPostHandler; +import io.streamthoughts.azkarra.http.handler.EnvironmentTypesGetListHandler; import io.streamthoughts.azkarra.http.spi.RoutingHandlerProvider; import io.undertow.Handlers; import io.undertow.server.RoutingHandler; @@ -29,7 +30,8 @@ public class ApiEnvironmentRoutes implements RoutingHandlerProvider { - public static final String PATH = APIVersions.PATH_V1 + "/environments"; + public static final String PATH_ENVIRONMENTS = APIVersions.PATH_V1 + "/environments"; + public static final String PATH_ENVIRONMENT_TYPES = APIVersions.PATH_V1 + "/environment-types"; /** * {@inheritDoc} @@ -37,7 +39,8 @@ public class ApiEnvironmentRoutes implements RoutingHandlerProvider { @Override public RoutingHandler handler(final AzkarraStreamsService service) { return Handlers.routing() - .get(PATH, new EnvironmentGetListHandler(service)) - .post(PATH, new BlockingHandler(new EnvironmentPostHandler(service))); + .get(PATH_ENVIRONMENTS, new EnvironmentGetListHandler(service)) + .get(PATH_ENVIRONMENT_TYPES, new EnvironmentTypesGetListHandler(service)) + .post(PATH_ENVIRONMENTS, new BlockingHandler(new EnvironmentPostHandler(service))); } } diff --git a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/AbstractConfigEntryLoader.java b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/AbstractConfigEntryLoader.java new file mode 100644 index 00000000..e9bcb650 --- /dev/null +++ b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/AbstractConfigEntryLoader.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamthoughts.azkarra.streams; + +import java.util.Objects; + +public abstract class AbstractConfigEntryLoader implements ApplicationConfigEntryLoader { + + private final String configEntryKey; + + /** + * Creates a new {@link AbstractConfigEntryLoader} instance. + * + * @param configEntryKey the configuration entry key accepted by this loader. + */ + public AbstractConfigEntryLoader(final String configEntryKey) { + this.configEntryKey = Objects.requireNonNull(configEntryKey, "configEntryKey cannot be null"); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean accept(final String configEntryKey) { + return this.configEntryKey.equalsIgnoreCase(configEntryKey); + } +} diff --git a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/ApplicationConfigEntryLoader.java b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/ApplicationConfigEntryLoader.java new file mode 100644 index 00000000..f432129b --- /dev/null +++ b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/ApplicationConfigEntryLoader.java @@ -0,0 +1,43 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamthoughts.azkarra.streams; + +import io.streamthoughts.azkarra.api.config.ConfEntry; + +public interface ApplicationConfigEntryLoader { + + /** + * Checks whether this loader supported the specified root config key. + * + * @param configKey the entry config key. + * @return {@code true} of the method {@link #load(ConfEntry, AzkarraApplication)} + * can be invoked with the value config for the given key + */ + boolean accept(final String configKey); + + /** + * Loads the configuration key/value entry. + * + * @param configEntryObject the configuration entry key/value object. + * @param application the {@link AzkarraApplication} instance. + */ + void load(final ConfEntry configEntryObject, + final AzkarraApplication application); +} diff --git a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/ApplicationConfigLoader.java b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/ApplicationConfigLoader.java new file mode 100644 index 00000000..55f8ea7b --- /dev/null +++ b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/ApplicationConfigLoader.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamthoughts.azkarra.streams; + +import io.streamthoughts.azkarra.api.config.Conf; +import io.streamthoughts.azkarra.api.config.ConfEntry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Objects; +import java.util.Set; + +import static io.streamthoughts.azkarra.streams.AzkarraApplication.AZKARRA_ROOT_CONFIG_KEY; + +public class ApplicationConfigLoader { + + private static final Logger LOG = LoggerFactory.getLogger(ApplicationConfigLoader.class); + + private final List applicationConfigEntryLoaders; + + public ApplicationConfigLoader(final List applicationConfigEntryLoaders) { + Objects.requireNonNull(applicationConfigEntryLoaders, "applicationConfigEntryLoaders cannot be null"); + this.applicationConfigEntryLoaders = applicationConfigEntryLoaders; + } + + public void load(final AzkarraApplication application) { + final Conf configuration = application + .getConfiguration() + .getSubConf(AZKARRA_ROOT_CONFIG_KEY); + + final Set keySet = configuration.keySet(); + for (String key : keySet) { + final ConfEntry entry = ConfEntry.of(key, configuration.getValue(key)); + applicationConfigEntryLoaders.stream() + .filter(applicationConfigEntryLoader -> applicationConfigEntryLoader.accept(key)) + .findFirst() + .ifPresentOrElse( + applicationConfigEntryLoader -> { + LOG.debug("Loading application config with key: '{}'", key); + applicationConfigEntryLoader.load(entry, application); + }, + () -> LOG.warn( + "Cannot found ApplicationConfigEntryLoader for key: {}. Configuration is ignored. ", key) + ); + } + } +} diff --git a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/AzkarraApplication.java b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/AzkarraApplication.java index fe8ba2d8..198b4b14 100644 --- a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/AzkarraApplication.java +++ b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/AzkarraApplication.java @@ -23,20 +23,26 @@ import io.streamthoughts.azkarra.api.StreamsExecutionEnvironment; import io.streamthoughts.azkarra.api.banner.Banner; import io.streamthoughts.azkarra.api.banner.BannerPrinter; +import io.streamthoughts.azkarra.api.components.Ordered; import io.streamthoughts.azkarra.api.config.ArgsConf; import io.streamthoughts.azkarra.api.config.Conf; import io.streamthoughts.azkarra.api.errors.AzkarraException; +import io.streamthoughts.azkarra.api.providers.TopologyDescriptor; import io.streamthoughts.azkarra.api.server.EmbeddedHttpServer; import io.streamthoughts.azkarra.api.server.ServerInfo; import io.streamthoughts.azkarra.api.spi.EmbeddedHttpServerProvider; import io.streamthoughts.azkarra.api.util.Network; import io.streamthoughts.azkarra.http.ServerConfig; +import io.streamthoughts.azkarra.http.ServerConfigBuilder; import io.streamthoughts.azkarra.runtime.streams.topology.InternalExecuted; import io.streamthoughts.azkarra.streams.autoconfigure.AutoConfigure; import io.streamthoughts.azkarra.streams.banner.AzkarraBanner; import io.streamthoughts.azkarra.streams.banner.BannerPrinterBuilder; import io.streamthoughts.azkarra.streams.components.ReflectiveComponentScanner; -import io.streamthoughts.azkarra.streams.context.AzkarraContextLoader; +import io.streamthoughts.azkarra.streams.config.loader.AutoStartConfigEntryLoader; +import io.streamthoughts.azkarra.streams.config.loader.ComponentConfigEntryLoader; +import io.streamthoughts.azkarra.streams.config.loader.EnvironmentsConfigEntryLoader; +import io.streamthoughts.azkarra.streams.config.loader.ServerConfigEntryLoader; import org.apache.kafka.streams.StreamsConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,10 +50,12 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.ServiceLoader; +import java.util.Set; /** * {@link AzkarraApplication} is the high-level class which can be used to deploy a simple server for managing multiple @@ -76,8 +84,19 @@ public class AzkarraApplication { private static final Logger LOG = LoggerFactory.getLogger(AzkarraApplication.class); - private static final String HTTP_SERVER_CONFIG = "azkarra.server"; - private static final String COMPONENT_PATHS_CONFIG = "azkarra.component.paths"; + public static final String AZKARRA_ROOT_CONFIG_KEY = "azkarra"; + private static final String COMPONENT_PATHS_CONFIG_KEY = "component.paths"; + private static final String CONTEXT_CONFIG_KEY = "context"; + + /** + * Helper method to prefix a key property with the root property prefix (i.e {@code "azkarra"}. + * + * @param key the key propery to prefix. + * @return the prefixed key. + */ + public static String withAzkarraPrefix(final String key) { + return AZKARRA_ROOT_CONFIG_KEY + "." + key; + } public static AzkarraContext run() { return run(new Class[0], new String[0]); @@ -97,9 +116,7 @@ public static AzkarraContext run(final Class[] sources, final String[] args) private boolean enableComponentScan; - private boolean enableHttpServer; - - private Conf httpServerConf; + private boolean isHttpServerEnable; private Banner banner; @@ -109,10 +126,14 @@ public static AzkarraContext run(final Class[] sources, final String[] args) private Conf configuration; - private Class mainApplicationClass; + private final Class mainApplicationClass; private AutoStart autoStart; + private Conf httpServerConf; + + private final List applicationConfigEntryLoaders; + /** * Creates a new {@link AzkarraApplication} instance. * @@ -120,29 +141,54 @@ public static AzkarraContext run(final Class[] sources, final String[] args) */ public AzkarraApplication(final Class... sources) { this.sources = new ArrayList<>(Arrays.asList(sources)); - this.enableHttpServer = false; + this.isHttpServerEnable = false; this.enableComponentScan = false; this.registerShutdownHook = true; this.autoStart = new AutoStart(false, null); this.banner = new AzkarraBanner(); this.bannerMode = Banner.Mode.CONSOLE; this.mainApplicationClass = deduceMainApplicationClass(); + this.applicationConfigEntryLoaders = new ArrayList<>(); this.configuration = Conf.empty(); - this.httpServerConf = ServerConfig.newBuilder().setListener(Network.HOSTNAME).build(); } - public AzkarraApplication enableHttpServer(final boolean enableHttpServer) { - this.enableHttpServer = enableHttpServer; + /** + * Sets whether the HTTP server should be enable. + * + * @param httpServerEnable set to {@code true} to enable the server. + */ + public AzkarraApplication setHttpServerEnable(boolean httpServerEnable) { + isHttpServerEnable = httpServerEnable; + if (this.isHttpServerEnable) { + this.httpServerConf = ServerConfig.newBuilder().setListener(Network.HOSTNAME).build(); + } return this; } - public AzkarraApplication enableHttpServer(final boolean enableHttpServer, - final Conf conf) { - this.enableHttpServer = enableHttpServer; - this.httpServerConf = conf; + /** + * Sets the HTTP server configuration. + * + * @param httpServerConf the http server configuration. + * @return this {@link AzkarraApplication} instance. + */ + public AzkarraApplication setHttpServerConf(final Conf httpServerConf) { + final ServerConfigBuilder builder = ServerConfig.newBuilder(httpServerConf); + if (!httpServerConf.hasPath(ServerConfig.HTTP_SERVER_LISTENER_CONFIG)) { + builder.setListener(Network.HOSTNAME); + } + this.httpServerConf = builder.build(); return this; } + /** + * Checks whether the HTTP Server is enable. + * + * @return {@code true} is the HTTP Server is enable, {@code false} otherwise. + */ + private boolean isHttpServerEnable() { + return isHttpServerEnable; + } + /** * Sets if the classpath must be scanned for searching components. * @@ -198,7 +244,7 @@ public AzkarraApplication setBanner(final Banner banner) { * @return this {@link AzkarraApplication} instance. */ public AzkarraApplication setAutoStart(final boolean enable) { - setAutoStart(enable, "__default"); + setAutoStart(enable, null); return this; } @@ -243,6 +289,18 @@ public AzkarraApplication setConfiguration(final Conf configuration) { return this; } + /** + * Adds a new {@link ApplicationConfigEntryLoader} to be used for reading the application configuration. + * + * @param configEntryLoader the {@link ApplicationConfigEntryLoader} to be added. + * @return this {@link AzkarraApplication} instance. + */ + public AzkarraApplication addConfigEntryLoader(final ApplicationConfigEntryLoader configEntryLoader) { + Objects.requireNonNull(configEntryLoader, "ApplicationConfigEntryLoader cannot be null"); + this.applicationConfigEntryLoaders.add(configEntryLoader); + return this; + } + /** * Gets the configuration for this {@link AzkarraApplication}. * @@ -270,45 +328,40 @@ public AzkarraContext run(final String[] args) { if (registerShutdownHook) { context.setRegisterShutdownHook(true); } + context.addConfiguration(configuration.getSubConf(withAzkarraPrefix(CONTEXT_CONFIG_KEY))); - if (enableComponentScan) { + if (isComponentScanEnable()) { var scanner = new ReflectiveComponentScanner(context.getComponentFactory()); context.registerSingleton(scanner); // Scan all sub-packages of the root package of Azkarra for declared components. scanner.scanForPackage("io.streamthoughts.azkarra"); - final Optional componentPaths = configuration.getOptionalString(COMPONENT_PATHS_CONFIG); - componentPaths.ifPresent(scanner::scan); + configuration.getOptionalString(withAzkarraPrefix(COMPONENT_PATHS_CONFIG_KEY)) + .ifPresent(scanner::scan); - for (Class source : sources) { - scanner.scanForPackage(source.getPackage()); - } + sources.stream().map(Class::getPackage).forEach(scanner::scanForPackage); } + final List configLoaders = new ArrayList<>(); + configLoaders.add(new NoopConfigEntryLoader(Set.of(CONTEXT_CONFIG_KEY, COMPONENT_PATHS_CONFIG_KEY))); + configLoaders.add(new ComponentConfigEntryLoader()); + configLoaders.add(new ServerConfigEntryLoader()); + configLoaders.add(new AutoStartConfigEntryLoader()); + configLoaders.add(new EnvironmentsConfigEntryLoader()); + configLoaders.addAll(applicationConfigEntryLoaders); + configLoaders.addAll(context.getAllComponents(ApplicationConfigEntryLoader.class)); + // Initializing context from the application configuration. - AzkarraContextLoader.load(context, configuration); + new ApplicationConfigLoader(configLoaders).load(this); - if (enableHttpServer) { - List result = loadEmbeddedHttpServerImplementations(); - if (result.isEmpty()) { - throw new AzkarraException( - "Cannot find implementation for service provider : " + EmbeddedHttpServerProvider.class.getName()); - } - final EmbeddedHttpServer embeddedHttpServer = result.get(0); - context.addListener(new EmbeddedServerLifecycle(embeddedHttpServer, loadHttpServerConf())); - } + // Automatically start all streams topology for the specified environment. + if (autoStart.isEnable()) + context.addListener(new AutoStartContextListener()); - if (autoStart.isEnable()) { - StreamsExecutionEnvironment target = context.getEnvironmentForNameOrCreate(autoStart.targetEnvironment()); - context.topologyProviders(target).forEach(desc -> - context.addTopology( - desc.className(), - desc.version().toString(), - target.name(), - new InternalExecuted() - ) - ); + if (isHttpServerEnable()) { + final EmbeddedHttpServer embeddedHttpServer = loadEmbeddedHttpServerImplementation(); + context.addListener(new EmbeddedServerLifecycle(embeddedHttpServer, buildHttpServerConfig())); } context.start(); @@ -316,25 +369,8 @@ public AzkarraContext run(final String[] args) { return context; } - private List loadEmbeddedHttpServerImplementations() { - ServiceLoader serviceLoader = ServiceLoader.load(EmbeddedHttpServerProvider.class); - List result = new ArrayList<>(); - for (EmbeddedHttpServerProvider embeddedServerImpl : serviceLoader) { - final EmbeddedHttpServer server = embeddedServerImpl.get(context); - result.add(server); - LOG.info( - "Loading io.streamthoughts.azkarra.api.server.EmbeddedHttpServer: {}", - server.getClass().getName() - ); - } - return result; - } - - private Conf loadHttpServerConf() { - if (configuration.hasPath(HTTP_SERVER_CONFIG)) { - httpServerConf = configuration.getSubConf(HTTP_SERVER_CONFIG).withFallback(httpServerConf); - } - return httpServerConf; + private boolean isComponentScanEnable() { + return enableComponentScan; } /** @@ -379,6 +415,11 @@ public AzkarraContext getContext() { return context; } + /** + * Gets the {@link Class} containing the main method. + * + * @return the main application {@link Class}. + */ public Class getMainApplicationClass() { return mainApplicationClass; } @@ -393,73 +434,165 @@ private void printBanner() { printer.print(banner); } - private static final class EmbeddedServerLifecycle implements AzkarraContextListener { + private Class deduceMainApplicationClass() { + try { + StackTraceElement[] stackTrace = new RuntimeException().getStackTrace(); + for (StackTraceElement stackTraceElement : stackTrace) { + if ("main".equals(stackTraceElement.getMethodName())) { + return Class.forName(stackTraceElement.getClassName()); + } + } + } + catch (final ClassNotFoundException ex) { + // ignore + } + return null; + } - private final Conf conf; - private final EmbeddedHttpServer embeddedHttpServer; + private Conf buildHttpServerConfig() { + // Get the user-defined configuration. + Conf applicationHttpServerConf = httpServerConf; - EmbeddedServerLifecycle(final EmbeddedHttpServer embeddedHttpServer, - final Conf conf) { - this.embeddedHttpServer = embeddedHttpServer; - this.conf = conf; + // Get all declared server configurations. + final Collection allServerConfigs = context.getAllComponents(ServerConfig.class); + + // Merge all configurations. + for (ServerConfig config : allServerConfigs) { + applicationHttpServerConf = applicationHttpServerConf.withFallback(config); + } + + return applicationHttpServerConf; + } + + private EmbeddedHttpServer loadEmbeddedHttpServerImplementation() { + ServiceLoader serviceLoader = ServiceLoader.load(EmbeddedHttpServerProvider.class); + List result = new ArrayList<>(); + for (EmbeddedHttpServerProvider embeddedServerImpl : serviceLoader) { + final EmbeddedHttpServer server = embeddedServerImpl.get(context); + result.add(server); + LOG.info( + "Loading io.streamthoughts.azkarra.api.server.EmbeddedHttpServer: {}", + server.getClass().getName() + ); + } + if (result.isEmpty()) { + throw new AzkarraException( + "Cannot find implementation for service provider : " + EmbeddedHttpServerProvider.class.getName()); } + return result.get(0); + } + + private class AutoStartContextListener implements AzkarraContextListener { /** * {@inheritDoc} */ @Override - public void onContextStart(final AzkarraContext context) { - embeddedHttpServer.configure(conf); - embeddedHttpServer.start(); - - // configure streams discovery if http server is enable - final ServerInfo info = embeddedHttpServer.info(); - final String server = info.getHost() + ":" + info.getPort(); - final Conf serverConfig = Conf.of(StreamsConfig.APPLICATION_SERVER_CONFIG, server); - context.addConfiguration(Conf.of("streams", serverConfig)); + public int order() { + return Ordered.LOWEST_ORDER - 1; } /** * {@inheritDoc} */ @Override - public void onContextStop(final AzkarraContext context) { - embeddedHttpServer.stop(); - } - - } - - private Class deduceMainApplicationClass() { - try { - StackTraceElement[] stackTrace = new RuntimeException().getStackTrace(); - for (StackTraceElement stackTraceElement : stackTrace) { - if ("main".equals(stackTraceElement.getMethodName())) { - return Class.forName(stackTraceElement.getClassName()); - } + public void onContextStart(final AzkarraContext context) { + final String environment = selectExecutionEnvironmentForAutoStart(context); + final Set topologies = context.getTopologyDescriptors(environment); + for (TopologyDescriptor desc : topologies) { + context.addTopology( + desc.className(), + desc.version().toString(), + environment, + new InternalExecuted() + ); } } - catch (final ClassNotFoundException ex) { - // ignore + + private String selectExecutionEnvironmentForAutoStart(final AzkarraContext context) { + return autoStart + .environment() + .or(() -> context.getAllEnvironments() + .stream().filter(StreamsExecutionEnvironment::isDefault) + .findFirst() + .map(StreamsExecutionEnvironment::name)) + .orElseThrow(() -> + new AzkarraException("Cannot auto-start topology, no default environment can be found") + ); } - return null; } + /** + * Wrapper class. + */ private static class AutoStart { private final boolean enable; - private final String targetEnvironment; + private final String environment; - AutoStart(final boolean enable, final String targetEnvironment) { + /** + * Creates a new {@link AutoStart} instance. + * + * @param enable is all topologies should be started automatically. + * @param environment the environment that should be used to topologies. + */ + AutoStart(final boolean enable, final String environment) { this.enable = enable; - this.targetEnvironment = targetEnvironment; + this.environment = environment; } boolean isEnable(){ return enable; } - String targetEnvironment() { - return targetEnvironment; + Optional environment() { + return Optional.ofNullable(environment); + } + } + + private static final class EmbeddedServerLifecycle implements AzkarraContextListener { + + private final Conf conf; + private final EmbeddedHttpServer embeddedHttpServer; + + EmbeddedServerLifecycle(final EmbeddedHttpServer embeddedHttpServer, + final Conf conf) { + this.embeddedHttpServer = embeddedHttpServer; + this.conf = conf; + } + + /** + * {@inheritDoc} + */ + @Override + public int order() { + return Ordered.LOWEST_ORDER; + } + + /** + * {@inheritDoc} + */ + @Override + public void onContextStart(final AzkarraContext context) { + + embeddedHttpServer.configure(conf); + embeddedHttpServer.start(); + + // configure streams discovery if http server is enable + final String streamsApplicationServerConfigKey = "streams." + StreamsConfig.APPLICATION_SERVER_CONFIG; + if (!context.getConfiguration().hasPath(streamsApplicationServerConfigKey)) { + final ServerInfo info = embeddedHttpServer.info(); + final String server = info.getHost() + ":" + info.getPort(); + context.addConfiguration(Conf.of(streamsApplicationServerConfigKey, server)); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void onContextStop(final AzkarraContext context) { + embeddedHttpServer.stop(); } } } diff --git a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/NoopConfigEntryLoader.java b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/NoopConfigEntryLoader.java new file mode 100644 index 00000000..8945e622 --- /dev/null +++ b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/NoopConfigEntryLoader.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamthoughts.azkarra.streams; + +import io.streamthoughts.azkarra.api.config.ConfEntry; + +import java.util.Set; + +public class NoopConfigEntryLoader implements ApplicationConfigEntryLoader { + + private final Set configEntryKeys; + /** + * Creates a new {@link AbstractConfigEntryLoader} instance. + * + * @param configEntryKeys the configuration entry keys accepted by this loader. + */ + public NoopConfigEntryLoader(final Set configEntryKeys) { + this.configEntryKeys = configEntryKeys; + } + + @Override + public boolean accept(final String configKey) { + return configEntryKeys.contains(configKey); + } + + /** + * {@inheritDoc} + */ + @Override + public void load(final ConfEntry configEntryObject, final AzkarraApplication application) { + /* noop */ + } +} diff --git a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/autoconfigure/AutoConfigure.java b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/autoconfigure/AutoConfigure.java index 5d56d19d..1df0d94e 100644 --- a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/autoconfigure/AutoConfigure.java +++ b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/autoconfigure/AutoConfigure.java @@ -55,7 +55,7 @@ public void load(final AzkarraApplication application) { LOG.info("Loading application configuration"); Objects.requireNonNull(application, "application cannot be null"); - AzkarraContext context = application.getContext(); + final AzkarraContext context = application.getContext(); if (context == null) { LOG.info("No AzkarraContext provided, initializing default provided implementation"); @@ -74,7 +74,7 @@ public void load(final AzkarraApplication application) { } }); isHttpServerEnable(mainApplicationClass) - .ifPresent(application::enableHttpServer); + .ifPresent(application::setHttpServerEnable); loadAutoStartEnvironmentNameIfEnable(mainApplicationClass) .ifPresent(env -> application.setAutoStart(true, env)); diff --git a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/AzkarraConf.java b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/AzkarraConf.java index 7aec8d72..87bf64f6 100644 --- a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/AzkarraConf.java +++ b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/AzkarraConf.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -108,6 +109,22 @@ protected AzkarraConf(final Config config) { this.config = config; } + /** + * {@inheritDoc} + */ + @Override + public Set keySet() { + return config.root().unwrapped().keySet(); + } + + /** + * {@inheritDoc} + */ + @Override + public Object getValue(String path) { + return config.getValue(path).unwrapped(); + } + /** * {@inheritDoc} */ diff --git a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/AutoStartConfigEntryLoader.java b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/AutoStartConfigEntryLoader.java new file mode 100644 index 00000000..ef6c5292 --- /dev/null +++ b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/AutoStartConfigEntryLoader.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamthoughts.azkarra.streams.config.loader; + +import io.streamthoughts.azkarra.api.config.Conf; +import io.streamthoughts.azkarra.api.config.ConfEntry; +import io.streamthoughts.azkarra.streams.AbstractConfigEntryLoader; +import io.streamthoughts.azkarra.streams.AzkarraApplication; + +public class AutoStartConfigEntryLoader extends AbstractConfigEntryLoader { + + public static final String CONFIG_ENTRY_KEY = "autoStart"; + + public AutoStartConfigEntryLoader() { + super(CONFIG_ENTRY_KEY); + } + + /** + * {@inheritDoc} + */ + @Override + public void load(final ConfEntry configEntryObject, + final AzkarraApplication application) { + final Conf autoStart = configEntryObject.asSubConf(); + application.setAutoStart( + autoStart.getBoolean("enable"), + autoStart.getOptionalString("environments").orElse(null) + ); + } +} diff --git a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/ComponentConfigEntryLoader.java b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/ComponentConfigEntryLoader.java new file mode 100644 index 00000000..69bbc952 --- /dev/null +++ b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/ComponentConfigEntryLoader.java @@ -0,0 +1,54 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamthoughts.azkarra.streams.config.loader; + +import io.streamthoughts.azkarra.api.config.ConfEntry; +import io.streamthoughts.azkarra.api.errors.AzkarraException; +import io.streamthoughts.azkarra.streams.AbstractConfigEntryLoader; +import io.streamthoughts.azkarra.streams.AzkarraApplication; + +import java.util.List; + +public class ComponentConfigEntryLoader extends AbstractConfigEntryLoader { + + private static final String CONFIG_ENTRY_KEY = "components"; + + /** + * Creates a new {@link ComponentConfigEntryLoader} instance. + */ + public ComponentConfigEntryLoader() { + super(CONFIG_ENTRY_KEY); + } + + /** + * {@inheritDoc} + */ + @Override + public void load(final ConfEntry configObject, final AzkarraApplication application) { + final List components = configObject.asStringList(); + for (final String componentType : components) { + try { + application.getContext().registerComponent(Class.forName(componentType)); + } catch (ClassNotFoundException e) { + throw new AzkarraException("Invalid configuration", e); + } + } + } +} diff --git a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/EnvironmentsConfigEntryLoader.java b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/EnvironmentsConfigEntryLoader.java new file mode 100644 index 00000000..eab78e1d --- /dev/null +++ b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/EnvironmentsConfigEntryLoader.java @@ -0,0 +1,97 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamthoughts.azkarra.streams.config.loader; + +import io.streamthoughts.azkarra.api.AzkarraContext; +import io.streamthoughts.azkarra.api.AzkarraContextListener; +import io.streamthoughts.azkarra.api.Executed; +import io.streamthoughts.azkarra.api.StreamsExecutionEnvironment; +import io.streamthoughts.azkarra.api.config.ConfEntry; +import io.streamthoughts.azkarra.runtime.StreamsExecutionEnvironmentAbstractFactory; +import io.streamthoughts.azkarra.streams.AbstractConfigEntryLoader; +import io.streamthoughts.azkarra.streams.AzkarraApplication; +import io.streamthoughts.azkarra.streams.config.loader.internal.EnvironmentConfig; +import io.streamthoughts.azkarra.streams.config.loader.internal.TopologyConfig; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +public class EnvironmentsConfigEntryLoader extends AbstractConfigEntryLoader { + + private static final String CONFIG_ENTRY_KEY = "environments"; + + /** + * Creates a new {@link EnvironmentsConfigEntryLoader} instance. + */ + public EnvironmentsConfigEntryLoader() { + super(CONFIG_ENTRY_KEY); + } + + /** + * {@inheritDoc} + */ + @Override + public void load(final ConfEntry configEntryObject, final AzkarraApplication application) { + final List environmentConfigEntries = loadConfiguredEnvironments(configEntryObject); + application.getContext().addListener(new EnvironmentContextLoader(environmentConfigEntries)); + } + + private List loadConfiguredEnvironments(final ConfEntry configEntryObject) { + return configEntryObject.asSubConfList() + .stream() + .map(EnvironmentConfig::read) + .collect(Collectors.toList()); + } + + private static class EnvironmentContextLoader implements AzkarraContextListener { + + final List environmentConfigEntries; + + public EnvironmentContextLoader(final List environmentConfigEntries) { + this.environmentConfigEntries = environmentConfigEntries; + } + + /** + * {@inheritDoc} + */ + @Override + public void onContextStart(final AzkarraContext context) { + var factory = context.getComponent(StreamsExecutionEnvironmentAbstractFactory.class); + for (EnvironmentConfig environmentConfigEntry : environmentConfigEntries) { + final StreamsExecutionEnvironment environment = factory.create( + environmentConfigEntry.type(), + environmentConfigEntry.name(), + environmentConfigEntry.config() + ); + context.addExecutionEnvironment(environment); + List topologyConfigs = environmentConfigEntry.topologyStreamConfigs(); + for (TopologyConfig topology : topologyConfigs) { + Executed executed = Executed.as(topology.name().orElse(null)); + executed = topology.config().map(executed::withConfig).orElse(executed); + executed = topology.description().map(executed::withDescription).orElse(executed); + + Optional version = topology.version(); + context.addTopology(topology.type(), version.orElse(null), environmentConfigEntry.name(), executed); + } + } + } + } +} diff --git a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/ServerConfigEntryLoader.java b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/ServerConfigEntryLoader.java new file mode 100644 index 00000000..3ea7fe00 --- /dev/null +++ b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/ServerConfigEntryLoader.java @@ -0,0 +1,48 @@ +/* + * Copyright 2019-2021 StreamThoughts. + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamthoughts.azkarra.streams.config.loader; + +import io.streamthoughts.azkarra.api.config.Conf; +import io.streamthoughts.azkarra.api.config.ConfEntry; +import io.streamthoughts.azkarra.streams.AbstractConfigEntryLoader; +import io.streamthoughts.azkarra.streams.AzkarraApplication; + +public class ServerConfigEntryLoader extends AbstractConfigEntryLoader { + + private static final String CONFIG_ENTRY_KEY = "server"; + + /** + * Creates a new {@link AbstractConfigEntryLoader} instance. + */ + public ServerConfigEntryLoader() { + super(CONFIG_ENTRY_KEY); + } + + /** + * {@inheritDoc} + */ + @Override + public void load(final ConfEntry configEntryObject, + final AzkarraApplication application) { + final Conf serverConf = configEntryObject.asSubConf(); + application.setHttpServerEnable(serverConf.getOptionalBoolean("enable").orElse(true)); + application.setHttpServerConf(serverConf); + } +} diff --git a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/context/internal/EnvironmentConfig.java b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/internal/EnvironmentConfig.java similarity index 84% rename from azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/context/internal/EnvironmentConfig.java rename to azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/internal/EnvironmentConfig.java index 220731ca..6417344e 100644 --- a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/context/internal/EnvironmentConfig.java +++ b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/internal/EnvironmentConfig.java @@ -16,10 +16,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.streamthoughts.azkarra.streams.context.internal; +package io.streamthoughts.azkarra.streams.config.loader.internal; import io.streamthoughts.azkarra.api.StreamsExecutionEnvironment; import io.streamthoughts.azkarra.api.config.Conf; +import io.streamthoughts.azkarra.runtime.env.LocalStreamsExecutionEnvironment; import java.util.Collections; import java.util.List; @@ -33,6 +34,7 @@ public class EnvironmentConfig { public final static String ENVIRONMENT_CONFIG = "config"; public final static String ENVIRONMENT_NAME_CONFIG = "name"; + public final static String ENVIRONMENT_TYPE_CONFIG = "type"; public static final String ENVIRONMENT_STREAMS_CONFIG = "jobs"; /** @@ -47,6 +49,7 @@ public static EnvironmentConfig read(final Conf conf) { } private final String name; + private final String type; private final Conf config; private final List topologyStreamSettings; @@ -54,18 +57,29 @@ public static EnvironmentConfig read(final Conf conf) { * Creates a new {@link } * * @param name the environment name. + * @param type the environment type. * @param topologyConfigs the environment topologies. * @param config the environment configuration. */ private EnvironmentConfig(final String name, + final String type, final List topologyConfigs, final Conf config) { - Objects.requireNonNull(name, "name cannot be null"); - this.name = name; + this.name = Objects.requireNonNull(name, "name cannot be null");; + this.type = Objects.requireNonNull(type, "type cannot be null");; this.config = config; this.topologyStreamSettings = topologyConfigs; } + /** + * Gets the environment type. + * + * @return a string type. + */ + public String type() { + return type; + } + /** * Gets the environment name. * @@ -98,6 +112,7 @@ public static final class Reader { EnvironmentConfig read(final Conf conf) { return new EnvironmentConfig( conf.getString(ENVIRONMENT_NAME_CONFIG), + conf.getOptionalString(ENVIRONMENT_TYPE_CONFIG).orElse(LocalStreamsExecutionEnvironment.TYPE), mayGetEnvironmentStreams(conf), mayGetConfiguredDefaultsConf(conf) ); diff --git a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/context/internal/TopologyConfig.java b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/internal/TopologyConfig.java similarity index 98% rename from azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/context/internal/TopologyConfig.java rename to azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/internal/TopologyConfig.java index 10386fd9..6fcbc790 100644 --- a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/context/internal/TopologyConfig.java +++ b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/config/loader/internal/TopologyConfig.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.streamthoughts.azkarra.streams.context.internal; +package io.streamthoughts.azkarra.streams.config.loader.internal; import io.streamthoughts.azkarra.api.config.Conf; import io.streamthoughts.azkarra.api.streams.TopologyProvider; diff --git a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/context/AzkarraContextLoader.java b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/context/AzkarraContextLoader.java deleted file mode 100644 index b83755c3..00000000 --- a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/context/AzkarraContextLoader.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2019-2020 StreamThoughts. - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.streamthoughts.azkarra.streams.context; - -import io.streamthoughts.azkarra.api.AzkarraContext; -import io.streamthoughts.azkarra.api.Executed; -import io.streamthoughts.azkarra.api.StreamsExecutionEnvironment; -import io.streamthoughts.azkarra.api.config.Conf; -import io.streamthoughts.azkarra.api.errors.AzkarraException; -import io.streamthoughts.azkarra.runtime.env.DefaultStreamsExecutionEnvironment; -import io.streamthoughts.azkarra.streams.context.internal.ApplicationConfig; -import io.streamthoughts.azkarra.streams.context.internal.EnvironmentConfig; -import io.streamthoughts.azkarra.streams.context.internal.TopologyConfig; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -/** - * Class which can be used for initializing an {@link AzkarraContext} instance from a specified {@link Conf} instance. - * - * @see ApplicationConfig - * @see EnvironmentConfig - * @see TopologyConfig - */ -public class AzkarraContextLoader { - - /** - * Initialize the specified {@link AzkarraContext} using the specified {@link Conf}. - * - * @param context the {@link AzkarraContext} to initialize. - * @param configuration the {@link Conf} instance. - * - * @return a new {@link AzkarraContext} instance. - */ - public static AzkarraContext load(final AzkarraContext context, final Conf configuration) { - return load(context, ApplicationConfig.read(configuration)); - } - - /** - * Initialize the specified {@link AzkarraContext} using the specified {@link ApplicationConfig}. - * - * @param context the {@link AzkarraContext} to initialize. - * @param configuration the {@link ApplicationConfig} instance. - * - * @return a new {@link AzkarraContext} instance. - */ - public static AzkarraContext load(final AzkarraContext context, final ApplicationConfig configuration) { - Objects.requireNonNull(context, "context cannot be null"); - Objects.requireNonNull(configuration, "contextConfig cannot be null"); - - loadConfiguration(context, configuration); - loadConfigurationDeclaredComponent(configuration, context); - loadConfigurationDeclaredEnvironments(configuration, context); - - return context; - } - - private static void loadConfiguration(final AzkarraContext context, - final ApplicationConfig config) { - - context.addConfiguration(config.context()); - } - - private static void loadConfigurationDeclaredComponent(final ApplicationConfig config, - final AzkarraContext context) { - - for (final String componentType : config.components()) { - try { - context.registerComponent(Class.forName(componentType)); - } catch (ClassNotFoundException e) { - throw new AzkarraException("Invalid configuration", e); - } - } - } - - private static void loadConfigurationDeclaredEnvironments(final ApplicationConfig config, - final AzkarraContext context) { - - config.environments().forEach(conf -> initializeStreamsEnvironment(conf, context)); - } - - private static void initializeStreamsEnvironment(final EnvironmentConfig envConfig, - final AzkarraContext context) { - - final String name = envConfig.name(); - - StreamsExecutionEnvironment env; - if (name == null) { - env = DefaultStreamsExecutionEnvironment.create(envConfig.config()); - } else { - env = DefaultStreamsExecutionEnvironment.create(envConfig.config(), name); - } - - context.addExecutionEnvironment(env); - - List topologyConfigs = envConfig.topologyStreamConfigs(); - for (TopologyConfig topology : topologyConfigs) { - - Executed executed = Executed.as(topology.name().orElse(null)); - executed = topology.config().map(executed::withConfig).orElse(executed); - executed = topology.description().map(executed::withDescription).orElse(executed); - - Optional version = topology.version(); - context.addTopology(topology.type(), version.orElse(null), env.name(), executed); - } - } -} diff --git a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/context/internal/ApplicationConfig.java b/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/context/internal/ApplicationConfig.java deleted file mode 100644 index 657e63bc..00000000 --- a/azkarra-streams/src/main/java/io/streamthoughts/azkarra/streams/context/internal/ApplicationConfig.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright 2019-2020 StreamThoughts. - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.streamthoughts.azkarra.streams.context.internal; - -import io.streamthoughts.azkarra.api.AzkarraContext; -import io.streamthoughts.azkarra.api.config.Conf; - -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Class which is used to initially configure a {@link AzkarraContext} instance. - */ -public class ApplicationConfig { - - public final static String CONTEXT_CONFIG = "azkarra.context"; - public final static String CONTEXT_COMPONENT_CONFIG = "azkarra.components"; - public final static String CONTEXT_ENVIRONMENTS_CONFIG = "azkarra.environments"; - - /** - * Static helper that can be used to creates a new {@link ApplicationConfig} instance - * from the provided {@link Conf}. - * - * @return a new {@link ApplicationConfig} instance. - */ - public static ApplicationConfig read(final Conf conf) { - Objects.requireNonNull(conf, "conf cannot be null"); - return new Reader().read(conf); - } - - private final Conf context; - private final Set components; - private final List environments; - - /** - * Creates a new {@link ApplicationConfig} instance. - * - * @param context the context configuration. - * @param components the components class. - * @param environments the environments configuration. - */ - private ApplicationConfig(final Conf context, - final Set components, - final List environments) { - this.context = context; - this.components = components; - this.environments = environments; - } - - /** - * Gets the context configuration. - * - * @return the {@link Conf} instance. - */ - public Conf context() { - return Optional.ofNullable(context).orElse(Conf.empty()); - } - - /** - * Gets all registered components class name. - * - * @return a set of string class name. - */ - public Set components() { - return components; - } - - /** - * Gets all registered environments. - * - * @return a list of {@link EnvironmentConfig} instances. - */ - public List environments() { - return environments; - } - - public static final class Reader { - - private static final String AZKARRA_METRICS = "azkarra.metrics"; - - public ApplicationConfig read(final Conf conf) { - - Objects.requireNonNull(conf, "conf cannot be null"); - - Conf context = conf.hasPath(CONTEXT_CONFIG) ? conf.getSubConf(CONTEXT_CONFIG) : Conf.empty(); - if (conf.hasPath(AZKARRA_METRICS)) { - // Lookup for configuration prefixed with 'metrics' to simplify the user-defined configuration. - // Note : this will be refactored in later version because this create a direct reference - // to the Azkarra Metrics module. - context = context.withFallback(Conf.of("metrics", conf.getSubConf(AZKARRA_METRICS))); - } - return new ApplicationConfig( - context, - mayGetConfiguredComponents(conf), - mayGetConfiguredEnvironments(conf) - ); - } - - private List mayGetConfiguredEnvironments(final Conf originals) { - if (!originals.hasPath(CONTEXT_ENVIRONMENTS_CONFIG)) { - return Collections.emptyList(); - } - - return originals.getSubConfList(CONTEXT_ENVIRONMENTS_CONFIG) - .stream() - .map(EnvironmentConfig::read) - .collect(Collectors.toList()); - } - - private Set mayGetConfiguredComponents(final Conf originals) { - return originals.hasPath(CONTEXT_COMPONENT_CONFIG) ? - new HashSet<>(originals.getStringList(CONTEXT_COMPONENT_CONFIG)) : - Collections.emptySet(); - } - } -} diff --git a/azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/context/internal/EnvironmentConfigTest.java b/azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/config/loader/internal/EnvironmentConfigTest.java similarity index 95% rename from azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/context/internal/EnvironmentConfigTest.java rename to azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/config/loader/internal/EnvironmentConfigTest.java index 45a338af..38797f65 100644 --- a/azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/context/internal/EnvironmentConfigTest.java +++ b/azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/config/loader/internal/EnvironmentConfigTest.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.streamthoughts.azkarra.streams.context.internal; +package io.streamthoughts.azkarra.streams.config.loader.internal; import io.streamthoughts.azkarra.api.config.Conf; import org.junit.jupiter.api.Assertions; diff --git a/azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/context/internal/TopologyConfigTest.java b/azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/config/loader/internal/TopologyConfigTest.java similarity index 97% rename from azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/context/internal/TopologyConfigTest.java rename to azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/config/loader/internal/TopologyConfigTest.java index 21f02cde..eab8acf6 100644 --- a/azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/context/internal/TopologyConfigTest.java +++ b/azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/config/loader/internal/TopologyConfigTest.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.streamthoughts.azkarra.streams.context.internal; +package io.streamthoughts.azkarra.streams.config.loader.internal; import io.streamthoughts.azkarra.api.config.Conf; import io.streamthoughts.azkarra.api.errors.MissingConfException; diff --git a/azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/context/AzkarraContextLoaderTest.java b/azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/context/AzkarraContextLoaderTest.java deleted file mode 100644 index 4310f0ec..00000000 --- a/azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/context/AzkarraContextLoaderTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2019-2020 StreamThoughts. - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.streamthoughts.azkarra.streams.context; - -import io.streamthoughts.azkarra.api.AzkarraContext; -import io.streamthoughts.azkarra.api.StreamsExecutionEnvironment; -import io.streamthoughts.azkarra.runtime.components.DefaultComponentDescriptorFactory; -import io.streamthoughts.azkarra.runtime.components.DefaultComponentFactory; -import io.streamthoughts.azkarra.streams.config.AzkarraConf; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.stream.Collectors; - -import static io.streamthoughts.azkarra.runtime.context.DefaultAzkarraContext.DEFAULT_ENV_NAME; -import static io.streamthoughts.azkarra.runtime.context.DefaultAzkarraContext.create; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class AzkarraContextLoaderTest { - - public static final String TEST_ENV_NAME = "test"; - private AzkarraContext context; - - @BeforeEach - public void setUp() { - context = create(new DefaultComponentFactory(new DefaultComponentDescriptorFactory())); - } - - @Test - public void shouldInitializeContextWithEnvironmentGivenValidConfig() { - AzkarraContextLoader.load(context, AzkarraConf.create("context-autoconfig.conf")); - - List environments = context.environments(); - - assertEquals(2, environments.size()); - - List names = environments.stream().map(StreamsExecutionEnvironment::name).collect(Collectors.toList()); - assertTrue(names.contains(DEFAULT_ENV_NAME)); - assertTrue(names.contains(TEST_ENV_NAME)); - } -} \ No newline at end of file diff --git a/azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/context/internal/ApplicationConfigTest.java b/azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/context/internal/ApplicationConfigTest.java deleted file mode 100644 index 547cc916..00000000 --- a/azkarra-streams/src/test/java/io/streamthoughts/azkarra/streams/context/internal/ApplicationConfigTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2019-2020 StreamThoughts. - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.streamthoughts.azkarra.streams.context.internal; - -import io.streamthoughts.azkarra.api.config.Conf; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.Collections; -import java.util.List; -import java.util.Set; - -public class ApplicationConfigTest { - - @Test - public void shouldCreateContextConfGivenEmptyConf() { - ApplicationConfig contextConfig = ApplicationConfig.read(Conf.empty()); - - Assertions.assertNotNull(contextConfig.context()); - Assertions.assertNotNull(contextConfig.environments()); - Assertions.assertNotNull(contextConfig.components()); - } - - @Test - public void shouldCreateContextGivenConfWithProviders() { - - final Conf conf = Conf.of( - ApplicationConfig.CONTEXT_COMPONENT_CONFIG, - Collections.singletonList("test") - ); - ApplicationConfig contextConfig = ApplicationConfig.read(conf); - - Set providers = contextConfig.components(); - Assertions.assertNotNull(providers); - Assertions.assertEquals(1, providers.size()); - } - - @Test - public void shouldCreateContextGivenConfWithEnv() { - - Conf envConf = Conf.of(EnvironmentConfig.ENVIRONMENT_NAME_CONFIG, "__default"); - - final Conf conf = Conf.of( - ApplicationConfig.CONTEXT_ENVIRONMENTS_CONFIG, - Collections.singletonList(envConf)); - - ApplicationConfig contextConfig = ApplicationConfig.read(conf); - - List environmentConfigs = contextConfig.environments(); - Assertions.assertNotNull(environmentConfigs); - Assertions.assertEquals(1, environmentConfigs.size()); - } - -} \ No newline at end of file diff --git a/azkarra-ui/src/components/EnvironmentsList.vue b/azkarra-ui/src/components/EnvironmentsList.vue index 94ee7b72..09b9dde6 100644 --- a/azkarra-ui/src/components/EnvironmentsList.vue +++ b/azkarra-ui/src/components/EnvironmentsList.vue @@ -32,7 +32,7 @@
- Create + Create Environment
@@ -58,7 +58,7 @@