diff --git a/apps/emqx_bridge_cassandra/docker-ct b/apps/emqx_bridge_cassandra/docker-ct new file mode 100644 index 0000000000..2626b40685 --- /dev/null +++ b/apps/emqx_bridge_cassandra/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +cassandra diff --git a/apps/emqx_bridge_cassandra/include/emqx_bridge_cassandra.hrl b/apps/emqx_bridge_cassandra/include/emqx_bridge_cassandra.hrl new file mode 100644 index 0000000000..eef7c5d2b5 --- /dev/null +++ b/apps/emqx_bridge_cassandra/include/emqx_bridge_cassandra.hrl @@ -0,0 +1,5 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-define(CASSANDRA_DEFAULT_PORT, 9042). diff --git a/apps/emqx_bridge_cassandra/rebar.config b/apps/emqx_bridge_cassandra/rebar.config new file mode 100644 index 0000000000..b8bfc7dd6f --- /dev/null +++ b/apps/emqx_bridge_cassandra/rebar.config @@ -0,0 +1,11 @@ +%% -*- mode: erlang; -*- +{erl_opts, [debug_info]}. +{deps, [ {ecql, {git, "https://github.com/emqx/ecql.git", {tag, "v0.5.1"}}} + , {emqx_connector, {path, "../../apps/emqx_connector"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + ]}. + +{shell, [ + {apps, [emqx_bridge_cassandra]} +]}. diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src index 1ed65ea9f0..58e4a1984b 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src @@ -2,7 +2,7 @@ {description, "EMQX Enterprise Cassandra Bridge"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, ecql]}, {env, []}, {modules, []}, {links, []} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl similarity index 97% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl rename to apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl index 26c6de04d6..e8f7d50ce8 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_cassa). +-module(emqx_bridge_cassandra). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -88,7 +88,7 @@ fields("config") -> #{desc => ?DESC("local_topic"), default => undefined} )} ] ++ emqx_resource_schema:fields("resource_opts") ++ - (emqx_ee_connector_cassa:fields(config) -- + (emqx_bridge_cassandra_connector:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); fields("post") -> fields("post", cassandra); diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl similarity index 99% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl rename to apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl index 397532f47f..cf6ddff9f2 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl @@ -2,12 +2,12 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_cassa). +-module(emqx_bridge_cassandra_connector). -behaviour(emqx_resource). -include_lib("emqx_connector/include/emqx_connector.hrl"). --include_lib("emqx_ee_connector/include/emqx_ee_connector.hrl"). +-include("emqx_bridge_cassandra.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("hocon/include/hoconsc.hrl"). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl similarity index 99% rename from lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl rename to apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl index 4711d1981e..7865f0415b 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_cassa_SUITE). +-module(emqx_bridge_cassandra_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -57,7 +57,7 @@ %% CASSA_TCP_HOST=127.0.0.1 CASSA_TCP_PORT=19042 \ %% CASSA_TLS_HOST=127.0.0.1 CASSA_TLS_PORT=19142 \ %% PROXY_HOST=127.0.0.1 ./rebar3 as test ct -c -v --name ct@127.0.0.1 \ -%% --suite lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +%% --suite apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl %% %%------------------------------------------------------------------------------ @@ -590,7 +590,7 @@ t_missing_data(Config) -> {ok, _}, create_bridge(Config) ), - %% emqx_ee_connector_cassa will send missed data as a `null` atom + %% emqx_bridge_cassandra_connector will send missed data as a `null` atom %% to ecql driver ?check_trace( begin diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl similarity index 97% rename from lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl rename to apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl index f2647d756b..f419283a8b 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl @@ -2,13 +2,13 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_cassa_SUITE). +-module(emqx_bridge_cassandra_connector_SUITE). -compile(nowarn_export_all). -compile(export_all). --include("emqx_connector.hrl"). --include("emqx_ee_connector.hrl"). +-include("emqx_bridge_cassandra.hrl"). +-include("emqx_connector/include/emqx_connector.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("stdlib/include/assert.hrl"). @@ -16,7 +16,7 @@ %% Cassandra server defined at `.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml` %% You can change it to `127.0.0.1`, if you run this SUITE locally -define(CASSANDRA_HOST, "cassandra"). --define(CASSANDRA_RESOURCE_MOD, emqx_ee_connector_cassa). +-define(CASSANDRA_RESOURCE_MOD, emqx_bridge_cassandra_connector). %% This test SUITE requires a running cassandra instance. If you don't want to %% bring up the whole CI infrastuctucture with the `scripts/ct/run.sh` script diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index 35d6b9d5bb..aa19a495f8 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -10,5 +10,4 @@ tdengine clickhouse dynamo rocketmq -cassandra sqlserver diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index b26df658a2..358ff3bc89 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -1,6 +1,5 @@ {erl_opts, [debug_info]}. -{deps, [ {ecql, {git, "https://github.com/emqx/ecql.git", {tag, "v0.5.1"}}} - , {emqx_connector, {path, "../../apps/emqx_connector"}} +{deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} , {emqx_utils, {path, "../emqx_utils"}} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 9465464d93..7fdfbba997 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -17,6 +17,7 @@ api_schemas(Method) -> ref(emqx_bridge_gcp_pubsub, Method), ref(emqx_bridge_kafka, Method ++ "_consumer"), ref(emqx_bridge_kafka, Method ++ "_producer"), + ref(emqx_bridge_cassandra, Method), ref(emqx_ee_bridge_mysql, Method), ref(emqx_ee_bridge_pgsql, Method), ref(emqx_ee_bridge_mongodb, Method ++ "_rs"), @@ -34,13 +35,13 @@ api_schemas(Method) -> ref(emqx_ee_bridge_clickhouse, Method), ref(emqx_ee_bridge_dynamo, Method), ref(emqx_ee_bridge_rocketmq, Method), - ref(emqx_ee_bridge_cassa, Method), ref(emqx_ee_bridge_sqlserver, Method) ]. schema_modules() -> [ emqx_bridge_kafka, + emqx_bridge_cassandra, emqx_ee_bridge_hstreamdb, emqx_bridge_gcp_pubsub, emqx_ee_bridge_influxdb, @@ -54,7 +55,6 @@ schema_modules() -> emqx_ee_bridge_clickhouse, emqx_ee_bridge_dynamo, emqx_ee_bridge_rocketmq, - emqx_ee_bridge_cassa, emqx_ee_bridge_sqlserver ]. @@ -75,6 +75,7 @@ resource_type(kafka_consumer) -> emqx_bridge_kafka_impl_consumer; %% TODO: rename this to `kafka_producer' after alias support is added %% to hocon; keeping this as just `kafka' for backwards compatibility. resource_type(kafka) -> emqx_bridge_kafka_impl_producer; +resource_type(cassandra) -> emqx_bridge_cassandra_connector; resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb; resource_type(gcp_pubsub) -> emqx_bridge_gcp_pubsub_connector; resource_type(mongodb_rs) -> emqx_ee_connector_mongodb; @@ -93,7 +94,6 @@ resource_type(tdengine) -> emqx_ee_connector_tdengine; resource_type(clickhouse) -> emqx_ee_connector_clickhouse; resource_type(dynamo) -> emqx_ee_connector_dynamo; resource_type(rocketmq) -> emqx_ee_connector_rocketmq; -resource_type(cassandra) -> emqx_ee_connector_cassa; resource_type(sqlserver) -> emqx_ee_connector_sqlserver. fields(bridges) -> @@ -148,7 +148,7 @@ fields(bridges) -> )}, {cassandra, mk( - hoconsc:map(name, ref(emqx_ee_bridge_cassa, "config")), + hoconsc:map(name, ref(emqx_bridge_cassandra, "config")), #{ desc => <<"Cassandra Bridge Config">>, required => false diff --git a/lib-ee/emqx_ee_connector/docker-ct b/lib-ee/emqx_ee_connector/docker-ct index fc8e75e689..cb2f6f0281 100644 --- a/lib-ee/emqx_ee_connector/docker-ct +++ b/lib-ee/emqx_ee_connector/docker-ct @@ -1,5 +1,4 @@ toxiproxy influxdb clickhouse -cassandra sqlserver diff --git a/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl b/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl index 2a91d2524a..4b6fbbd926 100644 --- a/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl +++ b/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl @@ -3,4 +3,3 @@ %%------------------------------------------------------------------- -define(INFLUXDB_DEFAULT_PORT, 8086). --define(CASSANDRA_DEFAULT_PORT, 9042). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index ced7ae86a8..82f556bdb4 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -12,7 +12,6 @@ clickhouse, erlcloud, rocketmq, - ecql, odbc ]}, {env, []}, diff --git a/mix.exs b/mix.exs index f3149a5843..c5d6df8043 100644 --- a/mix.exs +++ b/mix.exs @@ -155,7 +155,20 @@ defmodule EMQXUmbrella.MixProject do defp enterprise_umbrella_apps() do MapSet.new([ :emqx_bridge_kafka, - :emqx_bridge_gcp_pubsub + :emqx_bridge_gcp_pubsub, + :emqx_bridge_cassandra, + :emqx_bridge_clickhouse, + :emqx_bridge_dynamo, + :emqx_bridge_hstreamdb, + :emqx_bridge_influxdb, + :emqx_bridge_matrix, + :emqx_bridge_mongodb, + :emqx_bridge_mysql, + :emqx_bridge_pgsql, + :emqx_bridge_redis, + :emqx_bridge_rocketmq, + :emqx_bridge_tdengine, + :emqx_bridge_timescale ]) end @@ -346,6 +359,19 @@ defmodule EMQXUmbrella.MixProject do emqx_ee_bridge: :permanent, emqx_bridge_kafka: :permanent, emqx_bridge_gcp_pubsub: :permanent, + emqx_bridge_cassandra: :permanent, + emqx_bridge_clickhouse: :permanent, + emqx_bridge_dynamo: :permanent, + emqx_bridge_hstreamdb: :permanent, + emqx_bridge_influxdb: :permanent, + emqx_bridge_matrix: :permanent, + emqx_bridge_mongodb: :permanent, + emqx_bridge_mysql: :permanent, + emqx_bridge_pgsql: :permanent, + emqx_bridge_redis: :permanent, + emqx_bridge_rocketmq: :permanent, + emqx_bridge_tdengine: :permanent, + emqx_bridge_timescale: :permanent, emqx_ee_schema_registry: :permanent ], else: [] diff --git a/rebar.config.erl b/rebar.config.erl index 9bdbfb8481..88471c39d3 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -80,6 +80,19 @@ is_enterprise(ee) -> true. is_community_umbrella_app("apps/emqx_bridge_kafka") -> false; is_community_umbrella_app("apps/emqx_bridge_gcp_pubsub") -> false; +is_community_umbrella_app("apps/emqx_bridge_cassandra") -> false; +is_community_umbrella_app("apps/emqx_bridge_clickhouse") -> false; +is_community_umbrella_app("apps/emqx_bridge_dynamo") -> false; +is_community_umbrella_app("apps/emqx_bridge_hstreamdb") -> false; +is_community_umbrella_app("apps/emqx_bridge_influxdb") -> false; +is_community_umbrella_app("apps/emqx_bridge_matrix") -> false; +is_community_umbrella_app("apps/emqx_bridge_mongodb") -> false; +is_community_umbrella_app("apps/emqx_bridge_mysql") -> false; +is_community_umbrella_app("apps/emqx_bridge_pgsql") -> false; +is_community_umbrella_app("apps/emqx_bridge_redis") -> false; +is_community_umbrella_app("apps/emqx_bridge_rocketmq") -> false; +is_community_umbrella_app("apps/emqx_bridge_tdengine") -> false; +is_community_umbrella_app("apps/emqx_bridge_timescale") -> false; is_community_umbrella_app(_) -> true. is_jq_supported() -> @@ -441,6 +454,19 @@ relx_apps_per_edition(ee) -> emqx_ee_bridge, emqx_bridge_kafka, emqx_bridge_gcp_pubsub, + emqx_bridge_cassandra, + emqx_bridge_clickhouse, + emqx_bridge_dynamo, + emqx_bridge_hstreamdb, + emqx_bridge_influxdb, + emqx_bridge_matrix, + emqx_bridge_mongodb, + emqx_bridge_mysql, + emqx_bridge_pgsql, + emqx_bridge_redis, + emqx_bridge_rocketmq, + emqx_bridge_tdengine, + emqx_bridge_timescale, emqx_ee_schema_registry ]; relx_apps_per_edition(ce) -> diff --git a/rel/i18n/emqx_ee_bridge_cassa.hocon b/rel/i18n/emqx_bridge_cassandra.hocon similarity index 97% rename from rel/i18n/emqx_ee_bridge_cassa.hocon rename to rel/i18n/emqx_bridge_cassandra.hocon index d86c95a5fc..d598d3921c 100644 --- a/rel/i18n/emqx_ee_bridge_cassa.hocon +++ b/rel/i18n/emqx_bridge_cassandra.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_cassa { +emqx_bridge_cassandra { config_enable.desc: """Enable or disable this bridge""" diff --git a/rel/i18n/emqx_ee_connector_cassa.hocon b/rel/i18n/emqx_bridge_cassandra_connector.hocon similarity index 90% rename from rel/i18n/emqx_ee_connector_cassa.hocon rename to rel/i18n/emqx_bridge_cassandra_connector.hocon index bd5fb544c4..b149cce8a0 100644 --- a/rel/i18n/emqx_ee_connector_cassa.hocon +++ b/rel/i18n/emqx_bridge_cassandra_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_cassa { +emqx_bridge_cassandra_connector { keyspace.desc: """Keyspace name to connect to.""" diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 4e79476e05..ab7fff4443 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -108,9 +108,12 @@ case "${WHICH_APP}" in ## ensure enterprise profile when testing lib-ee applications export PROFILE='emqx-enterprise' ;; - apps/emqx_bridge_kafka) - ## ensure enterprise profile when testing ee applications - export PROFILE='emqx-enterprise' + apps/*) + if [[ -f "${WHICH_APP}/BSL.txt" ]]; then + export PROFILE='emqx-enterprise' + else + export PROFILE='emqx' + fi ;; *) export PROFILE="${PROFILE:-emqx}"