From befc11fb7b5450b6e06bee295745a6c03d1148ac Mon Sep 17 00:00:00 2001 From: Alan Chen Date: Fri, 4 Aug 2023 09:01:20 -0700 Subject: [PATCH] fix: armv7 null_smartengine updates to track sm/seng updates --- crates/fluvio-run/Cargo.toml | 1 - crates/fluvio-spu/Cargo.toml | 4 --- crates/fluvio-spu/src/smartengine/chain.rs | 6 ++-- crates/fluvio-spu/src/smartengine/context.rs | 2 +- crates/fluvio-spu/src/smartengine/mod.rs | 35 +++++++++++++++++--- justfile | 24 ++++++++++++-- makefiles/build.mk | 10 ++++-- 7 files changed, 63 insertions(+), 19 deletions(-) diff --git a/crates/fluvio-run/Cargo.toml b/crates/fluvio-run/Cargo.toml index b588a3a4d44..ef53f90f6d9 100644 --- a/crates/fluvio-run/Cargo.toml +++ b/crates/fluvio-run/Cargo.toml @@ -33,6 +33,5 @@ tracing-subscriber = { workspace = true } # regardless of TLS, sc and spu always use openssl_tls for now because we need cert API fluvio-future = { workspace = true, features = ["subscriber"] } fluvio-extension-common = { workspace = true } - fluvio-sc = { path = "../fluvio-sc", default-features = false } fluvio-spu = { path = "../fluvio-spu", default-features = false } diff --git a/crates/fluvio-spu/Cargo.toml b/crates/fluvio-spu/Cargo.toml index c59b60b4d8d..7358d931da9 100644 --- a/crates/fluvio-spu/Cargo.toml +++ b/crates/fluvio-spu/Cargo.toml @@ -79,7 +79,3 @@ portpicker = { workspace = true } flv-util = { workspace = true, features = ["fixture"] } fluvio-future = { workspace = true,features = ["fixture", "subscriber"] } fluvio-protocol = { workspace = true, features = ["fixture"] } - -# not used? -# [target.armv7-unknown-linux-gnueabi] -# linker="armv7-unknown-linux-gnueabi-gcc" diff --git a/crates/fluvio-spu/src/smartengine/chain.rs b/crates/fluvio-spu/src/smartengine/chain.rs index b119cb05277..b9ddfc26e40 100644 --- a/crates/fluvio-spu/src/smartengine/chain.rs +++ b/crates/fluvio-spu/src/smartengine/chain.rs @@ -4,18 +4,18 @@ use fluvio_protocol::link::ErrorCode; use fluvio_spu_schema::server::smartmodule::SmartModuleInvocation; #[cfg(feature = "smartengine")] -use fluvio_smartengine::{ - EngineError, SmartModuleChainBuilder, SmartModuleConfig, SmartModuleInitialData, -}; +use fluvio_smartengine::{EngineError, SmartModuleConfig, SmartModuleInitialData}; #[cfg(feature = "smartengine")] use fluvio_spu_schema::server::smartmodule::{SmartModuleContextData, SmartModuleKind}; +use crate::smartengine::SmartModuleChainBuilder; use crate::smartengine::SmartEngine; use crate::smartengine::SmartModuleChainInstance; #[cfg(not(feature = "smartengine"))] pub(crate) fn build_chain( + mut _chain_builder: SmartModuleChainBuilder, _invocations: Vec, _version: i16, _engine: SmartEngine, diff --git a/crates/fluvio-spu/src/smartengine/context.rs b/crates/fluvio-spu/src/smartengine/context.rs index 9e2363666b7..116dff778ae 100644 --- a/crates/fluvio-spu/src/smartengine/context.rs +++ b/crates/fluvio-spu/src/smartengine/context.rs @@ -3,7 +3,6 @@ use std::time::Duration; use async_rwlock::RwLock; use chrono::Utc; -use fluvio::SmartModuleChainBuilder; use fluvio_protocol::link::ErrorCode; use fluvio_smartmodule::Record; use fluvio_spu_schema::server::smartmodule::{SmartModuleInvocation, SmartModuleInvocationWasm}; @@ -18,6 +17,7 @@ use crate::replication::leader::LeaderReplicaState; use crate::smartengine::chain; use crate::smartengine::file_batch::{FileRecordIterator, FileBatchIterator}; use crate::smartengine::Lookback; +use crate::smartengine::SmartModuleChainBuilder; use crate::smartengine::SmartModuleChainInstance; use crate::smartengine::Version; diff --git a/crates/fluvio-spu/src/smartengine/mod.rs b/crates/fluvio-spu/src/smartengine/mod.rs index 940f1b067a9..2996aaff21a 100644 --- a/crates/fluvio-spu/src/smartengine/mod.rs +++ b/crates/fluvio-spu/src/smartengine/mod.rs @@ -14,8 +14,8 @@ mod chain; #[cfg(feature = "smartengine")] pub(crate) use fluvio_smartengine::{ - EngineError, metrics::SmartModuleChainMetrics, SmartEngine, SmartModuleChainInstance, Version, - Lookback, + EngineError, Lookback, SmartModuleChainBuilder, metrics::SmartModuleChainMetrics, SmartEngine, + SmartModuleChainInstance, Version, }; // Stub structures to support a null smartengine config @@ -23,6 +23,7 @@ pub(crate) use fluvio_smartengine::{ mod null_smartengine { use std::future::Future; use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::Duration; use anyhow::Result; use serde::{Deserialize, Serialize}; @@ -81,6 +82,14 @@ mod null_smartengine { } } + #[derive(Default)] + pub struct SmartModuleChainBuilder; + + impl SmartModuleChainBuilder { + pub fn set_store_memory_limit(&mut self, _max_memory_bytes: usize) {} + } + + #[derive(Debug)] pub struct SmartModuleChainInstance; impl SmartModuleChainInstance { @@ -101,7 +110,12 @@ mod null_smartengine { input: SmartModuleInput, _metric: &SmartModuleChainMetrics, ) -> Result { - let out = SmartModuleOutput::new(input.try_into()?); + use fluvio_smartmodule::SMARTMODULE_TIMESTAMPS_VERSION; + const DEFAULT_SMARTENGINE_VERSION: Version = SMARTMODULE_TIMESTAMPS_VERSION; + + #[allow(deprecated)] + let records = input.try_into_records(DEFAULT_SMARTENGINE_VERSION)?; + let out = SmartModuleOutput::new(records); Ok(out) } } @@ -117,8 +131,19 @@ mod null_smartengine { } #[allow(dead_code)] - #[derive(Debug)] - pub enum EngineError {} + #[derive(thiserror::Error, Debug)] + pub enum EngineError { + #[error("No valid smartmodule found")] + UnknownSmartModule, + #[error("Failed to instantiate: {0}")] + Instantiate(anyhow::Error), + #[error("Requested memory {requested}b exceeded max allowed {max}b")] + StoreMemoryExceeded { + current: usize, + requested: usize, + max: usize, + }, + } } #[cfg(not(feature = "smartengine"))] diff --git a/justfile b/justfile index a8b1163a428..d2e8f4d6539 100644 --- a/justfile +++ b/justfile @@ -2,13 +2,31 @@ default: just -l +# build smoketest for native and armv7 +check: + just build-spu-def + just build-spu + just build-run + +check-mk: + cargo clean -p fluvio-run + make build-cluster TARGET=armv7-unknown-linux-gnueabihf + +# arm7 spu build (works), odd part is the cmd line needs to add --no-default-features for the target +build-run: + cargo build --bin fluvio-run -p fluvio-run --release --target=armv7-unknown-linux-gnueabihf --no-default-features + # arm7 spu build (works) build-spu: - cargo build --target=armv7-unknown-linux-gnueabi -p fluvio-spu --no-default-features --release + cargo build --target=armv7-unknown-linux-gnueabihf -p fluvio-spu --no-default-features --release + +# default spu build +build-spu-def: + cargo build -p fluvio-spu --release # cross build armv7 (works) build-spu-cross: - cross build --target=armv7-unknown-linux-gnueabi -p fluvio-spu --no-default-features + cross build --target=armv7-unknown-linux-gnueabihf -p fluvio-spu --no-default-features # cross build armv7 musl (works) build-spu-cross-musl: @@ -20,7 +38,7 @@ build-spu-musl: # cargo zigbuild armv7 gnu (link issue) build-spu-zig-gnu: - cargo zigbuild --target=armv7-unknown-linux-gnueabi -p fluvio-spu --no-default-features --release + cargo zigbuild --target=armv7-unknown-linux-gnueabihf -p fluvio-spu --no-default-features --release # cargo zigbuild armv7 musl (issue) build-spu-zig-musl: diff --git a/makefiles/build.mk b/makefiles/build.mk index e5c97a0fb6d..5b2979fdc9a 100644 --- a/makefiles/build.mk +++ b/makefiles/build.mk @@ -16,8 +16,14 @@ build-cli-minimal: install_rustup_target cargo build --bin fluvio -p fluvio-cli $(RELEASE_FLAG) $(TARGET_FLAG) $(VERBOSE_FLAG) \ --no-default-features --features consumer,producer-file-io -build-cluster: install_rustup_target - cargo build --bin fluvio-run -p fluvio-run $(RELEASE_FLAG) $(TARGET_FLAG) $(VERBOSE_FLAG) $(DEBUG_SMARTMODULE_FLAG) +# note: careful that the if statement branches are leading spaces, tabs +ifeq ($(TARGET), armv7-unknown-linux-gnueabihf) + fluvio_run_extra=--no-default-features +else + fluvio_run_extra= +endif +build-cluster: + cargo build --bin fluvio-run -p fluvio-run $(RELEASE_FLAG) $(TARGET_FLAG) $(VERBOSE_FLAG) $(DEBUG_SMARTMODULE_FLAG) $(fluvio_run_extra) build-test: install_rustup_target cargo build --bin fluvio-test -p fluvio-test $(RELEASE_FLAG) $(TARGET_FLAG) $(VERBOSE_FLAG)