Skip to content

Commit

Permalink
feat(makefile): introduce local-env to run local aurora & ipfs (#1701)
Browse files Browse the repository at this point in the history 
* feat(local-env): introduce local-env to Makefile

* wip

* chore: remove serde(default) randomly

* chore: remove serde(default) randomly, more

* chore: update aurora version

* chore: freeze libp2p versions

* feat: introduce enable_logs_for

* feat: Add LogSpec::with_directives

* feat: default directive

* fix: typo

* chore: add old image

* chore: format makefile

* chore: revert config overrides in deployer.rs

* chore: fix clippy

* chore: fix typo

* Update docker-compose.yml

Co-authored-by: Aleksey Proshutisnkiy <justprosh@users.noreply.github.com>

---------

Co-authored-by: Aleksey Proshutisnkiy <justprosh@users.noreply.github.com>
  • Loading branch information
@folex @justprosh
folex and justprosh committed Jul 18, 2023
1 parent 15f1921 commit 9199ab1
Show file tree
Hide file tree
Showing 8 changed files with 239 additions and 97 deletions.
96 changes: 69 additions & 27 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,34 +8,76 @@ test:
cargo test --release

server:
RUST_LOG="info,tide=off,tracing=off,avm_server=off,run-console=debug,system_services=debug,sorcerer::spell_builtins=debug,sorcerer=debug" \
WASM_LOG="trace" \
RUST_LOG="debug,\
aquamarine::aqua_runtime=off,\
ipfs_effector=off,\
ipfs_pure=off,\
system_services=debug,\
marine_core::module::marine_module=info,\
aquamarine=warn,\
tokio_threadpool=info,\
tokio_reactor=info,\
mio=info,\
tokio_io=info,\
soketto=info,\
yamux=info,\
multistream_select=info,\
libp2p_secio=info,\
libp2p_websocket::framed=info,\
libp2p_ping=info,\
libp2p_core::upgrade::apply=info,\
libp2p_kad::kbucket=info,\
cranelift_codegen=info,\
wasmer_wasi=info,\
cranelift_codegen=info,\
wasmer_wasi=info,\
run-console=trace,\
wasmtime_cranelift=off,\
wasmtime_jit=off,\
particle_reap=off" \
cargo run --release -p nox

server-debug:
local-env:
docker compose -f docker-compose.yml up -d

local-env-logs:
docker compose -f docker-compose.yml logs -f

local-nox:
FLUENCE_ENV_AQUA_IPFS_EXTERNAL_API_MULTIADDR="/ip4/127.0.0.1/tcp/5001" \
FLUENCE_ENV_CONNECTOR_API_ENDPOINT=http://127.0.0.1:8545 \
FLUENCE_ENV_CONNECTOR_CONTRACT_ADDRESS="0xea6777e8c011E7968605fd012A9Dd49401ec386C" \
FLUENCE_ENV_CONNECTOR_FROM_BLOCK=earliest \
FLUENCE_ENV_AQUA_IPFS_LOCAL_API_MULTIADDR="/ip4/127.0.0.1/tcp/5001" \
WASM_LOG="trace" \
RUST_LOG="debug,\
tide=off,\
cranelift_codegen=info,\
yamux::connection::stream=info,\
tokio_threadpool=info,\
tokio_reactor=info,\
mio=info,\
tokio_io=info,\
soketto=info,\
yamux=info,\
multistream_select=info,\
libp2p_secio=info,\
libp2p_websocket::framed=info,\
libp2p_ping=info,\
libp2p_core::upgrade::apply=info,\
libp2p_plaintext=info,\
cranelift_codegen=info,\
wasmer_wasi=info,\
wasmer_interface_types_fl=info,\
async_std=info,\
async_io=info,\
polling=info, \
avm_server=off,\
tracing=off"\
cargo run --release -p nox -- -c ./deploy/Config.default.toml
aquamarine::aqua_runtime=error,\
ipfs_effector=off,\
ipfs_pure=off,\
system_services=debug,\
marine_core::module::marine_module=info,\
aquamarine=warn,\
tokio_threadpool=info,\
tokio_reactor=info,\
mio=info,\
tokio_io=info,\
soketto=info,\
yamux=info,\
multistream_select=info,\
libp2p_secio=info,\
libp2p_websocket::framed=info,\
libp2p_ping=info,\
libp2p_core::upgrade::apply=info,\
libp2p_kad::kbucket=info,\
cranelift_codegen=info,\
wasmer_wasi=info,\
cranelift_codegen=info,\
wasmer_wasi=info,\
run-console=trace,\
wasmtime_cranelift=off,\
wasmtime_jit=off,\
particle_reap=off" \
cargo run --release -p nox

.PHONY: server server-debug test release build deploy
.PHONY: server server-debug test release build deploy local-nox local-env local-env-logs
2 changes: 1 addition & 1 deletion aquamarine/src/aqua_runtime.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ impl AquaRuntime for AVM<DataStoreError> {
}
}
Ok((data, ..)) => {
tracing::warn!(
tracing::info!(
particle_id,
"Executed particle, next_peer_pks is empty, no call requests. Nothing to do.",
);
Expand Down
188 changes: 131 additions & 57 deletions crates/log-utils/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,69 +14,143 @@
* limitations under the License.
*/

use tracing_subscriber::filter::LevelFilter;
use log::Level;
use tracing_subscriber::filter::Directive;

fn default_directives() -> Vec<Directive> {
let namespaces = vec![
"script_storage=trace",
"run-console=trace",
"sorcerer=trace",
"key_manager=trace",
"spell_event_bus=trace",
"aquamarine=trace",
"network=trace",
"network_api=trace",
"aquamarine::actor=debug",
"nox::bootstrapper=info",
"yamux::connection::stream=info",
"tokio_threadpool=info",
"tokio_reactor=info",
"mio=info",
"tokio_io=info",
"soketto=info",
"yamix=info",
"multistream_select=info",
"libp2p_swarm=info",
"libp2p_secio=info",
"libp2p_websocket::framed=info",
"libp2p_ping=info",
"libp2p_core::upgrade::apply=info",
"libp2p_kad::kbucket=info",
"libp2p_kad=info",
"libp2p_kad::query=info",
"libp2p_kad::iterlog=info",
"libp2p_plaintext=info",
"libp2p_identify::protocol=info",
"cranelift_codegen=off",
"cranelift_codegen::context=off",
"wasmer_wasi=info",
"wasmer_interface_types_fl=info",
"polling=info",
"walrus=info",
"regalloc2=info",
"cranelift_wasm=info",
"wasmtime_cranelift=info",
"tokio=info",
"libp2p_noise=info",
"yamux=info",
"wasmtime_jit=info",
"wasi_common=info",
"particle_reap=info",
"marine_core::module::marine_module=info",
"runtime::resource=info",
];

namespaces
.into_iter()
.map(|ns| {
ns.parse()
.unwrap_or_else(|e| panic!("cannot parse {ns} to Directive: {e}"))
})
.collect()
}

#[allow(dead_code)]
// Enables logging, filtering out unnecessary details
pub fn enable_logs() {
std::env::set_var("WASM_LOG", "info");

tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::builder()
.with_default_directive(LevelFilter::TRACE.into())
.from_env_lossy()
.add_directive("script_storage=trace".parse().unwrap())
.add_directive("run-console=trace".parse().unwrap())
.add_directive("sorcerer=trace".parse().unwrap())
.add_directive("key_manager=trace".parse().unwrap())
.add_directive("spell_event_bus=trace".parse().unwrap())
.add_directive("aquamarine=trace".parse().unwrap())
.add_directive("network=trace".parse().unwrap())
.add_directive("network_api=trace".parse().unwrap())
.add_directive("aquamarine::actor=debug".parse().unwrap())
.add_directive("nox::bootstrapper=info".parse().unwrap())
.add_directive("yamux::connection::stream=info".parse().unwrap())
.add_directive("tokio_threadpool=info".parse().unwrap())
.add_directive("tokio_reactor=info".parse().unwrap())
.add_directive("mio=info".parse().unwrap())
.add_directive("tokio_io=info".parse().unwrap())
.add_directive("soketto=info".parse().unwrap())
.add_directive("yamix=info".parse().unwrap())
.add_directive("multistream_select=info".parse().unwrap())
.add_directive("libp2p_swarm=info".parse().unwrap())
.add_directive("libp2p_secio=info".parse().unwrap())
.add_directive("libp2p_websocket::framed=info".parse().unwrap())
.add_directive("libp2p_ping=info".parse().unwrap())
.add_directive("libp2p_core::upgrade::apply=info".parse().unwrap())
.add_directive("libp2p_kad::kbucket=info".parse().unwrap())
.add_directive("libp2p_kad=info".parse().unwrap())
.add_directive("libp2p_kad::query=info".parse().unwrap())
.add_directive("libp2p_kad::iterlog=info".parse().unwrap())
.add_directive("libp2p_plaintext=info".parse().unwrap())
.add_directive("libp2p_identify::protocol=info".parse().unwrap())
.add_directive("cranelift_codegen=off".parse().unwrap())
.add_directive("cranelift_codegen::context=off".parse().unwrap())
.add_directive("wasmer_wasi=info".parse().unwrap())
.add_directive("wasmer_interface_types_fl=info".parse().unwrap())
.add_directive("polling=info".parse().unwrap())
.add_directive("walrus=info".parse().unwrap())
.add_directive("regalloc2=info".parse().unwrap())
.add_directive("cranelift_wasm=info".parse().unwrap())
.add_directive("wasmtime_cranelift=info".parse().unwrap())
.add_directive("tokio=info".parse().unwrap())
.add_directive("libp2p_noise=info".parse().unwrap())
.add_directive("yamux=info".parse().unwrap())
.add_directive("wasmtime_jit=info".parse().unwrap())
.add_directive("wasi_common=info".parse().unwrap())
.add_directive("particle_reap=info".parse().unwrap())
.add_directive("marine_core::module::marine_module=info".parse().unwrap())
.add_directive("runtime::resource=info".parse().unwrap()),
)
.try_init()
.ok();
enable_logs_for(LogSpec::default())
}

pub fn enable_console() {
console_subscriber::init();
}

pub struct LogSpec {
level: tracing::metadata::Level,
directives: Vec<Directive>,
wasm_log: Level,
}

impl Default for LogSpec {
fn default() -> Self {
Self::new(vec![])
.with_defaults()
.with_level(tracing::metadata::Level::INFO)
.with_wasm_level(Level::Info)
}
}

impl LogSpec {
pub fn new(directives: Vec<Directive>) -> Self {
Self {
level: tracing::metadata::Level::INFO,
directives,
wasm_log: Level::Info,
}
}

pub fn with_level(mut self, level: tracing::metadata::Level) -> Self {
self.level = level;

self
}

pub fn with_defaults(mut self) -> Self {
self.directives = default_directives()
.into_iter()
.chain(self.directives)
.collect();

self
}

pub fn with_wasm_level(mut self, level: Level) -> Self {
self.wasm_log = level;

self
}

pub fn with_directives(mut self, directives: Vec<Directive>) -> Self {
self.directives = self.directives.into_iter().chain(directives).collect();

self
}
}

pub fn enable_logs_for(spec: LogSpec) {
std::env::set_var("WASM_LOG", spec.wasm_log.to_string().to_lowercase());

let mut filter = tracing_subscriber::EnvFilter::builder()
.with_default_directive(spec.level.into())
.from_env_lossy();

for d in spec.directives {
filter = filter.add_directive(d);
}

tracing_subscriber::fmt()
.with_env_filter(filter)
.try_init()
.ok();
}
6 changes: 0 additions & 6 deletions crates/server-config/src/dir_config.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,26 +29,20 @@ pub struct UnresolvedDirConfig {
pub base_dir: PathBuf,

/// Base directory for resources needed by application services
#[serde(default)]
pub services_base_dir: Option<PathBuf>,

/// Base directory for builtin services
#[serde(default)]
pub builtins_base_dir: Option<PathBuf>,

/// Base directory for resources needed by application services
#[serde(default)]
pub avm_base_dir: Option<PathBuf>,

/// Path to AIR interpreter .wasm file (aquamarine.wasm)
#[serde(default)]
pub air_interpreter_path: Option<PathBuf>,

/// Path to spell service files (wasms, configs)
#[serde(default)]
pub spell_base_dir: Option<PathBuf>,

#[serde(default)]
pub keypairs_base_dir: Option<PathBuf>,
}

Expand Down
6 changes: 1 addition & 5 deletions crates/server-config/src/resolved_config.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,11 @@ pub struct UnresolvedConfig {
dir_config: UnresolvedDirConfig,
#[serde(flatten)]
node_config: UnresolvedNodeConfig,
#[serde(default)]

pub log: Option<LogConfig>,
#[serde(default)]
pub tracing: Option<TracingConfig>,
#[serde(default)]
pub console: Option<ConsoleConfig>,

pub no_banner: Option<bool>,

pub print_config: Option<bool>,
}

Expand Down
5 changes: 5 additions & 0 deletions crates/system-services/src/deployer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ const DEPLOYER_TTL: u64 = 60_000;
const DEPLOYER_PARTICLE_ID: &str = "system-services-deployment";

// A status of a service/spell after deployment
#[derive(Clone, Debug)]
enum ServiceStatus {
// Id of a newly created service
Created(String),
Expand All @@ -30,6 +31,7 @@ enum ServiceStatus {
}

// Status of the service or spell before deployment
#[derive(Clone, Debug)]
enum ServiceUpdateStatus {
// A service is found and we need to update it
NeedUpdate(String),
Expand All @@ -40,19 +42,22 @@ enum ServiceUpdateStatus {
}

// This is supposed to be in a separate lib for all system services crates
#[derive(Clone, Debug)]
struct ServiceDistro {
modules: HashMap<&'static str, &'static [u8]>,
config: TomlMarineConfig,
name: String,
}

#[derive(Clone, Debug)]
struct SpellDistro {
name: String,
air: &'static str,
kv: HashMap<&'static str, JValue>,
trigger_config: TriggerConfig,
}

#[derive(Clone, Debug)]
pub struct Deployer {
// These fields are used for deploying system services
services: ParticleAppServices,
Expand Down
Loading

0 comments on commit 9199ab1

Please sign in to comment.