From 95ce9ba284c8014fdb3ccac3eba950b80931210a Mon Sep 17 00:00:00 2001 From: Adrien Guillo Date: Tue, 7 Sep 2021 19:46:17 -0700 Subject: [PATCH] Apply fmt --- quickwit-actors/src/actor.rs | 44 +++---- quickwit-actors/src/actor_handle.rs | 59 +++++---- quickwit-actors/src/actor_state.rs | 32 +++-- quickwit-actors/src/actor_with_state_tx.rs | 37 +++--- quickwit-actors/src/async_actor.rs | 40 +++--- quickwit-actors/src/channel_with_priority.rs | 46 ++++--- quickwit-actors/src/kill_switch.rs | 32 +++-- quickwit-actors/src/lib.rs | 55 ++++---- quickwit-actors/src/mailbox.rs | 39 +++--- quickwit-actors/src/observation.rs | 29 ++--- quickwit-actors/src/progress.rs | 32 +++-- quickwit-actors/src/scheduler.rs | 60 +++++---- quickwit-actors/src/spawn_builder.rs | 29 ++--- quickwit-actors/src/sync_actor.rs | 32 ++--- quickwit-actors/src/tests.rs | 57 ++++----- quickwit-actors/src/universe.rs | 51 +++----- quickwit-cli/build.rs | 39 +++--- quickwit-cli/src/lib.rs | 118 ++++++++--------- quickwit-cli/src/main.rs | 72 ++++++----- quickwit-cli/tests/cli.rs | 52 ++++---- quickwit-cli/tests/helpers.rs | 58 ++++----- quickwit-cluster/src/cluster.rs | 35 +++-- quickwit-cluster/src/error.rs | 29 ++--- quickwit-cluster/src/lib.rs | 29 ++--- quickwit-cluster/src/service.rs | 33 +++-- quickwit-cluster/src/test_utils.rs | 39 +++--- quickwit-common/src/coolid.rs | 32 ++--- quickwit-common/src/lib.rs | 52 ++++---- quickwit-core/src/index.rs | 54 ++++---- quickwit-core/src/lib.rs | 49 ++++--- quickwit-directories/src/bundle_directory.rs | 56 ++++---- quickwit-directories/src/caching_directory.rs | 56 ++++---- .../src/debug_proxy_directory.rs | 54 ++++---- quickwit-directories/src/hot_directory.rs | 61 ++++----- quickwit-directories/src/lib.rs | 57 ++++----- quickwit-directories/src/storage_directory.rs | 55 ++++---- .../src/all_flatten_config.rs | 56 ++++---- quickwit-index-config/src/config.rs | 52 ++++---- .../default_index_config/default_config.rs | 107 +++++++++------- .../field_mapping_entry.rs | 107 ++++++++++------ .../field_mapping_type.rs | 39 +++--- .../src/default_index_config/mod.rs | 39 +++--- quickwit-index-config/src/error.rs | 39 +++--- quickwit-index-config/src/lib.rs | 42 +++--- quickwit-index-config/src/query_builder.rs | 43 +++---- quickwit-index-config/src/wikipedia_config.rs | 46 ++++--- quickwit-indexing/failpoints/mod.rs | 46 +++---- quickwit-indexing/src/actors/indexer.rs | 74 +++++------ quickwit-indexing/src/actors/mod.rs | 36 +++--- quickwit-indexing/src/actors/packager.rs | 60 ++++----- .../src/actors/pipeline_supervisor.rs | 86 ++++++------- quickwit-indexing/src/actors/publisher.rs | 47 ++++--- quickwit-indexing/src/actors/uploader.rs | 66 ++++------ quickwit-indexing/src/lib.rs | 36 +++--- quickwit-indexing/src/merge_policy.rs | 37 +++--- quickwit-indexing/src/models/commit_policy.rs | 29 ++--- quickwit-indexing/src/models/indexed_split.rs | 32 +++-- .../src/models/indexer_message.rs | 29 ++--- .../src/models/indexing_statistics.rs | 45 +++---- quickwit-indexing/src/models/mod.rs | 29 ++--- .../src/models/packaged_split.rs | 35 ++--- quickwit-indexing/src/models/raw_doc_batch.rs | 28 ++-- .../src/models/scratch_directory.rs | 38 +++--- .../src/models/uploaded_split.rs | 29 ++--- quickwit-indexing/src/semaphore.rs | 42 +++--- quickwit-indexing/src/source/file_source.rs | 62 ++++----- quickwit-indexing/src/source/kafka_source.rs | 86 +++++++------ quickwit-indexing/src/source/mod.rs | 48 ++++--- .../src/source/source_factory.rs | 45 ++++--- quickwit-indexing/src/source/vec_source.rs | 51 ++++---- quickwit-indexing/src/test_utils.rs | 50 ++++---- quickwit-metastore/src/checkpoint.rs | 67 +++++----- quickwit-metastore/src/error.rs | 39 +++--- quickwit-metastore/src/lib.rs | 52 ++++---- quickwit-metastore/src/metastore.rs | 69 +++++----- .../src/metastore/single_file_metastore.rs | 54 ++++---- quickwit-metastore/src/metastore_resolver.rs | 39 +++--- quickwit-metastore/src/tests.rs | 42 +++--- quickwit-proto/build.rs | 29 ++--- quickwit-proto/src/lib.rs | 29 ++--- quickwit-search/src/client.rs | 57 ++++----- quickwit-search/src/client_pool.rs | 41 +++--- .../src/client_pool/search_client_pool.rs | 50 ++++---- quickwit-search/src/collector.rs | 70 +++++----- quickwit-search/src/error.rs | 46 ++++--- quickwit-search/src/fetch_docs.rs | 37 +++--- quickwit-search/src/filters.rs | 34 +++-- quickwit-search/src/leaf.rs | 48 +++---- quickwit-search/src/lib.rs | 60 ++++----- quickwit-search/src/rendezvous_hasher.rs | 41 +++--- quickwit-search/src/root.rs | 120 +++++++++--------- quickwit-search/src/search_result_json.rs | 29 ++--- .../src/search_stream/collector.rs | 42 +++--- quickwit-search/src/search_stream/leaf.rs | 58 ++++----- quickwit-search/src/search_stream/mod.rs | 37 +++--- quickwit-search/src/search_stream/root.rs | 69 +++++----- quickwit-search/src/service.rs | 53 ++++---- quickwit-serve/src/args.rs | 38 +++--- quickwit-serve/src/error.rs | 47 +++---- quickwit-serve/src/grpc.rs | 43 +++---- quickwit-serve/src/grpc_adapter.rs | 38 +++--- .../src/grpc_adapter/cluster_adapter.rs | 39 +++--- .../src/grpc_adapter/search_adapter.rs | 39 +++--- quickwit-serve/src/http_handler.rs | 38 +++--- quickwit-serve/src/http_handler/cluster.rs | 44 +++---- .../src/http_handler/health_check.rs | 38 +++--- quickwit-serve/src/lib.rs | 63 ++++----- quickwit-serve/src/quickwit_cache.rs | 46 ++++--- quickwit-serve/src/rest.rs | 94 ++++++++------ quickwit-storage/src/bundle_storage.rs | 59 ++++----- .../src/cache/in_ram_slice_cache.rs | 29 ++--- .../src/cache/memory_sized_cache.rs | 36 +++--- quickwit-storage/src/cache/mod.rs | 39 +++--- .../src/cache/storage_with_cache.rs | 40 +++--- quickwit-storage/src/error.rs | 49 +++---- quickwit-storage/src/lib.rs | 81 ++++++------ quickwit-storage/src/local_file_storage.rs | 52 ++++---- quickwit-storage/src/object_storage/error.rs | 45 +++---- .../src/object_storage/file_slice_stream.rs | 48 ++++--- quickwit-storage/src/object_storage/mod.rs | 39 +++--- quickwit-storage/src/object_storage/policy.rs | 39 +++--- .../object_storage/s3_compatible_storage.rs | 59 ++++----- .../s3_compatible_storage_uri_resolver.rs | 42 +++--- quickwit-storage/src/prefix_storage.rs | 48 ++++--- quickwit-storage/src/ram_storage.rs | 52 ++++---- quickwit-storage/src/retry.rs | 46 ++++--- quickwit-storage/src/storage.rs | 44 +++---- quickwit-storage/src/storage_resolver.rs | 53 ++++---- quickwit-storage/tests/s3_storage.rs | 39 +++--- quickwit-swim/src/cluster.rs | 18 +-- quickwit-swim/src/cluster_config.rs | 10 +- quickwit-swim/src/errors.rs | 6 +- quickwit-swim/src/member.rs | 35 ++--- quickwit-swim/src/membership.rs | 13 +- quickwit-swim/src/state.rs | 69 +++++----- quickwit-telemetry/src/lib.rs | 43 +++---- quickwit-telemetry/src/payload.rs | 42 +++--- quickwit-telemetry/src/sender.rs | 50 ++++---- quickwit-telemetry/src/sink.rs | 42 +++--- 139 files changed, 3140 insertions(+), 3446 deletions(-) diff --git a/quickwit-actors/src/actor.rs b/quickwit-actors/src/actor.rs index 6bf1ea35f3..f21799e77c 100644 --- a/quickwit-actors/src/actor.rs +++ b/quickwit-actors/src/actor.rs @@ -1,40 +1,39 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +use std::any::type_name; use std::fmt; use std::ops::Deref; +use std::sync::Arc; use std::time::Duration; -use std::{any::type_name, sync::Arc}; + use thiserror::Error; use tokio::sync::watch::Sender; use tracing::{debug, error}; +use crate::actor_state::{ActorState, AtomicState}; use crate::channel_with_priority::Priority; use crate::mailbox::{Command, CommandOrMessage}; +use crate::progress::{Progress, ProtectedZoneGuard}; use crate::scheduler::{Callback, SchedulerMessage}; use crate::spawn_builder::SpawnBuilder; -use crate::{ - actor_state::{ActorState, AtomicState}, - progress::{Progress, ProtectedZoneGuard}, - KillSwitch, Mailbox, QueueCapacity, SendError, -}; +use crate::{KillSwitch, Mailbox, QueueCapacity, SendError}; /// The actor exit status represents the outcome of the execution of an actor, /// after the end of the execution. @@ -58,7 +57,8 @@ pub enum ActorExitStatus { /// The actor was asked to gracefully shutdown. /// - /// (Semantically equivalent to exit status code 130, triggered by SIGINT aka Ctrl-C, or SIGQUIT) + /// (Semantically equivalent to exit status code 130, triggered by SIGINT aka Ctrl-C, or + /// SIGQUIT) #[error("Quit")] Quit, diff --git a/quickwit-actors/src/actor_handle.rs b/quickwit-actors/src/actor_handle.rs index 0471e75676..2f4b3044f2 100644 --- a/quickwit-actors/src/actor_handle.rs +++ b/quickwit-actors/src/actor_handle.rs @@ -1,37 +1,38 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -use crate::actor_state::ActorState; -use crate::channel_with_priority::Priority; -use crate::mailbox::Command; -use crate::observation::ObservationType; -use crate::{Actor, ActorContext, ActorExitStatus, Observation}; +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + use std::any::Any; use std::borrow::Borrow; use std::fmt; use std::sync::Arc; + use tokio::sync::{oneshot, watch}; use tokio::task::JoinHandle; use tokio::time::timeout; use tracing::error; +use crate::actor_state::ActorState; +use crate::channel_with_priority::Priority; +use crate::mailbox::Command; +use crate::observation::ObservationType; +use crate::{Actor, ActorContext, ActorExitStatus, Observation}; + /// An Actor Handle serves as an address to communicate with an actor. pub struct ActorHandle { actor_context: ActorContext, @@ -134,8 +135,8 @@ impl ActorHandle { "Failed to send observe message" ); } - // The timeout is required here. If the actor fails, its inbox is properly dropped but the send channel might actually - // prevent the onechannel Receiver from being dropped. + // The timeout is required here. If the actor fails, its inbox is properly dropped but the + // send channel might actually prevent the onechannel Receiver from being dropped. self.wait_for_observable_state_callback(rx).await } @@ -275,12 +276,10 @@ impl ActorHandle { #[cfg(test)] mod tests { - use crate::AsyncActor; - use crate::SyncActor; - use crate::Universe; use async_trait::async_trait; use super::*; + use crate::{AsyncActor, SyncActor, Universe}; #[derive(Default)] struct PanickingActor { diff --git a/quickwit-actors/src/actor_state.rs b/quickwit-actors/src/actor_state.rs index c37ca85301..9847ccb195 100644 --- a/quickwit-actors/src/actor_state.rs +++ b/quickwit-actors/src/actor_state.rs @@ -1,25 +1,23 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . -use std::sync::atomic::AtomicU32; -use std::sync::atomic::Ordering; +use std::sync::atomic::{AtomicU32, Ordering}; #[repr(u32)] #[derive(Clone, Debug, Copy, Eq, PartialEq)] diff --git a/quickwit-actors/src/actor_with_state_tx.rs b/quickwit-actors/src/actor_with_state_tx.rs index fe039ea3f3..a2c870dad0 100644 --- a/quickwit-actors/src/actor_with_state_tx.rs +++ b/quickwit-actors/src/actor_with_state_tx.rs @@ -1,26 +1,25 @@ -use tokio::sync::watch::Sender; - -use crate::Actor; - -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use tokio::sync::watch::Sender; + +use crate::Actor; pub struct ActorWithStateTx { pub actor: A, diff --git a/quickwit-actors/src/async_actor.rs b/quickwit-actors/src/async_actor.rs index 30c0075619..f702756d32 100644 --- a/quickwit-actors/src/async_actor.rs +++ b/quickwit-actors/src/async_actor.rs @@ -1,22 +1,27 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use anyhow::Context; +use async_trait::async_trait; +use tokio::sync::watch::{self, Sender}; +use tokio::task::JoinHandle; +use tracing::{debug, error, info}; use crate::actor::{process_command, ActorExitStatus}; use crate::actor_handle::ActorHandle; @@ -24,11 +29,6 @@ use crate::actor_state::ActorState; use crate::actor_with_state_tx::ActorWithStateTx; use crate::mailbox::{CommandOrMessage, Inbox}; use crate::{Actor, ActorContext, RecvError}; -use anyhow::Context; -use async_trait::async_trait; -use tokio::sync::watch::{self, Sender}; -use tokio::task::JoinHandle; -use tracing::{debug, error, info}; /// An async actor is executed on a regular tokio task. /// diff --git a/quickwit-actors/src/channel_with_priority.rs b/quickwit-actors/src/channel_with_priority.rs index 29ec3e2496..89f59e57c0 100644 --- a/quickwit-actors/src/channel_with_priority.rs +++ b/quickwit-actors/src/channel_with_priority.rs @@ -1,26 +1,25 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -use flume::RecvTimeoutError; -use flume::TryRecvError; +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + use std::time::Duration; + +use flume::{RecvTimeoutError, TryRecvError}; use thiserror::Error; #[derive(Debug, Error)] @@ -257,8 +256,7 @@ impl Receiver { #[cfg(test)] mod tests { - use std::time::Duration; - use std::time::Instant; + use std::time::{Duration, Instant}; use super::*; diff --git a/quickwit-actors/src/kill_switch.rs b/quickwit-actors/src/kill_switch.rs index f69a0e7bf0..abc6518a49 100644 --- a/quickwit-actors/src/kill_switch.rs +++ b/quickwit-actors/src/kill_switch.rs @@ -1,25 +1,23 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . -use std::sync::atomic::AtomicBool; -use std::sync::atomic::Ordering; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; #[derive(Clone)] diff --git a/quickwit-actors/src/lib.rs b/quickwit-actors/src/lib.rs index 78d18494b4..886216f4cd 100644 --- a/quickwit-actors/src/lib.rs +++ b/quickwit-actors/src/lib.rs @@ -1,33 +1,29 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . -/*! -quickwit-actors is a simplified actor framework for quickwit. - -It solves the following problem: -- have sync and async tasks communicate together. -- make these task observable -- make these task modular and testable -- detect when some task is stuck and does not progress anymore - -*/ +//! quickwit-actors is a simplified actor framework for quickwit. +//! +//! It solves the following problem: +//! - have sync and async tasks communicate together. +//! - make these task observable +//! - make these task modular and testable +//! - detect when some task is stuck and does not progress anymore use tokio::time::Duration; mod actor; @@ -47,10 +43,6 @@ mod sync_actor; mod tests; mod universe; -pub use self::actor::ActorContext; -pub use self::actor_state::ActorState; -pub use self::channel_with_priority::{QueueCapacity, RecvError, SendError}; -pub use self::mailbox::{create_mailbox, create_test_mailbox, Mailbox}; pub use actor::{Actor, ActorExitStatus}; pub use actor_handle::{ActorHandle, Health, Supervisable}; pub use async_actor::AsyncActor; @@ -60,6 +52,11 @@ pub(crate) use scheduler::Scheduler; pub use sync_actor::SyncActor; pub use universe::Universe; +pub use self::actor::ActorContext; +pub use self::actor_state::ActorState; +pub use self::channel_with_priority::{QueueCapacity, RecvError, SendError}; +pub use self::mailbox::{create_mailbox, create_test_mailbox, Mailbox}; + /// Heartbeat used to verify that actors are progressing. /// /// If an actor does not advertise a progress within an interval of duration `HEARTBEAT`, diff --git a/quickwit-actors/src/mailbox.rs b/quickwit-actors/src/mailbox.rs index 30385b19a1..3c31410897 100644 --- a/quickwit-actors/src/mailbox.rs +++ b/quickwit-actors/src/mailbox.rs @@ -1,36 +1,31 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::any::Any; use std::fmt; +use std::hash::Hash; use std::sync::Arc; -use std::hash::Hash; use tokio::sync::oneshot; -use crate::channel_with_priority::Priority; -use crate::channel_with_priority::Receiver; -use crate::channel_with_priority::Sender; -use crate::QueueCapacity; -use crate::RecvError; -use crate::SendError; +use crate::channel_with_priority::{Priority, Receiver, Sender}; +use crate::{QueueCapacity, RecvError, SendError}; /// A mailbox is the object that makes it possible to send a message /// to an actor. diff --git a/quickwit-actors/src/observation.rs b/quickwit-actors/src/observation.rs index 252acd2c15..2fb2b4bb5f 100644 --- a/quickwit-actors/src/observation.rs +++ b/quickwit-actors/src/observation.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::fmt; use std::ops::Deref; diff --git a/quickwit-actors/src/progress.rs b/quickwit-actors/src/progress.rs index 28fc641c8a..a4dd1027de 100644 --- a/quickwit-actors/src/progress.rs +++ b/quickwit-actors/src/progress.rs @@ -1,25 +1,23 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . -use std::sync::atomic::AtomicU32; -use std::sync::atomic::Ordering; +use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; /// Progress makes it possible to register some progress. diff --git a/quickwit-actors/src/scheduler.rs b/quickwit-actors/src/scheduler.rs index d6ae913db1..8594486a96 100644 --- a/quickwit-actors/src/scheduler.rs +++ b/quickwit-actors/src/scheduler.rs @@ -1,44 +1,40 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . -use async_trait::async_trait; use core::fmt; -use futures::Future; -use std::cmp::Ordering; -use std::cmp::Reverse; +use std::cmp::{Ordering, Reverse}; use std::collections::BinaryHeap; use std::pin::Pin; -use std::time::Duration; -use std::time::Instant; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use futures::Future; use tokio::sync::oneshot::Sender; use tokio::task::JoinHandle; use tracing::info; -use crate::Actor; -use crate::ActorContext; -use crate::AsyncActor; +use crate::{Actor, ActorContext, AsyncActor}; pub(crate) struct Callback(pub Pin + Sync + Send + 'static>>); -// A bug in the rustc requires wrapping Box<...> in order to use it as an argument in an async method. -// pub(crate) struct Callback(pub BoxFuture<'static, ()>); +// A bug in the rustc requires wrapping Box<...> in order to use it as an argument in an async +// method. pub(crate) struct Callback(pub BoxFuture<'static, ()>); struct TimeoutEvent { deadline: Instant, @@ -198,8 +194,8 @@ impl Scheduler { { self.advance_by_duration(next_evt_before_deadline - now, ctx) .await; - // We leave 100ms for actors to process their messages. A callback on process would not work here, - // as callbacks might create extra messages in turn. + // We leave 100ms for actors to process their messages. A callback on process would not + // work here, as callbacks might create extra messages in turn. // A good way could be to wait for the overall actors in the universe to be idle. tokio::time::sleep(Duration::from_millis(100)).await; let _ = ctx @@ -281,14 +277,16 @@ impl Scheduler { #[cfg(test)] mod tests { - use super::{Callback, Scheduler, SchedulerMessage}; - use crate::scheduler::{SchedulerCounters, TimeShift}; - use crate::Universe; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; + use tokio::sync::oneshot; + use super::{Callback, Scheduler, SchedulerMessage}; + use crate::scheduler::{SchedulerCounters, TimeShift}; + use crate::Universe; + fn create_test_callback() -> (Arc, Callback) { let cb_called = Arc::new(AtomicBool::default()); let cb_called_clone = cb_called.clone(); diff --git a/quickwit-actors/src/spawn_builder.rs b/quickwit-actors/src/spawn_builder.rs index 5dc9ba80b7..a78660580c 100644 --- a/quickwit-actors/src/spawn_builder.rs +++ b/quickwit-actors/src/spawn_builder.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use crate::async_actor::spawn_async_actor; use crate::mailbox::Inbox; diff --git a/quickwit-actors/src/sync_actor.rs b/quickwit-actors/src/sync_actor.rs index 6b7606e57f..1cf737aa90 100644 --- a/quickwit-actors/src/sync_actor.rs +++ b/quickwit-actors/src/sync_actor.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use tokio::sync::watch::{self, Sender}; use tokio::task::{spawn_blocking, JoinHandle}; @@ -89,7 +88,8 @@ pub(crate) fn spawn_sync_actor( /// Process a given message. /// -/// If some `ActorExitStatus` is returned, the actor will exit and no more message will be processed. +/// If some `ActorExitStatus` is returned, the actor will exit and no more message will be +/// processed. fn process_msg( actor: &mut A, inbox: &mut Inbox, diff --git a/quickwit-actors/src/tests.rs b/quickwit-actors/src/tests.rs index a5d36e89ee..51d0b5db30 100644 --- a/quickwit-actors/src/tests.rs +++ b/quickwit-actors/src/tests.rs @@ -1,33 +1,32 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::collections::HashSet; + +use async_trait::async_trait; + use crate::mailbox::Command; use crate::observation::ObservationType; -use crate::Actor; -use crate::ActorHandle; -use crate::Health; -use crate::Supervisable; -use crate::Universe; -use crate::{ActorContext, ActorExitStatus, AsyncActor, Mailbox, Observation, SyncActor}; -use async_trait::async_trait; -use std::collections::HashSet; +use crate::{ + Actor, ActorContext, ActorExitStatus, ActorHandle, AsyncActor, Health, Mailbox, Observation, + Supervisable, SyncActor, Universe, +}; // An actor that receives ping messages. #[derive(Default)] @@ -217,7 +216,7 @@ async fn test_ping_actor() { ping_recv_handle.process_pending_and_observe().await, Observation { obs_type: ObservationType::PostMortem, - state: 2, + state: 2 } ); assert_eq!( diff --git a/quickwit-actors/src/universe.rs b/quickwit-actors/src/universe.rs index 4358766f40..8efb4219a0 100644 --- a/quickwit-actors/src/universe.rs +++ b/quickwit-actors/src/universe.rs @@ -1,42 +1,37 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::time::Duration; -use crate::scheduler::SchedulerMessage; -use crate::scheduler::TimeShift; +use crate::scheduler::{SchedulerMessage, TimeShift}; use crate::spawn_builder::SpawnBuilder; -use crate::Actor; -use crate::KillSwitch; -use crate::Mailbox; -use crate::QueueCapacity; -use crate::Scheduler; +use crate::{Actor, KillSwitch, Mailbox, QueueCapacity, Scheduler}; /// Universe serves as the top-level context in which Actor can be spawned. -/// It is *not* a singleton. A typical application will usually have only one universe hosting all of the actors -/// but it is not a requirement. +/// It is *not* a singleton. A typical application will usually have only one universe hosting all +/// of the actors but it is not a requirement. /// /// In particular, unit test all have their own universe and hence can be executed in parallel. pub struct Universe { scheduler_mailbox: Mailbox<::Message>, - // This killswitch is used for the scheduler, and will be used by default for all spawned actors. + // This killswitch is used for the scheduler, and will be used by default for all spawned + // actors. kill_switch: KillSwitch, } @@ -109,11 +104,7 @@ mod tests { use async_trait::async_trait; - use crate::Actor; - use crate::ActorContext; - use crate::ActorExitStatus; - use crate::AsyncActor; - use crate::Universe; + use crate::{Actor, ActorContext, ActorExitStatus, AsyncActor, Universe}; #[derive(Default)] pub struct ActorWithSchedule { diff --git a/quickwit-cli/build.rs b/quickwit-cli/build.rs index 406d8d3822..fa90064c6b 100644 --- a/quickwit-cli/build.rs +++ b/quickwit-cli/build.rs @@ -1,23 +1,22 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + fn main() { let git_commit_hash = std::env::var_os("GITHUB_SHA") .map_or("Unknown".to_string(), |value_os_str| { diff --git a/quickwit-cli/src/lib.rs b/quickwit-cli/src/lib.rs index bbdf5566a0..968f4ba339 100644 --- a/quickwit-cli/src/lib.rs +++ b/quickwit-cli/src/lib.rs @@ -1,69 +1,51 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -use anyhow::bail; -use anyhow::Context; +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::collections::VecDeque; +use std::io::{stdout, Stdout, Write}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use std::{env, usize}; + +use anyhow::{bail, Context}; use byte_unit::Byte; -use crossterm::style::Print; -use crossterm::style::PrintStyledContent; -use crossterm::style::Stylize; +use crossterm::style::{Print, PrintStyledContent, Stylize}; use crossterm::QueueableCommand; use json_comments::StripComments; -use quickwit_actors::ActorExitStatus; -use quickwit_actors::ActorHandle; -use quickwit_actors::ObservationType; -use quickwit_actors::Universe; +use quickwit_actors::{ActorExitStatus, ActorHandle, ObservationType, Universe}; use quickwit_common::extract_index_id_from_index_uri; -use quickwit_core::reset_index; -use quickwit_index_config::DefaultIndexConfigBuilder; -use quickwit_index_config::IndexConfig; -use quickwit_indexing::actors::IndexerParams; -use quickwit_indexing::actors::{IndexingPipelineParams, IndexingPipelineSupervisor}; -use quickwit_indexing::models::CommitPolicy; -use quickwit_indexing::models::IndexingStatistics; -use quickwit_indexing::models::ScratchDirectory; -use quickwit_indexing::source::FileSourceParams; -use quickwit_indexing::source::SourceConfig; +use quickwit_core::{create_index, delete_index, garbage_collect_index, reset_index}; +use quickwit_index_config::{DefaultIndexConfigBuilder, IndexConfig}; +use quickwit_indexing::actors::{ + IndexerParams, IndexingPipelineParams, IndexingPipelineSupervisor, +}; +use quickwit_indexing::models::{CommitPolicy, IndexingStatistics, ScratchDirectory}; +use quickwit_indexing::source::{FileSourceParams, SourceConfig}; use quickwit_metastore::checkpoint::Checkpoint; -use quickwit_metastore::IndexMetadata; -use quickwit_metastore::MetastoreUriResolver; -use quickwit_proto::SearchRequest; -use quickwit_proto::SearchResult; -use quickwit_search::single_node_search; -use quickwit_search::SearchResultJson; +use quickwit_metastore::{IndexMetadata, MetastoreUriResolver}; +use quickwit_proto::{SearchRequest, SearchResult}; +use quickwit_search::{single_node_search, SearchResultJson}; use quickwit_storage::quickwit_storage_uri_resolver; use quickwit_telemetry::payload::TelemetryEvent; -use std::collections::VecDeque; -use std::env; -use std::io::Stdout; -use std::io::{stdout, Write}; -use std::path::PathBuf; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use std::usize; use tracing::debug; -use quickwit_core::{create_index, delete_index, garbage_collect_index}; - /// Throughput calculation window size. const THROUGHPUT_WINDOW_SIZE: usize = 5; @@ -217,14 +199,22 @@ pub async fn index_data_cli(args: IndexDataArgs) -> anyhow::Result<()> { "windows" => "CTRL+Z", _ => "CTRL+D", }; - println!("Please enter your new line delimited json documents one line at a time.\nEnd your input using {}.", eof_shortcut); + println!( + "Please enter your new line delimited json documents one line at a time.\nEnd your \ + input using {}.", + eof_shortcut + ); } let statistics = start_statistics_reporting_loop(supervisor_handler, args.input_path.clone()).await?; if statistics.num_published_splits > 0 { - println!("You can now query your index with `quickwit search --index-id {} --metastore-uri {} --query \"barack obama\"`" , args.index_id, args.metastore_uri); + println!( + "You can now query your index with `quickwit search --index-id {} --metastore-uri {} \ + --query \"barack obama\"`", + args.index_id, args.metastore_uri + ); } Ok(()) } @@ -344,8 +334,9 @@ pub async fn start_statistics_reporting_loop( let mut report_interval = tokio::time::interval(Duration::from_secs(1)); loop { - // TODO fixme. The way we wait today is a bit lame: if the indexing pipeline exits, we will stil - // wait up to an entire heartbeat... Ideally we should select between two futures. + // TODO fixme. The way we wait today is a bit lame: if the indexing pipeline exits, we will + // stil wait up to an entire heartbeat... Ideally we should select between two + // futures. report_interval.tick().await; // Try to receive with a timeout of 1 second. // 1 second is also the frequency at which we update statistic in the console @@ -395,7 +386,7 @@ pub async fn start_statistics_reporting_loop( is_tty, )?; } - //display end of task report + // display end of task report println!(); let elapsed_secs = start_time.elapsed().as_secs(); if elapsed_secs >= 60 { @@ -448,7 +439,8 @@ fn display_statistics( stdout_handle.queue(Print(format!("{}\n", elapsed_time)))?; } else { let report_line = format!( - "Num docs: {:>7} Parse errs: {:>5} Staged splits: {:>3} Input size: {:>5}MB Thrghput: {:>5.2}MB/s Time: {}\n", + "Num docs: {:>7} Parse errs: {:>5} Staged splits: {:>3} Input size: {:>5}MB Thrghput: \ + {:>5.2}MB/s Time: {}\n", statistics.num_docs, statistics.num_invalid_docs, statistics.num_staged_splits, diff --git a/quickwit-cli/src/main.rs b/quickwit-cli/src/main.rs index 8ee41a0861..a690df4dcd 100644 --- a/quickwit-cli/src/main.rs +++ b/quickwit-cli/src/main.rs @@ -1,38 +1,34 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +use std::env; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::time::Duration; use anyhow::{bail, Context}; use byte_unit::Byte; use clap::{load_yaml, value_t, App, AppSettings, ArgMatches}; use quickwit_cli::*; use quickwit_common::to_socket_addr; -use quickwit_serve::serve_cli; -use quickwit_serve::ServeArgs; +use quickwit_serve::{serve_cli, ServeArgs}; use quickwit_telemetry::payload::TelemetryEvent; -use std::env; -use std::net::SocketAddr; -use std::path::Path; -use std::path::PathBuf; -use std::time::Duration; use tracing::Level; use tracing_subscriber::fmt::Subscriber; @@ -306,7 +302,11 @@ async fn main() { /// Return the about text with telemetry info. fn about_text() -> String { - let mut about_text = format!("Indexing your large dataset on object storage & making it searchable from the command line.\nCommit hash: {}\n", env!("GIT_COMMIT_HASH")); + let mut about_text = format!( + "Indexing your large dataset on object storage & making it searchable from the command \ + line.\nCommit hash: {}\n", + env!("GIT_COMMIT_HASH") + ); if quickwit_telemetry::is_telemetry_enabled() { about_text += "Telemetry Enabled"; } @@ -340,18 +340,20 @@ pub fn parse_duration_with_unit(duration: &str) -> anyhow::Result { #[cfg(test)] mod tests { - use crate::{ - parse_duration_with_unit, CliCommand, CreateIndexArgs, DeleteIndexArgs, - GarbageCollectIndexArgs, IndexDataArgs, SearchIndexArgs, - }; - use clap::{load_yaml, App, AppSettings}; - use quickwit_common::to_socket_addr; - use quickwit_serve::ServeArgs; use std::io::Write; use std::path::{Path, PathBuf}; use std::time::Duration; + + use clap::{load_yaml, App, AppSettings}; + use quickwit_common::to_socket_addr; + use quickwit_serve::ServeArgs; use tempfile::NamedTempFile; + use crate::{ + parse_duration_with_unit, CliCommand, CreateIndexArgs, DeleteIndexArgs, + GarbageCollectIndexArgs, IndexDataArgs, SearchIndexArgs, + }; + #[test] fn test_parse_new_args() -> anyhow::Result<()> { let yaml = load_yaml!("cli.yaml"); diff --git a/quickwit-cli/tests/cli.rs b/quickwit-cli/tests/cli.rs index fa55120f51..2aaf1419f1 100644 --- a/quickwit-cli/tests/cli.rs +++ b/quickwit-cli/tests/cli.rs @@ -1,30 +1,29 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . #![allow(clippy::bool_assert_comparison)] mod helpers; -use crate::helpers::{create_test_env, make_command, spawn_command}; +use std::io::Read; +use std::path::Path; + use anyhow::Result; use helpers::{TestEnv, TestStorageType}; use predicates::prelude::*; @@ -32,9 +31,10 @@ use quickwit_cli::{create_index_cli, CreateIndexArgs}; use quickwit_metastore::{Metastore, MetastoreUriResolver, SplitState}; use serde_json::{Number, Value}; use serial_test::serial; -use std::{io::Read, path::Path}; use tokio::time::{sleep, Duration}; +use crate::helpers::{create_test_env, make_command, spawn_command}; + fn create_logs_index(test_env: &TestEnv, index_id: &str) { make_command( format!( @@ -222,9 +222,9 @@ fn test_cmd_search() -> Result<()> { // search with tags make_command( format!( - "search --metastore-uri {} --index-id {} --query level:info --tags city:paris device:rpi", - test_env.metastore_uri, - index_id, + "search --metastore-uri {} --index-id {} --query level:info --tags city:paris \ + device:rpi", + test_env.metastore_uri, index_id, ) .as_str(), ) diff --git a/quickwit-cli/tests/helpers.rs b/quickwit-cli/tests/helpers.rs index 09d613d257..8beedc5755 100644 --- a/quickwit-cli/tests/helpers.rs +++ b/quickwit-cli/tests/helpers.rs @@ -1,42 +1,34 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +use std::collections::HashMap; +use std::path::PathBuf; +use std::process::{Child, Stdio}; +use std::sync::Arc; +use std::{fs, io}; use assert_cmd::cargo::cargo_bin; use assert_cmd::Command; use predicates::str; use quickwit_metastore::SingleFileMetastore; -use quickwit_storage::LocalFileStorage; -use quickwit_storage::RegionProvider; -use quickwit_storage::S3CompatibleObjectStorage; -use quickwit_storage::Storage; -use std::collections::HashMap; -use std::fs; -use std::io; -use std::path::PathBuf; -use std::process::Child; -use std::process::Stdio; -use std::sync::Arc; -use tempfile::tempdir; -use tempfile::TempDir; +use quickwit_storage::{LocalFileStorage, RegionProvider, S3CompatibleObjectStorage, Storage}; +use tempfile::{tempdir, TempDir}; const PACKAGE_BIN_NAME: &str = "quickwit"; diff --git a/quickwit-cluster/src/cluster.rs b/quickwit-cluster/src/cluster.rs index 062d34d2d4..f4f36f5df0 100644 --- a/quickwit-cluster/src/cluster.rs +++ b/quickwit-cluster/src/cluster.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::fs; use std::net::SocketAddr; @@ -25,7 +24,6 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; -use crate::error::{ClusterError, ClusterResult}; use quickwit_swim::prelude::{ ArtilleryError, ArtilleryMember, ArtilleryMemberEvent, ArtilleryMemberState, Cluster as ArtilleryCluster, ClusterConfig as ArtilleryClusterConfig, @@ -35,6 +33,8 @@ use tokio_stream::wrappers::WatchStream; use tracing::{debug, error, info, warn}; use uuid::Uuid; +use crate::error::{ClusterError, ClusterResult}; + /// The ID that makes the cluster unique. const CLUSTER_ID: &str = "quickwit-cluster"; @@ -259,9 +259,8 @@ fn log_artillery_event(artillery_member_event: ArtilleryMemberEvent) { #[cfg(test)] mod tests { use std::net::SocketAddr; - use std::thread; - use std::time; use std::time::Duration; + use std::{thread, time}; use quickwit_swim::prelude::{ArtilleryMember, ArtilleryMemberState}; diff --git a/quickwit-cluster/src/error.rs b/quickwit-cluster/src/error.rs index d4d0cc359a..0d5cac2d15 100644 --- a/quickwit-cluster/src/error.rs +++ b/quickwit-cluster/src/error.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use serde::{Deserialize, Serialize}; use thiserror::Error; diff --git a/quickwit-cluster/src/lib.rs b/quickwit-cluster/src/lib.rs index 9d291ba09d..f1de4016a5 100644 --- a/quickwit-cluster/src/lib.rs +++ b/quickwit-cluster/src/lib.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . pub mod cluster; pub mod error; diff --git a/quickwit-cluster/src/service.rs b/quickwit-cluster/src/service.rs index c4a9b88285..af7e84d9bf 100644 --- a/quickwit-cluster/src/service.rs +++ b/quickwit-cluster/src/service.rs @@ -1,27 +1,25 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::sync::Arc; use async_trait::async_trait; - use quickwit_proto::{LeaveRequest, LeaveResult, Member as PMember, MembersRequest, MembersResult}; use crate::cluster::{Cluster, Member}; @@ -81,9 +79,8 @@ impl ClusterService for ClusterServiceImpl { mod tests { use std::net::ToSocketAddrs; - use uuid::Uuid; - use quickwit_proto::Member as PMember; + use uuid::Uuid; use crate::cluster::Member; diff --git a/quickwit-cluster/src/test_utils.rs b/quickwit-cluster/src/test_utils.rs index 348f52c86a..dfb7b7a1b1 100644 --- a/quickwit-cluster/src/test_utils.rs +++ b/quickwit-cluster/src/test_utils.rs @@ -1,23 +1,22 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + use std::io; use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener}; use std::path::Path; diff --git a/quickwit-common/src/coolid.rs b/quickwit-common/src/coolid.rs index 5f93c74485..deff232203 100644 --- a/quickwit-common/src/coolid.rs +++ b/quickwit-common/src/coolid.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use rand::distributions::Alphanumeric; use rand::prelude::*; @@ -102,9 +101,10 @@ pub fn new_coolid(name: &str) -> String { #[cfg(test)] mod tests { - use super::new_coolid; use std::collections::HashSet; + use super::new_coolid; + #[test] fn test_coolid() { let cool_ids: HashSet = std::iter::repeat_with(|| new_coolid("hello")) diff --git a/quickwit-common/src/lib.rs b/quickwit-common/src/lib.rs index b6661c8e58..996693297d 100644 --- a/quickwit-common/src/lib.rs +++ b/quickwit-common/src/lib.rs @@ -1,39 +1,37 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . mod coolid; -use once_cell::sync::Lazy; -use regex::Regex; use std::net::{SocketAddr, ToSocketAddrs}; pub use coolid::new_coolid; +use once_cell::sync::Lazy; +use regex::Regex; /// Filenames used for hotcache files. pub const HOTCACHE_FILENAME: &str = "hotcache"; /// This function takes such a index_url and breaks it into /// s3://my_bucket/some_path_containing_my_indices / my_index -/// \------/ -/// index_id +/// \------/ +/// index_id pub fn extract_index_id_from_index_uri(mut index_uri: &str) -> anyhow::Result<&str> { static INDEX_URI_PATTERN: Lazy = Lazy::new(|| Regex::new(r"^.+://.+/.+$").unwrap()); static INDEX_ID_PATTERN: Lazy = @@ -54,7 +52,11 @@ pub fn extract_index_id_from_index_uri(mut index_uri: &str) -> anyhow::Result<&s anyhow::bail!("Failed to parse the uri into a metastore_uri and an index_id."); } if !INDEX_ID_PATTERN.is_match(parts[0]) { - anyhow::bail!("Invalid index_id `{}`. Only alpha-numeric, `-` and `_` characters allowed. Cannot start with `-`, `_` or digit.", parts[0]); + anyhow::bail!( + "Invalid index_id `{}`. Only alpha-numeric, `-` and `_` characters allowed. Cannot \ + start with `-`, `_` or digit.", + parts[0] + ); } Ok(parts[0]) diff --git a/quickwit-core/src/index.rs b/quickwit-core/src/index.rs index 3f5c0af1ee..f06a4cbe24 100644 --- a/quickwit-core/src/index.rs +++ b/quickwit-core/src/index.rs @@ -1,35 +1,35 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +use std::path::Path; +use std::time::Duration; -use crate::FileEntry; use futures::StreamExt; -use quickwit_metastore::{IndexMetadata, Metastore, MetastoreUriResolver}; -use quickwit_metastore::{SplitMetadataAndFooterOffsets, SplitState}; +use quickwit_metastore::{ + IndexMetadata, Metastore, MetastoreUriResolver, SplitMetadataAndFooterOffsets, SplitState, +}; use quickwit_storage::{quickwit_storage_uri_resolver, StorageUriResolver}; -use std::path::Path; -use std::time::Duration; use tantivy::chrono::Utc; use tracing::warn; +use crate::FileEntry; + pub const MAX_CONCURRENT_SPLIT_TASKS: usize = if cfg!(test) { 2 } else { 10 }; /// Creates an index at `index-path` extracted from `metastore_uri`. The command fails if an index @@ -37,7 +37,6 @@ pub const MAX_CONCURRENT_SPLIT_TASKS: usize = if cfg!(test) { 2 } else { 10 }; /// /// * `metastore_uri` - The metastore URI for accessing the metastore. /// * `index_metadata` - The metadata used to create the target index. -/// pub async fn create_index( metastore_uri: &str, index_metadata: IndexMetadata, @@ -99,7 +98,6 @@ pub async fn delete_index( /// * `index_id` - The target index Id. /// * `grace_period` - Threshold period after which a staged split can be garbage collected. /// * `dry_run` - Should this only return a list of affected files without performing deletion. -/// pub async fn garbage_collect_index( metastore_uri: &str, index_id: &str, @@ -150,7 +148,6 @@ pub async fn garbage_collect_index( /// * `metastore_uri` - The target index metastore uri. /// * `index_id` - The target index id. /// * `storage_resolver` - The storage resolver object. -/// pub async fn delete_garbage_files( metastore: &dyn Metastore, index_id: &str, @@ -212,7 +209,6 @@ pub async fn delete_garbage_files( /// * `index_id` - The target index Id. /// * `storage_resolver` - A storage resolver object to access the storage. /// * `metastore` - A metastore object for interacting with the metastore. -/// pub async fn reset_index( metastore: &dyn Metastore, index_id: &str, diff --git a/quickwit-core/src/lib.rs b/quickwit-core/src/lib.rs index 66a6f09227..14ffd852c3 100644 --- a/quickwit-core/src/lib.rs +++ b/quickwit-core/src/lib.rs @@ -1,33 +1,30 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ #![warn(missing_docs)] #![allow(clippy::bool_assert_comparison)] -/*! `quickwit-core` provides all the core functions used in quickwit cli: -- `create_index` for creating a new index -- `index_data` for indexing new-line delimited json documents -- `search_index` for searching an index -- `delete_index` for deleting an index -*/ +//! `quickwit-core` provides all the core functions used in quickwit cli: +//! - `create_index` for creating a new index +//! - `index_data` for indexing new-line delimited json documents +//! - `search_index` for searching an index +//! - `delete_index` for deleting an index mod index; diff --git a/quickwit-directories/src/bundle_directory.rs b/quickwit-directories/src/bundle_directory.rs index cdb653c8ca..db3621068c 100644 --- a/quickwit-directories/src/bundle_directory.rs +++ b/quickwit-directories/src/bundle_directory.rs @@ -1,36 +1,32 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -use async_trait::async_trait; -use quickwit_storage::BundleStorageFileOffsets; use std::fmt::Debug; use std::ops::Range; use std::path::{Path, PathBuf}; use std::{fmt, io}; + +use async_trait::async_trait; +use quickwit_storage::BundleStorageFileOffsets; use tantivy::directory::error::{DeleteError, OpenReadError, OpenWriteError}; -use tantivy::directory::{FileHandle, WatchCallback, WatchHandle, WritePtr}; -use tantivy::directory::{FileSlice, OwnedBytes}; -use tantivy::HasLen; -use tantivy::{AsyncIoResult, Directory}; +use tantivy::directory::{FileHandle, FileSlice, OwnedBytes, WatchCallback, WatchHandle, WritePtr}; +use tantivy::{AsyncIoResult, Directory, HasLen}; use tracing::error; struct BundleDirectoryFileHandle { @@ -164,10 +160,8 @@ impl Directory for BundleDirectory { #[cfg(test)] mod tests { - use std::{ - fs::{self, File}, - io::Write, - }; + use std::fs::{self, File}; + use std::io::Write; use quickwit_storage::BundleStorageBuilder; diff --git a/quickwit-directories/src/caching_directory.rs b/quickwit-directories/src/caching_directory.rs index 7933a3de27..689dd210fb 100644 --- a/quickwit-directories/src/caching_directory.rs +++ b/quickwit-directories/src/caching_directory.rs @@ -1,35 +1,31 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::ops::{Deref, Range}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::{fmt, io}; use async_trait::async_trait; use bytes::Bytes; use quickwit_storage::SliceCache; use stable_deref_trait::StableDeref; -use std::fmt; -use std::io; -use std::ops::Deref; -use std::ops::Range; -use std::path::{Path, PathBuf}; -use std::sync::Arc; use tantivy::directory::error::{DeleteError, OpenReadError, OpenWriteError}; use tantivy::directory::{FileHandle, OwnedBytes, WatchHandle, WritePtr}; use tantivy::{AsyncIoResult, Directory, HasLen}; @@ -198,13 +194,15 @@ impl Directory for CachingDirectory { #[cfg(test)] mod tests { - use super::CachingDirectory; - use crate::DebugProxyDirectory; use std::path::Path; use std::sync::Arc; + use tantivy::directory::RamDirectory; use tantivy::Directory; + use super::CachingDirectory; + use crate::DebugProxyDirectory; + #[test] fn test_caching_directory() -> tantivy::Result<()> { let ram_directory = RamDirectory::default(); diff --git a/quickwit-directories/src/debug_proxy_directory.rs b/quickwit-directories/src/debug_proxy_directory.rs index 36875894bc..e76288d2e3 100644 --- a/quickwit-directories/src/debug_proxy_directory.rs +++ b/quickwit-directories/src/debug_proxy_directory.rs @@ -1,40 +1,38 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -use crate::StorageDirectory; -use async_trait::async_trait; -use bytes::Bytes; use std::ops::Range; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use std::{fmt, io, mem}; + +use async_trait::async_trait; +use bytes::Bytes; use tantivy::chrono::{DateTime, Utc}; use tantivy::directory::error::{DeleteError, LockError, OpenReadError, OpenWriteError}; use tantivy::directory::{ DirectoryLock, FileHandle, OwnedBytes, WatchCallback, WatchHandle, WritePtr, }; -use tantivy::Directory; -use tantivy::HasLen; +use tantivy::{Directory, HasLen}; + +use crate::StorageDirectory; #[derive(Clone, Default)] struct OperationBuffer(Arc>>); @@ -270,12 +268,14 @@ impl DebugProxyDirectory { #[cfg(test)] mod tests { - use super::DebugProxyDirectory; use std::io::Write; use std::path::Path; + use tantivy::directory::{RamDirectory, TerminatingWrite}; use tantivy::Directory; + use super::DebugProxyDirectory; + const TEST_PATH: &str = "test.file"; const TEST_PAYLOAD: &[u8] = b"hello happy tax payer"; diff --git a/quickwit-directories/src/hot_directory.rs b/quickwit-directories/src/hot_directory.rs index c292186e23..7529a7036a 100644 --- a/quickwit-directories/src/hot_directory.rs +++ b/quickwit-directories/src/hot_directory.rs @@ -1,40 +1,37 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -use async_trait::async_trait; -use bytes::Bytes; -use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; -use std::fmt; -use std::io; use std::ops::Range; use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::{fmt, io}; + +use async_trait::async_trait; +use bytes::Bytes; +use serde::{Deserialize, Serialize}; use tantivy::directory::error::{LockError, OpenReadError}; -use tantivy::directory::DirectoryLock; -use tantivy::directory::{FileSlice, OwnedBytes, WatchCallback}; +use tantivy::directory::{ + DirectoryLock, FileHandle, FileSlice, OwnedBytes, WatchCallback, WatchHandle, +}; use tantivy::error::DataCorruption; -use tantivy::{directory::FileHandle, directory::WatchHandle, HasLen}; -use tantivy::{AsyncIoResult, Directory, Index, IndexReader, ReloadPolicy}; +use tantivy::{AsyncIoResult, Directory, HasLen, Index, IndexReader, ReloadPolicy}; use crate::caching_directory::BytesWrapper; use crate::{CachingDirectory, DebugProxyDirectory}; @@ -130,9 +127,7 @@ impl StaticDirectoryCacheBuilder { } fn deserialize_cbor(bytes: &mut OwnedBytes) -> serde_cbor::Result -where - T: serde::de::DeserializeOwned, -{ +where T: serde::de::DeserializeOwned { let len = bytes.read_u64(); let value = serde_cbor::from_reader(&bytes.as_slice()[..len as usize]); bytes.advance(len as usize); diff --git a/quickwit-directories/src/lib.rs b/quickwit-directories/src/lib.rs index a8e0dd1bf9..12d7ab251c 100644 --- a/quickwit-directories/src/lib.rs +++ b/quickwit-directories/src/lib.rs @@ -1,33 +1,30 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -/*! -This crate contains all of the building pieces that make quickwit's IO possible. - -- The `StorageDirectory` justs wraps a `Storage` trait to make it compatible with tantivy's Directory API. -- The `HotDirectory` wraps another directory with a static cache. -- The `CachingDirectory` wraps a Directory with a dynamic cache. -- The `DebugDirectory` acts as a proxy to another directory to instrument it and record all of its IO. -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! This crate contains all of the building pieces that make quickwit's IO possible. +//! +//! - The `StorageDirectory` justs wraps a `Storage` trait to make it compatible with tantivy's +//! Directory API. +//! - The `HotDirectory` wraps another directory with a static cache. +//! - The `CachingDirectory` wraps a Directory with a dynamic cache. +//! - The `DebugDirectory` acts as a proxy to another directory to instrument it and record all of +//! its IO. #![warn(missing_docs)] mod bundle_directory; diff --git a/quickwit-directories/src/storage_directory.rs b/quickwit-directories/src/storage_directory.rs index f0d570d73f..7a93675aca 100644 --- a/quickwit-directories/src/storage_directory.rs +++ b/quickwit-directories/src/storage_directory.rs @@ -1,39 +1,34 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -use async_trait::async_trait; -use bytes::Bytes; -use quickwit_storage::Storage; -use std::fmt; use std::fmt::Debug; -use std::io; use std::ops::Range; use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::{fmt, io}; + +use async_trait::async_trait; +use bytes::Bytes; +use quickwit_storage::Storage; use tantivy::directory::error::{DeleteError, OpenReadError, OpenWriteError}; -use tantivy::directory::OwnedBytes; -use tantivy::directory::{FileHandle, WatchCallback, WatchHandle, WritePtr}; -use tantivy::HasLen; -use tantivy::{AsyncIoResult, Directory}; +use tantivy::directory::{FileHandle, OwnedBytes, WatchCallback, WatchHandle, WritePtr}; +use tantivy::{AsyncIoResult, Directory, HasLen}; use tracing::error; use crate::caching_directory::BytesWrapper; diff --git a/quickwit-index-config/src/all_flatten_config.rs b/quickwit-index-config/src/all_flatten_config.rs index ecc58289f5..36d1d7d34c 100644 --- a/quickwit-index-config/src/all_flatten_config.rs +++ b/quickwit-index-config/src/all_flatten_config.rs @@ -1,37 +1,31 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -use crate::{ - default_index_config::{SOURCE_FIELD_NAME, TAGS_FIELD_NAME}, - query_builder::build_query, - DocParsingError, IndexConfig, QueryParserError, -}; use quickwit_proto::SearchRequest; use serde::{Deserialize, Serialize}; -use tantivy::{ - query::Query, - schema::{Schema, SchemaBuilder, STORED, STRING}, - Document, -}; +use tantivy::query::Query; +use tantivy::schema::{Schema, SchemaBuilder, STORED, STRING}; +use tantivy::Document; + +use crate::default_index_config::{SOURCE_FIELD_NAME, TAGS_FIELD_NAME}; +use crate::query_builder::build_query; +use crate::{DocParsingError, IndexConfig, QueryParserError}; /// A config that flatten the document to have all fields at top level. #[derive(Clone, Serialize, Deserialize)] diff --git a/quickwit-index-config/src/config.rs b/quickwit-index-config/src/config.rs index 4a2e11e4f5..7ab06b0eb9 100644 --- a/quickwit-index-config/src/config.rs +++ b/quickwit-index-config/src/config.rs @@ -1,36 +1,32 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +use std::fmt::Debug; -use crate::default_index_config::TAGS_FIELD_NAME; -use crate::DocParsingError; -use dyn_clone::clone_trait_object; -use dyn_clone::DynClone; +use dyn_clone::{clone_trait_object, DynClone}; use quickwit_proto::SearchRequest; -use std::fmt::Debug; use tantivy::query::Query; use tantivy::schema::{Field, Schema}; use tantivy::Document; -use crate::QueryParserError; +use crate::default_index_config::TAGS_FIELD_NAME; +use crate::{DocParsingError, QueryParserError}; /// Sorted order (either Ascending or Descending). /// To get a regular top-K results search, use `SortOrder::Desc`. @@ -43,7 +39,8 @@ pub enum SortOrder { } /// Defines the way documents should be sorted. -/// In case of a tie, the documents are ordered according to descending `(split_id, segment_ord, doc_id)`. +/// In case of a tie, the documents are ordered according to descending `(split_id, segment_ord, +/// doc_id)`. #[derive(Clone, Debug)] pub enum SortBy { /// Sort by a specific field. @@ -65,7 +62,6 @@ pub enum SortBy { /// - a way to build a tantivy::Document from a json payload /// - a way to build a tantivy::Query from a SearchRequest /// - a way to build a tantivy:Schema -/// #[typetag::serde(tag = "type")] pub trait IndexConfig: Send + Sync + Debug + DynClone + 'static { /// Returns the document built from a json string. diff --git a/quickwit-index-config/src/default_index_config/default_config.rs b/quickwit-index-config/src/default_index_config/default_config.rs index ef6dd1b7a3..183a40d0f5 100644 --- a/quickwit-index-config/src/default_index_config/default_config.rs +++ b/quickwit-index-config/src/default_index_config/default_config.rs @@ -1,40 +1,41 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -use super::{default_as_true, SOURCE_FIELD_NAME, TAGS_FIELD_NAME}; -use super::{field_mapping_entry::DocParsingError, FieldMappingEntry, FieldMappingType}; -use crate::query_builder::build_query; -use crate::{IndexConfig, QueryParserError, SortBy, SortOrder}; +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::collections::HashSet; +use std::convert::TryFrom; + use anyhow::{bail, Context}; use quickwit_proto::SearchRequest; use serde::{Deserialize, Serialize}; use serde_json::{self, Value as JsonValue}; -use std::{collections::HashSet, convert::TryFrom}; -use tantivy::schema::{Cardinality, Value, STRING}; -use tantivy::{ - query::Query, - schema::{FieldEntry, FieldType, FieldValue, Schema, SchemaBuilder, STORED}, - Document, +use tantivy::query::Query; +use tantivy::schema::{ + Cardinality, FieldEntry, FieldType, FieldValue, Schema, SchemaBuilder, Value, STORED, STRING, +}; +use tantivy::Document; + +use super::field_mapping_entry::DocParsingError; +use super::{ + default_as_true, FieldMappingEntry, FieldMappingType, SOURCE_FIELD_NAME, TAGS_FIELD_NAME, }; +use crate::query_builder::build_query; +use crate::{IndexConfig, QueryParserError, SortBy, SortOrder}; /// DefaultIndexConfigBuilder is here /// to create a valid IndexConfig. @@ -87,14 +88,26 @@ impl DefaultIndexConfigBuilder { let timestamp_field_entry = schema.get_field_entry(timestamp_field); if !timestamp_field_entry.is_fast() { - bail!("Timestamp field must be a fast field, please add fast property to your field `{}`.", timestamp_field_name) + bail!( + "Timestamp field must be a fast field, please add fast property to your field \ + `{}`.", + timestamp_field_name + ) } if let FieldType::I64(options) = timestamp_field_entry.field_type() { if options.get_fastfield_cardinality() == Some(Cardinality::MultiValues) { - bail!("Timestamp field cannot be an array, please change your field `{}` from an array to a single value.", timestamp_field_name) + bail!( + "Timestamp field cannot be an array, please change your field `{}` from \ + an array to a single value.", + timestamp_field_name + ) } } else { - bail!("Timestamp field must be of type i64, please change your field type `{}` to i64.", timestamp_field_name) + bail!( + "Timestamp field must be of type i64, please change your field type `{}` to \ + i64.", + timestamp_field_name + ) } } @@ -306,13 +319,14 @@ impl IndexConfig for DefaultIndexConfig { #[cfg(test)] mod tests { - use crate::default_index_config::TAGS_FIELD_NAME; - use crate::{default_index_config::default_config::SOURCE_FIELD_NAME, IndexConfig}; - use crate::{DefaultIndexConfigBuilder, DocParsingError}; + use std::collections::HashMap; - use super::DefaultIndexConfig; use serde_json::{self, Value as JsonValue}; - use std::collections::HashMap; + + use super::DefaultIndexConfig; + use crate::default_index_config::default_config::SOURCE_FIELD_NAME; + use crate::default_index_config::TAGS_FIELD_NAME; + use crate::{DefaultIndexConfigBuilder, DocParsingError, IndexConfig}; const JSON_DOC_VALUE: &str = r#" { @@ -476,7 +490,9 @@ mod tests { ] }"#; let builder = serde_json::from_str::(index_config)?; - let expected_msg = "Timestamp field must be a fast field, please add fast property to your field `timestamp`.".to_string(); + let expected_msg = "Timestamp field must be a fast field, please add fast property to \ + your field `timestamp`." + .to_string(); assert_eq!(builder.build().unwrap_err().to_string(), expected_msg); Ok(()) } @@ -498,7 +514,9 @@ mod tests { }"#; let builder = serde_json::from_str::(index_config)?; - let expected_msg = "Timestamp field cannot be an array, please change your field `timestamp` from an array to a single value.".to_string(); + let expected_msg = "Timestamp field cannot be an array, please change your field \ + `timestamp` from an array to a single value." + .to_string(); assert_eq!(builder.build().unwrap_err().to_string(), expected_msg); Ok(()) } @@ -547,7 +565,8 @@ mod tests { "image": "invalid base64 data" }"#, ); - let expected_msg = "The field 'image' could not be parsed: Expected Base64 string, got 'invalid base64 data'" ; + let expected_msg = "The field 'image' could not be parsed: Expected Base64 string, got \ + 'invalid base64 data'"; assert_eq!(result.unwrap_err().to_string(), expected_msg); Ok(()) } @@ -582,7 +601,7 @@ mod tests { }"#; let document = index_config.doc_from_json(JSON_DOC_VALUE)?; - // 2 properties, + 1 value for "_source" + 2 values for "_tags" + // 2 properties, + 1 value for "_source" + 2 values for "_tags" assert_eq!(document.len(), 5); let expected_json_paths_and_values: HashMap = serde_json::from_str( r#"{ diff --git a/quickwit-index-config/src/default_index_config/field_mapping_entry.rs b/quickwit-index-config/src/default_index_config/field_mapping_entry.rs index 7f40e9bebf..fc873ee01c 100644 --- a/quickwit-index-config/src/default_index_config/field_mapping_entry.rs +++ b/quickwit-index-config/src/default_index_config/field_mapping_entry.rs @@ -1,43 +1,38 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +use std::convert::TryFrom; use anyhow::bail; use chrono::{FixedOffset, Utc}; use itertools::process_results; use serde::{Deserialize, Serialize}; use serde_json::{self, Value as JsonValue}; -use std::convert::TryFrom; -use tantivy::schema::{BytesOptions, FieldType}; use tantivy::schema::{ - Cardinality, DocParsingError as TantivyDocParser, IndexRecordOption, IntOptions, - TextFieldIndexing, TextOptions, Value, + BytesOptions, Cardinality, DocParsingError as TantivyDocParser, FieldType, IndexRecordOption, + IntOptions, TextFieldIndexing, TextOptions, Value, }; use thiserror::Error; +use super::{default_as_true, FieldMappingType}; use crate::default_index_config::is_valid_field_mapping_name; -use super::default_as_true; -use super::FieldMappingType; - /// A `FieldMappingEntry` defines how a field is indexed, stored, /// and mapped from a JSON document to the related index fields. #[derive(Debug, Clone, Deserialize, Serialize)] @@ -617,7 +612,11 @@ impl FieldMappingEntryForSerialization { } options = options.set_indexing_options(indexing_options); } else if self.record.is_some() || self.tokenizer.is_some() { - bail!("Error when parsing `{}`: `record` and `tokenizer` parameters are allowed only if indexed is true.", self.name) + bail!( + "Error when parsing `{}`: `record` and `tokenizer` parameters are allowed only if \ + indexed is true.", + self.name + ) } if self.stored { options = options.set_stored(); @@ -706,7 +705,11 @@ impl FieldMappingEntryForSerialization { fn check_no_text_options(&self) -> anyhow::Result<()> { if self.record.is_some() || self.tokenizer.is_some() { - bail!("Error when parsing `{}`: `record` and `tokenizer` parameters are for text field only.", self.name) + bail!( + "Error when parsing `{}`: `record` and `tokenizer` parameters are for text field \ + only.", + self.name + ) } Ok(()) } @@ -746,7 +749,6 @@ impl From for DocParsingError { #[cfg(test)] mod tests { - use crate::{default_index_config::FieldMappingType, DocParsingError}; use anyhow::bail; use chrono::{NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc}; use matches::matches; @@ -754,6 +756,8 @@ mod tests { use tantivy::schema::{Cardinality, Value}; use super::FieldMappingEntry; + use crate::default_index_config::FieldMappingType; + use crate::DocParsingError; const TEXT_MAPPING_ENTRY_VALUE: &str = r#" { @@ -810,7 +814,11 @@ mod tests { ); assert!(result.is_err()); let error = result.unwrap_err(); - assert_eq!(error.to_string(), "Error when parsing `my_field_name`: `record` and `tokenizer` parameters are allowed only if indexed is true."); + assert_eq!( + error.to_string(), + "Error when parsing `my_field_name`: `record` and `tokenizer` parameters are allowed \ + only if indexed is true." + ); Ok(()) } @@ -859,7 +867,11 @@ mod tests { ); assert!(result.is_err()); let error = result.unwrap_err(); - assert_eq!(error.to_string(), "Error when parsing field `my_field_name`: object type must have at least one field mapping."); + assert_eq!( + error.to_string(), + "Error when parsing field `my_field_name`: object type must have at least one field \ + mapping." + ); } #[test] @@ -910,7 +922,8 @@ mod tests { let error = result.unwrap_err(); assert_eq!( error.to_string(), - "Error when parsing `my_field_name`: `record` and `tokenizer` parameters are for text field only." + "Error when parsing `my_field_name`: `record` and `tokenizer` parameters are for text \ + field only." ); } @@ -973,7 +986,11 @@ mod tests { "#, )?; let entry_str = serde_json::to_string(&entry)?; - assert_eq!(entry_str, "{\"name\":\"my_field_name\",\"type\":\"i64\",\"stored\":true,\"fast\":false,\"indexed\":true}"); + assert_eq!( + entry_str, + "{\"name\":\"my_field_name\",\"type\":\"i64\",\"stored\":true,\"fast\":false,\"\ + indexed\":true}" + ); Ok(()) } @@ -1048,24 +1065,26 @@ mod tests { #[test] fn test_deserialize_u64_field_with_wrong_options() { let cases = vec![ - (r#" + ( + r#" { "name": "my_field_name", "type": "u64", "tokenizer": "basic" } "#, - "Error when parsing `my_field_name`: `record` and `tokenizer` parameters are for text field only." - ), - ( - r#" + "Error when parsing `my_field_name`: `record` and `tokenizer` parameters are for \ + text field only.", + ), + ( + r#" { "name": "this is not ok", "type": "i64" } "#, - "Invalid field name: `this is not ok`." - ) + "Invalid field name: `this is not ok`.", + ), ]; for (json_str, err_str) in cases { @@ -1135,7 +1154,11 @@ mod tests { "#, )?; let entry_str = serde_json::to_string(&entry)?; - assert_eq!(entry_str, "{\"name\":\"my_field_name\",\"type\":\"u64\",\"stored\":true,\"fast\":false,\"indexed\":true}"); + assert_eq!( + entry_str, + "{\"name\":\"my_field_name\",\"type\":\"u64\",\"stored\":true,\"fast\":false,\"\ + indexed\":true}" + ); Ok(()) } diff --git a/quickwit-index-config/src/default_index_config/field_mapping_type.rs b/quickwit-index-config/src/default_index_config/field_mapping_type.rs index e9c8bea590..f4fd74edf2 100644 --- a/quickwit-index-config/src/default_index_config/field_mapping_type.rs +++ b/quickwit-index-config/src/default_index_config/field_mapping_type.rs @@ -1,24 +1,21 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use tantivy::schema::{BytesOptions, Cardinality, IntOptions, TextOptions}; diff --git a/quickwit-index-config/src/default_index_config/mod.rs b/quickwit-index-config/src/default_index_config/mod.rs index b9b61a8669..ea0a0e39aa 100644 --- a/quickwit-index-config/src/default_index_config/mod.rs +++ b/quickwit-index-config/src/default_index_config/mod.rs @@ -1,24 +1,21 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . mod default_config; mod field_mapping_entry; diff --git a/quickwit-index-config/src/error.rs b/quickwit-index-config/src/error.rs index e192a7a165..3143a2d2ce 100644 --- a/quickwit-index-config/src/error.rs +++ b/quickwit-index-config/src/error.rs @@ -1,23 +1,22 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + use thiserror::Error; // TODO improve me and my error messages :) diff --git a/quickwit-index-config/src/lib.rs b/quickwit-index-config/src/lib.rs index 09fdd4044c..4e66e9ec12 100644 --- a/quickwit-index-config/src/lib.rs +++ b/quickwit-index-config/src/lib.rs @@ -1,24 +1,21 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . #![warn(missing_docs)] #![allow(clippy::bool_assert_comparison)] @@ -34,11 +31,10 @@ mod error; mod query_builder; mod wikipedia_config; -pub use error::QueryParserError; - pub use all_flatten_config::AllFlattenIndexConfig; pub use config::{IndexConfig, SortBy, SortOrder}; pub use default_index_config::{DefaultIndexConfig, DefaultIndexConfigBuilder, DocParsingError}; +pub use error::QueryParserError; pub use wikipedia_config::WikipediaIndexConfig; /// Returns a default `DefaultIndexConfig` for unit tests. diff --git a/quickwit-index-config/src/query_builder.rs b/quickwit-index-config/src/query_builder.rs index 403e9a9e5a..8bf3c4e880 100644 --- a/quickwit-index-config/src/query_builder.rs +++ b/quickwit-index-config/src/query_builder.rs @@ -1,24 +1,21 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use quickwit_proto::SearchRequest; use tantivy::query::{Query, QueryParser, QueryParserError as TantivyQueryParserError}; @@ -80,11 +77,11 @@ fn resolve_fields(schema: &Schema, field_names: &[String]) -> anyhow::Result. - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -use crate::default_index_config::TAGS_FIELD_NAME; -use crate::query_builder::build_query; -use crate::{DocParsingError, IndexConfig, QueryParserError}; use quickwit_proto::SearchRequest; use serde::{Deserialize, Serialize}; use tantivy::query::Query; @@ -30,6 +24,10 @@ use tantivy::schema::{Schema, TextFieldIndexing, TextOptions, STRING}; use tantivy::tokenizer::TokenizerManager; use tantivy::Document; +use crate::default_index_config::TAGS_FIELD_NAME; +use crate::query_builder::build_query; +use crate::{DocParsingError, IndexConfig, QueryParserError}; + /// A document config tailored for the wikipedia corpus. #[derive(Clone, Serialize, Deserialize)] pub struct WikipediaIndexConfig { diff --git a/quickwit-indexing/failpoints/mod.rs b/quickwit-indexing/failpoints/mod.rs index a874f65b51..1047d923da 100644 --- a/quickwit-indexing/failpoints/mod.rs +++ b/quickwit-indexing/failpoints/mod.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . //! Fail points are a form of code instrumentation that allow errors and other behaviors //! to be injected dynamically at runtime, primarily for testing purposes. Fail @@ -36,25 +35,20 @@ //! //! Below we test panics at different steps in the indexing pipeline. +use std::sync::{Arc, Mutex}; +use std::time::Duration; + use byte_unit::Byte; use fail::FailScenario; use quickwit_index_config::default_config_for_tests; use quickwit_indexing::actors::IndexerParams; use quickwit_indexing::index_data; -use quickwit_indexing::models::CommitPolicy; -use quickwit_indexing::models::ScratchDirectory; +use quickwit_indexing::models::{CommitPolicy, ScratchDirectory}; use quickwit_indexing::source::SourceConfig; use quickwit_metastore::checkpoint::Checkpoint; -use quickwit_metastore::IndexMetadata; -use quickwit_metastore::Metastore; -use quickwit_metastore::SingleFileMetastore; -use quickwit_metastore::SplitState; -use quickwit_storage::quickwit_storage_uri_resolver; -use quickwit_storage::StorageUriResolver; +use quickwit_metastore::{IndexMetadata, Metastore, SingleFileMetastore, SplitState}; +use quickwit_storage::{quickwit_storage_uri_resolver, StorageUriResolver}; use serde_json::json; -use std::sync::Arc; -use std::sync::Mutex; -use std::time::Duration; #[tokio::test] async fn test_failpoint_no_failure() -> anyhow::Result<()> { diff --git a/quickwit-indexing/src/actors/indexer.rs b/quickwit-indexing/src/actors/indexer.rs index 145b463f7c..ae29271114 100644 --- a/quickwit-indexing/src/actors/indexer.rs +++ b/quickwit-indexing/src/actors/indexer.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::io; use std::ops::RangeInclusive; @@ -25,25 +24,15 @@ use std::sync::Arc; use anyhow::Context; use byte_unit::Byte; use fail::fail_point; -use quickwit_actors::Actor; -use quickwit_actors::ActorContext; -use quickwit_actors::ActorExitStatus; -use quickwit_actors::Mailbox; -use quickwit_actors::QueueCapacity; -use quickwit_actors::SendError; -use quickwit_actors::SyncActor; +use quickwit_actors::{ + Actor, ActorContext, ActorExitStatus, Mailbox, QueueCapacity, SendError, SyncActor, +}; use quickwit_index_config::IndexConfig; -use tantivy::schema::Field; -use tantivy::schema::Value; +use tantivy::schema::{Field, Value}; use tantivy::Document; -use tracing::info; -use tracing::warn; +use tracing::{info, warn}; -use crate::models::CommitPolicy; -use crate::models::IndexedSplit; -use crate::models::IndexerMessage; -use crate::models::RawDocBatch; -use crate::models::ScratchDirectory; +use crate::models::{CommitPolicy, IndexedSplit, IndexerMessage, RawDocBatch, ScratchDirectory}; #[derive(Clone, Default, Debug, Eq, PartialEq)] pub struct IndexerCounters { @@ -400,19 +389,14 @@ mod tests { use std::sync::Arc; use std::time::Duration; - use crate::actors::indexer::record_timestamp; - use crate::actors::indexer::IndexerCounters; - use crate::actors::IndexerParams; - use crate::models::CommitPolicy; - use crate::models::IndexerMessage; - use crate::models::RawDocBatch; - use crate::models::ScratchDirectory; use byte_unit::Byte; - use quickwit_actors::create_test_mailbox; - use quickwit_actors::Universe; + use quickwit_actors::{create_test_mailbox, Universe}; use quickwit_metastore::checkpoint::CheckpointDelta; use super::Indexer; + use crate::actors::indexer::{record_timestamp, IndexerCounters}; + use crate::actors::IndexerParams; + use crate::models::{CommitPolicy, IndexerMessage, RawDocBatch, ScratchDirectory}; #[test] fn test_record_timestamp() { @@ -470,7 +454,7 @@ mod tests { num_valid_docs: 2, num_splits_emitted: 0, num_docs_in_split: 2, //< we have not reached the commit limit yet. - overall_num_bytes: 103, + overall_num_bytes: 103 } ); universe @@ -492,7 +476,7 @@ mod tests { num_valid_docs: 3, num_splits_emitted: 1, num_docs_in_split: 0, //< the num docs in split counter has been reset. - overall_num_bytes: 146, + overall_num_bytes: 146 } ); let output_messages = inbox.drain_available_message_for_test(); @@ -541,7 +525,7 @@ mod tests { num_valid_docs: 1, num_splits_emitted: 0, num_docs_in_split: 1, - overall_num_bytes: 42, + overall_num_bytes: 42 } ); universe.simulate_time_shift(Duration::from_secs(61)).await; @@ -554,7 +538,7 @@ mod tests { num_valid_docs: 1, num_splits_emitted: 1, num_docs_in_split: 0, - overall_num_bytes: 42, + overall_num_bytes: 42 } ); let output_messages = inbox.drain_available_message_for_test(); @@ -600,7 +584,7 @@ mod tests { num_valid_docs: 1, num_splits_emitted: 1, num_docs_in_split: 0, - overall_num_bytes: 42, + overall_num_bytes: 42 } ); let output_messages = inbox.drain_available_message_for_test(); diff --git a/quickwit-indexing/src/actors/mod.rs b/quickwit-indexing/src/actors/mod.rs index d6dc48d286..3cdc6bc2a2 100644 --- a/quickwit-indexing/src/actors/mod.rs +++ b/quickwit-indexing/src/actors/mod.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . mod indexer; mod packager; @@ -24,10 +23,11 @@ mod pipeline_supervisor; mod publisher; mod uploader; +pub use pipeline_supervisor::{ + IndexingPipelineHandler, IndexingPipelineParams, IndexingPipelineSupervisor, +}; + pub use self::indexer::{Indexer, IndexerCounters, IndexerParams}; pub use self::packager::Packager; pub use self::publisher::{Publisher, PublisherCounters}; pub use self::uploader::{Uploader, UploaderCounters}; -pub use pipeline_supervisor::{ - IndexingPipelineHandler, IndexingPipelineParams, IndexingPipelineSupervisor, -}; diff --git a/quickwit-indexing/src/actors/packager.rs b/quickwit-indexing/src/actors/packager.rs index 3d40510279..09ecd06849 100644 --- a/quickwit-indexing/src/actors/packager.rs +++ b/quickwit-indexing/src/actors/packager.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::collections::HashSet; use std::fs::File; @@ -25,24 +24,17 @@ use std::io::Write; use std::ops::Range; use std::path::PathBuf; -use crate::models::IndexedSplit; -use crate::models::PackagedSplit; -use crate::models::ScratchDirectory; use anyhow::Context; use fail::fail_point; -use quickwit_actors::Actor; -use quickwit_actors::ActorContext; -use quickwit_actors::Mailbox; -use quickwit_actors::QueueCapacity; -use quickwit_actors::SyncActor; +use quickwit_actors::{Actor, ActorContext, Mailbox, QueueCapacity, SyncActor}; use quickwit_directories::write_hotcache; -use quickwit_storage::BundleStorageBuilder; -use quickwit_storage::BUNDLE_FILENAME; +use quickwit_storage::{BundleStorageBuilder, BUNDLE_FILENAME}; use tantivy::common::CountingWriter; -use tantivy::SegmentId; -use tantivy::SegmentMeta; +use tantivy::{SegmentId, SegmentMeta}; use tracing::*; +use crate::models::{IndexedSplit, PackagedSplit, ScratchDirectory}; + /// The role of the packager is to get an index writer and /// produce a split file. /// @@ -276,19 +268,13 @@ mod tests { use std::ops::RangeInclusive; use std::time::Instant; - use quickwit_actors::create_test_mailbox; - use quickwit_actors::ObservationType; - use quickwit_actors::Universe; + use quickwit_actors::{create_test_mailbox, ObservationType, Universe}; use quickwit_metastore::checkpoint::CheckpointDelta; - use tantivy::doc; - use tantivy::schema::Schema; - use tantivy::schema::FAST; - use tantivy::schema::TEXT; - use tantivy::Index; - - use crate::models::ScratchDirectory; + use tantivy::schema::{Schema, FAST, TEXT}; + use tantivy::{doc, Index}; use super::*; + use crate::models::ScratchDirectory; fn make_indexed_split_for_test(segments_timestamps: &[&[i64]]) -> anyhow::Result { let split_scratch_directory = ScratchDirectory::try_new_temp()?; diff --git a/quickwit-indexing/src/actors/pipeline_supervisor.rs b/quickwit-indexing/src/actors/pipeline_supervisor.rs index d434d13405..dbd6c1942e 100644 --- a/quickwit-indexing/src/actors/pipeline_supervisor.rs +++ b/quickwit-indexing/src/actors/pipeline_supervisor.rs @@ -1,50 +1,38 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::sync::Arc; +use std::time::Duration; -use crate::actors::Indexer; -use crate::actors::IndexerParams; -use crate::actors::Packager; -use crate::actors::Publisher; -use crate::actors::Uploader; -use crate::models::IndexingStatistics; -use crate::source::quickwit_supported_sources; -use crate::source::SourceActor; -use crate::source::SourceConfig; use async_trait::async_trait; -use quickwit_actors::Actor; -use quickwit_actors::ActorContext; -use quickwit_actors::ActorExitStatus; -use quickwit_actors::ActorHandle; -use quickwit_actors::AsyncActor; -use quickwit_actors::Health; -use quickwit_actors::KillSwitch; -use quickwit_actors::Supervisable; +use quickwit_actors::{ + Actor, ActorContext, ActorExitStatus, ActorHandle, AsyncActor, Health, KillSwitch, Supervisable, +}; use quickwit_metastore::Metastore; use quickwit_storage::StorageUriResolver; use smallvec::SmallVec; -use std::sync::Arc; -use std::time::Duration; use tokio::join; -use tracing::debug; -use tracing::error; -use tracing::info; +use tracing::{debug, error, info}; + +use crate::actors::{Indexer, IndexerParams, Packager, Publisher, Uploader}; +use crate::models::IndexingStatistics; +use crate::source::{quickwit_supported_sources, SourceActor, SourceConfig}; pub struct IndexingPipelineHandler { pub source: ActorHandle, @@ -233,9 +221,9 @@ impl IndexingPipelineSupervisor { // TODO Accept errors in spawning. See #463. self.spawn_pipeline(ctx).await?; // if let Err(spawn_error) = self.spawn_pipeline(ctx).await { - // // only retry n-times. - // error!(err=?spawn_error, "Error while spawning"); - // self.terminate().await; + // // only retry n-times. + // error!(err=?spawn_error, "Error while spawning"); + // self.terminate().await; // } } else { match self.healthcheck() { @@ -302,19 +290,17 @@ pub struct IndexingPipelineParams { #[cfg(test)] mod tests { - use super::IndexingPipelineParams; - use super::IndexingPipelineSupervisor; - use crate::actors::IndexerParams; - use crate::source::SourceConfig; + use std::path::PathBuf; + use std::sync::Arc; + + use quickwit_actors::Universe; + use quickwit_metastore::{IndexMetadata, MockMetastore, SplitState}; use quickwit_storage::StorageUriResolver; use serde_json::json; - use quickwit_actors::Universe; - use quickwit_metastore::IndexMetadata; - use quickwit_metastore::MockMetastore; - use quickwit_metastore::SplitState; - use std::path::PathBuf; - use std::sync::Arc; + use super::{IndexingPipelineParams, IndexingPipelineSupervisor}; + use crate::actors::IndexerParams; + use crate::source::SourceConfig; #[tokio::test] async fn test_indexing_pipeline() -> anyhow::Result<()> { diff --git a/quickwit-indexing/src/actors/publisher.rs b/quickwit-indexing/src/actors/publisher.rs index ad699279b5..a1e4d4c35d 100644 --- a/quickwit-indexing/src/actors/publisher.rs +++ b/quickwit-indexing/src/actors/publisher.rs @@ -1,36 +1,33 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::sync::Arc; -use crate::models::UploadedSplit; use anyhow::Context; use async_trait::async_trait; use fail::fail_point; -use quickwit_actors::Actor; -use quickwit_actors::ActorContext; -use quickwit_actors::AsyncActor; -use quickwit_actors::QueueCapacity; +use quickwit_actors::{Actor, ActorContext, AsyncActor, QueueCapacity}; use quickwit_metastore::Metastore; use tokio::sync::oneshot::Receiver; +use crate::models::UploadedSplit; + #[derive(Debug, Clone, Default)] pub struct PublisherCounters { pub num_published_splits: u64, @@ -72,7 +69,8 @@ impl AsyncActor for Publisher { fail_point!("publisher:before"); let uploaded_split = uploaded_split_future .await - .with_context(|| "Upload apparently failed")?; //< splits must be published in order, so one uploaded failing means we should fail entirely. + .with_context(|| "Upload apparently failed")?; //< splits must be published in order, so one uploaded failing means we should fail + //< entirely. self.metastore .publish_splits( &uploaded_split.index_id, @@ -89,12 +87,13 @@ impl AsyncActor for Publisher { #[cfg(test)] mod tests { - use super::*; use quickwit_actors::Universe; use quickwit_metastore::checkpoint::CheckpointDelta; use quickwit_metastore::{MockMetastore, SplitMetadata, SplitMetadataAndFooterOffsets}; use tokio::sync::oneshot; + use super::*; + #[tokio::test] async fn test_publisher_publishes_in_order() { quickwit_common::setup_logging_for_tests(); @@ -141,7 +140,7 @@ mod tests { }, footer_offsets: 1000..1200 }, - checkpoint_delta: CheckpointDelta::from(3..7), + checkpoint_delta: CheckpointDelta::from(3..7) }) .is_ok()); assert!(split_future_tx1 @@ -154,7 +153,7 @@ mod tests { }, footer_offsets: 1000..1200 }, - checkpoint_delta: CheckpointDelta::from(1..3), + checkpoint_delta: CheckpointDelta::from(1..3) }) .is_ok()); let publisher_observation = publisher_handle.process_pending_and_observe().await.state; diff --git a/quickwit-indexing/src/actors/uploader.rs b/quickwit-indexing/src/actors/uploader.rs index 5c9358d48e..52771e5257 100644 --- a/quickwit-indexing/src/actors/uploader.rs +++ b/quickwit-indexing/src/actors/uploader.rs @@ -1,54 +1,40 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::mem; use std::path::PathBuf; -use std::sync::atomic::AtomicU64; -use std::sync::atomic::Ordering; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::Instant; -use crate::models::PackagedSplit; -use crate::models::UploadedSplit; -use crate::semaphore::Semaphore; -use anyhow::bail; -use anyhow::Context; +use anyhow::{bail, Context}; use async_trait::async_trait; use fail::fail_point; -use quickwit_actors::Actor; -use quickwit_actors::ActorContext; -use quickwit_actors::ActorExitStatus; -use quickwit_actors::AsyncActor; -use quickwit_actors::Mailbox; -use quickwit_actors::QueueCapacity; -use quickwit_metastore::Metastore; -use quickwit_metastore::SplitMetadata; -use quickwit_metastore::SplitMetadataAndFooterOffsets; -use quickwit_metastore::SplitState; -use quickwit_storage::PutPayload; -use quickwit_storage::Storage; -use quickwit_storage::BUNDLE_FILENAME; +use quickwit_actors::{Actor, ActorContext, ActorExitStatus, AsyncActor, Mailbox, QueueCapacity}; +use quickwit_metastore::{Metastore, SplitMetadata, SplitMetadataAndFooterOffsets, SplitState}; +use quickwit_storage::{PutPayload, Storage, BUNDLE_FILENAME}; use tantivy::chrono::Utc; use tokio::sync::oneshot::Receiver; -use tracing::info; -use tracing::warn; +use tracing::{info, warn}; + +use crate::models::{PackagedSplit, UploadedSplit}; +use crate::semaphore::Semaphore; pub const MAX_CONCURRENT_SPLIT_UPLOAD: usize = 3; @@ -219,16 +205,14 @@ impl AsyncActor for Uploader { #[cfg(test)] mod tests { - use crate::models::ScratchDirectory; - use quickwit_actors::create_test_mailbox; - use quickwit_actors::ObservationType; - use quickwit_actors::Universe; + use quickwit_actors::{create_test_mailbox, ObservationType, Universe}; use quickwit_metastore::checkpoint::CheckpointDelta; use quickwit_metastore::MockMetastore; use quickwit_storage::RamStorage; use tantivy::SegmentId; use super::*; + use crate::models::ScratchDirectory; #[tokio::test] async fn test_uploader() -> anyhow::Result<()> { diff --git a/quickwit-indexing/src/lib.rs b/quickwit-indexing/src/lib.rs index 7d91abf752..ae9b9be188 100644 --- a/quickwit-indexing/src/lib.rs +++ b/quickwit-indexing/src/lib.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::sync::Arc; @@ -25,9 +24,7 @@ use quickwit_actors::Universe; use quickwit_metastore::Metastore; use quickwit_storage::StorageUriResolver; -use crate::actors::IndexerParams; -use crate::actors::IndexingPipelineParams; -use crate::actors::IndexingPipelineSupervisor; +use crate::actors::{IndexerParams, IndexingPipelineParams, IndexingPipelineSupervisor}; use crate::models::IndexingStatistics; use crate::source::SourceConfig; @@ -38,8 +35,7 @@ pub(crate) mod semaphore; pub mod source; mod test_utils; -pub use test_utils::mock_split_meta; -pub use test_utils::TestSandbox; +pub use test_utils::{mock_split_meta, TestSandbox}; pub use self::merge_policy::{MergePolicy, StableMultitenantWithTimestampMergePolicy}; diff --git a/quickwit-indexing/src/merge_policy.rs b/quickwit-indexing/src/merge_policy.rs index 322878ff5c..126698d002 100644 --- a/quickwit-indexing/src/merge_policy.rs +++ b/quickwit-indexing/src/merge_policy.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::cmp::Reverse; use std::fmt; @@ -86,7 +85,8 @@ pub trait MergePolicy: Send + Sync { /// As a result, each level interval is at least 3 times larger than the previous one, /// forming a logscale over the number of documents. /// -/// Because we stop merging splits reaching a size larger than if it would result in a size larger than `target_num_docs`. +/// Because we stop merging splits reaching a size larger than if it would result in a size larger +/// than `target_num_docs`. #[derive(Clone)] pub struct StableMultitenantWithTimestampMergePolicy { pub target_demux_ops: usize, @@ -295,7 +295,8 @@ mod tests { fn create_splits(num_docs_vec: Vec) -> Vec { let num_records_with_timestamp = num_docs_vec .into_iter() - // we give the same timestamp to all of them and rely on stable sort to keep the split order. + // we give the same timestamp to all of them and rely on stable sort to keep the split + // order. .map(|num_records| (num_records, (1630563067..=1630564067))) .collect(); create_splits_with_timestamps(num_records_with_timestamp) @@ -453,7 +454,7 @@ mod tests { let merge_policy = StableMultitenantWithTimestampMergePolicy::default(); let mut splits = create_splits(vec![ 100_000, 100_000, 100_000, 100_000, 100_000, - 10_000_000, /* this split should not interfere with the merging of other splits */ + 10_000_000, // this split should not interfere with the merging of other splits 100_000, 100_000, 100_000, 100_000, 100_000, ]); let merge_ops = merge_policy.operations(&mut splits); diff --git a/quickwit-indexing/src/models/commit_policy.rs b/quickwit-indexing/src/models/commit_policy.rs index 4c216609bb..73980e85ef 100644 --- a/quickwit-indexing/src/models/commit_policy.rs +++ b/quickwit-indexing/src/models/commit_policy.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . const DEFAULT_COMMIT_TIMEOUT: Duration = Duration::from_secs(60); const DEFAULT_NUM_DOCS_COMMIT_THRESHOLD: u64 = 10_000_000; diff --git a/quickwit-indexing/src/models/indexed_split.rs b/quickwit-indexing/src/models/indexed_split.rs index f31076bb3e..7f39388d5f 100644 --- a/quickwit-indexing/src/models/indexed_split.rs +++ b/quickwit-indexing/src/models/indexed_split.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::fmt; use std::ops::RangeInclusive; @@ -25,8 +24,7 @@ use std::time::Instant; use quickwit_metastore::checkpoint::CheckpointDelta; use tantivy::merge_policy::NoMergePolicy; -use tantivy::schema::Field; -use tantivy::schema::Schema; +use tantivy::schema::{Field, Schema}; use crate::actors::IndexerParams; use crate::models::ScratchDirectory; diff --git a/quickwit-indexing/src/models/indexer_message.rs b/quickwit-indexing/src/models/indexer_message.rs index 448a075e25..99bd3012fc 100644 --- a/quickwit-indexing/src/models/indexer_message.rs +++ b/quickwit-indexing/src/models/indexer_message.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use crate::models::RawDocBatch; diff --git a/quickwit-indexing/src/models/indexing_statistics.rs b/quickwit-indexing/src/models/indexing_statistics.rs index c5ad318c7d..0a6d344ffa 100644 --- a/quickwit-indexing/src/models/indexing_statistics.rs +++ b/quickwit-indexing/src/models/indexing_statistics.rs @@ -1,30 +1,25 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::sync::atomic::Ordering; -use crate::actors::IndexerCounters; -use crate::actors::PublisherCounters; -use crate::actors::UploaderCounters; +use crate::actors::{IndexerCounters, PublisherCounters, UploaderCounters}; /// A Struct that holds all statistical data about indexing #[derive(Debug, Default, Clone)] @@ -39,7 +34,7 @@ pub struct IndexingStatistics { pub num_staged_splits: u64, /// Number of uploaded splits pub num_uploaded_splits: u64, - ///Number of published splits + /// Number of published splits pub num_published_splits: u64, /// Size in byte of document processed pub total_bytes_processed: u64, diff --git a/quickwit-indexing/src/models/mod.rs b/quickwit-indexing/src/models/mod.rs index e707abeabe..a3304a54b8 100644 --- a/quickwit-indexing/src/models/mod.rs +++ b/quickwit-indexing/src/models/mod.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . mod commit_policy; mod indexed_split; diff --git a/quickwit-indexing/src/models/packaged_split.rs b/quickwit-indexing/src/models/packaged_split.rs index cde974f3e9..d28d708e6e 100644 --- a/quickwit-indexing/src/models/packaged_split.rs +++ b/quickwit-indexing/src/models/packaged_split.rs @@ -1,29 +1,30 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . -use crate::models::ScratchDirectory; -use quickwit_metastore::checkpoint::CheckpointDelta; use std::collections::HashSet; use std::ops::{Range, RangeInclusive}; + +use quickwit_metastore::checkpoint::CheckpointDelta; use tantivy::SegmentId; +use crate::models::ScratchDirectory; + #[derive(Debug)] pub struct PackagedSplit { pub split_id: String, diff --git a/quickwit-indexing/src/models/raw_doc_batch.rs b/quickwit-indexing/src/models/raw_doc_batch.rs index d7d543f206..30e4ea63bb 100644 --- a/quickwit-indexing/src/models/raw_doc_batch.rs +++ b/quickwit-indexing/src/models/raw_doc_batch.rs @@ -1,21 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use quickwit_metastore::checkpoint::CheckpointDelta; diff --git a/quickwit-indexing/src/models/scratch_directory.rs b/quickwit-indexing/src/models/scratch_directory.rs index 483fceaffe..a20ddcb99e 100644 --- a/quickwit-indexing/src/models/scratch_directory.rs +++ b/quickwit-indexing/src/models/scratch_directory.rs @@ -1,28 +1,25 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . -use std::fmt; -use std::io; -use std::path::Path; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::{fmt, io}; enum ScratchDirectoryType { Path(PathBuf), TempDir(tempfile::TempDir), @@ -108,9 +105,10 @@ impl ScratchDirectory { #[cfg(test)] mod tests { - use super::*; use std::mem; + use super::*; + #[test] fn test_scratch_directory() -> io::Result<()> { let pa = ScratchDirectory::try_new_temp()?; diff --git a/quickwit-indexing/src/models/uploaded_split.rs b/quickwit-indexing/src/models/uploaded_split.rs index 0d4a3f96b0..93a3b9d19b 100644 --- a/quickwit-indexing/src/models/uploaded_split.rs +++ b/quickwit-indexing/src/models/uploaded_split.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use quickwit_metastore::checkpoint::CheckpointDelta; use quickwit_metastore::SplitMetadataAndFooterOffsets; diff --git a/quickwit-indexing/src/semaphore.rs b/quickwit-indexing/src/semaphore.rs index 3ef939b3d8..df7021bf0a 100644 --- a/quickwit-indexing/src/semaphore.rs +++ b/quickwit-indexing/src/semaphore.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::sync::Arc; @@ -67,10 +66,10 @@ impl Drop for SemaphoreGuard { #[cfg(test)] mod tests { - use super::*; use std::mem; - use std::sync::atomic::AtomicUsize; - use std::sync::atomic::Ordering; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; #[tokio::test] async fn test_semaphore() { @@ -79,9 +78,10 @@ mod tests { let semaphore = Semaphore::new(NUM_CONCURRENT_TASKS); for i in 1..1_000 { let guard = semaphore.acquire().await; - // Thanks to the semaphore we have the guarantee that at most NUM_CONCURRENT_TASKS tasks are running - // at the same time. - // In other words, upon acquisition of guard, we know that at least (i - 3) tasks have terminated. + // Thanks to the semaphore we have the guarantee that at most NUM_CONCURRENT_TASKS tasks + // are running at the same time. + // In other words, upon acquisition of guard, we know that at least (i - 3) tasks have + // terminated. assert!(finished_task.load(Ordering::SeqCst) + NUM_CONCURRENT_TASKS >= i); let finished_task_clone = finished_task.clone(); tokio::task::spawn(async move { diff --git a/quickwit-indexing/src/source/file_source.rs b/quickwit-indexing/src/source/file_source.rs index 413d9533c1..a0a9c7581a 100644 --- a/quickwit-indexing/src/source/file_source.rs +++ b/quickwit-indexing/src/source/file_source.rs @@ -1,45 +1,38 @@ -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . -use crate::models::IndexerMessage; -use crate::models::RawDocBatch; -use crate::source::Source; -use crate::source::SourceContext; -use crate::source::TypedSourceFactory; -use anyhow::Context; -use async_trait::async_trait; -use quickwit_actors::ActorExitStatus; -use quickwit_actors::Mailbox; -use quickwit_metastore::checkpoint::CheckpointDelta; -use quickwit_metastore::checkpoint::PartitionId; -use quickwit_metastore::checkpoint::Position; -use serde::{Deserialize, Serialize}; use std::io; use std::io::SeekFrom; use std::path::PathBuf; + +use anyhow::Context; +use async_trait::async_trait; +use quickwit_actors::{ActorExitStatus, Mailbox}; +use quickwit_metastore::checkpoint::{CheckpointDelta, PartitionId, Position}; +use serde::{Deserialize, Serialize}; use tokio::fs::File; -use tokio::io::AsyncBufReadExt; -use tokio::io::AsyncRead; -use tokio::io::AsyncSeekExt; -use tokio::io::BufReader; +use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncSeekExt, BufReader}; use tracing::info; +use crate::models::{IndexerMessage, RawDocBatch}; +use crate::source::{Source, SourceContext, TypedSourceFactory}; + /// Cut a new batch as soon as we have read BATCH_NUM_BYTES_THRESHOLD. const BATCH_NUM_BYTES_THRESHOLD: u64 = 500_000u64; @@ -186,12 +179,11 @@ impl TypedSourceFactory for FileSourceFactory { mod tests { use std::io::Write; - use crate::source::SourceActor; + use quickwit_actors::{create_test_mailbox, Universe}; + use quickwit_metastore::checkpoint::Checkpoint; use super::*; - use quickwit_actors::create_test_mailbox; - use quickwit_actors::Universe; - use quickwit_metastore::checkpoint::Checkpoint; + use crate::source::SourceActor; #[tokio::test] async fn test_file_source() -> anyhow::Result<()> { diff --git a/quickwit-indexing/src/source/kafka_source.rs b/quickwit-indexing/src/source/kafka_source.rs index daab3305a5..e8f885d4e1 100644 --- a/quickwit-indexing/src/source/kafka_source.rs +++ b/quickwit-indexing/src/source/kafka_source.rs @@ -1,4 +1,4 @@ -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // // Quickwit is offered under the AGPL v3.0 and as commercial software. // For commercial licensing, contact us at hello@quickwit.io. @@ -17,17 +17,15 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use crate::models::RawDocBatch; -use crate::source::IndexerMessage; -use crate::source::Source; -use crate::source::SourceContext; -use crate::source::TypedSourceFactory; +use std::collections::HashMap; +use std::fmt; +use std::time::Duration; + use anyhow::{bail, Context}; use async_trait::async_trait; use futures::StreamExt; use itertools::Itertools; -use quickwit_actors::ActorExitStatus; -use quickwit_actors::Mailbox; +use quickwit_actors::{ActorExitStatus, Mailbox}; use quickwit_metastore::checkpoint::{Checkpoint, CheckpointDelta, PartitionId, Position}; use rdkafka::config::{ClientConfig, RDKafkaLogLevel}; use rdkafka::consumer::stream_consumer::StreamConsumer; @@ -39,23 +37,24 @@ use rdkafka::util::Timeout; use rdkafka::{ClientContext, Message, Offset}; use serde::{Deserialize, Serialize}; use serde_json::json; -use std::collections::HashMap; -use std::fmt; -use std::time::Duration; use tracing::{debug, info, warn}; +use crate::models::RawDocBatch; +use crate::source::{IndexerMessage, Source, SourceContext, TypedSourceFactory}; + /// Required parameters for instantiating a `KafkaSource`. #[derive(Clone, Deserialize, Serialize)] pub struct KafkaSourceParams { - /// Comma-separated list of host and port pairs that provide the initial addresses of Kafka brokers that act as the - /// starting point for a Kafka client to discover the full set of alive servers in the cluster. + /// Comma-separated list of host and port pairs that provide the initial addresses of Kafka + /// brokers that act as the starting point for a Kafka client to discover the full set of + /// alive servers in the cluster. pub bootstrap_servers: String, /// Specifies the name of the consumer group a Kafka consumer belongs to. pub group_id: String, /// Name of the topic that the source consumes. pub topic: String, - /// When set to `true`, the source will terminate after reading the last message of each assigned partition. - /// Otherwise, it will keep waiting for new incoming messages. + /// When set to `true`, the source will terminate after reading the last message of each + /// assigned partition. Otherwise, it will keep waiting for new incoming messages. pub enable_partition_eof: Option, } @@ -209,7 +208,8 @@ impl Source for KafkaSource { ); continue; } - // FIXME: This is assuming that Kafka errors are not recoverable, it may not be the case. + // FIXME: This is assuming that Kafka errors are not recoverable, it may not be the + // case. Err(err) => return Err(ActorExitStatus::from(anyhow::anyhow!(err))), }; if let Some(doc) = parse_message_payload(&message) { @@ -226,7 +226,8 @@ impl Source for KafkaSource { .get(&message.partition()) .ok_or_else(|| { anyhow::anyhow!( - "Received unexpected message from partition `{}`. Assigned partitions: `{{{}}}`.", + "Received unexpected message from partition `{}`. Assigned partitions: \ + `{{{}}}`.", message.partition(), self.state.assigned_partition_ids.keys().join(", "), ) @@ -322,7 +323,8 @@ fn create_consumer( Ok(consumer) } -/// Represents a checkpoint with the Kafka native types: `i32` for partition IDs and `i64` for offsets. +/// Represents a checkpoint with the Kafka native types: `i32` for partition IDs and `i64` for +/// offsets. fn kafka_checkpoint_from_checkpoint(checkpoint: &Checkpoint) -> anyhow::Result> { let mut kafka_checkpoint = HashMap::with_capacity(checkpoint.num_partitions()); for (partition_id, position) in checkpoint.iter() { @@ -372,11 +374,12 @@ async fn fetch_partition_ids( /// Fetches the low and high watermarks for the given topic and partition IDs. /// -/// The low watermark is the offset of the earliest message in the partition. If no messages have been written to the -/// topic, the low watermark offset is set to 0. The low watermark will also be 0 if one message has been written to -/// the partition (with offset 0). +/// The low watermark is the offset of the earliest message in the partition. If no messages have +/// been written to the topic, the low watermark offset is set to 0. The low watermark will also be +/// 0 if one message has been written to the partition (with offset 0). /// -/// The high watermark is the offset of the latest message in the partition available for consumption + 1. +/// The high watermark is the offset of the latest message in the partition available for +/// consumption + 1. async fn fetch_watermarks( consumer: &KafkaSourceConsumer, topic: &str, @@ -401,8 +404,8 @@ async fn fetch_watermarks( Ok(watermarks) } -/// Given a checkpoint, computes the next offset from which to start reading messages for the provided partition IDs. -/// See `compute_next_offset` for further explanation. +/// Given a checkpoint, computes the next offset from which to start reading messages for the +/// provided partition IDs. See `compute_next_offset` for further explanation. fn compute_assignment( topic: &str, partition_ids: &[i32], @@ -417,10 +420,11 @@ fn compute_assignment( Ok(assignment) } -/// Given a checkpoint, computes the next offset from which to start reading messages. In most cases, it should be the -/// offset of the last checkpointed record + 1. However, when that offset no longer exists in the partition (data loss, -/// retention, ...), the next offset is the low watermark. If a partition ID is not covered by a checkpoint, the -/// partition is read from the beginning. +/// Given a checkpoint, computes the next offset from which to start reading messages. In most +/// cases, it should be the offset of the last checkpointed record + 1. However, when that offset no +/// longer exists in the partition (data loss, retention, ...), the next offset is the low +/// watermark. If a partition ID is not covered by a checkpoint, the partition is read from the +/// beginning. fn compute_next_offset( partition_id: i32, checkpoint: &HashMap, @@ -434,7 +438,8 @@ fn compute_next_offset( Some(&watermarks) => watermarks, None => bail!("Missing watermarks for partition `{}`.", partition_id), }; - // We found a gap between the last checkpoint and the low watermark, so we resume from the low watermark. + // We found a gap between the last checkpoint and the low watermark, so we resume from the low + // watermark. if checkpoint_offset < low_watermark { return Ok(Offset::Offset(low_watermark)); } @@ -450,7 +455,8 @@ fn compute_next_offset( ); } -/// Converts the raw bytes of the message payload to a `String` skipping corrupted or empty messages. +/// Converts the raw bytes of the message payload to a `String` skipping corrupted or empty +/// messages. fn parse_message_payload(message: &BorrowedMessage) -> Option { match message.payload_view::() { Some(Ok(payload)) if payload.len() > 0 => { @@ -581,18 +587,16 @@ mod tests { #[cfg(all(test, feature = "kafka-broker-external-service"))] mod kafka_broker_tests { - use super::*; - use crate::source::SourceActor; - use quickwit_actors::create_test_mailbox; - use quickwit_actors::Universe; + use quickwit_actors::{create_test_mailbox, Universe}; use rand::distributions::Alphanumeric; use rand::Rng; - use rdkafka::admin::TopicReplication; - use rdkafka::admin::{AdminClient, AdminOptions, NewTopic}; + use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication}; use rdkafka::client::DefaultClientContext; use rdkafka::message::ToBytes; - use rdkafka::producer::FutureProducer; - use rdkafka::producer::FutureRecord; + use rdkafka::producer::{FutureProducer, FutureRecord}; + + use super::*; + use crate::source::SourceActor; fn append_random_suffix(string: &str) -> String { let rng = rand::thread_rng(); @@ -899,9 +903,9 @@ mod kafka_broker_tests { create_topic(&admin_client, &topic, 2).await?; let consumer = create_consumer(&bootstrap_servers, &group_id, true)?; - // Force metadata update for the consumer. Otherwise, `fetch_watermarks` may return `UnknownPartition` - // if the broker hasn't received a metadata update since the topic was created. - // See also https://issues.apache.org/jira/browse/KAFKA-6829. + // Force metadata update for the consumer. Otherwise, `fetch_watermarks` may return + // `UnknownPartition` if the broker hasn't received a metadata update since the + // topic was created. See also https://issues.apache.org/jira/browse/KAFKA-6829. consumer.fetch_metadata(Some(&topic), Duration::from_secs(5))?; assert!(fetch_watermarks(&consumer, "topic-does-not-exist", &[0]) .await diff --git a/quickwit-indexing/src/source/mod.rs b/quickwit-indexing/src/source/mod.rs index fab25291dc..5fdbc926f1 100644 --- a/quickwit-indexing/src/source/mod.rs +++ b/quickwit-indexing/src/source/mod.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . mod file_source; #[cfg(feature = "kafka")] @@ -24,22 +23,19 @@ mod kafka_source; mod source_factory; mod vec_source; -use crate::models::IndexerMessage; -use async_trait::async_trait; -use once_cell::sync::OnceCell; -use quickwit_actors::Actor; -use quickwit_actors::ActorContext; -use quickwit_actors::ActorExitStatus; -use quickwit_actors::AsyncActor; -use quickwit_actors::Mailbox; use std::fmt; +use async_trait::async_trait; pub use file_source::{FileSource, FileSourceFactory, FileSourceParams}; #[cfg(feature = "kafka")] pub use kafka_source::{KafkaSource, KafkaSourceFactory, KafkaSourceParams}; +use once_cell::sync::OnceCell; +use quickwit_actors::{Actor, ActorContext, ActorExitStatus, AsyncActor, Mailbox}; pub use source_factory::{SourceFactory, SourceLoader, TypedSourceFactory}; pub use vec_source::{VecSource, VecSourceFactory, VecSourceParams}; +use crate::models::IndexerMessage; + pub type SourceContext = ActorContext; /// A source is a trait that is mounted in a light wrapping Actor called `SourceActor`. @@ -55,9 +51,9 @@ pub type SourceContext = ActorContext; /// ```ignore /// source.initialize(ctx)? /// let exit_status = loop { -/// if let Err(exit_status) = source.emit_batches()? { -/// break exit_status; -//// } +/// if let Err(exit_status) = source.emit_batches()? { +/// break exit_status; +//// } /// }; /// source.finalize(exit_status)?; /// ``` diff --git a/quickwit-indexing/src/source/source_factory.rs b/quickwit-indexing/src/source/source_factory.rs index a692a6e17a..8928c9f5bc 100644 --- a/quickwit-indexing/src/source/source_factory.rs +++ b/quickwit-indexing/src/source/source_factory.rs @@ -1,31 +1,32 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::collections::HashMap; -use super::Source; -use crate::source::SourceConfig; use async_trait::async_trait; use itertools::Itertools; use quickwit_metastore::checkpoint::Checkpoint; -use std::collections::HashMap; use thiserror::Error; +use super::Source; +use crate::source::SourceConfig; + #[async_trait] pub trait SourceFactory: 'static + Send + Sync { async fn create_source( @@ -65,7 +66,10 @@ pub struct SourceLoader { #[derive(Error, Debug)] pub enum SourceLoaderError { - #[error("Unknown source type `{requested_source_type}` (available source types are {available_source_types}).")] + #[error( + "Unknown source type `{requested_source_type}` (available source types are \ + {available_source_types})." + )] UnknownSourceType { requested_source_type: String, available_source_types: String, //< a comma separated list with the available source_type. @@ -115,9 +119,10 @@ impl SourceLoader { #[cfg(test)] mod tests { + use serde_json::json; + use super::*; use crate::source::quickwit_supported_sources; - use serde_json::json; #[tokio::test] async fn test_source_loader_success() -> anyhow::Result<()> { diff --git a/quickwit-indexing/src/source/vec_source.rs b/quickwit-indexing/src/source/vec_source.rs index 542d0fa721..f095808c78 100644 --- a/quickwit-indexing/src/source/vec_source.rs +++ b/quickwit-indexing/src/source/vec_source.rs @@ -1,38 +1,31 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . -use crate::models::IndexerMessage; -use crate::models::RawDocBatch; -use crate::source::Source; -use crate::source::SourceContext; -use crate::source::TypedSourceFactory; use async_trait::async_trait; -use quickwit_actors::ActorExitStatus; -use quickwit_actors::Mailbox; -use quickwit_metastore::checkpoint::Checkpoint; -use quickwit_metastore::checkpoint::CheckpointDelta; -use quickwit_metastore::checkpoint::PartitionId; -use quickwit_metastore::checkpoint::Position; +use quickwit_actors::{ActorExitStatus, Mailbox}; +use quickwit_metastore::checkpoint::{Checkpoint, CheckpointDelta, PartitionId, Position}; use serde::{Deserialize, Serialize}; use tracing::info; +use crate::models::{IndexerMessage, RawDocBatch}; +use crate::source::{Source, SourceContext, TypedSourceFactory}; + #[derive(Deserialize, Serialize)] pub struct VecSourceParams { pub items: Vec, @@ -120,11 +113,11 @@ impl Source for VecSource { #[cfg(test)] mod tests { + use quickwit_actors::{create_test_mailbox, Universe}; + use serde_json::json; + use super::*; use crate::source::SourceActor; - use quickwit_actors::create_test_mailbox; - use quickwit_actors::Universe; - use serde_json::json; #[tokio::test] async fn test_vec_source() -> anyhow::Result<()> { diff --git a/quickwit-indexing/src/test_utils.rs b/quickwit-indexing/src/test_utils.rs index 2c372ae7b7..e423889ca2 100644 --- a/quickwit-indexing/src/test_utils.rs +++ b/quickwit-indexing/src/test_utils.rs @@ -1,32 +1,26 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; -use crate::actors::IndexerParams; -use crate::index_data; -use crate::models::{CommitPolicy, IndexingStatistics, ScratchDirectory}; -use crate::source::{SourceConfig, VecSourceParams}; use byte_unit::Byte; use quickwit_index_config::IndexConfig; use quickwit_metastore::checkpoint::Checkpoint; @@ -36,6 +30,11 @@ use quickwit_metastore::{ }; use quickwit_storage::StorageUriResolver; +use crate::actors::IndexerParams; +use crate::index_data; +use crate::models::{CommitPolicy, IndexingStatistics, ScratchDirectory}; +use crate::source::{SourceConfig, VecSourceParams}; + /// Creates a Test environment. /// /// It makes it easy to create a test index, perfect for unit testing. @@ -150,9 +149,10 @@ pub fn mock_split_meta(split_id: &str) -> SplitMetadataAndFooterOffsets { mod tests { use std::sync::Arc; - use super::TestSandbox; use quickwit_index_config::WikipediaIndexConfig; + use super::TestSandbox; + #[tokio::test] async fn test_test_sandbox() -> anyhow::Result<()> { quickwit_common::setup_logging_for_tests(); diff --git a/quickwit-metastore/src/checkpoint.rs b/quickwit-metastore/src/checkpoint.rs index 6839c0c1d8..bd76c00020 100644 --- a/quickwit-metastore/src/checkpoint.rs +++ b/quickwit-metastore/src/checkpoint.rs @@ -1,26 +1,22 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . -use serde::ser::SerializeMap; -use serde::Deserialize; -use serde::Serialize; use std::cmp::Ordering; use std::collections::btree_map::Entry; use std::collections::BTreeMap; @@ -28,9 +24,11 @@ use std::fmt; use std::iter::FromIterator; use std::ops::Range; use std::sync::Arc; + +use serde::ser::SerializeMap; +use serde::{Deserialize, Serialize}; use thiserror::Error; -use tracing::info; -use tracing::warn; +use tracing::{info, warn}; /// PartitionId identifies a partition for a given source. #[derive(Debug, Default, Clone, Eq, PartialEq, Ord, PartialOrd)] @@ -172,9 +170,7 @@ impl Checkpoint { /// ``` impl FromIterator<(PartitionId, Position)> for Checkpoint { fn from_iter(iter: I) -> Checkpoint - where - I: IntoIterator, - { + where I: IntoIterator { Checkpoint { per_partition: iter.into_iter().collect(), } @@ -183,9 +179,7 @@ impl FromIterator<(PartitionId, Position)> for Checkpoint { impl Serialize for Checkpoint { fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { + where S: serde::Serializer { let mut map = serializer.serialize_map(Some(self.per_partition.len()))?; for (partition, position) in &self.per_partition { map.serialize_entry(&*partition.0, &*position.as_str())?; @@ -196,9 +190,7 @@ impl Serialize for Checkpoint { impl<'de> Deserialize<'de> for Checkpoint { fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { + where D: serde::Deserializer<'de> { let string_to_string_map: BTreeMap = BTreeMap::deserialize(deserializer)?; let per_partition: BTreeMap = string_to_string_map .into_iter() @@ -214,7 +206,10 @@ impl<'de> Deserialize<'de> for Checkpoint { /// compatible. ie: the checkpoint delta starts from a point anterior to /// the checkpoint. #[derive(Error, Debug, PartialEq)] -#[error("IncompatibleChkptDelta at partition: {partition_id:?} cur_pos:{current_position:?} delta_pos:{delta_position_from:?}")] +#[error( + "IncompatibleChkptDelta at partition: {partition_id:?} cur_pos:{current_position:?} \ + delta_pos:{delta_position_from:?}" +)] pub struct IncompatibleCheckpointDelta { /// One PartitionId for which the incompatibility has been detected. pub partition_id: PartitionId, @@ -274,11 +269,11 @@ impl Checkpoint { /// as gaps may happen. For instance, assuming a Kafka source, if the indexing /// pipeline is down for more than the retention period. /// - /// | Checkpoint & Delta | Outcome | - /// |------------------------------|-----------------------------| - /// | (..a] (b..c] with a = b | Compatible | - /// | (..a] (b..c] with b > a | Compatible | - /// | (..a] (b..c] with b < a | Incompatible | + /// | Checkpoint & Delta | Outcome | + /// |------------------------------|-----------------------------| + /// | (..a] (b..c] with a = b | Compatible | + /// | (..a] (b..c] with b > a | Compatible | + /// | (..a] (b..c] with b < a | Incompatible | /// /// If the delta is compatible, returns an error without modifying the original checkpoint. pub fn try_apply_delta( @@ -613,7 +608,7 @@ mod tests { Err(IncompatibleCheckpointDelta { partition_id: PartitionId::from("a"), current_position: Position::from("00128"), - delta_position_from: Position::from("00130"), + delta_position_from: Position::from("00130") }) ); Ok(()) diff --git a/quickwit-metastore/src/error.rs b/quickwit-metastore/src/error.rs index 76c3dc6362..627c091ac2 100644 --- a/quickwit-metastore/src/error.rs +++ b/quickwit-metastore/src/error.rs @@ -1,24 +1,21 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::io; diff --git a/quickwit-metastore/src/lib.rs b/quickwit-metastore/src/lib.rs index 26bcadc390..0602425b2a 100644 --- a/quickwit-metastore/src/lib.rs +++ b/quickwit-metastore/src/lib.rs @@ -1,32 +1,29 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . #![warn(missing_docs)] #![allow(clippy::bool_assert_comparison)] -/*! `quickwit-metastore` is the abstraction used in quickwit to interface itself to different metastore: -- single file metastore -etc. -*/ +//! `quickwit-metastore` is the abstraction used in quickwit to interface itself to different +//! metastore: +//! - single file metastore +//! etc. #[macro_use] mod tests; @@ -39,10 +36,9 @@ mod metastore_resolver; pub use error::{MetastoreError, MetastoreResolverError, MetastoreResult}; pub use metastore::single_file_metastore::SingleFileMetastore; +#[cfg(feature = "testsuite")] +pub use metastore::MockMetastore; pub use metastore::{ IndexMetadata, MetadataSet, Metastore, SplitMetadata, SplitMetadataAndFooterOffsets, SplitState, }; pub use metastore_resolver::{MetastoreFactory, MetastoreUriResolver}; - -#[cfg(feature = "testsuite")] -pub use metastore::MockMetastore; diff --git a/quickwit-metastore/src/metastore.rs b/quickwit-metastore/src/metastore.rs index 9db63ff3be..a840b7d953 100644 --- a/quickwit-metastore/src/metastore.rs +++ b/quickwit-metastore/src/metastore.rs @@ -1,31 +1,28 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . pub mod single_file_metastore; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::fmt::Debug; use std::ops::{Range, RangeInclusive}; -use std::{collections::HashMap, sync::Arc}; +use std::sync::Arc; use async_trait::async_trait; use chrono::Utc; @@ -144,13 +141,13 @@ pub struct MetadataSet { /// /// The split state goes through the following life cycle: /// 1. `New` -/// - Create new split and start indexing. +/// - Create new split and start indexing. /// 2. `Staged` -/// - Start uploading the split files. +/// - Start uploading the split files. /// 3. `Published` -/// - Uploading the split files is complete and the split is searchable. +/// - Uploading the split files is complete and the split is searchable. /// 4. `ScheduledForDeletion` -/// - Mark the split for deletion. +/// - Mark the split for deletion. /// /// If a split has a file in the storage, it MUST be registered in the metastore, /// and its state can be as follows: @@ -158,10 +155,11 @@ pub struct MetadataSet { /// - `Published`: The split is ready and published. /// - `ScheduledForDeletion`: The split is scheduled for deletion. /// -/// Before creating any file, we need to stage the split. If there is a failure, upon recovery, we schedule for deletion all the staged splits. -/// A client may not necessarily remove files from storage right after marking it as deleted. -/// A CLI client may delete files right away, but a more serious deployment should probably -/// only delete those files after a grace period so that the running search queries can complete. +/// Before creating any file, we need to stage the split. If there is a failure, upon recovery, we +/// schedule for deletion all the staged splits. A client may not necessarily remove files from +/// storage right after marking it as deleted. A CLI client may delete files right away, but a more +/// serious deployment should probably only delete those files after a grace period so that the +/// running search queries can complete. #[cfg_attr(any(test, feature = "testsuite"), mockall::automock)] #[async_trait] pub trait Metastore: Send + Sync + 'static { @@ -233,8 +231,8 @@ pub trait Metastore: Send + Sync + 'static { ) -> MetastoreResult>; /// Marks a list of splits as deleted. - /// This API will change the state to `ScheduledForDeletion` so that it is not referenced by the client. - /// It actually does not remove the split from storage. + /// This API will change the state to `ScheduledForDeletion` so that it is not referenced by the + /// client. It actually does not remove the split from storage. /// An error will occur if you specify an index or split that does not exist in the storage. async fn mark_splits_as_deleted<'a>( &self, @@ -244,8 +242,9 @@ pub trait Metastore: Send + Sync + 'static { /// Deletes a list of splits. /// This API only takes a split that is in `Staged` or `ScheduledForDeletion` state. - /// This removes the split metadata from the metastore, but does not remove the split from storage. - /// An error will occur if you specify an index or split that does not exist in the storage. + /// This removes the split metadata from the metastore, but does not remove the split from + /// storage. An error will occur if you specify an index or split that does not exist in the + /// storage. async fn delete_splits<'a>(&self, index_id: &str, split_ids: &[&'a str]) -> MetastoreResult<()>; diff --git a/quickwit-metastore/src/metastore/single_file_metastore.rs b/quickwit-metastore/src/metastore/single_file_metastore.rs index ce62fb35bf..cdf9fac136 100644 --- a/quickwit-metastore/src/metastore/single_file_metastore.rs +++ b/quickwit-metastore/src/metastore/single_file_metastore.rs @@ -1,24 +1,21 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::collections::HashMap; use std::ops::{Range, RangeInclusive}; @@ -27,18 +24,17 @@ use std::sync::Arc; use async_trait::async_trait; use chrono::Utc; +use quickwit_storage::{ + quickwit_storage_uri_resolver, PutPayload, Storage, StorageErrorKind, StorageResolverError, + StorageUriResolver, +}; use tokio::sync::RwLock; -use quickwit_storage::StorageUriResolver; -use quickwit_storage::{quickwit_storage_uri_resolver, StorageResolverError}; -use quickwit_storage::{PutPayload, Storage, StorageErrorKind}; - use crate::checkpoint::CheckpointDelta; -use crate::MetastoreFactory; -use crate::MetastoreResolverError; use crate::{ - IndexMetadata, MetadataSet, Metastore, MetastoreError, MetastoreResult, SplitMetadata, - SplitMetadataAndFooterOffsets, SplitState, + IndexMetadata, MetadataSet, Metastore, MetastoreError, MetastoreFactory, + MetastoreResolverError, MetastoreResult, SplitMetadata, SplitMetadataAndFooterOffsets, + SplitState, }; /// Metadata file managed by [`SingleFileMetastore`]. diff --git a/quickwit-metastore/src/metastore_resolver.rs b/quickwit-metastore/src/metastore_resolver.rs index a63d773432..366d3d111c 100644 --- a/quickwit-metastore/src/metastore_resolver.rs +++ b/quickwit-metastore/src/metastore_resolver.rs @@ -1,24 +1,21 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::collections::HashMap; use std::sync::Arc; diff --git a/quickwit-metastore/src/tests.rs b/quickwit-metastore/src/tests.rs index aeb387a10f..10984551b9 100644 --- a/quickwit-metastore/src/tests.rs +++ b/quickwit-metastore/src/tests.rs @@ -1,24 +1,21 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::collections::HashSet; use std::ops::{Range, RangeInclusive}; @@ -26,9 +23,8 @@ use std::sync::Arc; use async_trait::async_trait; use chrono::Utc; -use tokio::time::{sleep, Duration}; - use quickwit_index_config::AllFlattenIndexConfig; +use tokio::time::{sleep, Duration}; use crate::checkpoint::{Checkpoint, CheckpointDelta}; use crate::{ diff --git a/quickwit-proto/build.rs b/quickwit-proto/build.rs index e0249cb2d7..eb1646d139 100644 --- a/quickwit-proto/build.rs +++ b/quickwit-proto/build.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . fn main() -> Result<(), Box> { println!("cargo:rerun-if-changed=proto/cluster.proto"); diff --git a/quickwit-proto/src/lib.rs b/quickwit-proto/src/lib.rs index 5f093740e7..f554d27ebc 100644 --- a/quickwit-proto/src/lib.rs +++ b/quickwit-proto/src/lib.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . mod cluster; mod quickwit; diff --git a/quickwit-search/src/client.rs b/quickwit-search/src/client.rs index e8f9bfb8b3..4059f56a4e 100644 --- a/quickwit-search/src/client.rs +++ b/quickwit-search/src/client.rs @@ -1,39 +1,35 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -use futures::StreamExt; -use futures::TryStreamExt; -use http::Uri; -use quickwit_proto::LeafSearchStreamResult; +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + use std::fmt; use std::net::SocketAddr; use std::sync::Arc; + +use futures::{StreamExt, TryStreamExt}; +use http::Uri; +use quickwit_proto::LeafSearchStreamResult; use tokio_stream::wrappers::UnboundedReceiverStream; -use tonic::transport::Channel; -use tonic::transport::Endpoint; +use tonic::transport::{Channel, Endpoint}; use tonic::Request; use crate::error::parse_grpc_error; -use crate::SearchError; -use crate::SearchService; +use crate::{SearchError, SearchService}; /// Impl is an enumeration that meant to manage Quickwit's search service client types. #[derive(Clone)] @@ -43,7 +39,8 @@ enum SearchServiceClientImpl { } /// A search service client. -/// It contains the client implementation and the gRPC address of the node to which the client connects. +/// It contains the client implementation and the gRPC address of the node to which the client +/// connects. #[derive(Clone)] pub struct SearchServiceClient { client_impl: SearchServiceClientImpl, diff --git a/quickwit-search/src/client_pool.rs b/quickwit-search/src/client_pool.rs index 66237a15a9..b40baa7298 100644 --- a/quickwit-search/src/client_pool.rs +++ b/quickwit-search/src/client_pool.rs @@ -1,27 +1,26 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . pub mod search_client_pool; -use std::{collections::HashSet, net::SocketAddr}; +use std::collections::HashSet; +use std::net::SocketAddr; use async_trait::async_trait; use quickwit_metastore::SplitMetadataAndFooterOffsets; diff --git a/quickwit-search/src/client_pool/search_client_pool.rs b/quickwit-search/src/client_pool/search_client_pool.rs index e027f24f2e..30efae2984 100644 --- a/quickwit-search/src/client_pool/search_client_pool.rs +++ b/quickwit-search/src/client_pool/search_client_pool.rs @@ -1,23 +1,21 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::cmp::Ordering; use std::collections::hash_map::Entry; @@ -26,17 +24,15 @@ use std::net::SocketAddr; use std::sync::Arc; use async_trait::async_trait; +use quickwit_cluster::cluster::Cluster; use tokio::sync::RwLock; use tokio_stream::StreamExt; use tracing::*; -use quickwit_cluster::cluster::Cluster; - use crate::client::create_search_service_client; use crate::client_pool::{ClientPool, Job}; use crate::rendezvous_hasher::{sort_by_rendez_vous_hash, Node}; -use crate::swim_addr_to_grpc_addr; -use crate::SearchServiceClient; +use crate::{swim_addr_to_grpc_addr, SearchServiceClient}; /// Search client pool implementation. #[derive(Clone)] @@ -233,8 +229,7 @@ mod tests { use std::collections::HashSet; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::sync::Arc; - use std::thread; - use std::time; + use std::{thread, time}; use quickwit_cluster::cluster::{read_host_key, Cluster}; use quickwit_cluster::test_utils::{available_port, test_cluster}; @@ -242,8 +237,7 @@ mod tests { use crate::client_pool::search_client_pool::create_search_service_client; use crate::client_pool::{ClientPool, Job}; - use crate::swim_addr_to_grpc_addr; - use crate::SearchClientPool; + use crate::{swim_addr_to_grpc_addr, SearchClientPool}; #[tokio::test] async fn test_search_client_pool_single_node() -> anyhow::Result<()> { diff --git a/quickwit-search/src/collector.rs b/quickwit-search/src/collector.rs index e68beebead..8b83e87b22 100644 --- a/quickwit-search/src/collector.rs +++ b/quickwit-search/src/collector.rs @@ -1,50 +1,38 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . -use itertools::Itertools; use std::cmp::Ordering; -use std::collections::BinaryHeap; -use std::collections::HashSet; -use tantivy::schema::Schema; - -use quickwit_index_config::IndexConfig; -use quickwit_index_config::SortBy; -use quickwit_index_config::SortOrder; -use quickwit_proto::LeafSearchResult; -use quickwit_proto::PartialHit; -use quickwit_proto::SearchRequest; -use tantivy::collector::Collector; -use tantivy::collector::SegmentCollector; -use tantivy::fastfield::DynamicFastFieldReader; -use tantivy::fastfield::FastFieldReader; -use tantivy::schema::Field; -use tantivy::DocId; -use tantivy::Score; -use tantivy::SegmentOrdinal; -use tantivy::SegmentReader; +use std::collections::{BinaryHeap, HashSet}; + +use itertools::Itertools; +use quickwit_index_config::{IndexConfig, SortBy, SortOrder}; +use quickwit_proto::{LeafSearchResult, PartialHit, SearchRequest}; +use tantivy::collector::{Collector, SegmentCollector}; +use tantivy::fastfield::{DynamicFastFieldReader, FastFieldReader}; +use tantivy::schema::{Field, Schema}; +use tantivy::{DocId, Score, SegmentOrdinal, SegmentReader}; use crate::filters::TimestampFilter; use crate::partial_hit_sorting_key; -/// The `SortingFieldComputer` can be seen as the specialization of `SortBy` applied to a specific `SegmentReader`. -/// Its role is to compute the sorting field given a `DocId`. +/// The `SortingFieldComputer` can be seen as the specialization of `SortBy` applied to a specific +/// `SegmentReader`. Its role is to compute the sorting field given a `DocId`. enum SortingFieldComputer { SortByFastField { fast_field_reader: DynamicFastFieldReader, @@ -66,7 +54,8 @@ impl SortingFieldComputer { match order { // Descending is our most common case. SortOrder::Desc => field_val, - // We get Ascending order by using a decreasing mapping over u64 as the sorting_field. + // We get Ascending order by using a decreasing mapping over u64 as the + // sorting_field. SortOrder::Asc => u64::MAX - field_val, } } @@ -402,11 +391,12 @@ pub fn make_merge_collector(search_request: &SearchRequest) -> QuickwitCollector #[cfg(test)] mod tests { - use crate::collector::top_k_partial_hits; + use std::cmp::Ordering; + use quickwit_proto::PartialHit; use super::PartialHitHeapItem; - use std::cmp::Ordering; + use crate::collector::top_k_partial_hits; #[test] fn test_partial_hit_ordered_by_sorting_field() { diff --git a/quickwit-search/src/error.rs b/quickwit-search/src/error.rs index 0460acfe39..a33fef8715 100644 --- a/quickwit-search/src/error.rs +++ b/quickwit-search/src/error.rs @@ -1,31 +1,29 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -use serde::{Deserialize, Serialize}; -use tantivy::TantivyError; -use thiserror::Error; -use tokio::task::JoinError; +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use quickwit_index_config::QueryParserError; use quickwit_metastore::MetastoreError; use quickwit_storage::StorageResolverError; +use serde::{Deserialize, Serialize}; +use tantivy::TantivyError; +use thiserror::Error; +use tokio::task::JoinError; /// Possible SearchError #[allow(missing_docs)] diff --git a/quickwit-search/src/fetch_docs.rs b/quickwit-search/src/fetch_docs.rs index 7d65fdc754..cd09565ece 100644 --- a/quickwit-search/src/fetch_docs.rs +++ b/quickwit-search/src/fetch_docs.rs @@ -1,35 +1,30 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::collections::HashMap; use std::sync::Arc; use anyhow::Context; use itertools::Itertools; -use quickwit_proto::FetchDocsResult; -use quickwit_proto::Hit; -use quickwit_proto::PartialHit; -use quickwit_proto::SplitIdAndFooterOffsets; +use quickwit_proto::{FetchDocsResult, Hit, PartialHit, SplitIdAndFooterOffsets}; use quickwit_storage::Storage; -use tantivy::IndexReader; -use tantivy::ReloadPolicy; +use tantivy::{IndexReader, ReloadPolicy}; use crate::leaf::open_index; use crate::GlobalDocAddress; diff --git a/quickwit-search/src/filters.rs b/quickwit-search/src/filters.rs index fd0a56bd09..bc08dd3ab2 100644 --- a/quickwit-search/src/filters.rs +++ b/quickwit-search/src/filters.rs @@ -1,29 +1,27 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::ops::{Bound, RangeBounds}; use tantivy::fastfield::{DynamicFastFieldReader, FastFieldReader}; -use tantivy::schema::Type; -use tantivy::DocId; -use tantivy::{schema::Field, SegmentReader, TantivyError}; +use tantivy::schema::{Field, Type}; +use tantivy::{DocId, SegmentReader, TantivyError}; /// A filter that only retains docs within a time range. #[derive(Clone)] diff --git a/quickwit-search/src/leaf.rs b/quickwit-search/src/leaf.rs index 8f42e34c2f..fd1c5d21d8 100644 --- a/quickwit-search/src/leaf.rs +++ b/quickwit-search/src/leaf.rs @@ -1,25 +1,27 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::collections::{BTreeMap, HashSet}; +use std::convert::TryInto; +use std::path::PathBuf; +use std::sync::Arc; -use crate::collector::{make_collector_for_split, make_merge_collector, GenericQuickwitCollector}; -use crate::SearchError; use anyhow::Context; use bytes::Bytes; use futures::future::try_join_all; @@ -29,14 +31,14 @@ use quickwit_directories::{CachingDirectory, HotDirectory, StorageDirectory}; use quickwit_index_config::IndexConfig; use quickwit_proto::{LeafSearchResult, SearchRequest, SplitIdAndFooterOffsets, SplitSearchError}; use quickwit_storage::{BundleStorage, MemorySizedCache, Storage}; -use std::collections::{BTreeMap, HashSet}; -use std::convert::TryInto; -use std::path::PathBuf; - -use std::sync::Arc; -use tantivy::{collector::Collector, query::Query, Index, ReloadPolicy, Searcher, Term}; +use tantivy::collector::Collector; +use tantivy::query::Query; +use tantivy::{Index, ReloadPolicy, Searcher, Term}; use tokio::task::spawn_blocking; +use crate::collector::{make_collector_for_split, make_merge_collector, GenericQuickwitCollector}; +use crate::SearchError; + fn global_split_footer_cache() -> &'static MemorySizedCache { static INSTANCE: OnceCell> = OnceCell::new(); INSTANCE.get_or_init(|| MemorySizedCache::with_capacity_in_bytes(500_000_000)) diff --git a/quickwit-search/src/lib.rs b/quickwit-search/src/lib.rs index 8cfc42f404..13db56b65c 100644 --- a/quickwit-search/src/lib.rs +++ b/quickwit-search/src/lib.rs @@ -1,23 +1,21 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . //! This projects implements quickwit's search API. #![warn(missing_docs)] @@ -36,7 +34,6 @@ mod search_result_json; mod search_stream; mod service; -/// /// Refer to this as `crate::Result`. pub type Result = std::result::Result; @@ -45,16 +42,12 @@ use std::net::SocketAddr; use std::ops::Range; use anyhow::Context; -use tantivy::DocAddress; - -use quickwit_metastore::SplitState; -use quickwit_metastore::{Metastore, MetastoreResult, SplitMetadataAndFooterOffsets}; -use quickwit_proto::{PartialHit, SearchResult}; -use quickwit_proto::{SearchRequest, SplitIdAndFooterOffsets}; +use quickwit_metastore::{Metastore, MetastoreResult, SplitMetadataAndFooterOffsets, SplitState}; +use quickwit_proto::{PartialHit, SearchRequest, SearchResult, SplitIdAndFooterOffsets}; use quickwit_storage::StorageUriResolver; +use tantivy::DocAddress; -pub use crate::client::create_search_service_client; -pub use crate::client::SearchServiceClient; +pub use crate::client::{create_search_service_client, SearchServiceClient}; pub use crate::client_pool::search_client_pool::SearchClientPool; pub use crate::client_pool::ClientPool; pub use crate::error::SearchError; @@ -63,8 +56,7 @@ use crate::leaf::leaf_search; pub use crate::root::root_search; pub use crate::search_result_json::SearchResultJson; pub use crate::search_stream::root_search_stream; -pub use crate::service::MockSearchService; -pub use crate::service::{SearchService, SearchServiceImpl}; +pub use crate::service::{MockSearchService, SearchService, SearchServiceImpl}; /// Compute the SWIM port from the HTTP port. /// Add 1 to the HTTP port to get the SWIM port. @@ -203,9 +195,9 @@ mod tests { use assert_json_diff::assert_json_include; use quickwit_index_config::{DefaultIndexConfigBuilder, WikipediaIndexConfig}; use quickwit_indexing::TestSandbox; + use serde_json::json; use super::*; - use serde_json::json; #[tokio::test] async fn test_single_node_simple() -> anyhow::Result<()> { @@ -245,9 +237,7 @@ mod tests { // TODO remove me once `Iterator::is_sorted_by_key` is stabilized. fn is_sorted>(mut it: I) -> bool - where - E: Ord, - { + where E: Ord { let mut previous_el = if let Some(first_el) = it.next() { first_el } else { diff --git a/quickwit-search/src/rendezvous_hasher.rs b/quickwit-search/src/rendezvous_hasher.rs index 008b740f71..58ffbe17cc 100644 --- a/quickwit-search/src/rendezvous_hasher.rs +++ b/quickwit-search/src/rendezvous_hasher.rs @@ -1,23 +1,21 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; @@ -64,8 +62,7 @@ pub fn sort_by_rendez_vous_hash(nodes: &mut [Node], key: &str) { #[cfg(test)] mod tests { - use std::net::IpAddr; - use std::net::Ipv4Addr; + use std::net::{IpAddr, Ipv4Addr}; use super::*; diff --git a/quickwit-search/src/root.rs b/quickwit-search/src/root.rs index 311afc5888..ab9942b789 100644 --- a/quickwit-search/src/root.rs +++ b/quickwit-search/src/root.rs @@ -1,59 +1,47 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -use std::collections::HashMap; -use std::collections::HashSet; - +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::collections::{HashMap, HashSet}; use std::net::SocketAddr; use std::sync::Arc; use itertools::Itertools; -use quickwit_metastore::IndexMetadata; -use quickwit_metastore::SplitMetadataAndFooterOffsets; -use quickwit_proto::SplitIdAndFooterOffsets; -use quickwit_proto::SplitSearchError; -use tantivy::collector::Collector; -use tantivy::TantivyError; -use tokio::task::spawn_blocking; -use tokio::task::JoinHandle; -use tracing::*; - -use quickwit_metastore::{Metastore, SplitMetadata}; +use quickwit_metastore::{IndexMetadata, Metastore, SplitMetadata, SplitMetadataAndFooterOffsets}; use quickwit_proto::{ FetchDocsRequest, FetchDocsResult, Hit, LeafSearchRequest, LeafSearchResult, PartialHit, - SearchRequest, SearchResult, + SearchRequest, SearchResult, SplitIdAndFooterOffsets, SplitSearchError, }; +use tantivy::collector::Collector; +use tantivy::TantivyError; +use tokio::task::{spawn_blocking, JoinHandle}; +use tracing::*; use crate::client_pool::Job; use crate::collector::make_merge_collector; -use crate::extract_split_and_footer_offsets; -use crate::list_relevant_splits; -use crate::ClientPool; -use crate::SearchClientPool; -use crate::SearchError; -use crate::SearchServiceClient; +use crate::{ + extract_split_and_footer_offsets, list_relevant_splits, ClientPool, SearchClientPool, + SearchError, SearchServiceClient, +}; // Measure the cost associated to searching in a given split metadata. fn compute_split_cost(_split_metadata: &SplitMetadata) -> u32 { - //TODO: Have a smarter cost, by smoothing the number of docs. + // TODO: Have a smarter cost, by smoothing the number of docs. 1 } @@ -194,7 +182,8 @@ pub(crate) fn job_for_splits( split_ids: &HashSet<&String>, split_metadata_map: &HashMap, ) -> Vec { - // Create a job for fetching docs and assign the splits that the node is responsible for based on the job. + // Create a job for fetching docs and assign the splits that the node is responsible for based + // on the job. let leaf_search_jobs: Vec = split_metadata_map .iter() .filter(|(split_id, _)| split_ids.contains(split_id)) @@ -210,11 +199,10 @@ pub(crate) fn job_for_splits( /// It sends a search request over gRPC to multiple leaf nodes and merges the search results. /// /// Retry Logic: -/// After a first round of leaf_search requests, we identify the list of failed but retryable splits, -/// and retry them. Complete failure against nodes are also considered retryable splits. +/// After a first round of leaf_search requests, we identify the list of failed but retryable +/// splits, and retry them. Complete failure against nodes are also considered retryable splits. /// The leaf nodes which did not return any result are suspected to be unhealthy and are excluded /// from this retry round. If all nodes are unhealthy the retry will not exclude any nodes. -/// pub async fn root_search( search_request: &SearchRequest, metastore: &dyn Metastore, @@ -222,7 +210,8 @@ pub async fn root_search( ) -> Result { let start_instant = tokio::time::Instant::now(); - // Create a job for leaf node search and assign the splits that the node is responsible for based on the job. + // Create a job for leaf node search and assign the splits that the node is responsible for + // based on the job. let split_metadata_list = list_relevant_splits(search_request, metastore).await?; let index_metadata = metastore.index_metadata(&search_request.index_id).await?; @@ -232,7 +221,8 @@ pub async fn root_search( .map(|metadata| (metadata.split_metadata.split_id.clone(), metadata)) .collect(); - // Create a job for fetching docs and assign the splits that the node is responsible for based on the job. + // Create a job for fetching docs and assign the splits that the node is responsible for based + // on the job. // let leaf_search_jobs: Vec = job_for_splits(&split_metadata_map.keys().collect(), &split_metadata_map); @@ -257,7 +247,8 @@ pub async fn root_search( let retry_action_opt = analyze_errors(&result_per_node_addr); if let Some(retry_action) = retry_action_opt.as_ref() { - // Create a job for fetching docs and assign the splits that the node is responsible for based on the job. + // Create a job for fetching docs and assign the splits that the node is responsible for + // based on the job. // let leaf_search_jobs: Vec = job_for_splits( &retry_action.retry_split_ids.iter().collect(), @@ -291,11 +282,16 @@ pub async fn root_search( result_per_node_addr.remove(&err_addr); } // Remove partial, retryable errors - // In this step we have only retryable errors, since it aborts with non-retryable errors, so we can just replace them. + // In this step we have only retryable errors, since it aborts with non-retryable errors, so + // we can just replace them. for result in result_per_node_addr.values_mut().flatten() { let contains_non_retryable_errors = result.failed_splits.iter().any(|err| !err.retryable_error); - assert!(!contains_non_retryable_errors, "Result still contains non-retryable errors, but logic expects to have aborted with non-retryable errors. (this may change if we add partial results) "); + assert!( + !contains_non_retryable_errors, + "Result still contains non-retryable errors, but logic expects to have aborted \ + with non-retryable errors. (this may change if we add partial results) " + ); result.failed_splits = vec![]; } @@ -331,7 +327,8 @@ pub async fn root_search( } } - // Find the sum of the number of hits and merge multiple partial hits into a single partial hits. + // Find the sum of the number of hits and merge multiple partial hits into a single partial + // hits. let mut leaf_search_results = Vec::new(); for (_addr, leaf_search_response) in result_per_node_addr.into_iter() { match leaf_search_response { @@ -462,16 +459,17 @@ pub async fn root_search( #[cfg(test)] mod tests { - use super::*; - use std::ops::Range; - use crate::{MockSearchService, SearchResultJson}; use quickwit_index_config::WikipediaIndexConfig; use quickwit_indexing::mock_split_meta; - use quickwit_metastore::{checkpoint::Checkpoint, IndexMetadata, MockMetastore, SplitState}; + use quickwit_metastore::checkpoint::Checkpoint; + use quickwit_metastore::{IndexMetadata, MockMetastore, SplitState}; use quickwit_proto::SplitSearchError; + use super::*; + use crate::{MockSearchService, SearchResultJson}; + fn mock_partial_hit( split_id: &str, sorting_field_value: u64, @@ -768,7 +766,7 @@ mod tests { ); let search_result = root_search(&search_request, &metastore, &client_pool).await?; - //println!("search_result={:?}", search_result); + // println!("search_result={:?}", search_result); let search_result_json = SearchResultJson::from(search_result.clone()); let search_result_json = serde_json::to_string_pretty(&search_result_json)?; @@ -900,7 +898,7 @@ mod tests { ); let search_result = root_search(&search_request, &metastore, &client_pool).await?; - //println!("search_result={:?}", search_result); + // println!("search_result={:?}", search_result); let search_result_json = SearchResultJson::from(search_result.clone()); let search_result_json = serde_json::to_string_pretty(&search_result_json)?; @@ -985,7 +983,7 @@ mod tests { Arc::new(SearchClientPool::from_mocks(vec![Arc::new(mock_search_service1)]).await?); let search_result = root_search(&search_request, &metastore, &client_pool).await?; - //println!("search_result={:?}", search_result); + // println!("search_result={:?}", search_result); let search_result_json = SearchResultJson::from(search_result.clone()); let search_result_json = serde_json::to_string_pretty(&search_result_json)?; @@ -1211,7 +1209,7 @@ mod tests { ); let search_result = root_search(&search_request, &metastore, &client_pool).await?; - //println!("search_result={:?}", search_result); + // println!("search_result={:?}", search_result); let search_result_json = SearchResultJson::from(search_result.clone()); let search_result_json = serde_json::to_string_pretty(&search_result_json)?; diff --git a/quickwit-search/src/search_result_json.rs b/quickwit-search/src/search_result_json.rs index 6a27f8ecfb..110c35aad2 100644 --- a/quickwit-search/src/search_result_json.rs +++ b/quickwit-search/src/search_result_json.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use serde::Serialize; use tracing::error; diff --git a/quickwit-search/src/search_stream/collector.rs b/quickwit-search/src/search_stream/collector.rs index 5b77e50734..9c9d2748ea 100644 --- a/quickwit-search/src/search_stream/collector.rs +++ b/quickwit-search/src/search_stream/collector.rs @@ -1,36 +1,32 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::collections::HashSet; use std::marker::PhantomData; +use tantivy::collector::{Collector, SegmentCollector}; +use tantivy::fastfield::{DynamicFastFieldReader, FastFieldReader, FastValue}; +use tantivy::schema::{Field, Type}; +use tantivy::{DocId, Score, SegmentOrdinal, SegmentReader, TantivyError}; + use crate::filters::TimestampFilter; use crate::SearchError; -use tantivy::fastfield::FastFieldReader; -use tantivy::schema::Type; -use tantivy::{ - collector::{Collector, SegmentCollector}, - fastfield::{DynamicFastFieldReader, FastValue}, - schema::Field, - DocId, Score, SegmentOrdinal, SegmentReader, TantivyError, -}; #[derive(Clone)] pub struct FastFieldSegmentCollector { diff --git a/quickwit-search/src/search_stream/leaf.rs b/quickwit-search/src/search_stream/leaf.rs index 8c135d059e..4bb10bb670 100644 --- a/quickwit-search/src/search_stream/leaf.rs +++ b/quickwit-search/src/search_stream/leaf.rs @@ -1,45 +1,42 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::sync::Arc; -use super::FastFieldCollectorBuilder; -use crate::leaf::open_index; -use crate::leaf::warmup; -use crate::SearchError; use futures::{FutureExt, StreamExt}; use quickwit_index_config::IndexConfig; -use quickwit_proto::LeafSearchStreamResult; -use quickwit_proto::OutputFormat; -use quickwit_proto::SearchRequest; -use quickwit_proto::SearchStreamRequest; -use quickwit_proto::SplitIdAndFooterOffsets; +use quickwit_proto::{ + LeafSearchStreamResult, OutputFormat, SearchRequest, SearchStreamRequest, + SplitIdAndFooterOffsets, +}; use quickwit_storage::Storage; -use std::sync::Arc; use tantivy::query::Query; use tantivy::schema::Type; -use tantivy::LeasedItem; -use tantivy::ReloadPolicy; -use tantivy::Searcher; +use tantivy::{LeasedItem, ReloadPolicy, Searcher}; use tokio::task::spawn_blocking; use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::error; +use super::FastFieldCollectorBuilder; +use crate::leaf::{open_index, warmup}; +use crate::SearchError; + // TODO: buffer of 5 seems to be sufficient to do the job locally, needs to be tested on a cluster. const CONCURRENT_SPLIT_SEARCH_STREAM: usize = 5; @@ -197,13 +194,14 @@ fn collect_fast_field_values( #[cfg(test)] mod tests { - use std::{str::from_utf8, sync::Arc}; + use std::str::from_utf8; + use std::sync::Arc; use quickwit_index_config::DefaultIndexConfigBuilder; use quickwit_indexing::TestSandbox; + use serde_json::json; use super::*; - use serde_json::json; #[tokio::test] async fn test_leaf_search_stream_to_csv_output_with_filtering() -> anyhow::Result<()> { diff --git a/quickwit-search/src/search_stream/mod.rs b/quickwit-search/src/search_stream/mod.rs index d6246ba254..be75fef73e 100644 --- a/quickwit-search/src/search_stream/mod.rs +++ b/quickwit-search/src/search_stream/mod.rs @@ -1,35 +1,34 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . mod collector; mod leaf; mod root; +use std::fmt::Display; +use std::io; +use std::io::Write; + pub use collector::{FastFieldCollector, FastFieldCollectorBuilder}; pub use leaf::leaf_search_stream; use quickwit_proto::OutputFormat; pub use root::root_search_stream; - -use std::fmt::Display; -use std::io; -use std::io::Write; use tantivy::fastfield::FastValue; /// Serialize the values into the `buffer` as bytes. diff --git a/quickwit-search/src/search_stream/root.rs b/quickwit-search/src/search_stream/root.rs index 1c20c68d53..bc5055d219 100644 --- a/quickwit-search/src/search_stream/root.rs +++ b/quickwit-search/src/search_stream/root.rs @@ -1,47 +1,37 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . -use std::collections::HashMap; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use bytes::Bytes; use futures::StreamExt; -use itertools::Either; -use itertools::Itertools; -use quickwit_metastore::SplitMetadataAndFooterOffsets; -use quickwit_proto::LeafSearchStreamRequest; -use quickwit_proto::SearchStreamRequest; -use quickwit_proto::SplitIdAndFooterOffsets; +use itertools::{Either, Itertools}; +use quickwit_metastore::{Metastore, SplitMetadataAndFooterOffsets}; +use quickwit_proto::{ + LeafSearchStreamRequest, SearchRequest, SearchStreamRequest, SplitIdAndFooterOffsets, +}; use tracing::*; -use quickwit_metastore::Metastore; -use quickwit_proto::SearchRequest; - use crate::client_pool::Job; -use crate::list_relevant_splits; -use crate::root::job_for_splits; -use crate::root::NodeSearchError; -use crate::ClientPool; -use crate::SearchClientPool; -use crate::SearchError; +use crate::root::{job_for_splits, NodeSearchError}; +use crate::{list_relevant_splits, ClientPool, SearchClientPool, SearchError}; /// Perform a distributed search stream. pub async fn root_search_stream( @@ -138,17 +128,18 @@ pub async fn root_search_stream( #[cfg(test)] mod tests { - use super::*; - use std::ops::Range; - use crate::MockSearchService; use quickwit_index_config::WikipediaIndexConfig; use quickwit_indexing::mock_split_meta; - use quickwit_metastore::{checkpoint::Checkpoint, IndexMetadata, MockMetastore, SplitState}; + use quickwit_metastore::checkpoint::Checkpoint; + use quickwit_metastore::{IndexMetadata, MockMetastore, SplitState}; use quickwit_proto::OutputFormat; use tokio_stream::wrappers::UnboundedReceiverStream; + use super::*; + use crate::MockSearchService; + #[tokio::test] async fn test_root_search_stream_single_split() -> anyhow::Result<()> { let request = quickwit_proto::SearchStreamRequest { @@ -248,7 +239,11 @@ mod tests { Arc::new(SearchClientPool::from_mocks(vec![Arc::new(mock_search_service)]).await?); let result = root_search_stream(&request, &metastore, &client_pool).await; assert_eq!(result.is_err(), true); - assert_eq!(result.unwrap_err().to_string(), "Internal error: `[NodeSearchError { search_error: InternalError(\"error\"), split_ids: [\"split1\"] }]`."); + assert_eq!( + result.unwrap_err().to_string(), + "Internal error: `[NodeSearchError { search_error: InternalError(\"error\"), \ + split_ids: [\"split1\"] }]`." + ); Ok(()) } } diff --git a/quickwit-search/src/service.rs b/quickwit-search/src/service.rs index efdb9a714f..f12af8319a 100644 --- a/quickwit-search/src/service.rs +++ b/quickwit-search/src/service.rs @@ -1,44 +1,39 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::sync::Arc; use async_trait::async_trait; use bytes::Bytes; use quickwit_index_config::IndexConfig; use quickwit_metastore::Metastore; use quickwit_proto::{ - FetchDocsRequest, FetchDocsResult, LeafSearchRequest, LeafSearchResult, SearchRequest, - SearchResult, + FetchDocsRequest, FetchDocsResult, LeafSearchRequest, LeafSearchResult, + LeafSearchStreamRequest, LeafSearchStreamResult, SearchRequest, SearchResult, + SearchStreamRequest, }; -use quickwit_proto::{LeafSearchStreamRequest, LeafSearchStreamResult, SearchStreamRequest}; use quickwit_storage::StorageUriResolver; -use std::sync::Arc; use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::info; -use crate::fetch_docs; -use crate::leaf_search; -use crate::root_search; use crate::search_stream::{leaf_search_stream, root_search_stream}; -use crate::SearchClientPool; -use crate::SearchError; +use crate::{fetch_docs, leaf_search, root_search, SearchClientPool, SearchError}; #[derive(Clone)] /// The search service implementation. diff --git a/quickwit-serve/src/args.rs b/quickwit-serve/src/args.rs index 9c7ac5d43d..87353a54a4 100644 --- a/quickwit-serve/src/args.rs +++ b/quickwit-serve/src/args.rs @@ -1,23 +1,21 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::net::SocketAddr; use std::path::PathBuf; diff --git a/quickwit-serve/src/error.rs b/quickwit-serve/src/error.rs index 1725657baf..aac7e91def 100644 --- a/quickwit-serve/src/error.rs +++ b/quickwit-serve/src/error.rs @@ -1,32 +1,29 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +use quickwit_cluster::error::ClusterError; +use quickwit_search::SearchError; use serde::ser::SerializeMap; use thiserror::Error; use warp::http; use warp::hyper::StatusCode; -use quickwit_cluster::error::ClusterError; -use quickwit_search::SearchError; - #[derive(Debug, Error)] pub enum ApiError { #[error("InvalidArgument: {0}.")] @@ -67,9 +64,7 @@ impl ApiError { // TODO implement nicer serialization of errors. impl serde::Serialize for ApiError { fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { + where S: serde::Serializer { let mut map = serializer.serialize_map(Some(2))?; map.serialize_key("error")?; map.serialize_value(&self.message())?; diff --git a/quickwit-serve/src/grpc.rs b/quickwit-serve/src/grpc.rs index f2143c7a28..67f0a9e1e6 100644 --- a/quickwit-serve/src/grpc.rs +++ b/quickwit-serve/src/grpc.rs @@ -1,31 +1,28 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::net::SocketAddr; -use tonic::transport::Server; -use tracing::*; - use quickwit_proto::cluster_service_server::ClusterServiceServer; use quickwit_proto::search_service_server::SearchServiceServer; +use tonic::transport::Server; +use tracing::*; use crate::grpc_adapter::cluster_adapter::GrpcClusterAdapter; use crate::grpc_adapter::search_adapter::GrpcSearchAdapter; diff --git a/quickwit-serve/src/grpc_adapter.rs b/quickwit-serve/src/grpc_adapter.rs index b33d85e78b..2535fcabc6 100644 --- a/quickwit-serve/src/grpc_adapter.rs +++ b/quickwit-serve/src/grpc_adapter.rs @@ -1,23 +1,21 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . pub mod cluster_adapter; pub mod search_adapter; diff --git a/quickwit-serve/src/grpc_adapter/cluster_adapter.rs b/quickwit-serve/src/grpc_adapter/cluster_adapter.rs index 9f8e83b285..07c2706d18 100644 --- a/quickwit-serve/src/grpc_adapter/cluster_adapter.rs +++ b/quickwit-serve/src/grpc_adapter/cluster_adapter.rs @@ -1,28 +1,25 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::sync::Arc; use async_trait::async_trait; - use quickwit_cluster::error::ClusterError; use quickwit_cluster::service::{ClusterService, ClusterServiceImpl}; use quickwit_proto::cluster_service_server as grpc; diff --git a/quickwit-serve/src/grpc_adapter/search_adapter.rs b/quickwit-serve/src/grpc_adapter/search_adapter.rs index a6f24e7301..961bfbae3b 100644 --- a/quickwit-serve/src/grpc_adapter/search_adapter.rs +++ b/quickwit-serve/src/grpc_adapter/search_adapter.rs @@ -1,29 +1,26 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::sync::Arc; use async_trait::async_trait; use futures::TryStreamExt; - use quickwit_proto::{ search_service_server as grpc, LeafSearchStreamRequest, LeafSearchStreamResult, }; diff --git a/quickwit-serve/src/http_handler.rs b/quickwit-serve/src/http_handler.rs index 4f0237a105..a5aba0f15f 100644 --- a/quickwit-serve/src/http_handler.rs +++ b/quickwit-serve/src/http_handler.rs @@ -1,23 +1,21 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . pub mod cluster; pub mod health_check; diff --git a/quickwit-serve/src/http_handler/cluster.rs b/quickwit-serve/src/http_handler/cluster.rs index 7203dfd161..30b42a940e 100644 --- a/quickwit-serve/src/http_handler/cluster.rs +++ b/quickwit-serve/src/http_handler/cluster.rs @@ -1,32 +1,28 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::convert::Infallible; use std::sync::Arc; -use serde::Deserialize; -use warp::Filter; -use warp::Rejection; - use quickwit_cluster::service::ClusterService; +use serde::Deserialize; +use warp::{Filter, Rejection}; use crate::rest::Format; use crate::ApiError; diff --git a/quickwit-serve/src/http_handler/health_check.rs b/quickwit-serve/src/http_handler/health_check.rs index 83b6c4c4b9..46e80792ea 100644 --- a/quickwit-serve/src/http_handler/health_check.rs +++ b/quickwit-serve/src/http_handler/health_check.rs @@ -1,23 +1,21 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::fmt; diff --git a/quickwit-serve/src/lib.rs b/quickwit-serve/src/lib.rs index b8dd952cb4..17ddc04df5 100644 --- a/quickwit-serve/src/lib.rs +++ b/quickwit-serve/src/lib.rs @@ -1,23 +1,21 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . mod args; mod error; @@ -27,14 +25,11 @@ mod http_handler; mod quickwit_cache; mod rest; -use quickwit_cache::QuickwitCache; use std::io::Write; use std::net::SocketAddr; use std::sync::Arc; -use termcolor::{self, Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; -use tracing::debug; - +use quickwit_cache::QuickwitCache; use quickwit_cluster::cluster::{read_host_key, Cluster}; use quickwit_cluster::service::ClusterServiceImpl; use quickwit_metastore::MetastoreUriResolver; @@ -46,13 +41,14 @@ use quickwit_storage::{ StorageWithCacheFactory, }; use quickwit_telemetry::payload::{ServeEvent, TelemetryEvent}; +use termcolor::{self, Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; +use tracing::debug; pub use crate::args::ServeArgs; pub use crate::error::ApiError; use crate::grpc::start_grpc_service; use crate::grpc_adapter::cluster_adapter::GrpcClusterAdapter; use crate::grpc_adapter::search_adapter::GrpcSearchAdapter; - use crate::rest::start_rest_service; fn display_help_message( @@ -153,12 +149,17 @@ pub async fn serve_cli(args: ServeArgs) -> anyhow::Result<()> { #[cfg(test)] mod tests { - use std::{array::IntoIter, collections::HashMap, ops::Range, sync::Arc}; + use std::array::IntoIter; + use std::collections::HashMap; + use std::ops::Range; + use std::sync::Arc; use quickwit_index_config::WikipediaIndexConfig; use quickwit_indexing::mock_split_meta; - use quickwit_metastore::{checkpoint::Checkpoint, IndexMetadata, MockMetastore, SplitState}; - use quickwit_proto::{search_service_server::SearchServiceServer, OutputFormat}; + use quickwit_metastore::checkpoint::Checkpoint; + use quickwit_metastore::{IndexMetadata, MockMetastore, SplitState}; + use quickwit_proto::search_service_server::SearchServiceServer; + use quickwit_proto::OutputFormat; use quickwit_search::{ create_search_service_client, root_search_stream, MockSearchService, SearchError, SearchService, @@ -237,7 +238,11 @@ mod tests { }); let search_result = root_search_stream(&request, &metastore, &client_pool).await; assert!(search_result.is_err()); - assert_eq!(search_result.unwrap_err().to_string(), "Internal error: `[NodeSearchError { search_error: InternalError(\"error\"), split_ids: [\"split1\"] }]`."); + assert_eq!( + search_result.unwrap_err().to_string(), + "Internal error: `[NodeSearchError { search_error: InternalError(\"error\"), \ + split_ids: [\"split1\"] }]`." + ); Ok(()) } } diff --git a/quickwit-serve/src/quickwit_cache.rs b/quickwit-serve/src/quickwit_cache.rs index 249e36d019..7c5e940ba7 100644 --- a/quickwit-serve/src/quickwit_cache.rs +++ b/quickwit-serve/src/quickwit_cache.rs @@ -1,32 +1,30 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::ops::Range; +use std::path::{Path, PathBuf}; +use std::sync::Arc; use async_trait::async_trait; use bytes::Bytes; use quickwit_common::HOTCACHE_FILENAME; -use quickwit_storage::Cache; -use quickwit_storage::SliceCache; -use std::ops::Range; -use std::path::Path; -use std::path::PathBuf; -use std::sync::Arc; +use quickwit_storage::{Cache, SliceCache}; const FULL_SLICE: Range = 0..usize::MAX; @@ -151,10 +149,10 @@ mod tests { use std::path::Path; use std::sync::Arc; - use super::QuickwitCache; use bytes::Bytes; - use quickwit_storage::Cache; - use quickwit_storage::MockCache; + use quickwit_storage::{Cache, MockCache}; + + use super::QuickwitCache; #[tokio::test] async fn test_quickwit_cache_get_all() { diff --git a/quickwit-serve/src/rest.rs b/quickwit-serve/src/rest.rs index d38b9316d5..fe583ff3f8 100644 --- a/quickwit-serve/src/rest.rs +++ b/quickwit-serve/src/rest.rs @@ -1,23 +1,21 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::convert::Infallible; use std::net::SocketAddr; @@ -25,16 +23,15 @@ use std::sync::Arc; use bytes::Bytes; use futures::stream::{self, StreamExt}; +use quickwit_cluster::service::ClusterServiceImpl; +use quickwit_proto::OutputFormat; +use quickwit_search::{SearchResultJson, SearchService, SearchServiceImpl}; use serde::{Deserialize, Deserializer}; use tracing::*; use warp::hyper::header::CONTENT_TYPE; use warp::hyper::StatusCode; use warp::{reply, Filter, Rejection, Reply}; -use quickwit_cluster::service::ClusterServiceImpl; -use quickwit_proto::OutputFormat; -use quickwit_search::{SearchResultJson, SearchService, SearchServiceImpl}; - use crate::http_handler::cluster::cluster_handler; use crate::http_handler::health_check::liveness_check_handler; use crate::ApiError; @@ -309,9 +306,7 @@ async fn recover_fn(rejection: Rejection) -> Result { } fn from_simple_list<'de, D>(deserializer: D) -> Result>, D::Error> -where - D: Deserializer<'de>, -{ +where D: Deserializer<'de> { let str_sequence = String::deserialize(deserializer)?; Ok(Some( str_sequence @@ -324,12 +319,13 @@ where #[cfg(test)] mod tests { - use super::*; use assert_json_diff::assert_json_include; use mockall::predicate; use quickwit_search::{MockSearchService, SearchError}; use serde_json::json; + use super::*; + #[test] fn test_serialize_search_results() -> anyhow::Result<()> { let search_results = SearchResultJson { @@ -351,10 +347,13 @@ mod tests { async fn test_rest_search_api_route_simple() { let rest_search_api_filter = search_filter(); let (index, req) = warp::test::request() - .path("/api/v1/quickwit-demo-index/search?query=*&endTimestamp=1450720000&maxHits=10&startOffset=22") - .filter(&rest_search_api_filter) - .await - .unwrap(); + .path( + "/api/v1/quickwit-demo-index/search?query=*&endTimestamp=1450720000&maxHits=10&\ + startOffset=22", + ) + .filter(&rest_search_api_filter) + .await + .unwrap(); assert_eq!(&index, "quickwit-demo-index"); assert_eq!( &req, @@ -366,7 +365,7 @@ mod tests { max_hits: 10, start_offset: 22, format: Format::default(), - tags: None, + tags: None } ); } @@ -375,7 +374,10 @@ mod tests { async fn test_rest_search_api_route_simple_default_num_hits_default_offset() { let rest_search_api_filter = search_filter(); let (index, req) = warp::test::request() - .path("/api/v1/quickwit-demo-index/search?query=*&endTimestamp=1450720000&searchField=title,body") + .path( + "/api/v1/quickwit-demo-index/search?query=*&endTimestamp=1450720000&\ + searchField=title,body", + ) .filter(&rest_search_api_filter) .await .unwrap(); @@ -390,7 +392,7 @@ mod tests { max_hits: 20, start_offset: 0, format: Format::default(), - tags: None, + tags: None } ); } @@ -414,7 +416,7 @@ mod tests { start_offset: 0, format: Format::Json, search_fields: None, - tags: None, + tags: None } ); } @@ -587,7 +589,7 @@ mod tests { end_timestamp: None, fast_field: "external_id".to_string(), output_format: OutputFormat::Csv, - tags: None, + tags: None } ); } @@ -595,7 +597,10 @@ mod tests { #[tokio::test] async fn test_rest_search_stream_api_click_house_row_binary() { let (index, req) = warp::test::request() - .path("/api/v1/my-index/search/stream?query=obama&fastField=external_id&outputFormat=clickHouseRowBinary&tags=lang:english") + .path( + "/api/v1/my-index/search/stream?query=obama&fastField=external_id&\ + outputFormat=clickHouseRowBinary&tags=lang:english", + ) .filter(&super::search_stream_filter()) .await .unwrap(); @@ -609,7 +614,7 @@ mod tests { end_timestamp: None, fast_field: "external_id".to_string(), output_format: OutputFormat::ClickHouseRowBinary, - tags: Some(vec!["lang:english".to_string()]), + tags: Some(vec!["lang:english".to_string()]) } ); } @@ -617,11 +622,18 @@ mod tests { #[tokio::test] async fn test_rest_search_stream_api_error() { let rejection = warp::test::request() - .path("/api/v1/my-index/search/stream?query=obama&fastField=external_id&outputFormat=click_house_row_binary") + .path( + "/api/v1/my-index/search/stream?query=obama&fastField=external_id&\ + outputFormat=click_house_row_binary", + ) .filter(&super::search_stream_filter()) .await .unwrap_err(); let parse_error = rejection.find::().unwrap(); - assert_eq!(parse_error.to_string(), "failed with reason: unknown variant `click_house_row_binary`, expected `csv` or `clickHouseRowBinary`"); + assert_eq!( + parse_error.to_string(), + "failed with reason: unknown variant `click_house_row_binary`, expected `csv` or \ + `clickHouseRowBinary`" + ); } } diff --git a/quickwit-storage/src/bundle_storage.rs b/quickwit-storage/src/bundle_storage.rs index d6af567300..3a2bd318f1 100644 --- a/quickwit-storage/src/bundle_storage.rs +++ b/quickwit-storage/src/bundle_storage.rs @@ -1,31 +1,22 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -use crate::Storage; -use crate::{StorageError, StorageResult}; -use async_trait::async_trait; -use bytes::Bytes; -use serde::Deserialize; -use serde::Serialize; +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + use std::collections::HashMap; use std::convert::TryInto; use std::fmt; @@ -35,12 +26,18 @@ use std::io::{self, ErrorKind, Read, Write}; use std::ops::Range; use std::path::{Path, PathBuf}; use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use serde::{Deserialize, Serialize}; use tantivy::common::CountingWriter; use tantivy::directory::FileSlice; use tantivy::HasLen; use thiserror::Error; use tracing::error; +use crate::{Storage, StorageError, StorageResult}; + /// Filename used for the bundle. pub const BUNDLE_FILENAME: &str = "bundle"; @@ -55,7 +52,8 @@ pub struct BundleStorage { impl BundleStorage { /// Creates a new BundleStorage. /// - /// The provided data must include the footer_bytes at the end of the slice, but it can have more up front. + /// The provided data must include the footer_bytes at the end of the slice, but it can have + /// more up front. pub fn new( storage: Arc, bundle_filepath: PathBuf, @@ -268,9 +266,8 @@ impl BundleStorageBuilder { mod tests { use std::fs; - use crate::RamStorageBuilder; - use super::*; + use crate::RamStorageBuilder; #[tokio::test] async fn bundle_storage_file_offsets() -> anyhow::Result<()> { diff --git a/quickwit-storage/src/cache/in_ram_slice_cache.rs b/quickwit-storage/src/cache/in_ram_slice_cache.rs index 608169fa00..af6a46884c 100644 --- a/quickwit-storage/src/cache/in_ram_slice_cache.rs +++ b/quickwit-storage/src/cache/in_ram_slice_cache.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::hash::Hash; use std::ops::Range; diff --git a/quickwit-storage/src/cache/memory_sized_cache.rs b/quickwit-storage/src/cache/memory_sized_cache.rs index c963d928ef..f2cb514f5f 100644 --- a/quickwit-storage/src/cache/memory_sized_cache.rs +++ b/quickwit-storage/src/cache/memory_sized_cache.rs @@ -1,22 +1,21 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::borrow::Borrow; use std::hash::Hash; @@ -25,6 +24,7 @@ use std::sync::Mutex; use bytes::Bytes; use lru::{KeyRef, LruCache}; use tracing::{error, warn}; + #[derive(Clone, Copy, Debug)] enum Capacity { Unlimited, @@ -86,7 +86,11 @@ impl NeedMutMemorySizedCache { if let Some((_, bytes)) = self.lru_cache.pop_lru() { self.num_bytes -= bytes.len(); } else { - error!("Logical error. Even after removing all of the items in the cache the capacity is insufficient. This case is guarded against and should never happen."); + error!( + "Logical error. Even after removing all of the items in the cache the \ + capacity is insufficient. This case is guarded against and should never \ + happen." + ); return; } } diff --git a/quickwit-storage/src/cache/mod.rs b/quickwit-storage/src/cache/mod.rs index 4e91f4822b..bd1ef709de 100644 --- a/quickwit-storage/src/cache/mod.rs +++ b/quickwit-storage/src/cache/mod.rs @@ -1,34 +1,35 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . mod in_ram_slice_cache; mod memory_sized_cache; mod storage_with_cache; +use std::ops::Range; +use std::path::{Path, PathBuf}; + +use async_trait::async_trait; +use bytes::Bytes; + pub use self::in_ram_slice_cache::SliceCache; pub use self::memory_sized_cache::MemorySizedCache; pub use self::storage_with_cache::StorageWithCacheFactory; -use async_trait::async_trait; -use bytes::Bytes; -use std::ops::Range; -use std::path::{Path, PathBuf}; /// The `Cache` trait is the abstraction used to describe the caching logic /// used in front of a storage. See `StorageWithCache`. diff --git a/quickwit-storage/src/cache/storage_with_cache.rs b/quickwit-storage/src/cache/storage_with_cache.rs index 07d935eb65..ab7652b5a5 100644 --- a/quickwit-storage/src/cache/storage_with_cache.rs +++ b/quickwit-storage/src/cache/storage_with_cache.rs @@ -1,31 +1,30 @@ -// Quickwit -// Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . -use async_trait::async_trait; -use bytes::Bytes; use std::ops::Range; use std::path::Path; use std::sync::Arc; -use crate::Cache; -use crate::{PutPayload, Storage, StorageFactory, StorageResult}; +use async_trait::async_trait; +use bytes::Bytes; + +use crate::{Cache, PutPayload, Storage, StorageFactory, StorageResult}; /// Use with care, StorageWithCache is read-only. struct StorageWithCache { @@ -121,8 +120,7 @@ mod tests { use std::sync::Mutex; use super::*; - use crate::MockCache; - use crate::MockStorage; + use crate::{MockCache, MockStorage}; #[tokio::test] async fn put_in_cache_test() { diff --git a/quickwit-storage/src/error.rs b/quickwit-storage/src/error.rs index f9803b8d86..9b66db65d1 100644 --- a/quickwit-storage/src/error.rs +++ b/quickwit-storage/src/error.rs @@ -1,28 +1,25 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +use std::{fmt, io}; use serde::{Deserialize, Serialize}; -use std::fmt; -use std::io; use thiserror::Error; /// Storage error kind. @@ -65,9 +62,7 @@ pub enum StorageResolverError { impl StorageErrorKind { /// Creates a StorageError. pub fn with_error(self, source: E) -> StorageError - where - anyhow::Error: From, - { + where anyhow::Error: From { StorageError { kind: self, source: From::from(source), @@ -101,9 +96,7 @@ pub type StorageResult = Result; impl StorageError { /// Add some context to the wrapper error. pub fn add_context(self, ctx: C) -> Self - where - C: fmt::Display + Send + Sync + 'static, - { + where C: fmt::Display + Send + Sync + 'static { StorageError { kind: self.kind, source: self.source.context(ctx), diff --git a/quickwit-storage/src/lib.rs b/quickwit-storage/src/lib.rs index ab9305587e..409971086a 100644 --- a/quickwit-storage/src/lib.rs +++ b/quickwit-storage/src/lib.rs @@ -1,37 +1,33 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + #![warn(missing_docs)] #![allow(clippy::bool_assert_comparison)] -/*! `quickwit-storage` is the abstraction used in quickwit to interface itself -to different storage: -- object storages (S3) -- local filesystem -- distributed filesystems. -etc. - -- The `BundleStorage` bundles together multiple files into a single file. - -*/ +//! `quickwit-storage` is the abstraction used in quickwit to interface itself +//! to different storage: +//! - object storages (S3) +//! - local filesystem +//! - distributed filesystems. +//! etc. +//! +//! - The `BundleStorage` bundles together multiple files into a single file. mod cache; mod storage; pub use self::storage::{PutPayload, Storage}; @@ -48,37 +44,34 @@ mod storage_resolver; pub use self::bundle_storage::{ BundleStorage, BundleStorageBuilder, BundleStorageFileOffsets, BUNDLE_FILENAME, }; +#[cfg(any(test, feature = "testsuite"))] +pub use self::cache::MockCache; pub use self::local_file_storage::{LocalFileStorage, LocalFileStorageFactory}; pub use self::object_storage::{ MultiPartPolicy, RegionProvider, S3CompatibleObjectStorage, S3CompatibleObjectStorageFactory, }; pub use self::prefix_storage::add_prefix_to_storage; pub use self::ram_storage::{RamStorage, RamStorageBuilder}; -pub use self::storage_resolver::{ - quickwit_storage_uri_resolver, StorageFactory, StorageUriResolver, -}; -pub use crate::cache::{Cache, MemorySizedCache, SliceCache, StorageWithCacheFactory}; -pub use crate::error::{StorageError, StorageErrorKind, StorageResolverError, StorageResult}; - -#[cfg(any(test, feature = "testsuite"))] -pub use self::cache::MockCache; #[cfg(any(test, feature = "testsuite"))] pub use self::storage::MockStorage; #[cfg(any(test, feature = "testsuite"))] pub use self::storage_resolver::MockStorageFactory; - +pub use self::storage_resolver::{ + quickwit_storage_uri_resolver, StorageFactory, StorageUriResolver, +}; #[cfg(feature = "testsuite")] pub use self::tests::storage_test_suite; +pub use crate::cache::{Cache, MemorySizedCache, SliceCache, StorageWithCacheFactory}; +pub use crate::error::{StorageError, StorageErrorKind, StorageResolverError, StorageResult}; #[cfg(any(test, feature = "testsuite"))] pub(crate) mod tests { - use anyhow::Context; - - use crate::PutPayload; use std::path::Path; - use crate::{Storage, StorageErrorKind}; + use anyhow::Context; + + use crate::{PutPayload, Storage, StorageErrorKind}; async fn test_get_inexistent_file(storage: &mut dyn Storage) -> anyhow::Result<()> { let err = storage diff --git a/quickwit-storage/src/local_file_storage.rs b/quickwit-storage/src/local_file_storage.rs index e8256acb00..a4a98cd2c2 100644 --- a/quickwit-storage/src/local_file_storage.rs +++ b/quickwit-storage/src/local_file_storage.rs @@ -1,38 +1,37 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -use crate::{PutPayload, Storage, StorageErrorKind, StorageFactory, StorageResult}; -use async_trait::async_trait; -use bytes::Bytes; -use futures::future::{BoxFuture, FutureExt}; use std::io::{ErrorKind, SeekFrom}; use std::ops::Range; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::{fmt, io}; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::future::{BoxFuture, FutureExt}; use tokio::fs; use tokio::io::{AsyncReadExt, AsyncSeekExt}; use tracing::warn; +use crate::{PutPayload, Storage, StorageErrorKind, StorageFactory, StorageResult}; + /// File system compatible storage implementation. #[derive(Clone)] pub struct LocalFileStorage { @@ -237,7 +236,8 @@ mod tests { use tempfile::tempdir; use super::*; - use crate::{tests::storage_test_suite, StorageError}; + use crate::tests::storage_test_suite; + use crate::StorageError; #[tokio::test] async fn test_storage() -> anyhow::Result<()> { diff --git a/quickwit-storage/src/object_storage/error.rs b/quickwit-storage/src/object_storage/error.rs index d429818bfc..4159c73eb4 100644 --- a/quickwit-storage/src/object_storage/error.rs +++ b/quickwit-storage/src/object_storage/error.rs @@ -1,28 +1,24 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::error::Error as StdError; -use std::fmt; -use std::io; +use std::{fmt, io}; use rusoto_core::RusotoError; use rusoto_s3::{ @@ -77,8 +73,7 @@ impl IsRetryable for RusotoErrorWrapper { } impl From> for StorageError -where - T: Send + Sync + std::error::Error + 'static + ToStorageErrorKind, +where T: Send + Sync + std::error::Error + 'static + ToStorageErrorKind { fn from(err: RusotoErrorWrapper) -> StorageError { let error_kind = match &err.0 { diff --git a/quickwit-storage/src/object_storage/file_slice_stream.rs b/quickwit-storage/src/object_storage/file_slice_stream.rs index da0b71110d..11d73db37c 100644 --- a/quickwit-storage/src/object_storage/file_slice_stream.rs +++ b/quickwit-storage/src/object_storage/file_slice_stream.rs @@ -1,24 +1,21 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::io::{self, SeekFrom}; use std::ops::Range; @@ -40,8 +37,7 @@ pub struct FileSliceStream { } impl FileSliceStream -where - R: AsyncRead + AsyncSeek + Unpin, +where R: AsyncRead + AsyncSeek + Unpin { pub async fn try_new(mut reader: R, range: Range) -> io::Result { if range.end < range.start { @@ -62,8 +58,7 @@ where } impl Stream for FileSliceStream -where - R: AsyncRead + Unpin, +where R: AsyncRead + Unpin { type Item = io::Result; @@ -85,9 +80,10 @@ where #[cfg(test)] mod tests { + use std::io::Cursor; + use bytes::Bytes; use futures::StreamExt; - use std::io::Cursor; use crate::object_storage::file_slice_stream::FileSliceStream; diff --git a/quickwit-storage/src/object_storage/mod.rs b/quickwit-storage/src/object_storage/mod.rs index 25eb83e391..3b48e9e9bd 100644 --- a/quickwit-storage/src/object_storage/mod.rs +++ b/quickwit-storage/src/object_storage/mod.rs @@ -1,24 +1,21 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . mod error; mod file_slice_stream; diff --git a/quickwit-storage/src/object_storage/policy.rs b/quickwit-storage/src/object_storage/policy.rs index 63af97ee0c..91f5f84971 100644 --- a/quickwit-storage/src/object_storage/policy.rs +++ b/quickwit-storage/src/object_storage/policy.rs @@ -1,24 +1,21 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . /// The multipart policy defines when and how multipart upload / download should happen. /// diff --git a/quickwit-storage/src/object_storage/s3_compatible_storage.rs b/quickwit-storage/src/object_storage/s3_compatible_storage.rs index 8ac2e0799f..eb25f2be40 100644 --- a/quickwit-storage/src/object_storage/s3_compatible_storage.rs +++ b/quickwit-storage/src/object_storage/s3_compatible_storage.rs @@ -1,24 +1,21 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . use std::fmt::{self, Debug}; use std::io; @@ -29,14 +26,9 @@ use std::time::Duration; use anyhow::Context; use async_trait::async_trait; use bytes::Bytes; -use futures::stream; -use futures::StreamExt; +use futures::{stream, StreamExt}; use once_cell::sync::OnceCell; use regex::Regex; -use tokio::fs::File; -use tokio_util::io::ReaderStream; -use tracing::warn; - use rusoto_core::credential::{AutoRefreshingProvider, ChainProvider}; use rusoto_core::{ByteStream, HttpClient, HttpConfig, Region, RusotoError}; use rusoto_s3::{ @@ -45,15 +37,16 @@ use rusoto_s3::{ GetObjectRequest, HeadObjectError, HeadObjectRequest, PutObjectError, PutObjectRequest, S3Client, UploadPartRequest, S3, }; +use tokio::fs::File; use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader}; +use tokio_util::io::ReaderStream; +use tracing::warn; use super::error::RusotoErrorWrapper; - use crate::object_storage::file_slice_stream::FileSliceStream; use crate::object_storage::MultiPartPolicy; use crate::retry::{retry, IsRetryable, Retry}; -use crate::{PutPayload, Storage, StorageErrorKind}; -use crate::{StorageError, StorageResult}; +use crate::{PutPayload, Storage, StorageError, StorageErrorKind, StorageResult}; /// A credential timeout. const CREDENTIAL_TIMEOUT: u64 = 5; @@ -276,7 +269,8 @@ impl S3CompatibleObjectStorage { ) -> io::Result> { assert!(len > 0); let chunks = split_range_into_chunks(len, part_len); - // Note that it should really be the first chunk, but who knows... and it is very cheap to compute this anyway. + // Note that it should really be the first chunk, but who knows... and it is very cheap to + // compute this anyway. let largest_chunk_num_bytes = chunks .iter() .map(|chunk| chunk.end - chunk.start) @@ -619,9 +613,10 @@ impl Storage for S3CompatibleObjectStorage { #[cfg(test)] mod tests { - use crate::object_storage::s3_compatible_storage::split_range_into_chunks; use std::path::PathBuf; + use crate::object_storage::s3_compatible_storage::split_range_into_chunks; + #[test] fn test_split_range_into_chunks_inexact() { assert_eq!( diff --git a/quickwit-storage/src/object_storage/s3_compatible_storage_uri_resolver.rs b/quickwit-storage/src/object_storage/s3_compatible_storage_uri_resolver.rs index 4c7802bba2..c0d1a57ce8 100644 --- a/quickwit-storage/src/object_storage/s3_compatible_storage_uri_resolver.rs +++ b/quickwit-storage/src/object_storage/s3_compatible_storage_uri_resolver.rs @@ -1,31 +1,29 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +use std::str::FromStr; +use std::sync::Arc; use ec2_instance_metadata::{InstanceMetadata, InstanceMetadataClient}; use once_cell::sync::OnceCell; use quickwit_common::{get_quickwit_env, QuickwitEnv}; pub use rusoto_core::Region; -use std::str::FromStr; -use std::sync::Arc; use crate::{S3CompatibleObjectStorage, StorageFactory}; diff --git a/quickwit-storage/src/prefix_storage.rs b/quickwit-storage/src/prefix_storage.rs index 98137e1873..7b150ddde7 100644 --- a/quickwit-storage/src/prefix_storage.rs +++ b/quickwit-storage/src/prefix_storage.rs @@ -1,32 +1,28 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::ops::Range; +use std::path::{Path, PathBuf}; +use std::sync::Arc; use async_trait::async_trait; use bytes::Bytes; -use std::{ - ops::Range, - path::{Path, PathBuf}, - sync::Arc, -}; use crate::Storage; diff --git a/quickwit-storage/src/ram_storage.rs b/quickwit-storage/src/ram_storage.rs index 0a37d1ea87..55c9d6f3b2 100644 --- a/quickwit-storage/src/ram_storage.rs +++ b/quickwit-storage/src/ram_storage.rs @@ -1,38 +1,38 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -use crate::add_prefix_to_storage; -use crate::{PutPayload, Storage, StorageErrorKind, StorageFactory, StorageResult}; -use async_trait::async_trait; -use bytes::Bytes; +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + use std::collections::HashMap; use std::ops::Range; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::{fmt, io}; + +use async_trait::async_trait; +use bytes::Bytes; use tokio::fs::File; use tokio::io::AsyncWriteExt; use tokio::sync::RwLock; +use crate::{ + add_prefix_to_storage, PutPayload, Storage, StorageErrorKind, StorageFactory, StorageResult, +}; + /// In Ram implementation of quickwit's storage. /// /// This implementation is mostly useful in unit tests. diff --git a/quickwit-storage/src/retry.rs b/quickwit-storage/src/retry.rs index 1839bea9b3..451df7d04d 100644 --- a/quickwit-storage/src/retry.rs +++ b/quickwit-storage/src/retry.rs @@ -1,29 +1,27 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +use std::fmt::Display; +use std::time::Duration; use futures::Future; use rand::Rng; -use std::fmt::Display; -use std::time::Duration; use tracing::{debug, warn}; const MAX_RETRY_ATTEMPTS: usize = 30; @@ -74,8 +72,8 @@ impl IsRetryable for Retry { } // TODO define retry strategy -/// Retry with exponential backoff and full jitter. Implementation and default values originate from the Java SDK. -/// See also: https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/. +/// Retry with exponential backoff and full jitter. Implementation and default values originate from +/// the Java SDK. See also: https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/. pub async fn retry(f: F) -> Result where F: Fn() -> Fut, diff --git a/quickwit-storage/src/storage.rs b/quickwit-storage/src/storage.rs index 2360926267..487d3e6432 100644 --- a/quickwit-storage/src/storage.rs +++ b/quickwit-storage/src/storage.rs @@ -1,31 +1,29 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -use async_trait::async_trait; -use bytes::Bytes; use std::io; use std::ops::Range; use std::path::{Path, PathBuf}; +use async_trait::async_trait; +use bytes::Bytes; + use crate::{StorageErrorKind, StorageResult}; /// Payload argument of a put request. diff --git a/quickwit-storage/src/storage_resolver.rs b/quickwit-storage/src/storage_resolver.rs index 2414811d59..9e516c5d9e 100644 --- a/quickwit-storage/src/storage_resolver.rs +++ b/quickwit-storage/src/storage_resolver.rs @@ -1,32 +1,32 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ -use crate::local_file_storage::LocalFileStorageFactory; -use crate::ram_storage::RamStorageFactory; -use crate::{RegionProvider, S3CompatibleObjectStorageFactory, Storage, StorageResolverError}; -use once_cell::sync::OnceCell; +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + use std::collections::HashMap; use std::error::Error; use std::sync::Arc; +use once_cell::sync::OnceCell; + +use crate::local_file_storage::LocalFileStorageFactory; +use crate::ram_storage::RamStorageFactory; +use crate::{RegionProvider, S3CompatibleObjectStorageFactory, Storage, StorageResolverError}; + /// Quickwit supported storage resolvers. pub fn quickwit_storage_uri_resolver() -> &'static StorageUriResolver { static STORAGE_URI_RESOLVER: OnceCell = OnceCell::new(); @@ -131,9 +131,8 @@ impl StorageUriResolver { mod tests { use std::path::Path; - use crate::RamStorage; - use super::*; + use crate::RamStorage; #[tokio::test] async fn test_storage_resolver_simple() -> anyhow::Result<()> { diff --git a/quickwit-storage/tests/s3_storage.rs b/quickwit-storage/tests/s3_storage.rs index 70ce2ccd7d..acc74ba907 100644 --- a/quickwit-storage/tests/s3_storage.rs +++ b/quickwit-storage/tests/s3_storage.rs @@ -1,24 +1,21 @@ -/* - Quickwit - Copyright (C) 2021 Quickwit Inc. - - Quickwit is offered under the AGPL v3.0 and as commercial software. - For commercial licensing, contact us at hello@quickwit.io. - - AGPL: - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . // This file is an integration test that assumes that the environement // makes it possible to connect to Amazon S3's quickwit-integration-test bucket. diff --git a/quickwit-swim/src/cluster.rs b/quickwit-swim/src/cluster.rs index b32056e3d4..441a56cc67 100644 --- a/quickwit-swim/src/cluster.rs +++ b/quickwit-swim/src/cluster.rs @@ -1,22 +1,24 @@ -use super::state::ArtilleryEpidemic; -use crate::cluster_config::ClusterConfig; -use crate::errors::*; -use crate::state::{ArtilleryClusterEvent, ArtilleryClusterRequest}; -use flume; use std::convert::AsRef; use std::net::SocketAddr; + +use flume; use tracing::debug; use uuid::Uuid; +use super::state::ArtilleryEpidemic; +use crate::cluster_config::ClusterConfig; +use crate::errors::*; +use crate::state::{ArtilleryClusterEvent, ArtilleryClusterRequest}; + #[derive(Debug)] pub struct Cluster { - comm: flume::Sender, + comm: flume::Sender } impl Cluster { pub fn create_and_start( host_key: Uuid, - config: ClusterConfig, + config: ClusterConfig ) -> Result<(Cluster, flume::Receiver)> { let (event_tx, event_rx) = flume::unbounded::(); let (internal_tx, mut internal_rx) = flume::unbounded::(); @@ -42,7 +44,7 @@ impl Cluster { self.comm .send(ArtilleryClusterRequest::Payload( id, - msg.as_ref().to_string(), + msg.as_ref().to_string() )) .unwrap(); } diff --git a/quickwit-swim/src/cluster_config.rs b/quickwit-swim/src/cluster_config.rs index d1f2ec8ced..f99e808886 100644 --- a/quickwit-swim/src/cluster_config.rs +++ b/quickwit-swim/src/cluster_config.rs @@ -1,7 +1,5 @@ -use std::{ - net::{SocketAddr, ToSocketAddrs}, - time::Duration, -}; +use std::net::{SocketAddr, ToSocketAddrs}; +use std::time::Duration; // ARTIL = 27845 /// Default Epidemic Port @@ -19,7 +17,7 @@ pub struct ClusterConfig { pub network_mtu: usize, pub ping_request_host_count: usize, pub ping_timeout: Duration, - pub listen_addr: SocketAddr, + pub listen_addr: SocketAddr } impl Default for ClusterConfig { @@ -32,7 +30,7 @@ impl Default for ClusterConfig { network_mtu: CONST_PACKET_SIZE, ping_request_host_count: 3, ping_timeout: Duration::from_secs(3), - listen_addr: directed.to_socket_addrs().unwrap().next().unwrap(), + listen_addr: directed.to_socket_addrs().unwrap().next().unwrap() } } } diff --git a/quickwit-swim/src/errors.rs b/quickwit-swim/src/errors.rs index ca23c7139c..c45b5896ad 100644 --- a/quickwit-swim/src/errors.rs +++ b/quickwit-swim/src/errors.rs @@ -1,5 +1,5 @@ -use std::io; -use std::result; +use std::{io, result}; + use thiserror::Error; /// Result type for operations that could result in an `ArtilleryError` @@ -21,7 +21,7 @@ pub enum ArtilleryError { #[error("Artillery :: Decoding Error: {}", _0)] Decoding(String), #[error("Artillery :: Numeric Cast Error: {}", _0)] - NumericCast(String), + NumericCast(String) } impl From for ArtilleryError { diff --git a/quickwit-swim/src/member.rs b/quickwit-swim/src/member.rs index ac26ef91bc..d02a684e99 100644 --- a/quickwit-swim/src/member.rs +++ b/quickwit-swim/src/member.rs @@ -1,9 +1,10 @@ -use serde::*; use std::cmp::Ordering; use std::fmt; use std::fmt::{Debug, Formatter}; use std::net::SocketAddr; use std::time::{Duration, Instant}; + +use serde::*; use uuid::Uuid; #[derive(Serialize, Deserialize, Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Copy)] @@ -19,7 +20,7 @@ pub enum ArtilleryMemberState { Down, /// Left the cluster #[serde(rename = "l")] - Left, + Left } #[derive(Serialize, Deserialize, Clone, PartialEq, Eq)] @@ -33,12 +34,12 @@ pub struct ArtilleryMember { #[serde(rename = "m")] member_state: ArtilleryMemberState, #[serde(rename = "t", skip, default = "Instant::now")] - last_state_change: Instant, + last_state_change: Instant } #[derive(Serialize, Deserialize, Debug, Clone, PartialOrd, Ord, PartialEq, Eq)] pub struct ArtilleryStateChange { - member: ArtilleryMember, + member: ArtilleryMember } impl ArtilleryMember { @@ -46,14 +47,14 @@ impl ArtilleryMember { host_key: Uuid, remote_host: SocketAddr, incarnation_number: u64, - known_state: ArtilleryMemberState, + known_state: ArtilleryMemberState ) -> Self { ArtilleryMember { host_key, remote_host: Some(remote_host), incarnation_number, member_state: known_state, - last_state_change: Instant::now(), + last_state_change: Instant::now() } } @@ -63,7 +64,7 @@ impl ArtilleryMember { remote_host: None, incarnation_number: 0, member_state: ArtilleryMemberState::Alive, - last_state_change: Instant::now(), + last_state_change: Instant::now() } } @@ -130,14 +131,14 @@ impl PartialOrd for ArtilleryMember { self.host_key.as_bytes(), format!("{:?}", self.remote_host), self.incarnation_number, - self.member_state, + self.member_state ); let t2 = ( rhs.host_key.as_bytes(), format!("{:?}", rhs.remote_host), rhs.incarnation_number, - rhs.member_state, + rhs.member_state ); t1.partial_cmp(&t2) @@ -158,14 +159,14 @@ impl Debug for ArtilleryMember { .field("state", &self.member_state) .field( "drift_time_ms", - &self.last_state_change.elapsed().as_millis(), + &self.last_state_change.elapsed().as_millis() ) .field( "remote_host", &self .remote_host .map_or(String::from("(current)"), |r| format!("{}", r)) - .as_str(), + .as_str() ) .finish() } @@ -173,7 +174,7 @@ impl Debug for ArtilleryMember { pub fn most_uptodate_member_data<'a>( lhs: &'a ArtilleryMember, - rhs: &'a ArtilleryMember, + rhs: &'a ArtilleryMember ) -> &'a ArtilleryMember { // Don't apply clippy here. // It's important bit otherwise we won't understand. @@ -183,7 +184,7 @@ pub fn most_uptodate_member_data<'a>( lhs.member_state, lhs.incarnation_number, rhs.member_state, - rhs.incarnation_number, + rhs.incarnation_number ) { (ArtilleryMemberState::Alive, i, ArtilleryMemberState::Suspect, j) => i > j, (ArtilleryMemberState::Alive, i, ArtilleryMemberState::Alive, j) => i > j, @@ -192,7 +193,7 @@ pub fn most_uptodate_member_data<'a>( (ArtilleryMemberState::Down, _, ArtilleryMemberState::Alive, _) => true, (ArtilleryMemberState::Down, _, ArtilleryMemberState::Suspect, _) => true, (ArtilleryMemberState::Left, _, _, _) => true, - _ => false, + _ => false }; if lhs_overrides { @@ -204,11 +205,13 @@ pub fn most_uptodate_member_data<'a>( #[cfg(test)] mod test { - use super::{ArtilleryMember, ArtilleryMemberState}; use std::str::FromStr; use std::time::{Duration, Instant}; + use uuid; + use super::{ArtilleryMember, ArtilleryMemberState}; + #[test] fn test_member_encode_decode() { let member = ArtilleryMember { @@ -216,7 +219,7 @@ mod test { remote_host: Some(FromStr::from_str("127.0.0.1:1337").unwrap()), incarnation_number: 123, member_state: ArtilleryMemberState::Alive, - last_state_change: Instant::now() - Duration::from_secs(3600), + last_state_change: Instant::now() - Duration::from_secs(3600) }; let encoded = bincode::serialize(&member).unwrap(); diff --git a/quickwit-swim/src/membership.rs b/quickwit-swim/src/membership.rs index b89aa39557..38a4433c26 100644 --- a/quickwit-swim/src/membership.rs +++ b/quickwit-swim/src/membership.rs @@ -10,14 +10,14 @@ use crate::member::{self, ArtilleryMember, ArtilleryMemberState, ArtilleryStateC pub struct ArtilleryMemberList { members: Vec, - periodic_index: usize, + periodic_index: usize } impl ArtilleryMemberList { pub fn new(current: ArtilleryMember) -> Self { ArtilleryMemberList { members: vec![current], - periodic_index: 0, + periodic_index: 0 } } @@ -79,7 +79,7 @@ impl ArtilleryMemberList { pub fn time_out_nodes( &mut self, - expired_hosts: &HashSet, + expired_hosts: &HashSet ) -> (Vec, Vec) { let mut suspect_members = Vec::new(); let mut down_members = Vec::new(); @@ -129,7 +129,7 @@ impl ArtilleryMemberList { pub fn apply_state_changes( &mut self, state_changes: Vec, - from: &SocketAddr, + from: &SocketAddr ) -> (Vec, Vec) { let mut current_members = self.to_map(); @@ -179,13 +179,11 @@ impl ArtilleryMemberList { (new_nodes, changed_nodes) } - /// - /// /// Random ping enqueuing pub fn hosts_for_indirect_ping( &self, host_count: usize, - target: &SocketAddr, + target: &SocketAddr ) -> Vec { let mut possible_members: Vec<_> = self .members @@ -217,7 +215,6 @@ impl ArtilleryMemberList { self.members.push(member) } - /// /// `get_member` will return artillery member if the given uuid is matches with any of the /// member in the cluster. pub fn get_member(&self, id: &Uuid) -> Option { diff --git a/quickwit-swim/src/state.rs b/quickwit-swim/src/state.rs index 213052a1f3..19243fd153 100644 --- a/quickwit-swim/src/state.rs +++ b/quickwit-swim/src/state.rs @@ -1,21 +1,22 @@ -use super::cluster_config::ClusterConfig; -use super::membership::ArtilleryMemberList; -use crate::errors::*; -use crate::member::{ArtilleryMember, ArtilleryMemberState, ArtilleryStateChange}; -use crate::prelude::CONST_PACKET_SIZE; -use mio::net::UdpSocket; -use mio::{Events, Interest, Poll, Token}; -use serde::*; use std::collections::hash_map::Entry; use std::collections::{HashMap, HashSet}; use std::io; use std::net::SocketAddr; -use std::sync::atomic::AtomicBool; -use std::sync::atomic::Ordering; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; + +use mio::net::UdpSocket; +use mio::{Events, Interest, Poll, Token}; +use serde::*; use tracing::{debug, error, info, warn}; use uuid::Uuid; +use super::cluster_config::ClusterConfig; +use super::membership::ArtilleryMemberList; +use crate::errors::*; +use crate::member::{ArtilleryMember, ArtilleryMemberState, ArtilleryStateChange}; +use crate::prelude::CONST_PACKET_SIZE; + pub type ArtilleryClusterEvent = (Vec, ArtilleryMemberEvent); pub type WaitList = HashMap>; @@ -26,7 +27,7 @@ pub enum ArtilleryMemberEvent { SuspectedDown(ArtilleryMember), WentDown(ArtilleryMember), Left(ArtilleryMember), - Payload(ArtilleryMember, String), + Payload(ArtilleryMember, String) } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -34,7 +35,7 @@ pub struct ArtilleryMessage { sender: Uuid, cluster_key: Vec, request: Request, - state_changes: Vec, + state_changes: Vec } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] @@ -46,13 +47,13 @@ enum Request { Ack, Ping(EncSocketAddr), AckHost(ArtilleryMember), - Payload(Uuid, String), + Payload(Uuid, String) } #[derive(Debug, Clone)] pub struct TargetedRequest { request: Request, - target: SocketAddr, + target: SocketAddr } #[derive(Clone)] @@ -62,7 +63,7 @@ pub enum ArtilleryClusterRequest { React(TargetedRequest), LeaveCluster, Exit, - Payload(Uuid, String), + Payload(Uuid, String) } const UDP_SERVER: Token = Token(0); @@ -78,7 +79,7 @@ pub struct ArtilleryEpidemic { server_socket: UdpSocket, request_tx: flume::Sender, event_tx: flume::Sender, - running: AtomicBool, + running: AtomicBool } pub type ClusterReactor = (Poll, ArtilleryEpidemic); @@ -88,7 +89,7 @@ impl ArtilleryEpidemic { host_key: Uuid, config: ClusterConfig, event_tx: flume::Sender, - internal_tx: flume::Sender, + internal_tx: flume::Sender ) -> Result { let poll: Poll = Poll::new()?; @@ -110,7 +111,7 @@ impl ArtilleryEpidemic { server_socket, request_tx: internal_tx, event_tx, - running: AtomicBool::new(true), + running: AtomicBool::new(true) }; Ok((poll, state)) @@ -119,7 +120,7 @@ impl ArtilleryEpidemic { pub(crate) fn event_loop( receiver: &mut flume::Receiver, mut poll: Poll, - mut state: ArtilleryEpidemic, + mut state: ArtilleryEpidemic ) -> Result<()> { let mut events = Events::with_capacity(1); let mut buf = [0_u8; CONST_PACKET_SIZE]; @@ -163,7 +164,7 @@ impl ArtilleryEpidemic { let message = serde_json::from_slice(&buf[..packet_size])?; state.request_tx.send(ArtilleryClusterRequest::Respond( source_address, - message, + message ))?; } Err(e) if e.kind() == io::ErrorKind::WouldBlock => { @@ -203,7 +204,7 @@ impl ArtilleryEpidemic { &self.config.cluster_key, &request.request, &self.state_changes, - self.config.network_mtu, + self.config.network_mtu ); if should_add_pending { @@ -224,7 +225,7 @@ impl ArtilleryEpidemic { self.request_tx .send(ArtilleryClusterRequest::React(TargetedRequest { request: Request::Heartbeat, - target: *seed_node, + target: *seed_node })) .unwrap(); } @@ -235,7 +236,7 @@ impl ArtilleryEpidemic { self.request_tx .send(ArtilleryClusterRequest::React(TargetedRequest { request: Request::Heartbeat, - target: member.remote_host().unwrap(), + target: member.remote_host().unwrap() })) .unwrap(); } @@ -278,7 +279,7 @@ impl ArtilleryEpidemic { self.request_tx .send(ArtilleryClusterRequest::React(TargetedRequest { request: Request::Ping(EncSocketAddr::from_addr(&target_host)), - target: relay, + target: relay })) .unwrap(); } @@ -310,7 +311,7 @@ impl ArtilleryEpidemic { request: Request::Payload(id, msg), target: target_peer .remote_host() - .expect("Expected target peer addr"), + .expect("Expected target peer addr") }); return; } @@ -337,7 +338,7 @@ impl ArtilleryEpidemic { let response = match message.request { Heartbeat => Some(TargetedRequest { request: Ack, - target: src_addr, + target: src_addr }), Ack => { self.ack_response(src_addr); @@ -349,7 +350,7 @@ impl ArtilleryEpidemic { add_to_wait_list(&mut self.wait_list, &dest_addr, &src_addr); Some(TargetedRequest { request: Heartbeat, - target: dest_addr, + target: dest_addr }) } AckHost(member) => { @@ -418,7 +419,7 @@ impl ArtilleryEpidemic { WentUp(ref m) => assert_eq!(m.state(), ArtilleryMemberState::Alive), WentDown(ref m) => assert_eq!(m.state(), ArtilleryMemberState::Down), SuspectedDown(ref m) => assert_eq!(m.state(), ArtilleryMemberState::Suspect), - Left(ref m) => assert_eq!(m.state(), ArtilleryMemberState::Left), + Left(ref m) => assert_eq!(m.state(), ArtilleryMemberState::Left) }; // If an error is returned, no one is listening to events anymore. This is normal. @@ -447,7 +448,7 @@ impl ArtilleryEpidemic { self.request_tx .send(ArtilleryClusterRequest::React(TargetedRequest { request: Request::AckHost(member.clone()), - target: *remote, + target: *remote })) .unwrap(); } @@ -466,13 +467,13 @@ fn build_message( cluster_key: &[u8], request: &Request, state_changes: &[ArtilleryStateChange], - network_mtu: usize, + network_mtu: usize ) -> ArtilleryMessage { let mut message = ArtilleryMessage { sender: *sender, cluster_key: cluster_key.into(), request: request.clone(), - state_changes: Vec::new(), + state_changes: Vec::new() }; for i in 0..=state_changes.len() { @@ -480,7 +481,7 @@ fn build_message( sender: *sender, cluster_key: cluster_key.into(), request: request.clone(), - state_changes: (&state_changes[..i]).to_vec(), + state_changes: (&state_changes[..i]).to_vec() }; let encoded = serde_json::to_string(&message).unwrap(); @@ -512,13 +513,13 @@ fn determine_member_event(member: ArtilleryMember) -> ArtilleryMemberEvent { ArtilleryMemberState::Alive => ArtilleryMemberEvent::WentUp(member), ArtilleryMemberState::Suspect => ArtilleryMemberEvent::SuspectedDown(member), ArtilleryMemberState::Down => ArtilleryMemberEvent::WentDown(member), - ArtilleryMemberState::Left => ArtilleryMemberEvent::Left(member), + ArtilleryMemberState::Left => ArtilleryMemberEvent::Left(member) } } fn enqueue_state_change( state_changes: &mut Vec, - members: &[ArtilleryMember], + members: &[ArtilleryMember] ) { for member in members { for state_change in state_changes.iter_mut() { diff --git a/quickwit-telemetry/src/lib.rs b/quickwit-telemetry/src/lib.rs index d5c981fa78..b1012ed70d 100644 --- a/quickwit-telemetry/src/lib.rs +++ b/quickwit-telemetry/src/lib.rs @@ -1,34 +1,33 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + #![allow(clippy::bool_assert_comparison)] pub mod payload; /// This crate contains the code responsible for sending usage data to Quickwit inc's server. mod sender; pub(crate) mod sink; -use crate::payload::TelemetryEvent; -use crate::sender::{TelemetryLoopHandle, TelemetrySender}; use once_cell::sync::OnceCell; +use crate::payload::TelemetryEvent; pub use crate::sender::is_telemetry_enabled; +use crate::sender::{TelemetryLoopHandle, TelemetrySender}; pub fn start_telemetry_loop() -> TelemetryLoopHandle { get_telemetry_sender_singleton().start_loop() diff --git a/quickwit-telemetry/src/payload.rs b/quickwit-telemetry/src/payload.rs index e1161552ea..a933d158a9 100644 --- a/quickwit-telemetry/src/payload.rs +++ b/quickwit-telemetry/src/payload.rs @@ -1,26 +1,26 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -use serde::{Deserialize, Serialize}; +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + use std::env; use std::time::UNIX_EPOCH; + +use serde::{Deserialize, Serialize}; use uuid::Uuid; /// Represents the payload of the request sent with telemetry requests. diff --git a/quickwit-telemetry/src/sender.rs b/quickwit-telemetry/src/sender.rs index e2842d48f8..f5d058cfb4 100644 --- a/quickwit-telemetry/src/sender.rs +++ b/quickwit-telemetry/src/sender.rs @@ -1,39 +1,35 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + use std::mem; -use std::sync::atomic::AtomicBool; -use std::sync::atomic::Ordering; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; + use tokio::sync::mpsc::{Receiver, Sender}; -use tokio::sync::oneshot; -use tokio::sync::Mutex; -use tokio::sync::RwLock; +use tokio::sync::{oneshot, Mutex, RwLock}; use tokio::task::JoinHandle; use tokio::time::Interval; use tracing::info; use crate::payload::{ClientInformation, EventWithTimestamp, TelemetryEvent, TelemetryPayload}; -use crate::sink::HttpClient; -use crate::sink::Sink; +use crate::sink::{HttpClient, Sink}; /// At most 1 Request per minutes. const TELEMETRY_PUSH_COOLDOWN: Duration = Duration::from_secs(60); diff --git a/quickwit-telemetry/src/sink.rs b/quickwit-telemetry/src/sink.rs index 38d951ded2..222e4767e9 100644 --- a/quickwit-telemetry/src/sink.rs +++ b/quickwit-telemetry/src/sink.rs @@ -1,27 +1,27 @@ -/* - * Copyright (C) 2021 Quickwit Inc. - * - * Quickwit is offered under the AGPL v3.0 and as commercial software. - * For commercial licensing, contact us at hello@quickwit.io. - * - * AGPL: - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Copyright (C) 2021 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::time::Duration; + use async_trait::async_trait; use reqwest::redirect::Policy; use reqwest::Client; -use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; use crate::payload::TelemetryPayload;