Merge branch 'quickwit-oss:main' into add-time-series-dimension

distribution/ecs/quickwit/iam.tf

@@ -3,7 +3,6 @@ data "aws_iam_policy_document" "quickwit_task_permission" {
   statement {
     actions = [
       "s3:ListBucket",
-      "s3:ListObjects",
       "s3:GetObject",
       "s3:PutObject",
       "s3:DeleteObject"

docs/configuration/node-config.md

@@ -208,16 +208,16 @@ This section contains the configuration options for a Searcher.
 | `partial_request_cache_capacity` | Partial request in memory cache capacity on a Searcher. Cache intermediate state for a request, possibly making subsequent requests faster. It can be disabled by setting the size to `0`. | `64M` |
 | `max_num_concurrent_split_searches` | Maximum number of concurrent split search requests running on a Searcher. | `100` |
 | `max_num_concurrent_split_streams` | Maximum number of concurrent split stream requests running on a Searcher. | `100` |
-| `split_cache` | Searcher split cache configuration options defined in the section below. | |
+| `split_cache` | Searcher split cache configuration options defined in the section below. Cache disabled if unspecified. | |
 
 
 ### Searcher split cache configuration
 
-This section contains the configuration options for the searcher split cache.
+This section contains the configuration options for the on disk searcher split cache.
 
 | Property | Description | Default value |
 | --- | --- | --- |
-| `max_num_bytes` | Maximum size in bytes allowed in the split cache. | `1G` |
+| `max_num_bytes` | Maximum disk size in bytes allowed in the split cache. Can be exceeded by the size of one split. | |
 | `max_num_splits` | Maximum number of splits allowed in the split cache.   | `10000` |
 | `num_concurrent_downloads` | Maximum number of concurrent download of splits. | `1` |
 

docs/guides/aws-setup.md

@@ -46,12 +46,11 @@ AWS credentials or region resolution may take a few seconds, especially if the A
 
 Required authorized actions:
 - `ListBucket` (on the bucket directly)
-- `AbortMultipartUpload`
-- `DeleteObject`
 - `GetObject`
-- `ListMultipartUploadParts`
-- `ListObjects`
 - `PutObject`
+- `DeleteObject`
+- `ListMultipartUploadParts`
+- `AbortMultipartUpload`
 
 Here is an example of a bucket policy:
 ```json
@@ -70,12 +69,11 @@ Here is an example of a bucket policy:
     {
       "Effect": "Allow",
       "Action": [
-        "s3:AbortMultipartUpload",
-        "s3:DeleteObject",
         "s3:GetObject",
+        "s3:PutObject",
+        "s3:DeleteObject",
         "s3:ListMultipartUploadParts",
-        "S3:ListObjects",
-        "s3:PutObject"
+        "s3:AbortMultipartUpload"
       ],
       "Resource": [
         "arn:aws:s3:::my-bucket/*"

quickwit/quickwit-serve/src/elasticsearch_api/filter.rs

@@ -74,7 +74,7 @@ pub(crate) fn elasticsearch_filter(
 pub(crate) fn elastic_bulk_filter(
 ) -> impl Filter<Extract = (Body, ElasticBulkOptions), Error = Rejection> + Clone {
     warp::path!("_elastic" / "_bulk")
-        .and(warp::post())
+        .and(warp::post().or(warp::put()).unify())
         .and(warp::body::content_length_limit(
             CONTENT_LENGTH_LIMIT.as_u64(),
         ))
@@ -97,7 +97,7 @@ pub(crate) fn elastic_bulk_filter(
 pub(crate) fn elastic_index_bulk_filter(
 ) -> impl Filter<Extract = (String, Body, ElasticBulkOptions), Error = Rejection> + Clone {
     warp::path!("_elastic" / String / "_bulk")
-        .and(warp::post())
+        .and(warp::post().or(warp::put()).unify())
         .and(warp::body::content_length_limit(
             CONTENT_LENGTH_LIMIT.as_u64(),
         ))

quickwit/quickwit-storage/src/split_cache/split_table.rs

@@ -100,7 +100,8 @@ pub struct SplitInfo {
 /// - downloading_splits
 /// - candidate_splits.
 ///
-/// It is possible for the split table to exceed its limits, by at most one split.
+/// It is possible for the split table size in bytes to exceed its limits, by at
+/// most one split.
 pub struct SplitTable {
     on_disk_splits: BTreeSet<SplitKey>,
     downloading_splits: BTreeSet<SplitKey>,
@@ -360,7 +361,7 @@ impl SplitTable {
             return false;
         }
         if self.on_disk_splits.len() + self.downloading_splits.len()
-            > self.limits.max_num_splits.get() as usize
+            >= self.limits.max_num_splits.get() as usize
         {
             return true;
         }
@@ -608,7 +609,7 @@ mod tests {
             splits_to_delete,
             split_to_download,
         } = split_table.find_download_opportunity().unwrap();
-        assert_eq!(&splits_to_delete[..], &[splits[0].0][..]);
+        assert_eq!(&splits_to_delete[..], &[splits[0].0, splits[1].0]);
         assert_eq!(split_to_download.split_ulid, new_ulid);
     }
 

quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0004-put-request.yaml

@@ -0,0 +1,4 @@
+method: PUT
+ndjson:
+  - index: { "_index": "test-index", "_id": "1" }
+  - message: Hello, World!

quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.elasticsearch.yaml

@@ -0,0 +1,18 @@
+# Delete possibly remaining index
+method: DELETE
+endpoint: test-index
+status_code: null
+---
+method: PUT
+endpoint: test-index
+json: {
+  "mappings": {
+    "properties": {
+      "message": {
+        "type": "text",
+        "store": true
+      }
+    }
+  }
+}
+

quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.quickwit.yaml

@@ -0,0 +1,17 @@
+# Delete possibly remaining index
+method: DELETE
+api_root: http://localhost:7280/api/v1/
+endpoint: indexes/test-index
+status_code: null
+---
+method: POST
+api_root: http://localhost:7280/api/v1/
+endpoint: indexes/
+json:
+  version: "0.7"
+  index_id: test-index
+  doc_mapping:
+    field_mappings:
+        - name: message
+          type: text
+sleep_after: 3

quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_teardown.elasticsearch.yaml

@@ -0,0 +1,3 @@
+method: DELETE
+endpoint: test-index
+---

quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_teardown.quickwit.yaml

@@ -0,0 +1,3 @@
+method: DELETE
+api_root: http://localhost:7280/api/v1/
+endpoint: indexes/test-index

quickwit/rest-api-tests/scenarii/sort_orders/_teardown.quickwit.yaml

@@ -1,3 +1,3 @@
 # # Delete index
-# method: DELETE
-# endpoint: indexes/simple
+method: DELETE
+endpoint: indexes/sortorder