From c3f0df3a4bc983362cc0bea002ab9ccf741ea1da Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Fri, 15 Feb 2019 08:23:31 -0800 Subject: [PATCH 01/15] [DOCS] Updates methods for upgrading machine learning (#38876) --- docs/reference/upgrade/close-ml.asciidoc | 32 +++++++++++++++++++ .../upgrade/cluster_restart.asciidoc | 11 +++++-- docs/reference/upgrade/open-ml.asciidoc | 13 ++++++++ .../upgrade/rolling_upgrade.asciidoc | 12 +++++-- 4 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 docs/reference/upgrade/close-ml.asciidoc create mode 100644 docs/reference/upgrade/open-ml.asciidoc diff --git a/docs/reference/upgrade/close-ml.asciidoc b/docs/reference/upgrade/close-ml.asciidoc new file mode 100644 index 0000000000000..c4efddca759c9 --- /dev/null +++ b/docs/reference/upgrade/close-ml.asciidoc @@ -0,0 +1,32 @@ +[testenv="platinum"] + +If your {ml} indices were created earlier than the previous major version, they +must be reindexed. In those circumstances, there must be no machine learning +jobs running during the upgrade. + +In all other circumstances, there is no requirement to close your {ml} jobs. +There are, however, advantages to doing so. If you choose to leave your jobs +running during the upgrade, they are affected when you stop the {ml} nodes. The +jobs move to another {ml} node and restore the model states. This scenario has +the least disruption to the active {ml} jobs but incurs the highest load on the +cluster. + +To close all {ml} jobs before you upgrade, see +{stack-ov}/stopping-ml.html[Stopping {ml}]. This method persists the model +state at the moment of closure, which means that when you open your jobs after +the upgrade, they use the exact same model. This scenario takes the most time, +however, especially if you have many jobs or jobs with large model states. + +To temporarily halt the tasks associated with your {ml} jobs and {dfeeds} and +prevent new jobs from opening, use the <>: + +[source,js] +-------------------------------------------------- +POST _ml/set_upgrade_mode?enabled=true +-------------------------------------------------- +// CONSOLE + +This method does not persist the absolute latest model state, rather it uses the +last model state that was automatically saved. By halting the tasks, you avoid +incurring the cost of managing active jobs during the upgrade and it's quicker +than stopping {dfeeds} and closing jobs. \ No newline at end of file diff --git a/docs/reference/upgrade/cluster_restart.asciidoc b/docs/reference/upgrade/cluster_restart.asciidoc index 4c229e373f505..a8552a82bb8d2 100644 --- a/docs/reference/upgrade/cluster_restart.asciidoc +++ b/docs/reference/upgrade/cluster_restart.asciidoc @@ -26,8 +26,11 @@ recovery. include::synced-flush.asciidoc[] -- -. *Stop any machine learning jobs that are running.* See -{xpack-ref}/stopping-ml.html[Stopping Machine Learning]. +. *Stop any machine learning jobs that are running.* ++ +-- +include::close-ml.asciidoc[] +-- . *Shutdown all nodes.* + @@ -132,3 +135,7 @@ GET _cat/recovery -- . *Restart machine learning jobs.* ++ +-- +include::open-ml.asciidoc[] +-- diff --git a/docs/reference/upgrade/open-ml.asciidoc b/docs/reference/upgrade/open-ml.asciidoc new file mode 100644 index 0000000000000..b9b6b772bbe8d --- /dev/null +++ b/docs/reference/upgrade/open-ml.asciidoc @@ -0,0 +1,13 @@ +[testenv="platinum"] +If you closed all {ml} jobs before the upgrade, you must open them. Use {kib} or +the <>. + +Alternatively, if you temporarily halted the tasks associated with your {ml} jobs, +use the <> to return them to active +states: + +[source,js] +-------------------------------------------------- +POST _ml/set_upgrade_mode?enabled=false +-------------------------------------------------- +// CONSOLE diff --git a/docs/reference/upgrade/rolling_upgrade.asciidoc b/docs/reference/upgrade/rolling_upgrade.asciidoc index dff3895ac4c1d..e62bd9348f1ab 100644 --- a/docs/reference/upgrade/rolling_upgrade.asciidoc +++ b/docs/reference/upgrade/rolling_upgrade.asciidoc @@ -43,8 +43,11 @@ include::synced-flush.asciidoc[] -- -. *Stop any machine learning jobs that are running.* See -{xpack-ref}/stopping-ml.html[Stopping Machine Learning]. +. *Stop any machine learning jobs that are running.* ++ +-- +include::close-ml.asciidoc[] +-- . [[upgrade-node]] *Shut down a single node*. + @@ -160,6 +163,11 @@ for each node that needs to be updated. -- . *Restart machine learning jobs.* ++ +-- +include::open-ml.asciidoc[] +-- + [IMPORTANT] ==================================================== From 9339fc8a12ab799a8c4bbaa58e6e9aeae89d4ac8 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 15 Feb 2019 17:30:57 +0100 Subject: [PATCH 02/15] Fix serialization bug in ShardFollowTask after cutting this class over to extend from ImmutableFollowParameters. Prior to #38910 readVInt and writeVInt was used, #38910 accidentally changed ShardFollowTask to use readOptionalVInt / writeVOptionalVInt. This changes changes ShardFollowTask back to use vint. --- .../xpack/ccr/action/ShardFollowTask.java | 39 ++++++++++++++----- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTask.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTask.java index 123bd2c996dae..a82670a52a0c4 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTask.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTask.java @@ -93,15 +93,24 @@ public class ShardFollowTask extends ImmutableFollowParameters implements XPackP } public static ShardFollowTask readFrom(StreamInput in) throws IOException { - return new ShardFollowTask(in.readString(), ShardId.readShardId(in), ShardId.readShardId(in), in); - } - - private ShardFollowTask(String remoteCluster, ShardId followShardId, ShardId leaderShardId, StreamInput in) throws IOException { - super(in); - this.remoteCluster = remoteCluster; - this.followShardId = followShardId; - this.leaderShardId = leaderShardId; - this.headers = Collections.unmodifiableMap(in.readMap(StreamInput::readString, StreamInput::readString)); + String remoteCluster = in.readString(); + ShardId followShardId = ShardId.readShardId(in); + ShardId leaderShardId = ShardId.readShardId(in); + // TODO: use ImmutableFollowParameters(StreamInput) constructor + int maxReadRequestOperationCount = in.readVInt(); + ByteSizeValue maxReadRequestSize = new ByteSizeValue(in); + int maxOutstandingReadRequests = in.readVInt(); + int maxWriteRequestOperationCount = in.readVInt(); + ByteSizeValue maxWriteRequestSize = new ByteSizeValue(in); + int maxOutstandingWriteRequests = in.readVInt(); + int maxWriteBufferCount = in.readVInt(); + ByteSizeValue maxWriteBufferSize = new ByteSizeValue(in); + TimeValue maxRetryDelay = in.readTimeValue(); + TimeValue readPollTimeout = in.readTimeValue(); + Map headers = Collections.unmodifiableMap(in.readMap(StreamInput::readString, StreamInput::readString)); + return new ShardFollowTask(remoteCluster, followShardId, leaderShardId, maxReadRequestOperationCount, + maxWriteRequestOperationCount, maxOutstandingReadRequests, maxOutstandingWriteRequests, maxReadRequestSize, + maxWriteRequestSize, maxWriteBufferCount, maxWriteBufferSize, maxRetryDelay, readPollTimeout, headers); } public String getRemoteCluster() { @@ -130,7 +139,17 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(remoteCluster); followShardId.writeTo(out); leaderShardId.writeTo(out); - super.writeTo(out); + // TODO: use super.writeTo() + out.writeVLong(getMaxReadRequestOperationCount()); + getMaxReadRequestSize().writeTo(out); + out.writeVInt(getMaxOutstandingReadRequests()); + out.writeVLong(getMaxWriteRequestOperationCount()); + getMaxWriteRequestSize().writeTo(out); + out.writeVInt(getMaxOutstandingWriteRequests()); + out.writeVInt(getMaxWriteBufferCount()); + getMaxWriteBufferSize().writeTo(out); + out.writeTimeValue(getMaxRetryDelay()); + out.writeTimeValue(getReadPollTimeout()); out.writeMap(headers, StreamOutput::writeString, StreamOutput::writeString); } From faa838ca691ac7ca50f10551166f484e0ef517a4 Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Fri, 15 Feb 2019 09:37:34 -0800 Subject: [PATCH 03/15] [DOCS] Edits warning in put watch API (#38582) --- .../en/rest-api/watcher/put-watch.asciidoc | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/x-pack/docs/en/rest-api/watcher/put-watch.asciidoc b/x-pack/docs/en/rest-api/watcher/put-watch.asciidoc index 89b79b5680056..1349e8def05d9 100644 --- a/x-pack/docs/en/rest-api/watcher/put-watch.asciidoc +++ b/x-pack/docs/en/rest-api/watcher/put-watch.asciidoc @@ -5,7 +5,7 @@ Put watch ++++ -The PUT watch API either registers a new watch in {watcher} or update an +The PUT watch API either registers a new watch in {watcher} or updates an existing one. [float] @@ -21,13 +21,13 @@ the `.watches` index and its trigger is immediately registered with the relevant trigger engine. Typically for the `schedule` trigger, the scheduler is the trigger engine. -IMPORTANT: Putting a watch must be done via this API only. Do not put a watch - directly to the `.watches` index using the Elasticsearch Index API. - If {es} {security-features} are enabled, make sure no `write` - privileges are granted to anyone over the `.watches` index. +IMPORTANT: You must use {kib} or this API to create a watch. Do not put a watch + directly to the `.watches` index using the Elasticsearch index API. + If {es} {security-features} are enabled, do not give users `write` + privileges on the `.watches` index. When adding a watch you can also define its initial -{xpack-ref}/how-watcher-works.html#watch-active-state[active state]. You do that +{stack-ov}/how-watcher-works.html#watch-active-state[active state]. You do that by setting the `active` parameter. [float] @@ -52,16 +52,16 @@ A watch has the following fields: |====== | Name | Description -| `trigger` | The {xpack-ref}/trigger.html[trigger] that defines when +| `trigger` | The {stack-ov}/trigger.html[trigger] that defines when the watch should run. -| `input` | The {xpack-ref}/input.html[input] that defines the input +| `input` | The {stack-ov}/input.html[input] that defines the input that loads the data for the watch. -| `condition` | The {xpack-ref}/condition.html[condition] that defines if +| `condition` | The {stack-ov}/condition.html[condition] that defines if the actions should be run. -| `actions` | The list of {xpack-ref}/actions.html[actions] that will be +| `actions` | The list of {stack-ov}/actions.html[actions] that will be run if the condition matches | `metadata` | Metadata json that will be copied into the history entries. @@ -75,7 +75,7 @@ A watch has the following fields: ==== Authorization You must have `manage_watcher` cluster privileges to use this API. For more -information, see {xpack-ref}/security-privileges.html[Security Privileges]. +information, see {stack-ov}/security-privileges.html[Security Privileges]. [float] ==== Security Integration @@ -148,7 +148,7 @@ PUT _watcher/watch/my-watch // CONSOLE When you add a watch you can also define its initial -{xpack-ref}/how-watcher-works.html#watch-active-state[active state]. You do that +{stack-ov}/how-watcher-works.html#watch-active-state[active state]. You do that by setting the `active` parameter. The following command adds a watch and sets it to be inactive by default: From fadad3c2f247910d76ee89fb6d44569378fe2e8e Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Fri, 15 Feb 2019 12:48:56 -0500 Subject: [PATCH 04/15] Relax testStressMaybeFlushOrRollTranslogGeneration (#38918) The predicate shouldPeriodicallyFlush is determined by the uncommitted translog size and the local checkpoint. The uncommitted translog size depends on the local checkpoint. The condition shouldPeriodicallyFlush can be true twice in in the test in the following scenario: 1. Index doc-0 and advances the local checkpoint to 0, the condition shouldPeriodicallyFlush remains false. 2. Index doc-1 and add it to translog, but the local checkpoint is not advanced yet (still 0). The condition shouldPeriodicallyFlush becomes true because the uncommitted translog size is 216bytes (2ops + gen-1 + gen-2) > 180bytes and the translog generation of the new index commit would advance from 1 to 2. > [2019-02-13T23:33:58,257][TRACE][o.e.i.e.Engine ] [node_s_0] > [test][0] committing writer with commit data [{local_checkpoint=0, > max_unsafe_auto_id_timestamp=-1, translog_uuid=fFp1Yqd4QiqKDD4ZrC8F-g, > min_retained_seq_no=0, history_uuid=cn31yrwVQk-Vs7qcg4bi_Q, > retention_leases=primary_term:1;version:0;, translog_generation=2, > max_seq_no=1}] 1. The shouldPeriodicallyFlush becomes true again after the local checkpoint is advanced to 1 because the uncommitted translog size is 216bytes (2ops + gen-2 + gen-3) > 180bytes and the translog generation of the new index commit would advance from 2 to 4. > [2019-02-13T23:33:58,264][TRACE][o.e.i.e.Engine ] [node_s_0] > [test][0] committing writer with commit data [{local_checkpoint=1, > max_unsafe_auto_id_timestamp=-1, translog_uuid=fFp1Yqd4QiqKDD4ZrC8F-g, > min_retained_seq_no=0, history_uuid=cn31yrwVQk-Vs7qcg4bi_Q, > retention_leases=primary_term:1;version:0;, translog_generation=4, > max_seq_no=1}] We need to relax the assertion in this test to cover this situation. Closes #31629 --- .../elasticsearch/index/shard/IndexShardIT.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java index 674c252d780f3..fb8574594a874 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -120,6 +120,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoSearchHits; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -474,18 +475,22 @@ public void testStressMaybeFlushOrRollTranslogGeneration() throws Exception { final FlushStats initialStats = shard.flushStats(); client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get(); check = () -> { + assertFalse(shard.shouldPeriodicallyFlush()); final FlushStats currentStats = shard.flushStats(); String msg = String.format(Locale.ROOT, "flush stats: total=[%d vs %d], periodic=[%d vs %d]", initialStats.getTotal(), currentStats.getTotal(), initialStats.getPeriodic(), currentStats.getPeriodic()); - assertThat(msg, currentStats.getPeriodic(), equalTo(initialStats.getPeriodic() + 1)); - assertThat(msg, currentStats.getTotal(), equalTo(initialStats.getTotal() + 1)); + assertThat(msg, currentStats.getPeriodic(), + either(equalTo(initialStats.getPeriodic() + 1)).or(equalTo(initialStats.getPeriodic() + 2))); + assertThat(msg, currentStats.getTotal(), + either(equalTo(initialStats.getTotal() + 1)).or(equalTo(initialStats.getTotal() + 2))); }; } else { final long generation = getTranslog(shard).currentFileGeneration(); client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get(); - check = () -> assertEquals( - generation + 1, - getTranslog(shard).currentFileGeneration()); + check = () -> { + assertFalse(shard.shouldRollTranslogGeneration()); + assertEquals(generation + 1, getTranslog(shard).currentFileGeneration()); + }; } assertBusy(check); running.set(false); From b88760b704ba7a80ecb550b6c360d2b340210e4c Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Thu, 12 Jul 2018 11:12:37 -0700 Subject: [PATCH 05/15] [test] disable packaging tests for suse boxes --- .../gradle/vagrant/VagrantTestPlugin.groovy | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy index 603e217ecda86..210fb939c7113 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy @@ -547,7 +547,11 @@ class VagrantTestPlugin implements Plugin { project.gradle.removeListener(batsPackagingReproListener) } if (project.extensions.esvagrant.boxes.contains(box)) { - packagingTest.dependsOn(batsPackagingTest) + // these tests are temporarily disabled for suse boxes while we debug an issue + // https://github.com/elastic/elasticsearch/issues/30295 + if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) { + packagingTest.dependsOn(batsPackagingTest) + } } } @@ -586,7 +590,11 @@ class VagrantTestPlugin implements Plugin { project.gradle.removeListener(javaPackagingReproListener) } if (project.extensions.esvagrant.boxes.contains(box)) { - packagingTest.dependsOn(javaPackagingTest) + // these tests are temporarily disabled for suse boxes while we debug an issue + // https://github.com/elastic/elasticsearch/issues/30295 + if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) { + packagingTest.dependsOn(javaPackagingTest) + } } /* From 4ffa9ae6e94ab9af2abcc941a3588cc5e69708ed Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Fri, 15 Feb 2019 13:40:17 -0500 Subject: [PATCH 06/15] Reduce global checkpoint sync interval in disruption tests (#38931) We verify seq_no_stats is aligned between copies at the end of some disruption tests. Sometimes, the assertion `assertSeqNos` is tripped due to a lagged global checkpoint on replicas. The global checkpoint on replicas is lagged because we sync the global checkpoint 30 seconds (by default) after the last replication operation. This change reduces the global checkpoint sync-internal to 1s in the disruption tests. Closes #38318 Closes #36789 --- .../cluster/routing/PrimaryAllocationIT.java | 7 +++++-- .../discovery/AbstractDisruptionTestCase.java | 11 ++++++++++- .../elasticsearch/recovery/RelocationIT.java | 17 +++++++++++------ .../elasticsearch/xpack/CcrIntegTestCase.java | 5 ++++- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java b/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java index 9a7e25d29bb08..f4b834e4d29a6 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java @@ -46,6 +46,7 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.disruption.NetworkDisruption; import org.elasticsearch.test.disruption.NetworkDisruption.NetworkDisconnect; @@ -84,7 +85,7 @@ public class PrimaryAllocationIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { // disruption tests need MockTransportService - return Arrays.asList(MockTransportService.TestPlugin.class); + return Arrays.asList(MockTransportService.TestPlugin.class, InternalSettingsPlugin.class); } public void testBulkWeirdScenario() throws Exception { @@ -92,7 +93,9 @@ public void testBulkWeirdScenario() throws Exception { internalCluster().startDataOnlyNodes(2); assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder() - .put("index.number_of_shards", 1).put("index.number_of_replicas", 1)).get()); + .put("index.number_of_shards", 1).put("index.number_of_replicas", 1) + .put("index.global_checkpoint_sync.interval", "1s")) + .get()); ensureGreen(); BulkResponse bulkResponse = client().prepareBulk() diff --git a/server/src/test/java/org/elasticsearch/discovery/AbstractDisruptionTestCase.java b/server/src/test/java/org/elasticsearch/discovery/AbstractDisruptionTestCase.java index 97ba76b822020..bd89ceb64e6df 100644 --- a/server/src/test/java/org/elasticsearch/discovery/AbstractDisruptionTestCase.java +++ b/server/src/test/java/org/elasticsearch/discovery/AbstractDisruptionTestCase.java @@ -30,8 +30,10 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.IndexService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.disruption.NetworkDisruption; import org.elasticsearch.test.disruption.NetworkDisruption.Bridge; @@ -65,6 +67,13 @@ protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(DEFAULT_SETTINGS).build(); } + @Override + public Settings indexSettings() { + return Settings.builder().put(super.indexSettings()) + // sync global checkpoint quickly so we can verify seq_no_stats aligned between all copies after tests. + .put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "1s").build(); + } + @Override protected int numberOfShards() { return 3; @@ -128,7 +137,7 @@ List startCluster(int numberOfNodes) { @Override protected Collection> nodePlugins() { - return Arrays.asList(MockTransportService.TestPlugin.class); + return Arrays.asList(MockTransportService.TestPlugin.class, InternalSettingsPlugin.class); } ClusterState getNodeClusterState(String node) { diff --git a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java index fb455f37d76f3..c2d35279bdff4 100644 --- a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java +++ b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java @@ -110,6 +110,13 @@ protected void beforeIndexDeletion() throws Exception { internalCluster().assertSameDocIdsOnShards(); } + @Override + public Settings indexSettings() { + return Settings.builder().put(super.indexSettings()) + // sync global checkpoint quickly so we can verify seq_no_stats aligned between all copies after tests. + .put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "1s").build(); + } + public void testSimpleRelocationNoIndexing() { logger.info("--> starting [node1] ..."); final String node_1 = internalCluster().startNode(); @@ -279,8 +286,7 @@ public void testRelocationWhileRefreshing() throws Exception { .put("index.number_of_shards", 1) .put("index.number_of_replicas", numberOfReplicas) .put("index.refresh_interval", -1) // we want to control refreshes - .put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "100ms")) - .get(); + ).get(); for (int i = 1; i < numberOfNodes; i++) { logger.info("--> starting [node_{}] ...", i); @@ -465,8 +471,7 @@ public void testIndexAndRelocateConcurrently() throws ExecutionException, Interr final Settings.Builder settings = Settings.builder() .put("index.routing.allocation.exclude.color", "blue") .put(indexSettings()) - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomInt(halfNodes - 1)) - .put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "100ms"); + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomInt(halfNodes - 1)); assertAcked(prepareCreate("test", settings)); assertAllShardsOnNodes("test", redNodes); int numDocs = randomIntBetween(100, 150); @@ -518,8 +523,8 @@ public void testRelocateWhileWaitingForRefresh() { prepareCreate("test", Settings.builder() .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) - .put("index.refresh_interval", -1) // we want to control refreshes - ).get(); + // we want to control refreshes + .put("index.refresh_interval", -1)).get(); logger.info("--> index 10 docs"); for (int i = 0; i < 10; i++) { diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java index 81b90a3ff60b0..561b76e83154e 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java @@ -43,6 +43,7 @@ import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.DocIdSeqNoAndTerm; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.seqno.SeqNoStats; @@ -62,6 +63,7 @@ import org.elasticsearch.test.BackgroundIndexer; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.MockHttpTransport; import org.elasticsearch.test.NodeConfigurationSource; @@ -125,7 +127,7 @@ public final void startClusters() throws Exception { stopClusters(); Collection> mockPlugins = Arrays.asList(ESIntegTestCase.TestSeedPlugin.class, MockHttpTransport.TestPlugin.class, MockTransportService.TestPlugin.class, - MockNioTransportPlugin.class); + MockNioTransportPlugin.class, InternalSettingsPlugin.class); InternalTestCluster leaderCluster = new InternalTestCluster(randomLong(), createTempDir(), true, true, numberOfNodesPerCluster(), numberOfNodesPerCluster(), "leader_cluster", createNodeConfigurationSource(null), 0, "leader", mockPlugins, @@ -390,6 +392,7 @@ protected String getIndexSettings(final int numberOfShards, final int numberOfRe builder.startObject("settings"); { builder.field(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), 0); + builder.field(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "1s"); builder.field("index.number_of_shards", numberOfShards); builder.field("index.number_of_replicas", numberOfReplicas); for (final Map.Entry additionalSetting : additionalIndexSettings.entrySet()) { From 5624eee282bb2a252e07dd6e30a9123aeb6bc681 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Fri, 15 Feb 2019 13:42:02 -0500 Subject: [PATCH 07/15] Advance max_seq_no before add operation to Lucene (#38879) Today when processing an operation on a replica engine (or the following engine), we first add it to Lucene, then add it to translog, then finally marks its seq_no as completed. If a flush occurs after step1, but before step-3, the max_seq_no in the commit's user_data will be smaller than the seq_no of some documents in the Lucene commit. --- .../index/engine/InternalEngine.java | 10 ++++ .../index/seqno/LocalCheckpointTracker.java | 9 ++++ .../index/engine/InternalEngineTests.java | 38 +++++++++++++++ .../index/engine/EngineTestCase.java | 42 ++++++++++++----- .../ccr/index/engine/FollowingEngine.java | 2 + .../index/engine/FollowingEngineTests.java | 46 +++++++++++++++++++ 6 files changed, 136 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 832df83fe0f5c..7ddd6348f5245 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -948,6 +948,7 @@ protected final IndexingStrategy planIndexingAsNonPrimary(Index index) throws IO } } } + markSeqNoAsSeen(index.seqNo()); return plan; } @@ -1301,6 +1302,7 @@ protected final DeletionStrategy planDeletionAsNonPrimary(Delete delete) throws delete.seqNo(), delete.version()); } } + markSeqNoAsSeen(delete.seqNo()); return plan; } @@ -1455,6 +1457,7 @@ public void maybePruneDeletes() { public NoOpResult noOp(final NoOp noOp) { NoOpResult noOpResult; try (ReleasableLock ignored = readLock.acquire()) { + markSeqNoAsSeen(noOp.seqNo()); noOpResult = innerNoOp(noOp); } catch (final Exception e) { noOpResult = new NoOpResult(getPrimaryTerm(), noOp.seqNo(), e); @@ -2434,6 +2437,13 @@ public void waitForOpsToComplete(long seqNo) throws InterruptedException { localCheckpointTracker.waitForOpsToComplete(seqNo); } + /** + * Marks the given seq_no as seen and advances the max_seq_no of this engine to at least that value. + */ + protected final void markSeqNoAsSeen(long seqNo) { + localCheckpointTracker.advanceMaxSeqNo(seqNo); + } + /** * Checks if the given operation has been processed in this engine or not. * @return true if the given operation was processed; otherwise false. diff --git a/server/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointTracker.java b/server/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointTracker.java index 50f4e311c8d81..a19d9ac4abb94 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointTracker.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointTracker.java @@ -81,6 +81,15 @@ public synchronized long generateSeqNo() { return nextSeqNo++; } + /** + * Marks the provided sequence number as seen and updates the max_seq_no if needed. + */ + public synchronized void advanceMaxSeqNo(long seqNo) { + if (seqNo >= nextSeqNo) { + nextSeqNo = seqNo + 1; + } + } + /** * Marks the processing of the provided sequence number as completed as updates the checkpoint if possible. * diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index f23665d201206..d9ed5cd2c719e 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -5653,4 +5653,42 @@ public void testStoreHonorsLuceneVersion() throws IOException { } } } + + public void testMaxSeqNoInCommitUserData() throws Exception { + AtomicBoolean running = new AtomicBoolean(true); + Thread rollTranslog = new Thread(() -> { + while (running.get() && engine.getTranslog().currentFileGeneration() < 500) { + engine.rollTranslogGeneration(); // make adding operations to translog slower + } + }); + rollTranslog.start(); + + Thread indexing = new Thread(() -> { + long seqNo = 0; + while (running.get() && seqNo <= 1000) { + try { + String id = Long.toString(between(1, 50)); + if (randomBoolean()) { + ParsedDocument doc = testParsedDocument(id, null, testDocumentWithTextField(), SOURCE, null); + engine.index(replicaIndexForDoc(doc, 1L, seqNo, false)); + } else { + engine.delete(replicaDeleteForDoc(id, 1L, seqNo, 0L)); + } + seqNo++; + } catch (IOException e) { + throw new AssertionError(e); + } + } + }); + indexing.start(); + + int numCommits = between(5, 20); + for (int i = 0; i < numCommits; i++) { + engine.flush(false, true); + } + running.set(false); + indexing.join(); + rollTranslog.join(); + assertMaxSeqNoInCommitUserData(engine); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index 005bfb42f8a22..a6765e4e44fa6 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -27,6 +27,8 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReader; @@ -126,6 +128,7 @@ import static org.elasticsearch.index.engine.Engine.Operation.Origin.REPLICA; import static org.elasticsearch.index.translog.TranslogDeletionPolicies.createTranslogDeletionPolicy; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; public abstract class EngineTestCase extends ESTestCase { @@ -254,18 +257,20 @@ public EngineConfig copy(EngineConfig config, MergePolicy mergePolicy) { @After public void tearDown() throws Exception { super.tearDown(); - if (engine != null && engine.isClosed.get() == false) { - engine.getTranslog().getDeletionPolicy().assertNoOpenTranslogRefs(); - assertConsistentHistoryBetweenTranslogAndLuceneIndex(engine, createMapperService("test")); - } - if (replicaEngine != null && replicaEngine.isClosed.get() == false) { - replicaEngine.getTranslog().getDeletionPolicy().assertNoOpenTranslogRefs(); - assertConsistentHistoryBetweenTranslogAndLuceneIndex(replicaEngine, createMapperService("test")); + try { + if (engine != null && engine.isClosed.get() == false) { + engine.getTranslog().getDeletionPolicy().assertNoOpenTranslogRefs(); + assertConsistentHistoryBetweenTranslogAndLuceneIndex(engine, createMapperService("test")); + assertMaxSeqNoInCommitUserData(engine); + } + if (replicaEngine != null && replicaEngine.isClosed.get() == false) { + replicaEngine.getTranslog().getDeletionPolicy().assertNoOpenTranslogRefs(); + assertConsistentHistoryBetweenTranslogAndLuceneIndex(replicaEngine, createMapperService("test")); + assertMaxSeqNoInCommitUserData(replicaEngine); + } + } finally { + IOUtils.close(replicaEngine, storeReplica, engine, store, () -> terminate(threadPool)); } - IOUtils.close( - replicaEngine, storeReplica, - engine, store); - terminate(threadPool); } @@ -1067,6 +1072,21 @@ public static void assertConsistentHistoryBetweenTranslogAndLuceneIndex(Engine e } } + /** + * Asserts that the max_seq_no stored in the commit's user_data is never smaller than seq_no of any document in the commit. + */ + public static void assertMaxSeqNoInCommitUserData(Engine engine) throws Exception { + List commits = DirectoryReader.listCommits(engine.store.directory()); + for (IndexCommit commit : commits) { + try (DirectoryReader reader = DirectoryReader.open(commit)) { + AtomicLong maxSeqNoFromDocs = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); + Lucene.scanSeqNosInReader(reader, 0, Long.MAX_VALUE, n -> maxSeqNoFromDocs.set(Math.max(n, maxSeqNoFromDocs.get()))); + assertThat(Long.parseLong(commit.getUserData().get(SequenceNumbers.MAX_SEQ_NO)), + greaterThanOrEqualTo(maxSeqNoFromDocs.get())); + } + } + } + public static MapperService createMapperService(String type) throws IOException { IndexMetaData indexMetaData = IndexMetaData.builder("test") .settings(Settings.builder() diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngine.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngine.java index c779b491d581e..e0b0734912b97 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngine.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngine.java @@ -68,6 +68,7 @@ private void preFlight(final Operation operation) { @Override protected InternalEngine.IndexingStrategy indexingStrategyForOperation(final Index index) throws IOException { preFlight(index); + markSeqNoAsSeen(index.seqNo()); // NOTES: refer Engine#getMaxSeqNoOfUpdatesOrDeletes for the explanation of the optimization using sequence numbers. final long maxSeqNoOfUpdatesOrDeletes = getMaxSeqNoOfUpdatesOrDeletes(); assert maxSeqNoOfUpdatesOrDeletes != SequenceNumbers.UNASSIGNED_SEQ_NO : "max_seq_no_of_updates is not initialized"; @@ -103,6 +104,7 @@ protected InternalEngine.IndexingStrategy indexingStrategyForOperation(final Ind @Override protected InternalEngine.DeletionStrategy deletionStrategyForOperation(final Delete delete) throws IOException { preFlight(delete); + markSeqNoAsSeen(delete.seqNo()); if (delete.origin() == Operation.Origin.PRIMARY && hasBeenProcessedBefore(delete)) { // See the comment in #indexingStrategyForOperation for the explanation why we can safely skip this operation. final AlreadyProcessedFollowingEngineException error = new AlreadyProcessedFollowingEngineException( diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java index 67d31ff39007f..69fa23bd3fbcd 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java @@ -59,6 +59,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.index.engine.EngineTestCase.getDocIds; +import static org.elasticsearch.index.engine.EngineTestCase.getTranslog; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -659,4 +660,49 @@ public void testVerifyShardBeforeIndexClosingIsNoOp() throws IOException { } }); } + + public void testMaxSeqNoInCommitUserData() throws Exception { + final Settings settings = Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0) + .put("index.version.created", Version.CURRENT).put("index.xpack.ccr.following_index", true) + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).build(); + final IndexMetaData indexMetaData = IndexMetaData.builder(index.getName()).settings(settings).build(); + final IndexSettings indexSettings = new IndexSettings(indexMetaData, settings); + try (Store store = createStore(shardId, indexSettings, newDirectory())) { + final EngineConfig engineConfig = engineConfig(shardId, indexSettings, threadPool, store, logger, xContentRegistry()); + try (FollowingEngine engine = createEngine(store, engineConfig)) { + AtomicBoolean running = new AtomicBoolean(true); + Thread rollTranslog = new Thread(() -> { + while (running.get() && getTranslog(engine).currentFileGeneration() < 500) { + engine.rollTranslogGeneration(); // make adding operations to translog slower + } + }); + rollTranslog.start(); + + Thread indexing = new Thread(() -> { + List ops = EngineTestCase.generateSingleDocHistory(true, VersionType.EXTERNAL, 2, 50, 500, "id"); + engine.advanceMaxSeqNoOfUpdatesOrDeletes(ops.stream().mapToLong(Engine.Operation::seqNo).max().getAsLong()); + for (Engine.Operation op : ops) { + if (running.get() == false) { + return; + } + try { + EngineTestCase.applyOperation(engine, op); + } catch (IOException e) { + throw new AssertionError(e); + } + } + }); + indexing.start(); + + int numCommits = between(5, 20); + for (int i = 0; i < numCommits; i++) { + engine.flush(false, true); + } + running.set(false); + indexing.join(); + rollTranslog.join(); + EngineTestCase.assertMaxSeqNoInCommitUserData(engine); + } + } + } } From ad012c834804d9ab2b0879424806d9deb909981e Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Fri, 15 Feb 2019 11:07:22 -0800 Subject: [PATCH 08/15] Improve testcluster distribution artifact handling (#38933) This commit moves validation logic for ensuring our testclusters configuration doesn't contain unexpected artifacts into the plugin itself. This change allows us to remove the custom copy task implementation altogether. Additionally, the error message has been improved to display component ids in addition to the artifacts to make it easier to figure out what actual dependency is at fault. --- .../SyncTestClustersConfiguration.java | 77 ------------------- .../testclusters/TestClustersPlugin.java | 34 +++++++- 2 files changed, 30 insertions(+), 81 deletions(-) delete mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/SyncTestClustersConfiguration.java diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/SyncTestClustersConfiguration.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/SyncTestClustersConfiguration.java deleted file mode 100644 index d1a86a38c66ff..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/SyncTestClustersConfiguration.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.gradle.testclusters; - -import org.gradle.api.DefaultTask; -import org.gradle.api.Project; -import org.gradle.api.file.FileCollection; -import org.gradle.api.tasks.InputFiles; -import org.gradle.api.tasks.OutputDirectory; -import org.gradle.api.tasks.TaskAction; - -import java.io.File; -import java.util.Set; -import java.util.stream.Collectors; - -public class SyncTestClustersConfiguration extends DefaultTask { - - @InputFiles - public FileCollection getDependencies() { - Set nonZip = getProject().getConfigurations() - .getByName(TestClustersPlugin.HELPER_CONFIGURATION_NAME) - .getFiles() - .stream() - .filter(file -> file.getName().endsWith(".zip") == false) - .collect(Collectors.toSet()); - if(nonZip.isEmpty() == false) { - throw new IllegalStateException("Expected only zip files in configuration : " + - TestClustersPlugin.HELPER_CONFIGURATION_NAME + " but found " + - nonZip - ); - } - return getProject().files( - getProject().getConfigurations() - .getByName(TestClustersPlugin.HELPER_CONFIGURATION_NAME) - .getFiles() - ); - } - - @OutputDirectory - public File getOutputDir() { - return getTestClustersConfigurationExtractDir(getProject()); - } - - @TaskAction - public void doExtract() { - File outputDir = getOutputDir(); - getProject().delete(outputDir); - outputDir.mkdirs(); - getDependencies().forEach(dep -> - getProject().copy(spec -> { - spec.from(getProject().zipTree(dep)); - spec.into(new File(outputDir, "zip")); - }) - ); - } - - static File getTestClustersConfigurationExtractDir(Project project) { - return new File(TestClustersPlugin.getTestClustersBuildDir(project), "extract"); - } - -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java index 3abc9a6a6177e..56ff501a3885c 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java @@ -25,11 +25,14 @@ import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.component.ComponentArtifactIdentifier; import org.gradle.api.execution.TaskActionListener; import org.gradle.api.execution.TaskExecutionListener; +import org.gradle.api.file.FileTree; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; import org.gradle.api.plugins.ExtraPropertiesExtension; +import org.gradle.api.tasks.Sync; import org.gradle.api.tasks.TaskState; import java.io.File; @@ -39,6 +42,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -87,6 +91,20 @@ public void apply(Project project) { "Internal helper configuration used by cluster configuration to download " + "ES distributions and plugins." ); + helperConfiguration.getIncoming().afterResolve(resolvableDependencies -> { + Set nonZipComponents = resolvableDependencies.getArtifacts() + .getArtifacts() + .stream() + .filter(artifact -> artifact.getFile().getName().endsWith(".zip") == false) + .map(artifact -> artifact.getId()) + .collect(Collectors.toSet()); + + if(nonZipComponents.isEmpty() == false) { + throw new IllegalStateException("Dependencies with non-zip artifacts found in configuration '" + + TestClustersPlugin.HELPER_CONFIGURATION_NAME + "': " + nonZipComponents + ); + } + }); // When running in the Daemon it's possible for this to hold references to past usedClusters.clear(); @@ -98,7 +116,15 @@ public void apply(Project project) { // the clusters will look for artifacts there based on the naming conventions. // Tasks that use a cluster will add this as a dependency automatically so it's guaranteed to run early in // the build. - rootProject.getTasks().create(SYNC_ARTIFACTS_TASK_NAME, SyncTestClustersConfiguration.class); + rootProject.getTasks().create(SYNC_ARTIFACTS_TASK_NAME, Sync.class, sync -> { + sync.from((Callable>) () -> + helperConfiguration.getFiles() + .stream() + .map(project::zipTree) + .collect(Collectors.toList()) + ); + sync.into(new File(getTestClustersConfigurationExtractDir(project), "zip")); + }); // When we know what tasks will run, we claim the clusters of those task to differentiate between clusters // that are defined in the build script and the ones that will actually be used in this invocation of gradle @@ -129,7 +155,7 @@ private NamedDomainObjectContainer createTestClustersContaine project.getPath(), name, GradleServicesAdapter.getInstance(project), - SyncTestClustersConfiguration.getTestClustersConfigurationExtractDir(project), + getTestClustersConfigurationExtractDir(project), new File(project.getBuildDir(), "testclusters") ) ); @@ -249,8 +275,8 @@ public void beforeExecute(Task task) {} ); } - static File getTestClustersBuildDir(Project project) { - return new File(project.getRootProject().getBuildDir(), "testclusters"); + static File getTestClustersConfigurationExtractDir(Project project) { + return new File(project.getRootProject().getBuildDir(), "testclusters/extract"); } /** From 8ee96576b83ba7604d3bbb237b31d4f91ee1d83c Mon Sep 17 00:00:00 2001 From: James Baiera Date: Fri, 15 Feb 2019 14:08:58 -0500 Subject: [PATCH 09/15] Add an exception throw if waiting on transport port file fails (#37574) In the ClusterConfiguration class of the build source, there is an Ant waitfor block that runs to ensure that the seed node's transport ports file is created before trying to read it. If the wait times out, the file read fails with not much helpful info. This just adds a timeout property to the waitfor block and throws a descriptive exception instead. --- .../elasticsearch/gradle/test/ClusterConfiguration.groovy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index 0a53787c10597..e64f30d48bd3f 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -118,11 +118,16 @@ class ClusterConfiguration { if (seedNode == node) { return null } - ant.waitfor(maxwait: '40', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond') { + ant.waitfor(maxwait: '40', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', + timeoutproperty: "failed.${seedNode.transportPortsFile.path}") { resourceexists { file(file: seedNode.transportPortsFile.toString()) } } + if (ant.properties.containsKey("failed.${seedNode.transportPortsFile.path}".toString())) { + throw new GradleException("Failed to locate seed node transport file [${seedNode.transportPortsFile}]: " + + "timed out waiting for it to be created after ${waitSeconds} seconds") + } return seedNode.transportUri() } From 7bb2da197dcabba8cbd0244cb5557b8ebb88e462 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 15 Feb 2019 20:15:05 +0100 Subject: [PATCH 10/15] Remove `nGram` and `edgeNGram` token filter names (#38911) In #30209 we deprecated the camel case `nGram` filter name in favour of `ngram` and did the same for `edgeNGram` and `edge_ngram`. Using these names has been deprecated since 6.4 and is issuing deprecation warnings since then. I think we can remove these filters in 8.0. In a backport of this PR I would change what was a dreprecation warning from 6.4. to an error starting with new indices created in 7.0. --- .../edgengram-tokenfilter.asciidoc | 4 +- .../tokenfilters/ngram-tokenfilter.asciidoc | 4 +- docs/reference/migration/migrate_8_0.asciidoc | 6 +- .../migration/migrate_8_0/mappings.asciidoc | 10 +++ .../analysis/common/CommonAnalysisPlugin.java | 16 ---- .../common/CommonAnalysisFactoryTests.java | 2 - .../common/CommonAnalysisPluginTests.java | 86 ------------------- .../common/HighlighterWithAnalyzersTests.java | 2 +- .../test/analysis-common/30_tokenizers.yml | 60 +++++-------- .../test/indices.analyze/10_analyze.yml | 2 +- 10 files changed, 43 insertions(+), 149 deletions(-) create mode 100644 docs/reference/migration/migrate_8_0/mappings.asciidoc diff --git a/docs/reference/analysis/tokenfilters/edgengram-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/edgengram-tokenfilter.asciidoc index be37d24f7dd7c..e460725523cf6 100644 --- a/docs/reference/analysis/tokenfilters/edgengram-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/edgengram-tokenfilter.asciidoc @@ -1,9 +1,9 @@ [[analysis-edgengram-tokenfilter]] === Edge NGram Token Filter -A token filter of type `edgeNGram`. +A token filter of type `edge_ngram`. -The following are settings that can be set for a `edgeNGram` token +The following are settings that can be set for a `edge_ngram` token filter type: [cols="<,<",options="header",] diff --git a/docs/reference/analysis/tokenfilters/ngram-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/ngram-tokenfilter.asciidoc index acc178a2741fa..53bda23d12bf9 100644 --- a/docs/reference/analysis/tokenfilters/ngram-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/ngram-tokenfilter.asciidoc @@ -1,9 +1,9 @@ [[analysis-ngram-tokenfilter]] === NGram Token Filter -A token filter of type `nGram`. +A token filter of type `ngram`. -The following are settings that can be set for a `nGram` token filter +The following are settings that can be set for a `ngram` token filter type: [cols="<,<",options="header",] diff --git a/docs/reference/migration/migrate_8_0.asciidoc b/docs/reference/migration/migrate_8_0.asciidoc index 4477090dc16b6..ea1166e36d3b9 100644 --- a/docs/reference/migration/migrate_8_0.asciidoc +++ b/docs/reference/migration/migrate_8_0.asciidoc @@ -9,4 +9,8 @@ your application to {es} 8.0. See also <> and <>. -coming[8.0.0] \ No newline at end of file +coming[8.0.0] + +* <> + +include::migrate_8_0/mappings.asciidoc[] \ No newline at end of file diff --git a/docs/reference/migration/migrate_8_0/mappings.asciidoc b/docs/reference/migration/migrate_8_0/mappings.asciidoc new file mode 100644 index 0000000000000..ea36309447e4c --- /dev/null +++ b/docs/reference/migration/migrate_8_0/mappings.asciidoc @@ -0,0 +1,10 @@ +[float] +[[breaking_80_mappings_changes]] +=== Mapping changes + +[float] +==== The `nGram` and `edgeNGram` token filter names have been removed + +The `nGram` and `edgeNGram` token filter names that have been deprecated since +version 6.4 have been removed. Both token filters should be used by their +alternative names `ngram` and `edge_ngram` instead. \ No newline at end of file diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java index ac439f4bae227..e05305a8ebd39 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -414,14 +414,6 @@ public List getPreConfiguredTokenFilters() { filters.add(PreConfiguredTokenFilter.singleton("dutch_stem", false, input -> new SnowballFilter(input, new DutchStemmer()))); filters.add(PreConfiguredTokenFilter.singleton("edge_ngram", false, false, input -> new EdgeNGramTokenFilter(input, 1))); - filters.add(PreConfiguredTokenFilter.singletonWithVersion("edgeNGram", false, false, (reader, version) -> { - if (version.onOrAfter(org.elasticsearch.Version.V_6_4_0)) { - deprecationLogger.deprecatedAndMaybeLog("edgeNGram_deprecation", - "The [edgeNGram] token filter name is deprecated and will be removed in a future version. " - + "Please change the filter name to [edge_ngram] instead."); - } - return new EdgeNGramTokenFilter(reader, 1); - })); filters.add(PreConfiguredTokenFilter.singleton("elision", true, input -> new ElisionFilter(input, FrenchAnalyzer.DEFAULT_ARTICLES))); filters.add(PreConfiguredTokenFilter.singleton("french_stem", false, input -> new SnowballFilter(input, new FrenchStemmer()))); @@ -438,14 +430,6 @@ public List getPreConfiguredTokenFilters() { LimitTokenCountFilterFactory.DEFAULT_MAX_TOKEN_COUNT, LimitTokenCountFilterFactory.DEFAULT_CONSUME_ALL_TOKENS))); filters.add(PreConfiguredTokenFilter.singleton("ngram", false, false, reader -> new NGramTokenFilter(reader, 1, 2, false))); - filters.add(PreConfiguredTokenFilter.singletonWithVersion("nGram", false, false, (reader, version) -> { - if (version.onOrAfter(org.elasticsearch.Version.V_6_4_0)) { - deprecationLogger.deprecatedAndMaybeLog("nGram_deprecation", - "The [nGram] token filter name is deprecated and will be removed in a future version. " - + "Please change the filter name to [ngram] instead."); - } - return new NGramTokenFilter(reader, 1, 2, false); - })); filters.add(PreConfiguredTokenFilter.singleton("persian_normalization", true, PersianNormalizationFilter::new)); filters.add(PreConfiguredTokenFilter.singleton("porter_stem", false, PorterStemFilter::new)); filters.add(PreConfiguredTokenFilter.singleton("reverse", false, ReverseStringFilter::new)); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java index 99e882c622085..acb7f2213f641 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java @@ -185,7 +185,6 @@ protected Map> getPreConfiguredTokenFilters() { filters.put("delimited_payload", org.apache.lucene.analysis.payloads.DelimitedPayloadTokenFilterFactory.class); filters.put("dutch_stem", SnowballPorterFilterFactory.class); filters.put("edge_ngram", null); - filters.put("edgeNGram", null); filters.put("elision", null); filters.put("french_stem", SnowballPorterFilterFactory.class); filters.put("german_stem", null); @@ -197,7 +196,6 @@ protected Map> getPreConfiguredTokenFilters() { filters.put("length", null); filters.put("limit", LimitTokenCountFilterFactory.class); filters.put("ngram", null); - filters.put("nGram", null); filters.put("persian_normalization", null); filters.put("porter_stem", null); filters.put("reverse", ReverseStringFilterFactory.class); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisPluginTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisPluginTests.java index c52c78ffe27e3..f128c07361c45 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisPluginTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisPluginTests.java @@ -20,8 +20,6 @@ package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.analysis.Tokenizer; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; @@ -29,98 +27,14 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.test.VersionUtils; import java.io.IOException; -import java.io.StringReader; -import java.util.Map; public class CommonAnalysisPluginTests extends ESTestCase { - /** - * Check that the deprecated name "nGram" issues a deprecation warning for indices created since 6.3.0 - */ - public void testNGramDeprecationWarning() throws IOException { - Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) - .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_6_4_0, Version.CURRENT)) - .build(); - - IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings); - try (CommonAnalysisPlugin commonAnalysisPlugin = new CommonAnalysisPlugin()) { - Map tokenFilters = createTestAnalysis(idxSettings, settings, commonAnalysisPlugin).tokenFilter; - TokenFilterFactory tokenFilterFactory = tokenFilters.get("nGram"); - Tokenizer tokenizer = new MockTokenizer(); - tokenizer.setReader(new StringReader("foo bar")); - assertNotNull(tokenFilterFactory.create(tokenizer)); - assertWarnings( - "The [nGram] token filter name is deprecated and will be removed in a future version. " - + "Please change the filter name to [ngram] instead."); - } - } - - /** - * Check that the deprecated name "nGram" does NOT issues a deprecation warning for indices created before 6.4.0 - */ - public void testNGramNoDeprecationWarningPre6_4() throws IOException { - Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) - .put(IndexMetaData.SETTING_VERSION_CREATED, - VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.V_6_3_0)) - .build(); - - IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings); - try (CommonAnalysisPlugin commonAnalysisPlugin = new CommonAnalysisPlugin()) { - Map tokenFilters = createTestAnalysis(idxSettings, settings, commonAnalysisPlugin).tokenFilter; - TokenFilterFactory tokenFilterFactory = tokenFilters.get("nGram"); - Tokenizer tokenizer = new MockTokenizer(); - tokenizer.setReader(new StringReader("foo bar")); - assertNotNull(tokenFilterFactory.create(tokenizer)); - } - } - - /** - * Check that the deprecated name "edgeNGram" issues a deprecation warning for indices created since 6.3.0 - */ - public void testEdgeNGramDeprecationWarning() throws IOException { - Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) - .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_6_4_0, Version.CURRENT)) - .build(); - - IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings); - try (CommonAnalysisPlugin commonAnalysisPlugin = new CommonAnalysisPlugin()) { - Map tokenFilters = createTestAnalysis(idxSettings, settings, commonAnalysisPlugin).tokenFilter; - TokenFilterFactory tokenFilterFactory = tokenFilters.get("edgeNGram"); - Tokenizer tokenizer = new MockTokenizer(); - tokenizer.setReader(new StringReader("foo bar")); - assertNotNull(tokenFilterFactory.create(tokenizer)); - assertWarnings( - "The [edgeNGram] token filter name is deprecated and will be removed in a future version. " - + "Please change the filter name to [edge_ngram] instead."); - } - } - - /** - * Check that the deprecated name "edgeNGram" does NOT issues a deprecation warning for indices created before 6.4.0 - */ - public void testEdgeNGramNoDeprecationWarningPre6_4() throws IOException { - Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) - .put(IndexMetaData.SETTING_VERSION_CREATED, - VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.V_6_3_0)) - .build(); - - IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings); - try (CommonAnalysisPlugin commonAnalysisPlugin = new CommonAnalysisPlugin()) { - Map tokenFilters = createTestAnalysis(idxSettings, settings, commonAnalysisPlugin).tokenFilter; - TokenFilterFactory tokenFilterFactory = tokenFilters.get("edgeNGram"); - Tokenizer tokenizer = new MockTokenizer(); - tokenizer.setReader(new StringReader("foo bar")); - assertNotNull(tokenFilterFactory.create(tokenizer)); - } - } - - /** * Check that the deprecated analyzer name "standard_html_strip" throws exception for indices created since 7.0.0 */ diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java index e96243efc4254..8f58a074cf102 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java @@ -81,7 +81,7 @@ public void testNgramHighlightingWithBrokenPositions() throws IOException { .put("analysis.tokenizer.autocomplete.max_gram", 20) .put("analysis.tokenizer.autocomplete.min_gram", 1) .put("analysis.tokenizer.autocomplete.token_chars", "letter,digit") - .put("analysis.tokenizer.autocomplete.type", "nGram") + .put("analysis.tokenizer.autocomplete.type", "ngram") .put("analysis.filter.wordDelimiter.type", "word_delimiter") .putList("analysis.filter.wordDelimiter.type_table", "& => ALPHANUM", "| => ALPHANUM", "! => ALPHANUM", diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/30_tokenizers.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/30_tokenizers.yml index 9a7c158fc4734..4fe5162e68743 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/30_tokenizers.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/30_tokenizers.yml @@ -23,38 +23,6 @@ - match: { detail.tokenizer.tokens.0.token: Foo Bar! } --- -"nGram": - - do: - indices.analyze: - body: - text: good - explain: true - tokenizer: - type: nGram - min_gram: 2 - max_gram: 2 - - length: { detail.tokenizer.tokens: 3 } - - match: { detail.tokenizer.name: _anonymous_tokenizer } - - match: { detail.tokenizer.tokens.0.token: go } - - match: { detail.tokenizer.tokens.1.token: oo } - - match: { detail.tokenizer.tokens.2.token: od } - ---- -"nGram_exception": - - skip: - version: " - 6.99.99" - reason: only starting from version 7.x this throws an error - - do: - catch: /The difference between max_gram and min_gram in NGram Tokenizer must be less than or equal to[:] \[1\] but was \[2\]\. This limit can be set by changing the \[index.max_ngram_diff\] index level setting\./ - indices.analyze: - body: - text: good - explain: true - tokenizer: - type: nGram - min_gram: 2 - max_gram: 4 ---- "simple_pattern": - do: indices.analyze: @@ -133,7 +101,7 @@ text: "foobar" explain: true tokenizer: - type: nGram + type: ngram min_gram: 3 max_gram: 3 - length: { detail.tokenizer.tokens: 4 } @@ -162,15 +130,31 @@ body: text: "foo" explain: true - tokenizer: nGram + tokenizer: ngram - length: { detail.tokenizer.tokens: 5 } - - match: { detail.tokenizer.name: nGram } + - match: { detail.tokenizer.name: ngram } - match: { detail.tokenizer.tokens.0.token: f } - match: { detail.tokenizer.tokens.1.token: fo } - match: { detail.tokenizer.tokens.2.token: o } - match: { detail.tokenizer.tokens.3.token: oo } - match: { detail.tokenizer.tokens.4.token: o } +--- +"ngram_exception": + - skip: + version: " - 6.99.99" + reason: only starting from version 7.x this throws an error + - do: + catch: /The difference between max_gram and min_gram in NGram Tokenizer must be less than or equal to[:] \[1\] but was \[2\]\. This limit can be set by changing the \[index.max_ngram_diff\] index level setting\./ + indices.analyze: + body: + text: good + explain: true + tokenizer: + type: ngram + min_gram: 2 + max_gram: 4 + --- "edge_ngram": - do: @@ -194,7 +178,7 @@ text: "foo" explain: true tokenizer: - type: edgeNGram + type: edge_ngram min_gram: 1 max_gram: 3 - length: { detail.tokenizer.tokens: 3 } @@ -219,9 +203,9 @@ body: text: "foo" explain: true - tokenizer: edgeNGram + tokenizer: edge_ngram - length: { detail.tokenizer.tokens: 2 } - - match: { detail.tokenizer.name: edgeNGram } + - match: { detail.tokenizer.name: edge_ngram } - match: { detail.tokenizer.tokens.0.token: f } - match: { detail.tokenizer.tokens.1.token: fo } diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yml index ec00b6d41f1c5..56bbed7044e14 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yml @@ -76,7 +76,7 @@ analysis: tokenizer: trigram: - type: nGram + type: ngram min_gram: 3 max_gram: 3 filter: From 49c7b3b239cca1904182d34e9f952e5ed5b7adee Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Fri, 15 Feb 2019 21:15:40 +0200 Subject: [PATCH 11/15] SQL: Polish the rest chapter (#38971) Organize the text a bit and add tip on triple quotes in Kibana Console --- .../images/sql/rest/console-triple-quotes.png | Bin 0 -> 54729 bytes docs/reference/sql/endpoints/rest.asciidoc | 141 +++++++++++++++--- .../sql/endpoints/translate.asciidoc | 2 +- 3 files changed, 118 insertions(+), 25 deletions(-) create mode 100644 docs/reference/images/sql/rest/console-triple-quotes.png diff --git a/docs/reference/images/sql/rest/console-triple-quotes.png b/docs/reference/images/sql/rest/console-triple-quotes.png new file mode 100644 index 0000000000000000000000000000000000000000..c689b5902b580f334a640fe44bea641c1c5ae5c6 GIT binary patch literal 54729 zcmeG_cOaJE_s?r5B0DM?GD9KRugb{Ydo_^ktn9avL}rOHOR^<eFZbzMl{G5AS{MJ@?#m&U5d%=bn4+q`KFth@feO4 z5<8Kb#D|(Z=p1#z)b=;R`pq2f2j3jMOCN&WGv|3mq%PF}BUC9{%wvo6Qx5r0WRj{P zaf*x?F=~VD$Q>tH+8hMmE!EFzQNJ(;R0d9FMdYVrlUNQ*sg z1?^WPj7>c9iqZ!TMi8JQ-5~iF%i}AG>tfkkIijO>cG6qcu>Pim{km=^Auwm@QJv)_ zR+D5wx~3leL3^r0c$C#VNpX#UYxpQ*Y%hl$oGr0gQ;y#9zm)G@X2!C+OSd_lo@56n zB^TAqyAW@5CKqQDKa97Z>+WE{=E=efWTZTyH$=Wm8URvlqCefE06_;@+_hSOQgfA# zF`kBD;w%jG<^TYH5bLI`7xGE{%tjox@SV%jFw}cPrxo$eNUF~`?r8VC9J-LiMDshu zu>H))*pe{ll87_1wrHYjQ*=;3o~qu3spcqeJTDdi&;9W3)%I4lTW6)4nrX(D3 z0gZ!O-m(!p9}gWnnT1uiA0u;nGNJ8m(~YuwLM>#PEp6>YYXthu z13~z!R)#V40}ps}t=hX|fgMm2WK!=u03Zab8Ao71&hgZk^WPjIDVn357NyT?>a{-duh;Ea;6tK1Rd2d62 zcji3v&v1Q%`d|L|gV76g`rW6%;z_1#!@6?6kCp-kAZvV{UDI5bTlvxc>HBXkKC3YX zHM9QZ{1nF02-cPw{}1^o-@SrBkq09pYS{8qUKL-Ev1AF!F7!K^Zqakiyb9apcaxL5at+ww4azIf!MJRM&e%Iv$ zp%zf6{nT0z9N4F~c*kPAcGfud#ZsgiUUeLk7pUwHIh}$9j1lH7jvWR3X{;9)4!zb^ z8)~Md|IrKmuD5?}0UU2#(&@tXsuyT{vccuN3xpiKc{LGhm-mk*JpKgXEwY>B6mrm6 z`q=}T?YB7Cw@UydkC9LAN{05;pW6oWe>GJ<=H5W%?|IeSBw@nTtjlEGZvTsgSPEpK z-}Ch=PJBv_BtZqPXsf{b;1YBUA82eA`>lq88wWSehU`a!4_z*6@^m$|!YS<1=PYjg z_b2fh#+M%uzsKm|)td0@ho&!JMMF`4e%>m4Xg!=8pls9G|En(<}!qWjM|1xApzk82^*}pf<9Mgb|iqH}^TRzq?f?@u0psWY8nBDsaibTJGW<(L#ry zy$vtq5ZFv;5?DC}hxd=^j?CpZQUPSvRg=!9YANq7g&KLI8S_znJ;J=_eHl6~mgh|{ z1)|~D%z~i+E1^)Q@7%CaDGvJ5%IcPST)YI+yeLbTAjU%O=wbA+_pW4idRmgDHCAF> z_u&SXD9X-F=+m^T)&`VAY}ysmi>AFlc7?QYB%n*Co{0(laWfAd+@txI;L@V#LyqGQ zMGO6>g}M@-0y3#x#>eS7a3K8vsPX)SW*7lPqplNlF$DnL6}1l)n6s8{PSquGp#AAa z<>6HU1tkQDT-Hx9Zi0oOJQi8z?RDobYK29B6+v@!%}R1;AoVHQOwKBBl8vog{c@%h63EotdRg}V+QYka%E)rLQc_x zg}m1G!wk?QrVEm6&M>~I%)E5g20UyV7dNVou$_BO$vV|-VA9dXK(q$g<^zC|Cz)(> z{-8^I;=g=Y{gCaK@!-rEQTDLXX0*)5*8`P z=Evn%Q#~{+8zzF`I|2Nl;D>|k2k;xXc*UIEz#{lX)>n7)zt~{If~_At`@bX(B(jjQ zJ@1r{xZOnQpIxB!^*g=I+f$48@jJcyD|XAjEj{Go7DTlFQGzQlWT%5}RZi#;!DOBk zE#8;=S}b2(**U119+(#tN#=3Ge=y31NP|g>$je2d)+)nABC5yl#zU69r=o~-B&iN0 z8RN<1cW3&LGTk*MI3md8m{{VFIS?yS zMjkK4h7s2TPCe#)etMt%_wO57VWVM~#LB6I%~@-3=W$wv4LR_uW#41_NePD$M{rPC zIM*0?#f}Ptm+s&w5vF6eH^wf{hrBE7VqH=AqLQbx_u0V+o@1fAk9{J!R1|7c`de^(wgJ zCMlc43qd~%d5>NNQ_B${-WW7S^raS@RR_jAHPFhIvj{E9m=5YpXh(xWP%m&PQb3W?lC_=$abG6JEAF>N)ExMuU>h zdeF$@Zwi`f6lt*DS`tt0@;UOoR1oj;fO?&K05LvJDvN^JRv;>u4Q{K(2)JK{eUz-L z#{kZyY?W^(#|Q4|1NkJjnJe9VG-Y!A9B{9i;f5MBKL3lsn~rW18Pa_B?ol9`%Nce> zHh~cd+s?KNL}2N^y-Rv!#N=YP?Yx{ANG|eY?@eDhQ;-(iH1pkZ7y!Q({+WFqLg9gW zof)hT#%H$_i2J^>dFg&=T(=<|yaWXxFg1Am7`}2E$9m`5UAAi^E5BfEO`Ot*wv7QU zz#od=Y4REX?r+;I>E(9xQP1ZRn-zKyUej02s7W>VeJx1Uw}Y7_Yr?u(15chCuG=p1 z_$Eq4O?N&|m|)_-qOwHf=2ANiBWkw8fF{MTsnXZy zI>v)!%8zX;&NuKImlTJE6gby>zAn=rIrnsW(HoZP_}pZ1F=hL>?m}f~gvnQ}hRmkF z`MNEB-{sS<_H)?}c!Fn1bx)6<{7lqmE3goPzNy=4Ncp-U;YUuZ!h?6*-^=dWP9}BW zUapcjDDoetxRtSJ=xo;`WdO8wrZ88#43E^$Modq;u0Po7>aj!`3&md7q*AJ!zx-p| zU`(6^P2=?Hfv*9~P~0IE0=0P`wQVe_cz44Vb)i=sBkAd=&DEsw=UYgBU*;dtm@E zTa2!CaAy<}BFwIi^BS@1)#NQ=w+C?dF1b$yWfVYoFm>g7VPwz`J$h zt;LV@E*LPp?=?z0JWJya1#o-PR^>T@pFLSllN6PzucuR&15cH2SB&_Zn{UTKwn)A_ zTr+f1MqMjjT`W&9ewhzs;4v`t57B@d4UkdFU;uOWmE(^(?dRLFq+%l)@KcqnTs~zK zb{_$x|0KiCkvwTw==&l>cHyhR3XOXnEgz@kM3UP@=HUZH(?aoA4AW&)YvWn-yFVII6`; zhI=LRqhAE|x;6T8bcm&+tyBhLBpU?MM2e?RiGqc{l`K*#+|xd7@@V0{lnEEsa0)B+ z;@P@(3P4?vaZk(OfRy;s>Ep{9fk3`)sGSr({Rt->mwr#^h0YmOb?JE!n0Y*Cbu@?^ zu=|9&$8s{1m)1EgjtPU%(bc|622mEL~bE3=qw$Jt!n_W=}?lJuf}Ue5;zDb06{4cy~3G8f|( z{YndB+()HuP_`I3Q_MVI%W!6ex#eHD)f;jFTIgu(n>4#^z%0~ig0?Sg9xq3Ta8J8a zL()Ei-@vvO$N2?uV5f(e3Rt?}Xr8?*rII&0ydp@BNxR$O#i_C9{=KwJ_b*Y0#L0+9#OvAxI+eWMW^pHnHt(3N{AAypq0 zJ}z%CqCN{3d;UIHpz($Zx9azdK%gy})Nm~q94>Cdr&$f)635D(BkZ8kDZ#4d+gtXy zv50RF`Pssm7aJLU-TFdrQz;psKail!Yo~vD8=$Uh>DHY#fBdmFZ zQ+~jOkH{B0rP5sm+^Se2OTcCek3)B}!QNX8u#{ZT>m1_&5N+IMlIl4D>Q z^+s@|GZdyKIxF|45u!z5O(ssRj{<1CAv&R?FXIBJHeyBbk|KPX`2fIV05di308n`c zFs*2SfeMipQO^xKSB0g2CTnFSQL&HhQy?ptPm?*A2@KrjX=?tG(mn*l?H^{VCHDZ4 z06<0P9S3Qpb4t*-jS9@kmsX=1fXWv`<|ZEW_?Z-B4Jt*`+O!rHQo!41`o^EsCWBNO zk_4wlTcgIhM4#byX49(6Hs%!fp1o%U?=QPl(|>f$A_Wz7!rdnqYo0z8npQkp{o1&Z z8N`ja`eLf-PyuFuGgFP=(T{cGFZKe!`^AKuXw{|;D7XSXp2Zn0KECb5XpBVRVrs`$ z9^#f781TUOn7lIHB`z5}FZ|^x1sFc`IU`BdJ-Yw%apxDI^8wEB)la94h$28y zI=0?<6YYq*cUp0Oro+ZCV?C!BI#@Ju0f}1os5oK_8fl@A>q)Oi>4OQ*~x5~#<3?#xKk-|)Xzj8XU8y^FTXGSvu96;imsR>Xe4)$ z&M)@lxIF<4YJtbSr~Ep%IXl1S8IaSNd~y4lpesMnF>7DX?$=!B9O-73w}rzXZp40A zsuBZ;OUP>DjPaA?H(P#jW{(T7}>9j2i^-k4hitFo)75Il?LNnEN&gzD-oa?UQLMrGx2t+x<(3EjxUGk7?zM0R3 zCEV_hhHXch6=(VFn$_0`N=cS>-iTiaTBWnHCqBe6EEbCl%07U>O>JXuYJsyB#OAVS z8M)v9AohkX44!)9(h$riSL?SC<&e(x(rb_?-_Eze)G(y+nQK>YN z?_B$!ajTY-O&vJ9k?xlwh2jTCgP)|1h!Ac!OgT#!F?Ut4FYPk7j&6?dCEhDVO$M!& zZ3`uNrb$=;mDPj%l6z&U474Ukz6|J2K5XP|E~{NU{Gj(S8%=nPZAKS={@oa&q4DUb z+E)i{QN04dsKw5V;ShfbihCykeY4t|u4(tE*P-5W z`oRe8aIG*AAb+;DQ?UQedq%{kdfJ@HkM7|_w66`^t)EH!yvDsfa|4 zSh6E7|3-TxqkI7}YQXSXh|X?;={UD-5z%}4Of`-0n{%izcgr&t$ZZj;K<#q+6y zp~ZH>rXQUqXU4;0ZWQ$eQyRg5Fo#K(+4gs_M9%F?ttUkN*x$F?ix+IYeNIy+zD$L| z;}Hhp&RZfuo}$;$_Q}fNOJ7Z0vV-}d%I$6MJ~td7UoBoQCz;%C8JQVyQe4-yCMz=1 z#?={IR*>i3lYAH&RP1bzFU(hRv5%b@mc0hrW*2(I9y7&3!sWP|i{&_*S7&dZFfy#X z1Xp`+EARkE&sy`&#NwIdQO;q&#Y!9>6t{-D9OgwP5CvS>a-Y6JX87dD1w%Q%t#Vl= z#}~j$-pDc0YpxBk!X(+&m`gbsQ4jYZ}&5&`%%cX*je&If}i=Q-pSjr%F1|q0ge$&zv6?z#oFUeeD(JB zGoe>-KOW^tvD3((*OLJnt7)hrNVOh!%s`4mS;d>_=wjdHv-0~Wd=i3!=Fe75-+Xl4 z#h~Eq13E2zTP|>sKqc0MtS<{XRUbiOUfG$Ix`8Hq2qT`A`KRn{G9|XA&4WWOf@}iK zDTJ}Y;Xk7v(eJ%&K1t4Pzj@znhJu3FlB=zXM z`100dqR>WC=v-z|rtI=WhcASwJq(?n`fR3I7E7ReN4Ytpxj1HD#QZdq1B%!mW>{C{ z*JT~~Kz3@r%~5Z*HRtLNTL6XyE=)!Oiv5j5Z9P>l5fYERyiPAee;STCG{RQ+a!_m7 zMhZ(_JKk8S0Jp=X{LzB_hwJ>L09+%p$8Vv4+Z!fj-WYhX+&aFx5|C6){`6sxcXom6 zRYu%kY^#G5D)iIs4#n z+{v|bU%7RzV0Y-#kV4vdWJH=OLs2U$4uDXvz0v~1CmlU^T@(hur>=GK9S%<|iY_l^ zMvmR$sK7J&d=;MuK$@y1AcCwQ+=@ms^zbM5l2x6oPVautxQq|_@#Gea+ZDyJ$M3c}7!nsUEsg(TXwf{_M+H$No#z2! z0^&L6XZp&A<{H?B$^phZk=BGhBHFxkbI&by-H~nTd}DBiT;EZW%FsQ1>ey8paBDBy zI-{cNJbvqG{^1!{yEZMAWZk9?R!;yRbrVR6U)&2IvSfraW``94{brX44ID%@VzhPS zOT`cxSwF}k_pVgj)|~aRGkN=kDR=VP+@nQ{y86jZ%e@nbq9T_)D=^>vc9GKdTK32c zP@t?@B!kclRW0pw{(Lo97U?>`%nWojXqtc!CP#!#O*w(RTS8TdnkP$M-@(xH>z)yv zr~RZ~&J6mGeouni1Wov2wiJs$sp&MS#w~G9HfodN*P{1^9ah}Y)-qg?#a#xCf#^F@ z>P#9Cb^ZZmT@7|dS;-2ZMvkJ>f*WWPe;t80j0mf1sa^!3cvvC?Ii6Ifa{BuDrIk(k zfj9wU&2~QwfX+<+9D#sUTrFs(2jsZu8Ovh;cNDIi)6fLXI7G%%she0tEucXbGR=`p z+j`|ZI#Y%vKDsYQ%{p9jL?E?d`0NJE)MDt3g4}{%f!!EUzl)K!6fv%UQCI+FFo74S zyI(|B)5t&QX*gdKJ1#Y)E3>`qrTP8Ik0vcghJtR;2@fCKV#4y3hvjZUB>br4AMD!= znm})R6<&`9+0i5d#D?Q%oa6~5Xr6GL{1jDL4q8-1wKIWs7w%`D!A}pCVFks2{pdgBGB-M$u(AP1XYkwn zvVfuDtx_r-g$-&nJzg%uwQXyzv9E0vGWr?a$U&<#*C1CP$qTObyObC#G%CC|sIV|^ zRk5N0{v>-#xxRU>)NS%v+2I3n4!y#C(8Q03r4NvYwtxD^sANf!x0695p*Y>6&-mp2 zTWF&KdSOZ164po@Kya7=2St`_c)aJ^>rR3zRE{pOEG9q}U$7^|Z2XGuK#PGjA>dC` ztLT-aP7S^qx?n5UySIC7%V7G2T`%tJ*8tocQ`e$sVG zae0OvAjWCGboJ<=2u1+UdI!|+yTes-wP{aed4USbu%n#r?kzEhLF~6&06d_Drn^N| z_(M#J&XGotXrAu2OkOf8rR0{Epwb*&z+@AsU$Q1ver(~hhru4A_J5`ya#M?4`Boi@ zuXQr1Xo{cWHrdZ?z^G>jdAiw^iu-bTF*po9)G$Ek5KJS5Zsb7fi%LCYn;8!Ou+@q; z03@-VnTXA$Q+S)-%sHKSdv;yhzHk$6a4obbp%Ief+2vuU-R!aGtPcR#4sbT!_3+0^rGwIoJhe%IF2&kfxHLjEh&OGtlF1X=W&(X?zlJhA3C2*iCmd5ScCgFA5NekkbO!iu44?3 z!gg`TG*y0%K~;R0$$2$;_o}qpyG343L&M#qw9}RBu6>1dtN_jrDz=D_Ct1Jv^V__4r?^CW=>#tRqj)~O)t zlWN&%KfR9UG8nX()8S4x?qjbdc@eZUGxj7Tto4cGr(v@)*<)%~^0iM&Ve&35tbrmv zR6wbuf&5MIdfIvvF*9Qwo$T<921CiD5ktcCzb`f( zu`=Gf+SSGiZ-R+?H93tuImShoaXG7QA(;FoE%z6 z-xldyeR0gLT6t~1$b4A=+v1}Wi`B4B-Sm^#Yv}mqAq0F$)!TM``Uo28QQEB!RZUoc zhc`U1$vz%^y#i2&*VA$sZUqN$q~lY z_U3%;)FC+*74(4}MYpr0+no+luuF^bAx{=uL<^r;$5?l%XSh8N~Cou~J!5ODK_fc0JdXX)i#y&&xn-b?EtS0KVtU z#k<9HDLbRms#<7vOj$pAC|Cec51-34o(qW$Vums58utubDCqNTb^LS`K=|T1n(A!{ z*=_ailY;m|(h0Y0mFK4!>n%%|XWRx$Y1LX??3acqs?XobfJ^vtLR{T?Ft=ZIoMv}w zuSE~2W`CGY6n%)*BHhi3Y0i|G+Lt&v-7h%<1H9B`iV7X_-I#F`yrkxpz68&AwF7C07mZ2o0evhKxm;b8yIlB~~d>0Vk?R|)HKR2^V`obc5Omp7^{FiFzE53e<=$_pL&HV%ZDGcp1px$#fbIFgyg*A8||9aT49`NzP!?mS6rR zncFbsUVxtOfe;{!i>L09rhJ-9YQHd~;I!YbiluCCc;gxZ8s1f)f+xge&!Sdy;7x6E4# zM-+g&wrNSIio)d0zVNhWs*zn%w>nl*kiWBC^UQndgoKv>*-7(`-mAqbnq^)fCTQf9 z2yT-Rw>6z-04;jPqBA~6taSxG-~E!lrpw=YKwd6d7=PsUy>dYW1Z*cKU*EZfbXrWA z={+Q8>d@4a{k{2wyv51B>kCo8WYKV*n$psC>9@%EWA$A%hX!43CCeYuwz$|c{V^PW zPb<{Q^2g8{TFcHYxTg>`)n$nNodV>rrXLF8>A2;`rS2sDKP5mweXYsx`{dYF;HEY| zN`**BXdN>C#`ySsn0^j@<7irJ$vp?~oS?xX3o_jQhZ5RW`wvhR$ILdN|D3X*V*xr6 z`rA??!@ItF632U)^Ror|@34!nwSfM%ZhsLm>(rp$;*8q&&{R8iC@3iWZxQ=FmH$5_ zdw;9jtrVc8IfZ`9!bG11ZeWWV+Q!YZA; zq#ATS{9IN~eE3$%ut2Up;OD zgn|XXq><7}Un+%Jz{T zcubBgnic`_}8&h_bt(v9cqWNm{EH0^EjQZu(L2d4nY`xg$1q>p5n`Gb4h z)ff}ah7sFO<;&Z^uA5eUQ5Ry~k+}C%iS$aoqF=)yc7MfA@-3NCYYLYiKE2AZmd{nZ zu;tk@hIN3nnW(P5Bvh?iTSfMO(^?ty>wjyktPj1m?N6Up7pf@%>xO2chd+h>iz;&* zRXkhD7$Il*{M-HI14*95y0ZK3W#NOK^qnHwhZ!-X@t&UqJ3IK85jfSr3m~cw5IA3K zYl!F(gTcj^!Ep*7S_x$aNY(I1&uQy2cwA?}cHm-rM7m}ehg3?E6TkR=XAEKw$x-wd z?-}JgB(2R<8g1nezTVlFb|{BulUd^awUP&qjj^$oPNUI9g3R1<#j`z25<24fV+j@E zi0soa>ZImYQI5}=$$L|=*7fzraQ{}TTr14(2{n*U=uEtYdhiw_9((lOd*Fr$8OEW8 z48{rq&WM$4Z~s_d(gbk3K7_N7)LEbUEp(C9BpPRIK`f<_PL=7`B5f?7wTg{s*FwytF>vTz z_0|<|L>`}oVu11{?_fr_>@xw+LrHV3m_$ibdO?ICoKw!RGqhtTVZbxB$%GN1b5Vdb zPUML4Vhvu;@rZea;2A-u@j@P>_s?%m2-L^8>iJ#TiS*pp9-dXeS8UV3QoV?5oUj3- zq|$n(mBeCUKxtq4i3as>V04-_fLnFxR1ih@*9@Yq-fI7T6VQ<+J@+{kjJ*;E)uZKt z8njGOgUSV;vf$zv;@X5Q6nj-Pa*DprN z;RD2ld1@|`BAD0!c$RS~*ij(k5fs&BE4Rz$ooNH)R=l8A65yZ2w04+%L)K3pQS)1< zI?Q(B3e(H%q|8$W@qX1mZS@yb=CbrCCy|gn4|}TjF)jHMHyD~oaeV5n=rpw6bX{4F zc35(vgF#$4AzL_cUh9&AG6EoV%oRt7xWl4xg~9D@zrP1}CABvUJQzPge+4|dUP**n zPTBzcdBbi>@GQHqYpj~6B_IM6@t3!ud9JkfqmK;_Az$!=K@o(!L>v;UbMlajZ#C+;xe?2Ug+$0RNz}s01qbit!-VZ#`;NIsT?ltco;23hd@&bxeJpgLGvNFo#OIq66NJ1)jOKmwtr_*=K_^0{ zy11(a+_V?F{C(dOVgP6cmwYKz#6#$K3D^Ou!}}HJIXD6>w@Hm}nr0!c7xe_t2 zFJa#A;+juC@bY*8D;wqk{A2HM+)UuwCAZC52BRenuL;aw(Pek#yM(@OaD0({RMbxc z&U<(|WyJY4OBZ)0b$A`WE}RU}oTPlI$xAa6$W;OZ4a=f(d}oW|&1|)&_e%SHn78sX zVj;x<d>hxnXy3jF_WD%*rc{WLG{h2j>hK&bPN*Ie|p zoR$$^us&vLoEYtQWb0mpM8K#)r5zh)2X6t#Lhvb(wdqXHwcoC{3O3F3u=vCj%yqFL%g1 zTNm4*f$Mh1hR~-MTJ%t9RIM1yBhD$KA3a6oNSa~epBWu^0d&y5c~sh{28JI?g!eZJ z>!tKuDlyg5z=xlebPa^gDZDHBjoa1B_qtWP!Q6O(pNE-7?D$ zhLvMT%WZjn!;(TT&T<_i2u06tXY{v!g+cc9*UHAm-eS`liMV_9x7+=B2*hPm-?K`$ z`oHMrrq(>PsL!z*3!77szCnHKQ@ED*((JH`R4d9B0dK(&2z45?Mh@2DK=aw$%-*n- zddb)T7pByLOmT6+Y2k^MH^L7l!>taA>;xFk63E@uuI-JP{l>VX6R-+q(<;y`dDej5?^bVi3Q30gSOQ=BS|eg@q&D$rJn<>{q-lv!Aa z=YQm)abTB-8H*GQGU<8-!ShM~9o(L`V;~(vS7G3vx)(Ia4;y1P>F&T@FRl;0hu@oD zu||Zp2_M-?8}><980>RW_ECpDqt3)+;hT)VecO>Ew8yWhd+%FXUz#Mh*H!d8Tx!yI zpAl6{sW_iJ>;)=RSb5@0@V^el99E~MJOT7*fYWt9$w~;QXJRtOX?u&;c7~-9TP>tq zxY@PYV}XHnNpI}r$d+QPwt8KfE74eb2HbNAz&QJ# z+JwaUyj3w<8Vwb0Bp#o+X55;gdl;kCAT08sfkXHOfaC30Gf(f!dhk}vG{?c1(wP{@ z(@?SHIctES2pRC@jIP5OJaAYd$1ht_15pw~geug*+^ub+^shjg|4gFziIxN>Bl^5BlK6TQ@LLF0XWg zo3IS#Uv8kV6e+ zX_+gu6dFYWumjIBL_}3zM1!9`E-AQrI|>I* zx%xeaks(gVot-!NoZ2EHBLi{4XX==kvQt_5pL#KOJMKG3^d^o949L1!SC@bd?n$6v z708zaMTBctOX|??sPlv&l9X|{x1diVWYA^k!|eNCtPS9n>d`#wdH5PrF6EU_g3g<*BL)0uD~z1GW=6g2G@hoPJdWm!wqCbJM3?<+uW4 zVw$_}D;z?2;lrv5c=p>GtZ^bQzwh7mgE9<%SF1k^T$MU3Jbx(`1`Fw-w>HR$7Dhof=Gp|t~CRhpE+y)yoe=e1loR*;#3 zN-BdpozL$=JuYYDHOxfapL&<5m9}g z8MZd&4zZWs`;fD`3lBhOpV1=ZT2bpw!L)vC+qa$mu4g#3NBQT{g{!Qd?RigUVgOO| zIL0%svW1Y7ol(N*eVjxbcwLi&fnX^-7Q9&m5@m3?OcCHbRSc-&)&s~u_j(0qj?))+ zL+=HA`x0@D&f^W;O;gm2Ct z0taRBKooVUpOX~iZiBpHwt+BWL-F31^jG!*(0z{miT8ghN5uI|6#`v8rpas_yUtzM z06ep@)3i}n0qgY3`j*C#(VGn_l7ZxMF|h4P`BtCAj9bwEE?scL=%PQ80`9#(gc zK!3g6O7nni&1-k!S~^lu>_F^5mP*=<>mQbZ<&OGMj z&Kw<;z;@}DMDpdtZD$S}A3NJ<9{2I0fUlba-cNb_txAM{pTj!3Y{*XQwgGzEV|b$i ztEM&aG5@~R7)=E6Wr6Bdb?i?34hDmo?`M)gx z4;yR$GU=d`yBTqYE{&h&m%YS4h1slxBmjD8|DKe=C+L$KA!bB_$^h`h+S-soW1B3I zl?)6()uJ+g4>|uN*8R(!AMC5<)!O2ubLr$32StOOK2eZfpYr~3^j<4zirJ(I}OZV1=6tMg2C6K zaR{++4A8)+liC6xlBnQrdl0O!ftYpdNru(isnvCKfN`>E0_Y&4`pC(R z2iIf?O#lZU&a5;chJQoZL-d%3zr6o{2q8*-skxuzIJd&3%My;`3ObGLaQfy{_#dL@ z-_i}*##NxBIW_I|g)cJ)iFU8%V1lp0*XDf1}I`F&p)`F~~M|Dp4hCA6zbxT4|_7!x%9B0$MkWpiHs zC`HRq7Hj#2mC4T--<*U$lMZ^_<|IUy#!vHqn;Z70 zz5W|3e{ghH-f_OJ?!2Du`?>N17D;QO3IB~;{?#<2VyBR|UlMg%jaM4iRz5ULk8hq! z*Fx({|KzO2Bd2j(_UZ9!EC1I4t$Jf;QTv}ymL2?Vxx?n*wb|9AUz!^)7oKkH@IEcr zm9L*@Bx`GY+Za2p>n(57TrblEJKPNO#BtZY>P+D60!^jBFp+n0a zB1E7m?L)Vb@m!S+FTZaDu)5h&=bA0PQ3amu!k>QXr`_R}0)G3q`=vJy4i1o%oSb_B z{1>3-vRl(wp%cBXqQ)mPW%M`Q3UCYg9m+>sk79j;cY{B!EGqzGqc6>DixnEy~1fudATzV+>{g-p{iqemrUN?E4*#;w<9)%26trZ@A@$Xwff;Ukp3Lc(wvBPeq@>T{+0v*dqD?_bC6KN5Q9gtLRLx zV_0NfX|ZBHk+k;Cn}(|!ota5Zqq>iMmQ)B;5X$0{Zc)Ya|1(- za~C&d_Kj>lEn;*bK#k>oO5J)9M}bpOX$3&Bt{luxUE1 wBb0TEHlx7T70dU9h_x2#_W+wE%Mw6hW{Xm=1nzx43l*Fcl|7MiO#kx#0aOf3AOHXW literal 0 HcmV?d00001 diff --git a/docs/reference/sql/endpoints/rest.asciidoc b/docs/reference/sql/endpoints/rest.asciidoc index 1cc42c3071b44..e21071a49dbbc 100644 --- a/docs/reference/sql/endpoints/rest.asciidoc +++ b/docs/reference/sql/endpoints/rest.asciidoc @@ -4,8 +4,8 @@ == SQL REST API The SQL REST API accepts SQL in a JSON document, executes it, -and returns the results. For example: - +and returns the results. +For example: [source,js] -------------------------------------------------- @@ -32,19 +32,68 @@ James S.A. Corey |Leviathan Wakes |561 |2011-06-02T00:00:00.000Z // TESTRESPONSE[s/\|/\\|/ s/\+/\\+/] // TESTRESPONSE[_cat] -While the `text/plain` format is nice for humans, computers prefer something -more structured. You can replace the value of `format` with: -- `json` aka `application/json` -- `yaml` aka `application/yaml` -- `smile` aka `application/smile` -- `cbor` aka `application/cbor` -- `txt` aka `text/plain` -- `csv` aka `text/csv` -- `tsv` aka `text/tab-separated-values` +[[sql-kibana-console]] +.Using Kibana Console +If you are using {kibana-ref}/console-kibana.html[Kibana Console]. +(which is highly recommended), take advantage of the +triple quotes `"""` when creating the query. This not only automatically escapes double +quotes (`"`) inside the query string but also support multi-line as shown below: +image:images/sql/rest/console-triple-quotes.png[] + +[[sql-rest-format]] +[float] +=== Response Data Formats + +While the textual format is nice for humans, computers prefer something +more structured. + +{es-sql} can return the data in the following formats which can be set +either through the `format` property in the URL or by setting the `Accept` HTTP header: + +NOTE: The URL parameter takes precedence over the `Accept` HTTP header. +If neither is specified then the response is returned in the same format as the request. + +[cols="^m,^4m,^8"] + +|=== +s|format +s|`Accept` HTTP header +s|Description + +3+h| Human Readable + +|csv +|text/csv +|https://en.wikipedia.org/wiki/Comma-separated_values[Comma-separated values] + +|json +|application/json +|https://www.json.org/[JSON] (JavaScript Object Notation) human-readable format + +|tsv +|text/tab-separated-values +|https://en.wikipedia.org/wiki/Tab-separated_values[Tab-separated values] + +|txt +|text/plain +|CLI-like representation + +|yaml +|application/yaml +|https://en.wikipedia.org/wiki/YAML[YAML] (YAML Ain't Markup Language) human-readable format + +3+h| Binary Formats + +|cbor +|application/cbor +|http://cbor.io/[Concise Binary Object Representation] + +|smile +|application/smile +|https://en.wikipedia.org/wiki/Smile_(data_interchange_format)[Smile] binary data format similar to CBOR + +|=== -Alternatively you can set the `Accept` HTTP header to the appropriate media -format. The GET parameter takes precedence over the header. If neither is -specified then the response is returned in the same format as the request. [source,js] -------------------------------------------------- @@ -80,7 +129,11 @@ Which returns: -------------------------------------------------- // TESTRESPONSE[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl\+v\/\/\/w8=/$body.cursor/] -You can continue to the next page by sending back the `cursor` field. In +[[sql-pagination]] +[float] +=== Paginating through a large response + +Using the example above, onu can continue to the next page by sending back the `cursor` field. In case of text format the cursor is returned as `Cursor` http header. [source,js] @@ -111,7 +164,7 @@ Which looks like: -------------------------------------------------- // TESTRESPONSE[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWODRMaXBUaVlRN21iTlRyWHZWYUdrdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl9f\/\/\/w8=/$body.cursor/] -Note that the `column` object is only part of the first page. +Note that the `columns` object is only part of the first page. You've reached the last page when there is no `cursor` returned in the results. Like Elasticsearch's <>, @@ -145,9 +198,11 @@ Which will like return the [[sql-rest-filtering]] +[float] +=== Filtering using {es} query DSL You can filter the results that SQL will run on using a standard -Elasticsearch query DSL by specifying the query in the filter +{es} query DSL by specifying the query in the filter parameter. [source,js] @@ -181,10 +236,48 @@ Douglas Adams |The Hitchhiker's Guide to the Galaxy|180 |1979-10-12T // TESTRESPONSE[_cat] [[sql-rest-fields]] -In addition to the `query` and `cursor` fields, the request can -contain `fetch_size` and `time_zone`. `fetch_size` is a hint for how -many results to return in each page. SQL might chose to return more -or fewer results though. `time_zone` is the time zone to use for datetime -functions and datetime parsing. `time_zone` defaults to `utc` and can take -any values documented -http://www.joda.org/joda-time/apidocs/org/joda/time/DateTimeZone.html[here]. +[float] +=== Supported REST parameters + +In addition to the `query` and `fetch_size`, a request a number of user-defined fields for specifying +the request time-outs or localization information (such as timezone). + +The table below lists the supported parameters: + +[cols="^m,^m,^5"] + +|=== + +s|name +s|Default value +s|Description + +|query +|Mandatory +|SQL query to execute + +|fetch_size +|1000 +|The maximum number of rows (or entries) to return in one response + +|filter +|none +|Optional {es} query DSL for additional <>. + +|request_timeout +|90s +|The timeout before the request fails. + +|page_timeout +|45s +|The timeout before a pagination request fails. + +|time_zone +|`Z` (or `UTC`) +|Time-zone in ISO 8601 used for executing the query on the server. +More information available https://docs.oracle.com/javase/8/docs/api/java/time/ZoneId.html[here]. + +|=== + +Do note that most parameters (outside the timeout ones) make sense only during the initial query - any follow-up pagination request only requires the `cursor` parameter as explained in the <> chapter. +That's because the query has already been executed and the calls are simply about returning the found results - thus the parameters are simply ignored. \ No newline at end of file diff --git a/docs/reference/sql/endpoints/translate.asciidoc b/docs/reference/sql/endpoints/translate.asciidoc index cd7f1cc846e8a..6a347ff614af7 100644 --- a/docs/reference/sql/endpoints/translate.asciidoc +++ b/docs/reference/sql/endpoints/translate.asciidoc @@ -4,7 +4,7 @@ == SQL Translate API The SQL Translate API accepts SQL in a JSON document and translates it -into native Elasticsearch queries. For example: +into native {es} queries. For example: [source,js] -------------------------------------------------- From 528385f9bade7d6ceaa62af1149b34077ea7899e Mon Sep 17 00:00:00 2001 From: lcawl Date: Fri, 15 Feb 2019 11:09:49 -0800 Subject: [PATCH 12/15] [DOCS] Fixes broken formatting --- docs/reference/index-modules/allocation/filtering.asciidoc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/reference/index-modules/allocation/filtering.asciidoc b/docs/reference/index-modules/allocation/filtering.asciidoc index 9e7a67946a997..0ae331d0e446d 100644 --- a/docs/reference/index-modules/allocation/filtering.asciidoc +++ b/docs/reference/index-modules/allocation/filtering.asciidoc @@ -49,6 +49,7 @@ settings support three types of filters: `include`, `exclude`, and `require`. For example, to tell {es} to allocate shards from the `test` index to either `big` or `medium` nodes, use `index.routing.allocation.include`: + +-- [source,js] ------------------------ PUT test/_settings @@ -58,11 +59,11 @@ PUT test/_settings ------------------------ // CONSOLE // TEST[s/^/PUT test\n/] -+ + If you specify multiple filters, all conditions must be satisfied for shards to be relocated. For example, to move the `test` index to `big` nodes in `rack1`, you could specify: -+ + [source,js] ------------------------ PUT test/_settings @@ -73,6 +74,7 @@ PUT test/_settings ------------------------ // CONSOLE // TEST[s/^/PUT test\n/] +-- [float] [[index-allocation-settings]] From 4140bfc5f1dd3966dfcb4b4d7ed0c6145cfda1ab Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Fri, 15 Feb 2019 22:12:08 +0200 Subject: [PATCH 13/15] SQL: doc polishing --- .../images/sql/rest/console-triple-quotes.png | Bin 54729 -> 4879 bytes .../sql/language/syntax/lexic/index.asciidoc | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/images/sql/rest/console-triple-quotes.png b/docs/reference/images/sql/rest/console-triple-quotes.png index c689b5902b580f334a640fe44bea641c1c5ae5c6..4a13acb986114cac419b8e463183d9de617ad4f1 100644 GIT binary patch literal 4879 zcmZvgc|6oz`^Uec8p4q*PZ~6iNXZz8? z)QKu~0|4H97{tK(ks~cjCxPt-&|u3BNM>hQHIr0WTHk#{b6N9YU1VSu$e{l|5mUzrkV_#B}}Ro8tu zH7`j`Mz**64rHHadm1e-Tx~?7;>FpX*kQ!MPsG_=$d9Q{7qb0DQKcAPk&5igP6a%N*?k}R*`1=;k|q9i0c zR#n^!*!#(!_m-O>FLCe{RVhcLq#b1ARg@e(5I1AkQENx~(<^X|dQ~S_LP4Gl&kS^d zwsimlHA(_4qs(2n!aJAlQZm5qT*& zM#p7ZjX=#vIO(PZb{*mWweiKGXSjVf^YVto-!b~L;N2ir~nZC zhH9T?7#1T)*)lp&wm5H}l)K*N)xVVrVknXAQH6}IUF?~H+SW|&_Odvh9MtDYd*+3N9Bt4zhVtiMx^&q`Lc2k6tc_} z2#?-w)&3H}FNCrc6s~m?+?a@89rsbJegyHDIbG47q?^P7Y$+ydtD>{d;CHn{|B`>FK0j<$k_MWuN}~MgGehyX z;Asp!VzyhlhB2>r!e^mYrqBxzK=$i7SK~ zTu#=XCv_?h^lZhjvuWFmp+;35Bv0SpS)7B~>z1B6^>F2@*9Or()Y`wLq;OErR-^O~ z6+$;vuvZ@a$*~+-q15!VjnK3`r){-4DfXr1>Fdv4n8T)QkI@yPy+1kc{c=g4p|-u=b}n*vEUUZQI`snP`P_?R?K?rBezZ{9T^BG8Pi!_6ccO6oik!*nXu~a! z$Vd0Ukj5^ULoY?bQ!n&Ij3E8<-S=X-MG0dB_FW58y2aa9u~J!qt+O(&4NAU?5m@e& zQe~SC0UMUP|WOPvsuK&$IoGTgVJKJuQ4qZ)=JC`of8bfYS`^kBxlYW02Iy!9aR z_o)0!$9TM9>JH|0SP4y9pqdvvcr#F5N0WV#kR=5k5-ze=Cgwbtd@I!e<)!aJZ%t4V z-m471^^ThV+%@c3b6Gj!`}wxz@M4FDyz$%J#k*_1n8{bWW>-)z1IEO-)Y2Q2Xv(q6 zswEkFI$+5sk@WCig9opX!%w1mo8bzt%?yz?H*Co*!B*TUJfpuNz%jiZsb{36+X~q( z7^~~VBIQ9aTHw@Qu#&@`;B2@0I8!YQCq0?g`y+-Aw?g+*J~4$Q)p(%}GcdtE4Hs-H zi*6Y`uTn;*kH?%iY&6ldiK+GyOKXoGx{$9c@1tEigFDIw+DEB?vMyL18)=bUvVY$? z-^frAE+yc{LWXOs>FyPo^&M!VL^TdpuGN^jPS_JShr8t_2Ds+Gx8Dcf^03m_Cpd?k ziAODd|uordiydv1phj(JUmNp*snFyq28%h z(K4&2Ayiw!Z$Wq|W@3L_#K#UVf z!h`c#)~6tjttMy94T|I9>nIypzh6p7ctw<=8gusOV}x2v`KBjDXvHVN5KUO}%OWT* z1^;PAte=~kaTSj!(eWSO2~4o4?C-Ci-e2lr!7Mlcew%rsn>1!OqeM zNy=>0g1HNSJYG@lwJ?G(*{RUm*FATSXcvk5rZ?d@HlqKuEo6D&r1@{cQpJ8?kVdIR zmLh6bX8UwkN3G(OdeK7VynX&CxGw9It0S1Tuc=XQgHRf%t%_AN9w2uXF%-xXF7(lQ z%?g@mQv~SfIDCB;B?msNzjI9ijJnk^NxwB=)uc2I8_BlY$CW}&s5}N!++F+#)X;^# z#&_~N`P1L{LFEHGJC3NtNS#Y}!~%vTz9#gdWQnT~KB2?1Hb0 za^)E0l9_W#=M^fq}ZD9wqRE;zx9?0@!__OKey?-XfvExw?(K2ciYMYwNj}F+jQ~CKv z!ZXn*$Z$*R1)Ro9A)1l>;C(R`nz5$LwCgxLK~(JeZN^NferH#_r^4J}zVRT^Li-_k zHQEp@EdQq1RI2<{Y~I>MlPA=d_PLj0e~n0hgIxY-xcQWi?1=sN6juUwQ=zdt1L|^-#LzlGN^Z` zFff?g0lw^68siT0ItFwxkpjvrPIx}x1N1qg23+GT?OZbez(*f_&NJn`@E8k_$c$gJ z%i|yUQb?Z8&aO3w6Syv#Z`_`F%w;fmeLwfEecGMi)0$<;4(|T$`+ctaZNIy9$6HK? zprd;ZgLEfq88fM%I-hVib-3{50V=PG-6S;pso9H%cpvt%fbu;vd4ttwEt=;i>Ng*? zn$zs5seR?toZq_~y_MMtP`m@35nDs)I$y;{EFLBw6rg?^I-D zl+xomGPEXNolJF;&8{-!^t&wH`$?v%tZ3lEb>Gc3c~Lbfn*yp= zx&%6O+yldUuXBsWSY4W1sdY$#jPAL##p{vs{w-G~#_TVbC!atyJaF^#2_FQno4(Ro zos%)RS+c#gkWQ~3>4W)2Ip-+kJ1)7n4+Z8sxCua7+N(o8LT8aTUpfgIpja(r7|^|Jffo%~o?;xzF&3mOWRnyBe+)VE#+Y$$ zz>s`zHHJKsD)f_!A8a4|e0d|^?ATgwBSfguZ@_()oQ?oehXP9VJq>Lbj?<+SA7?(DL41KWjck~cN zj_r4Lq7rVCKDb=_x2vy`|I&QBfeW>zRPx4YZZ}KGOV1BNCv&x-5T2bcMu~Rg zIfa#_n5Ru2g?RW#%3w*gLnB+s^>(?6yKr&B}AiTtF0Uiiy+$G~s z$+#4|3zij8l2h^z+28rv4;vhpWrx6ZI{RjXOp=Fe8;~NfiNRT8K^m z&;Es6dx!fWl};e(DU6hdugIM4$UgsdEwFA`Rf+4R)R&J$HbgVvKEkM==}zUl+*zH% z1yvu_)w;EfMKI^Raqp$()$a|z==dZR9jK#nu&@|-4c0hHUUouR@Nb3<26SkQ!PX*TW>fvf7kE<12Cbxa^Dhm zP7m*z61?w@=H4_o6qGSXA3yd_x{v|FME;S&1k8j{wMS$l)auy*aaPD|wV_;DwHW|t zE+QNqUu%-^yMqabit%v#(4L4%*|nN_LSO&HM7F`LKY+xO_%(fWFF<0_hjC}I;_d%n zKma3^Yjaan88CB$7c%>p`f+@8rg4sO{%T-O{5)e#k;vK<+_cS+BO$VM$@@ zerrs9u*(?+f=Hhcq-+b#Tm1-9N@(Hfk$FNLvMZq;lf&Ul?_7~0{7^lzkR(npC+GRz z{+i0@INYLm<}g)pFc%~(>{*7uaOlB)_>Pl-i=jiOUNLHV6?cchFDWdf0CB>dFe>m$ zYI~U^ij_<=V<8nhrXH0sCMcDp$18rrmWm7LGwQSW2e-(XQ^!^P$H$pgy z!9)6I-at3ki-k$pod^PVYC#xvNFUA6(~}4ypyIbg5XYjB8b?2LvqQrlD2`3ODKkLd z69+Tgy3^Hyc`F~K!~uz9NH`TWz)VtPK62LpeJ%=lhB1xEvj3+o>BD`wP=5@Dv4wK+ zeFZbzMl{G5AS{MJ@?#m&U5d%=bn4+q`KFth@feO4 z5<8Kb#D|(Z=p1#z)b=;R`pq2f2j3jMOCN&WGv|3mq%PF}BUC9{%wvo6Qx5r0WRj{P zaf*x?F=~VD$Q>tH+8hMmE!EFzQNJ(;R0d9FMdYVrlUNQ*sg z1?^WPj7>c9iqZ!TMi8JQ-5~iF%i}AG>tfkkIijO>cG6qcu>Pim{km=^Auwm@QJv)_ zR+D5wx~3leL3^r0c$C#VNpX#UYxpQ*Y%hl$oGr0gQ;y#9zm)G@X2!C+OSd_lo@56n zB^TAqyAW@5CKqQDKa97Z>+WE{=E=efWTZTyH$=Wm8URvlqCefE06_;@+_hSOQgfA# zF`kBD;w%jG<^TYH5bLI`7xGE{%tjox@SV%jFw}cPrxo$eNUF~`?r8VC9J-LiMDshu zu>H))*pe{ll87_1wrHYjQ*=;3o~qu3spcqeJTDdi&;9W3)%I4lTW6)4nrX(D3 z0gZ!O-m(!p9}gWnnT1uiA0u;nGNJ8m(~YuwLM>#PEp6>YYXthu z13~z!R)#V40}ps}t=hX|fgMm2WK!=u03Zab8Ao71&hgZk^WPjIDVn357NyT?>a{-duh;Ea;6tK1Rd2d62 zcji3v&v1Q%`d|L|gV76g`rW6%;z_1#!@6?6kCp-kAZvV{UDI5bTlvxc>HBXkKC3YX zHM9QZ{1nF02-cPw{}1^o-@SrBkq09pYS{8qUKL-Ev1AF!F7!K^Zqakiyb9apcaxL5at+ww4azIf!MJRM&e%Iv$ zp%zf6{nT0z9N4F~c*kPAcGfud#ZsgiUUeLk7pUwHIh}$9j1lH7jvWR3X{;9)4!zb^ z8)~Md|IrKmuD5?}0UU2#(&@tXsuyT{vccuN3xpiKc{LGhm-mk*JpKgXEwY>B6mrm6 z`q=}T?YB7Cw@UydkC9LAN{05;pW6oWe>GJ<=H5W%?|IeSBw@nTtjlEGZvTsgSPEpK z-}Ch=PJBv_BtZqPXsf{b;1YBUA82eA`>lq88wWSehU`a!4_z*6@^m$|!YS<1=PYjg z_b2fh#+M%uzsKm|)td0@ho&!JMMF`4e%>m4Xg!=8pls9G|En(<}!qWjM|1xApzk82^*}pf<9Mgb|iqH}^TRzq?f?@u0psWY8nBDsaibTJGW<(L#ry zy$vtq5ZFv;5?DC}hxd=^j?CpZQUPSvRg=!9YANq7g&KLI8S_znJ;J=_eHl6~mgh|{ z1)|~D%z~i+E1^)Q@7%CaDGvJ5%IcPST)YI+yeLbTAjU%O=wbA+_pW4idRmgDHCAF> z_u&SXD9X-F=+m^T)&`VAY}ysmi>AFlc7?QYB%n*Co{0(laWfAd+@txI;L@V#LyqGQ zMGO6>g}M@-0y3#x#>eS7a3K8vsPX)SW*7lPqplNlF$DnL6}1l)n6s8{PSquGp#AAa z<>6HU1tkQDT-Hx9Zi0oOJQi8z?RDobYK29B6+v@!%}R1;AoVHQOwKBBl8vog{c@%h63EotdRg}V+QYka%E)rLQc_x zg}m1G!wk?QrVEm6&M>~I%)E5g20UyV7dNVou$_BO$vV|-VA9dXK(q$g<^zC|Cz)(> z{-8^I;=g=Y{gCaK@!-rEQTDLXX0*)5*8`P z=Evn%Q#~{+8zzF`I|2Nl;D>|k2k;xXc*UIEz#{lX)>n7)zt~{If~_At`@bX(B(jjQ zJ@1r{xZOnQpIxB!^*g=I+f$48@jJcyD|XAjEj{Go7DTlFQGzQlWT%5}RZi#;!DOBk zE#8;=S}b2(**U119+(#tN#=3Ge=y31NP|g>$je2d)+)nABC5yl#zU69r=o~-B&iN0 z8RN<1cW3&LGTk*MI3md8m{{VFIS?yS zMjkK4h7s2TPCe#)etMt%_wO57VWVM~#LB6I%~@-3=W$wv4LR_uW#41_NePD$M{rPC zIM*0?#f}Ptm+s&w5vF6eH^wf{hrBE7VqH=AqLQbx_u0V+o@1fAk9{J!R1|7c`de^(wgJ zCMlc43qd~%d5>NNQ_B${-WW7S^raS@RR_jAHPFhIvj{E9m=5YpXh(xWP%m&PQb3W?lC_=$abG6JEAF>N)ExMuU>h zdeF$@Zwi`f6lt*DS`tt0@;UOoR1oj;fO?&K05LvJDvN^JRv;>u4Q{K(2)JK{eUz-L z#{kZyY?W^(#|Q4|1NkJjnJe9VG-Y!A9B{9i;f5MBKL3lsn~rW18Pa_B?ol9`%Nce> zHh~cd+s?KNL}2N^y-Rv!#N=YP?Yx{ANG|eY?@eDhQ;-(iH1pkZ7y!Q({+WFqLg9gW zof)hT#%H$_i2J^>dFg&=T(=<|yaWXxFg1Am7`}2E$9m`5UAAi^E5BfEO`Ot*wv7QU zz#od=Y4REX?r+;I>E(9xQP1ZRn-zKyUej02s7W>VeJx1Uw}Y7_Yr?u(15chCuG=p1 z_$Eq4O?N&|m|)_-qOwHf=2ANiBWkw8fF{MTsnXZy zI>v)!%8zX;&NuKImlTJE6gby>zAn=rIrnsW(HoZP_}pZ1F=hL>?m}f~gvnQ}hRmkF z`MNEB-{sS<_H)?}c!Fn1bx)6<{7lqmE3goPzNy=4Ncp-U;YUuZ!h?6*-^=dWP9}BW zUapcjDDoetxRtSJ=xo;`WdO8wrZ88#43E^$Modq;u0Po7>aj!`3&md7q*AJ!zx-p| zU`(6^P2=?Hfv*9~P~0IE0=0P`wQVe_cz44Vb)i=sBkAd=&DEsw=UYgBU*;dtm@E zTa2!CaAy<}BFwIi^BS@1)#NQ=w+C?dF1b$yWfVYoFm>g7VPwz`J$h zt;LV@E*LPp?=?z0JWJya1#o-PR^>T@pFLSllN6PzucuR&15cH2SB&_Zn{UTKwn)A_ zTr+f1MqMjjT`W&9ewhzs;4v`t57B@d4UkdFU;uOWmE(^(?dRLFq+%l)@KcqnTs~zK zb{_$x|0KiCkvwTw==&l>cHyhR3XOXnEgz@kM3UP@=HUZH(?aoA4AW&)YvWn-yFVII6`; zhI=LRqhAE|x;6T8bcm&+tyBhLBpU?MM2e?RiGqc{l`K*#+|xd7@@V0{lnEEsa0)B+ z;@P@(3P4?vaZk(OfRy;s>Ep{9fk3`)sGSr({Rt->mwr#^h0YmOb?JE!n0Y*Cbu@?^ zu=|9&$8s{1m)1EgjtPU%(bc|622mEL~bE3=qw$Jt!n_W=}?lJuf}Ue5;zDb06{4cy~3G8f|( z{YndB+()HuP_`I3Q_MVI%W!6ex#eHD)f;jFTIgu(n>4#^z%0~ig0?Sg9xq3Ta8J8a zL()Ei-@vvO$N2?uV5f(e3Rt?}Xr8?*rII&0ydp@BNxR$O#i_C9{=KwJ_b*Y0#L0+9#OvAxI+eWMW^pHnHt(3N{AAypq0 zJ}z%CqCN{3d;UIHpz($Zx9azdK%gy})Nm~q94>Cdr&$f)635D(BkZ8kDZ#4d+gtXy zv50RF`Pssm7aJLU-TFdrQz;psKail!Yo~vD8=$Uh>DHY#fBdmFZ zQ+~jOkH{B0rP5sm+^Se2OTcCek3)B}!QNX8u#{ZT>m1_&5N+IMlIl4D>Q z^+s@|GZdyKIxF|45u!z5O(ssRj{<1CAv&R?FXIBJHeyBbk|KPX`2fIV05di308n`c zFs*2SfeMipQO^xKSB0g2CTnFSQL&HhQy?ptPm?*A2@KrjX=?tG(mn*l?H^{VCHDZ4 z06<0P9S3Qpb4t*-jS9@kmsX=1fXWv`<|ZEW_?Z-B4Jt*`+O!rHQo!41`o^EsCWBNO zk_4wlTcgIhM4#byX49(6Hs%!fp1o%U?=QPl(|>f$A_Wz7!rdnqYo0z8npQkp{o1&Z z8N`ja`eLf-PyuFuGgFP=(T{cGFZKe!`^AKuXw{|;D7XSXp2Zn0KECb5XpBVRVrs`$ z9^#f781TUOn7lIHB`z5}FZ|^x1sFc`IU`BdJ-Yw%apxDI^8wEB)la94h$28y zI=0?<6YYq*cUp0Oro+ZCV?C!BI#@Ju0f}1os5oK_8fl@A>q)Oi>4OQ*~x5~#<3?#xKk-|)Xzj8XU8y^FTXGSvu96;imsR>Xe4)$ z&M)@lxIF<4YJtbSr~Ep%IXl1S8IaSNd~y4lpesMnF>7DX?$=!B9O-73w}rzXZp40A zsuBZ;OUP>DjPaA?H(P#jW{(T7}>9j2i^-k4hitFo)75Il?LNnEN&gzD-oa?UQLMrGx2t+x<(3EjxUGk7?zM0R3 zCEV_hhHXch6=(VFn$_0`N=cS>-iTiaTBWnHCqBe6EEbCl%07U>O>JXuYJsyB#OAVS z8M)v9AohkX44!)9(h$riSL?SC<&e(x(rb_?-_Eze)G(y+nQK>YN z?_B$!ajTY-O&vJ9k?xlwh2jTCgP)|1h!Ac!OgT#!F?Ut4FYPk7j&6?dCEhDVO$M!& zZ3`uNrb$=;mDPj%l6z&U474Ukz6|J2K5XP|E~{NU{Gj(S8%=nPZAKS={@oa&q4DUb z+E)i{QN04dsKw5V;ShfbihCykeY4t|u4(tE*P-5W z`oRe8aIG*AAb+;DQ?UQedq%{kdfJ@HkM7|_w66`^t)EH!yvDsfa|4 zSh6E7|3-TxqkI7}YQXSXh|X?;={UD-5z%}4Of`-0n{%izcgr&t$ZZj;K<#q+6y zp~ZH>rXQUqXU4;0ZWQ$eQyRg5Fo#K(+4gs_M9%F?ttUkN*x$F?ix+IYeNIy+zD$L| z;}Hhp&RZfuo}$;$_Q}fNOJ7Z0vV-}d%I$6MJ~td7UoBoQCz;%C8JQVyQe4-yCMz=1 z#?={IR*>i3lYAH&RP1bzFU(hRv5%b@mc0hrW*2(I9y7&3!sWP|i{&_*S7&dZFfy#X z1Xp`+EARkE&sy`&#NwIdQO;q&#Y!9>6t{-D9OgwP5CvS>a-Y6JX87dD1w%Q%t#Vl= z#}~j$-pDc0YpxBk!X(+&m`gbsQ4jYZ}&5&`%%cX*je&If}i=Q-pSjr%F1|q0ge$&zv6?z#oFUeeD(JB zGoe>-KOW^tvD3((*OLJnt7)hrNVOh!%s`4mS;d>_=wjdHv-0~Wd=i3!=Fe75-+Xl4 z#h~Eq13E2zTP|>sKqc0MtS<{XRUbiOUfG$Ix`8Hq2qT`A`KRn{G9|XA&4WWOf@}iK zDTJ}Y;Xk7v(eJ%&K1t4Pzj@znhJu3FlB=zXM z`100dqR>WC=v-z|rtI=WhcASwJq(?n`fR3I7E7ReN4Ytpxj1HD#QZdq1B%!mW>{C{ z*JT~~Kz3@r%~5Z*HRtLNTL6XyE=)!Oiv5j5Z9P>l5fYERyiPAee;STCG{RQ+a!_m7 zMhZ(_JKk8S0Jp=X{LzB_hwJ>L09+%p$8Vv4+Z!fj-WYhX+&aFx5|C6){`6sxcXom6 zRYu%kY^#G5D)iIs4#n z+{v|bU%7RzV0Y-#kV4vdWJH=OLs2U$4uDXvz0v~1CmlU^T@(hur>=GK9S%<|iY_l^ zMvmR$sK7J&d=;MuK$@y1AcCwQ+=@ms^zbM5l2x6oPVautxQq|_@#Gea+ZDyJ$M3c}7!nsUEsg(TXwf{_M+H$No#z2! z0^&L6XZp&A<{H?B$^phZk=BGhBHFxkbI&by-H~nTd}DBiT;EZW%FsQ1>ey8paBDBy zI-{cNJbvqG{^1!{yEZMAWZk9?R!;yRbrVR6U)&2IvSfraW``94{brX44ID%@VzhPS zOT`cxSwF}k_pVgj)|~aRGkN=kDR=VP+@nQ{y86jZ%e@nbq9T_)D=^>vc9GKdTK32c zP@t?@B!kclRW0pw{(Lo97U?>`%nWojXqtc!CP#!#O*w(RTS8TdnkP$M-@(xH>z)yv zr~RZ~&J6mGeouni1Wov2wiJs$sp&MS#w~G9HfodN*P{1^9ah}Y)-qg?#a#xCf#^F@ z>P#9Cb^ZZmT@7|dS;-2ZMvkJ>f*WWPe;t80j0mf1sa^!3cvvC?Ii6Ifa{BuDrIk(k zfj9wU&2~QwfX+<+9D#sUTrFs(2jsZu8Ovh;cNDIi)6fLXI7G%%she0tEucXbGR=`p z+j`|ZI#Y%vKDsYQ%{p9jL?E?d`0NJE)MDt3g4}{%f!!EUzl)K!6fv%UQCI+FFo74S zyI(|B)5t&QX*gdKJ1#Y)E3>`qrTP8Ik0vcghJtR;2@fCKV#4y3hvjZUB>br4AMD!= znm})R6<&`9+0i5d#D?Q%oa6~5Xr6GL{1jDL4q8-1wKIWs7w%`D!A}pCVFks2{pdgBGB-M$u(AP1XYkwn zvVfuDtx_r-g$-&nJzg%uwQXyzv9E0vGWr?a$U&<#*C1CP$qTObyObC#G%CC|sIV|^ zRk5N0{v>-#xxRU>)NS%v+2I3n4!y#C(8Q03r4NvYwtxD^sANf!x0695p*Y>6&-mp2 zTWF&KdSOZ164po@Kya7=2St`_c)aJ^>rR3zRE{pOEG9q}U$7^|Z2XGuK#PGjA>dC` ztLT-aP7S^qx?n5UySIC7%V7G2T`%tJ*8tocQ`e$sVG zae0OvAjWCGboJ<=2u1+UdI!|+yTes-wP{aed4USbu%n#r?kzEhLF~6&06d_Drn^N| z_(M#J&XGotXrAu2OkOf8rR0{Epwb*&z+@AsU$Q1ver(~hhru4A_J5`ya#M?4`Boi@ zuXQr1Xo{cWHrdZ?z^G>jdAiw^iu-bTF*po9)G$Ek5KJS5Zsb7fi%LCYn;8!Ou+@q; z03@-VnTXA$Q+S)-%sHKSdv;yhzHk$6a4obbp%Ief+2vuU-R!aGtPcR#4sbT!_3+0^rGwIoJhe%IF2&kfxHLjEh&OGtlF1X=W&(X?zlJhA3C2*iCmd5ScCgFA5NekkbO!iu44?3 z!gg`TG*y0%K~;R0$$2$;_o}qpyG343L&M#qw9}RBu6>1dtN_jrDz=D_Ct1Jv^V__4r?^CW=>#tRqj)~O)t zlWN&%KfR9UG8nX()8S4x?qjbdc@eZUGxj7Tto4cGr(v@)*<)%~^0iM&Ve&35tbrmv zR6wbuf&5MIdfIvvF*9Qwo$T<921CiD5ktcCzb`f( zu`=Gf+SSGiZ-R+?H93tuImShoaXG7QA(;FoE%z6 z-xldyeR0gLT6t~1$b4A=+v1}Wi`B4B-Sm^#Yv}mqAq0F$)!TM``Uo28QQEB!RZUoc zhc`U1$vz%^y#i2&*VA$sZUqN$q~lY z_U3%;)FC+*74(4}MYpr0+no+luuF^bAx{=uL<^r;$5?l%XSh8N~Cou~J!5ODK_fc0JdXX)i#y&&xn-b?EtS0KVtU z#k<9HDLbRms#<7vOj$pAC|Cec51-34o(qW$Vums58utubDCqNTb^LS`K=|T1n(A!{ z*=_ailY;m|(h0Y0mFK4!>n%%|XWRx$Y1LX??3acqs?XobfJ^vtLR{T?Ft=ZIoMv}w zuSE~2W`CGY6n%)*BHhi3Y0i|G+Lt&v-7h%<1H9B`iV7X_-I#F`yrkxpz68&AwF7C07mZ2o0evhKxm;b8yIlB~~d>0Vk?R|)HKR2^V`obc5Omp7^{FiFzE53e<=$_pL&HV%ZDGcp1px$#fbIFgyg*A8||9aT49`NzP!?mS6rR zncFbsUVxtOfe;{!i>L09rhJ-9YQHd~;I!YbiluCCc;gxZ8s1f)f+xge&!Sdy;7x6E4# zM-+g&wrNSIio)d0zVNhWs*zn%w>nl*kiWBC^UQndgoKv>*-7(`-mAqbnq^)fCTQf9 z2yT-Rw>6z-04;jPqBA~6taSxG-~E!lrpw=YKwd6d7=PsUy>dYW1Z*cKU*EZfbXrWA z={+Q8>d@4a{k{2wyv51B>kCo8WYKV*n$psC>9@%EWA$A%hX!43CCeYuwz$|c{V^PW zPb<{Q^2g8{TFcHYxTg>`)n$nNodV>rrXLF8>A2;`rS2sDKP5mweXYsx`{dYF;HEY| zN`**BXdN>C#`ySsn0^j@<7irJ$vp?~oS?xX3o_jQhZ5RW`wvhR$ILdN|D3X*V*xr6 z`rA??!@ItF632U)^Ror|@34!nwSfM%ZhsLm>(rp$;*8q&&{R8iC@3iWZxQ=FmH$5_ zdw;9jtrVc8IfZ`9!bG11ZeWWV+Q!YZA; zq#ATS{9IN~eE3$%ut2Up;OD zgn|XXq><7}Un+%Jz{T zcubBgnic`_}8&h_bt(v9cqWNm{EH0^EjQZu(L2d4nY`xg$1q>p5n`Gb4h z)ff}ah7sFO<;&Z^uA5eUQ5Ry~k+}C%iS$aoqF=)yc7MfA@-3NCYYLYiKE2AZmd{nZ zu;tk@hIN3nnW(P5Bvh?iTSfMO(^?ty>wjyktPj1m?N6Up7pf@%>xO2chd+h>iz;&* zRXkhD7$Il*{M-HI14*95y0ZK3W#NOK^qnHwhZ!-X@t&UqJ3IK85jfSr3m~cw5IA3K zYl!F(gTcj^!Ep*7S_x$aNY(I1&uQy2cwA?}cHm-rM7m}ehg3?E6TkR=XAEKw$x-wd z?-}JgB(2R<8g1nezTVlFb|{BulUd^awUP&qjj^$oPNUI9g3R1<#j`z25<24fV+j@E zi0soa>ZImYQI5}=$$L|=*7fzraQ{}TTr14(2{n*U=uEtYdhiw_9((lOd*Fr$8OEW8 z48{rq&WM$4Z~s_d(gbk3K7_N7)LEbUEp(C9BpPRIK`f<_PL=7`B5f?7wTg{s*FwytF>vTz z_0|<|L>`}oVu11{?_fr_>@xw+LrHV3m_$ibdO?ICoKw!RGqhtTVZbxB$%GN1b5Vdb zPUML4Vhvu;@rZea;2A-u@j@P>_s?%m2-L^8>iJ#TiS*pp9-dXeS8UV3QoV?5oUj3- zq|$n(mBeCUKxtq4i3as>V04-_fLnFxR1ih@*9@Yq-fI7T6VQ<+J@+{kjJ*;E)uZKt z8njGOgUSV;vf$zv;@X5Q6nj-Pa*DprN z;RD2ld1@|`BAD0!c$RS~*ij(k5fs&BE4Rz$ooNH)R=l8A65yZ2w04+%L)K3pQS)1< zI?Q(B3e(H%q|8$W@qX1mZS@yb=CbrCCy|gn4|}TjF)jHMHyD~oaeV5n=rpw6bX{4F zc35(vgF#$4AzL_cUh9&AG6EoV%oRt7xWl4xg~9D@zrP1}CABvUJQzPge+4|dUP**n zPTBzcdBbi>@GQHqYpj~6B_IM6@t3!ud9JkfqmK;_Az$!=K@o(!L>v;UbMlajZ#C+;xe?2Ug+$0RNz}s01qbit!-VZ#`;NIsT?ltco;23hd@&bxeJpgLGvNFo#OIq66NJ1)jOKmwtr_*=K_^0{ zy11(a+_V?F{C(dOVgP6cmwYKz#6#$K3D^Ou!}}HJIXD6>w@Hm}nr0!c7xe_t2 zFJa#A;+juC@bY*8D;wqk{A2HM+)UuwCAZC52BRenuL;aw(Pek#yM(@OaD0({RMbxc z&U<(|WyJY4OBZ)0b$A`WE}RU}oTPlI$xAa6$W;OZ4a=f(d}oW|&1|)&_e%SHn78sX zVj;x<d>hxnXy3jF_WD%*rc{WLG{h2j>hK&bPN*Ie|p zoR$$^us&vLoEYtQWb0mpM8K#)r5zh)2X6t#Lhvb(wdqXHwcoC{3O3F3u=vCj%yqFL%g1 zTNm4*f$Mh1hR~-MTJ%t9RIM1yBhD$KA3a6oNSa~epBWu^0d&y5c~sh{28JI?g!eZJ z>!tKuDlyg5z=xlebPa^gDZDHBjoa1B_qtWP!Q6O(pNE-7?D$ zhLvMT%WZjn!;(TT&T<_i2u06tXY{v!g+cc9*UHAm-eS`liMV_9x7+=B2*hPm-?K`$ z`oHMrrq(>PsL!z*3!77szCnHKQ@ED*((JH`R4d9B0dK(&2z45?Mh@2DK=aw$%-*n- zddb)T7pByLOmT6+Y2k^MH^L7l!>taA>;xFk63E@uuI-JP{l>VX6R-+q(<;y`dDej5?^bVi3Q30gSOQ=BS|eg@q&D$rJn<>{q-lv!Aa z=YQm)abTB-8H*GQGU<8-!ShM~9o(L`V;~(vS7G3vx)(Ia4;y1P>F&T@FRl;0hu@oD zu||Zp2_M-?8}><980>RW_ECpDqt3)+;hT)VecO>Ew8yWhd+%FXUz#Mh*H!d8Tx!yI zpAl6{sW_iJ>;)=RSb5@0@V^el99E~MJOT7*fYWt9$w~;QXJRtOX?u&;c7~-9TP>tq zxY@PYV}XHnNpI}r$d+QPwt8KfE74eb2HbNAz&QJ# z+JwaUyj3w<8Vwb0Bp#o+X55;gdl;kCAT08sfkXHOfaC30Gf(f!dhk}vG{?c1(wP{@ z(@?SHIctES2pRC@jIP5OJaAYd$1ht_15pw~geug*+^ub+^shjg|4gFziIxN>Bl^5BlK6TQ@LLF0XWg zo3IS#Uv8kV6e+ zX_+gu6dFYWumjIBL_}3zM1!9`E-AQrI|>I* zx%xeaks(gVot-!NoZ2EHBLi{4XX==kvQt_5pL#KOJMKG3^d^o949L1!SC@bd?n$6v z708zaMTBctOX|??sPlv&l9X|{x1diVWYA^k!|eNCtPS9n>d`#wdH5PrF6EU_g3g<*BL)0uD~z1GW=6g2G@hoPJdWm!wqCbJM3?<+uW4 zVw$_}D;z?2;lrv5c=p>GtZ^bQzwh7mgE9<%SF1k^T$MU3Jbx(`1`Fw-w>HR$7Dhof=Gp|t~CRhpE+y)yoe=e1loR*;#3 zN-BdpozL$=JuYYDHOxfapL&<5m9}g z8MZd&4zZWs`;fD`3lBhOpV1=ZT2bpw!L)vC+qa$mu4g#3NBQT{g{!Qd?RigUVgOO| zIL0%svW1Y7ol(N*eVjxbcwLi&fnX^-7Q9&m5@m3?OcCHbRSc-&)&s~u_j(0qj?))+ zL+=HA`x0@D&f^W;O;gm2Ct z0taRBKooVUpOX~iZiBpHwt+BWL-F31^jG!*(0z{miT8ghN5uI|6#`v8rpas_yUtzM z06ep@)3i}n0qgY3`j*C#(VGn_l7ZxMF|h4P`BtCAj9bwEE?scL=%PQ80`9#(gc zK!3g6O7nni&1-k!S~^lu>_F^5mP*=<>mQbZ<&OGMj z&Kw<;z;@}DMDpdtZD$S}A3NJ<9{2I0fUlba-cNb_txAM{pTj!3Y{*XQwgGzEV|b$i ztEM&aG5@~R7)=E6Wr6Bdb?i?34hDmo?`M)gx z4;yR$GU=d`yBTqYE{&h&m%YS4h1slxBmjD8|DKe=C+L$KA!bB_$^h`h+S-soW1B3I zl?)6()uJ+g4>|uN*8R(!AMC5<)!O2ubLr$32StOOK2eZfpYr~3^j<4zirJ(I}OZV1=6tMg2C6K zaR{++4A8)+liC6xlBnQrdl0O!ftYpdNru(isnvCKfN`>E0_Y&4`pC(R z2iIf?O#lZU&a5;chJQoZL-d%3zr6o{2q8*-skxuzIJd&3%My;`3ObGLaQfy{_#dL@ z-_i}*##NxBIW_I|g)cJ)iFU8%V1lp0*XDf1}I`F&p)`F~~M|Dp4hCA6zbxT4|_7!x%9B0$MkWpiHs zC`HRq7Hj#2mC4T--<*U$lMZ^_<|IUy#!vHqn;Z70 zz5W|3e{ghH-f_OJ?!2Du`?>N17D;QO3IB~;{?#<2VyBR|UlMg%jaM4iRz5ULk8hq! z*Fx({|KzO2Bd2j(_UZ9!EC1I4t$Jf;QTv}ymL2?Vxx?n*wb|9AUz!^)7oKkH@IEcr zm9L*@Bx`GY+Za2p>n(57TrblEJKPNO#BtZY>P+D60!^jBFp+n0a zB1E7m?L)Vb@m!S+FTZaDu)5h&=bA0PQ3amu!k>QXr`_R}0)G3q`=vJy4i1o%oSb_B z{1>3-vRl(wp%cBXqQ)mPW%M`Q3UCYg9m+>sk79j;cY{B!EGqzGqc6>DixnEy~1fudATzV+>{g-p{iqemrUN?E4*#;w<9)%26trZ@A@$Xwff;Ukp3Lc(wvBPeq@>T{+0v*dqD?_bC6KN5Q9gtLRLx zV_0NfX|ZBHk+k;Cn}(|!ota5Zqq>iMmQ)B;5X$0{Zc)Ya|1(- za~C&d_Kj>lEn;*bK#k>oO5J)9M}bpOX$3&Bt{luxUE1 wBb0TEHlx7T70dU9h_x2#_W+wE%Mw6hW{Xm=1nzx43l*Fcl|7MiO#kx#0aOf3AOHXW diff --git a/docs/reference/sql/language/syntax/lexic/index.asciidoc b/docs/reference/sql/language/syntax/lexic/index.asciidoc index 423934b4c6bbd..014193a0469dd 100644 --- a/docs/reference/sql/language/syntax/lexic/index.asciidoc +++ b/docs/reference/sql/language/syntax/lexic/index.asciidoc @@ -20,7 +20,7 @@ Take the following example: SELECT * FROM table ---- -This query has four tokens: `SELECT`, `\*`, `FROM` and `table`. The first three, namely `SELECT`, `*` and `FROM` are __key words__ meaning words that have a fixed meaning in SQL. The token `table` is an _identifier_ meaning it identifies (by name) an entity inside SQL such as a table (in this case), a column, etc... +This query has four tokens: `SELECT`, `*`, `FROM` and `table`. The first three, namely `SELECT`, `*` and `FROM` are __key words__ meaning words that have a fixed meaning in SQL. The token `table` is an _identifier_ meaning it identifies (by name) an entity inside SQL such as a table (in this case), a column, etc... As one can see, both key words and identifiers have the _same_ lexical structure and thus one cannot know whether a token is one or the other without knowing the SQL language; the complete list of key words is available in the <>. Do note that key words are case-insensitive meaning the previous example can be written as: @@ -45,7 +45,7 @@ Identifiers can be of two types: __quoted__ and __unquoted__: SELECT ip_address FROM "hosts-*" ---- -This query has two identifiers, `ip_address` and `hosts-\*` (an <>). As `ip_address` does not clash with any key words it can be used verbatim, `hosts-*` on the other hand cannot as it clashes with `-` (minus operation) and `*` hence the double quotes. +This query has two identifiers, `ip_address` and `hosts-*` (an <>). As `ip_address` does not clash with any key words it can be used verbatim, `hosts-*` on the other hand cannot as it clashes with `-` (minus operation) and `*` hence the double quotes. Another example: @@ -213,7 +213,7 @@ s|Description Two styles are supported: Single Line:: Comments start with a double dash `--` and continue until the end of the line. -Multi line:: Comments that start with `/\*` and end with `*/` (also known as C-style). +Multi line:: Comments that start with `/*` and end with `*/` (also known as C-style). [source, sql] From 065fec0b012f9c55ae6e24a0cbf4efd5bdcd4dca Mon Sep 17 00:00:00 2001 From: Darren Meiss Date: Fri, 15 Feb 2019 14:39:46 -0500 Subject: [PATCH 14/15] Edits to text in Completion Suggester doc (#38980) --- .../search/suggesters/completion-suggest.asciidoc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/reference/search/suggesters/completion-suggest.asciidoc b/docs/reference/search/suggesters/completion-suggest.asciidoc index c0b527c06e550..b27e6f0ef0b54 100644 --- a/docs/reference/search/suggesters/completion-suggest.asciidoc +++ b/docs/reference/search/suggesters/completion-suggest.asciidoc @@ -43,6 +43,7 @@ PUT music Mapping supports the following parameters: +[horizontal] `analyzer`:: The index analyzer to use, defaults to `simple`. In case you are wondering why we did not opt for the `standard` @@ -70,7 +71,7 @@ Mapping supports the following parameters: Limits the length of a single input, defaults to `50` UTF-16 code points. This limit is only used at index time to reduce the total number of characters per input string in order to prevent massive inputs from - bloating the underlying datastructure. Most usecases won't be influenced + bloating the underlying datastructure. Most use cases won't be influenced by the default value since prefix completions seldom grow beyond prefixes longer than a handful of characters. @@ -97,6 +98,7 @@ PUT music/_doc/1?refresh The following parameters are supported: +[horizontal] `input`:: The input to store, this can be an array of strings or just a string. This field is mandatory. @@ -285,6 +287,7 @@ Which should look like: The basic completion suggester query supports the following parameters: +[horizontal] `field`:: The name of the field on which to run the query (required). `size`:: The number of suggestions to return (defaults to `5`). `skip_duplicates`:: Whether duplicate suggestions should be filtered out (defaults to `false`). @@ -326,13 +329,13 @@ POST music/_search?pretty -------------------------------------------------- // CONSOLE -WARNING: when set to true this option can slow down search because more suggestions +WARNING: When set to true, this option can slow down search because more suggestions need to be visited to find the top N. [[fuzzy]] ==== Fuzzy queries -The completion suggester also supports fuzzy queries - this means, +The completion suggester also supports fuzzy queries -- this means you can have a typo in your search and still get results back. [source,js] From 7342d5a90fba45910b4cf8f31aa78f6cacb76a40 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 15 Feb 2019 15:48:56 -0500 Subject: [PATCH 15/15] Address some CCR REST test case flakiness (#38975) The CCR REST tests that rely on these assertions are flaky. They are flaky since the introduction of recovery from the remote. The underlying problem is this: these tests are making assertions about the number of operations read by the shard following task. However, with recovery from remote, we no longer have guarantees that the assumptions these tests were relying on hold. Namely, these tests were assuming that the only way that a document could land in the follower index is via the shard following task. With recovery from remote, there is another way, which is via the files that are copied over during the recovery phase. Most of the time this will not be a problem because with the small number of documents that we are indexing in these tests, it is usally not the case that a flush would occur and so there would not be any documents in the files copied over. However, a flush can occur any time at which point all of the indexed documents could end up in a safe commit and copied over during recovery from remote. This commit modifies these assertions to ones that are not prone to this issue, yet still validate the health of the follower shard. --- .../xpack/ccr/ESCCRRestTestCase.java | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java index 656328d5ead9e..4891c51049b62 100644 --- a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java +++ b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java @@ -28,6 +28,7 @@ import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; public class ESCCRRestTestCase extends ESRestTestCase { @@ -139,8 +140,9 @@ protected static void verifyCcrMonitoring(final String expectedLeaderIndex, fina throw new AssertionError("error while searching", e); } - int numberOfOperationsReceived = 0; - int numberOfOperationsIndexed = 0; + int followerMaxSeqNo = 0; + int followerMappingVersion = 0; + int followerSettingsVersion = 0; List hits = (List) XContentMapValues.extractValue("hits.hits", response); assertThat(hits.size(), greaterThanOrEqualTo(1)); @@ -153,16 +155,20 @@ protected static void verifyCcrMonitoring(final String expectedLeaderIndex, fina final String followerIndex = (String) XContentMapValues.extractValue("_source.ccr_stats.follower_index", hit); assertThat(followerIndex, equalTo(expectedFollowerIndex)); - int foundNumberOfOperationsReceived = - (int) XContentMapValues.extractValue("_source.ccr_stats.operations_read", hit); - numberOfOperationsReceived = Math.max(numberOfOperationsReceived, foundNumberOfOperationsReceived); - int foundNumberOfOperationsIndexed = - (int) XContentMapValues.extractValue("_source.ccr_stats.operations_written", hit); - numberOfOperationsIndexed = Math.max(numberOfOperationsIndexed, foundNumberOfOperationsIndexed); + int foundFollowerMaxSeqNo = + (int) XContentMapValues.extractValue("_source.ccr_stats.follower_max_seq_no", hit); + followerMaxSeqNo = Math.max(followerMaxSeqNo, foundFollowerMaxSeqNo); + int foundFollowerMappingVersion = + (int) XContentMapValues.extractValue("_source.ccr_stats.follower_mapping_version", hit); + followerMappingVersion = Math.max(followerMappingVersion, foundFollowerMappingVersion); + int foundFollowerSettingsVersion = + (int) XContentMapValues.extractValue("_source.ccr_stats.follower_settings_version", hit); + followerSettingsVersion = Math.max(followerSettingsVersion, foundFollowerSettingsVersion); } - assertThat(numberOfOperationsReceived, greaterThanOrEqualTo(1)); - assertThat(numberOfOperationsIndexed, greaterThanOrEqualTo(1)); + assertThat(followerMaxSeqNo, greaterThan(0)); + assertThat(followerMappingVersion, greaterThan(0)); + assertThat(followerSettingsVersion, greaterThan(0)); } protected static void verifyAutoFollowMonitoring() throws IOException {