Skip to content

Commit

Permalink
Merge branch '6.x' into ccr-6.x
Browse files Browse the repository at this point in the history 
* 6.x:
  Force execution of fetch tasks (#31974)
  [TEST] Mute SlackMessageTests.testTemplateRender
  Docs: Explain closing the high level client
  [test] disable java packaging tests for suse
  XContentTests : Insert random fields at random positions (#30867)
  Add Get Snapshots High Level REST API (#31980)
  Fix unreachable error condition in AmazonS3Fixture (#32005)
  [6.x][ML] Ensure immutability of MlMetadata (#31994)
  Add Expected Reciprocal Rank metric (#31891)
  SQL: Add support for single parameter text manipulating functions (#31874)
  muted tests due to #31940
  Work around reported problem in eclipse (#31960)
  Move build integration tests out of :buildSrc project (#31961)
  [Test] Reactive 3rd party tests on CI (#31919)
  Fix assertIngestDocument wrongfully passing (#31913) (#31951)
  SQL: Support for escape sequences (#31884)
  SQL: HAVING clause should accept only aggregates (#31872)
  Docs: fix typo in datehistogram (#31972)
  Switch url repository rest tests to new style requests (#31944)
  Switch reindex tests to new style requests (#31941)
  Switch test framework to new style requests (#31939)
  Docs: Added note about cloud service to installation and getting started
  [DOCS] Removes alternative docker pull example (#31934)
  ingest: date_index_name processor template resolution (#31841)
  Test: fix null failure in watcher test (#31968)
  Watcher: Slack message empty text (#31596)
  Switch low level rest tests to new style Requests (#31938)
  Switch high level rest tests to new style requests (#31937)
  HLREST: Bundle the x-pack protocol project (#31904)
  [ML] Mute test failing due to Java 11 date time format parsing bug (#31899)
  Increase logging level for testStressMaybeFlush
  rolling upgrade should use a replica to prevent relocations while running a scroll
  [test] port archive distribution packaging tests (#31314)
  HLRest: Move xPackInfo() to xPack().info() (#31905)
  Increase logging level for :qa:rolling-upgrade
  Backport: Add template config for Beat state to X-Pack Monitoring (#31809) (#31893)
  Fix building AD URL from domain name (#31849)
  Fix broken NaN check in MovingFunctions#stdDev() (#31888)
  Change trappy float comparison (#31889)
  Add opaque_id to audit logging (#31878)
  add support for is_write_index in put-alias body parsing (#31674)
  Ingest: Enable Templated Fieldnames in Rename (#31690) (#31896)
  Ingest: Add ignore_missing option to RemoveProc (#31693) (#31892)
  [Docs] Fix typo in the Rollup API Quick Reference (#31855)
  Watcher: Add ssl.trust email account setting (#31684)
  [PkiRealm] Invalidate cache on role mappings change (#31510)
  [Security] Check auth scheme case insensitively (#31490)
  HLREST: Add x-pack-info API (#31870)
  Remove link to oss-MSI (#31844)
  Painless: Restructure Definition/Whitelist (#31879)
  • Loading branch information
@dnhatn
dnhatn committed Jul 12, 2018
2 parents 04b5681 + 71380bd commit 4363363
Show file tree
Hide file tree
Showing 259 changed files with 10,512 additions and 4,204 deletions.
3 changes: 3 additions & 0 deletions build.gradle
View file
Expand Up @@ -429,6 +429,9 @@ allprojects {
if (isEclipse) {
// set this so generated dirs will be relative to eclipse build
project.buildDir = eclipseBuild
// Work around https://docs.gradle.org/current/userguide/java_gradle_plugin.html confusing Eclipse by the metadata
// it adds to the classpath
project.file("$buildDir/pluginUnderTestMetadata").mkdirs()
}
eclipse.classpath.file.whenMerged { classpath ->
// give each source folder a unique corresponding output folder
Expand Down
16 changes: 16 additions & 0 deletions buildSrc/build.gradle
View file
Expand Up @@ -128,6 +128,10 @@ if (project == rootProject) {
}
mavenCentral()
}
test {
include "**/*Tests.class"
exclude "**/*IT.class"
}
}

/*****************************************************************************
Expand All @@ -152,6 +156,18 @@ if (project != rootProject) {
jarHell.enabled = false
thirdPartyAudit.enabled = false

// tests can't be run with randomized test runner
// it's fine as we run them as part of :buildSrc
test.enabled = false
task integTest(type: Test) {
exclude "**/*Tests.class"
include "**/*IT.class"
testClassesDirs = sourceSets.test.output.classesDirs
classpath = sourceSets.test.runtimeClasspath
inputs.dir(file("src/testKit"))
}
check.dependsOn(integTest)

// TODO: re-enable once randomizedtesting gradle code is published and removed from here
licenseHeaders.enabled = false

Expand Down
16 changes: 13 additions & 3 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
View file
Expand Up @@ -550,9 +550,15 @@ class VagrantTestPlugin implements Plugin<Project> {
javaPackagingTest.command = 'ssh'
javaPackagingTest.args = ['--command', 'sudo bash "$PACKAGING_TESTS/run-tests.sh"']
} else {
// powershell sessions run over winrm always run as administrator, whether --elevated is passed or not. however
// remote sessions have some restrictions on what they can do, such as impersonating another user (or the same user
// without administrator elevation), which we need to do for these tests. passing --elevated runs the session
// as a scheduled job locally on the vm as a true administrator to get around this limitation
//
// https://github.com/hashicorp/vagrant/blob/9c299a2a357fcf87f356bb9d56e18a037a53d138/plugins/communicators/winrm/communicator.rb#L195-L225
// https://devops-collective-inc.gitbooks.io/secrets-of-powershell-remoting/content/manuscript/accessing-remote-computers.html
javaPackagingTest.command = 'winrm'
// winrm commands run as administrator
javaPackagingTest.args = ['--command', 'powershell -File "$Env:PACKAGING_TESTS/run-tests.ps1"']
javaPackagingTest.args = ['--elevated', '--command', 'powershell -File "$Env:PACKAGING_TESTS/run-tests.ps1"']
}

TaskExecutionAdapter javaPackagingReproListener = createReproListener(project, javaPackagingTest.path)
Expand All @@ -563,7 +569,11 @@ class VagrantTestPlugin implements Plugin<Project> {
project.gradle.removeListener(javaPackagingReproListener)
}
if (project.extensions.esvagrant.boxes.contains(box)) {
packagingTest.dependsOn(javaPackagingTest)
// these tests are temporarily disabled for suse boxes while we debug an issue
// https://github.com/elastic/elasticsearch/issues/30295
if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) {
packagingTest.dependsOn(javaPackagingTest)
}
}

/*
Expand Down
122 changes: 111 additions & 11 deletions client/rest-high-level/build.gradle
View file
@@ -1,5 +1,3 @@
import org.elasticsearch.gradle.precommit.PrecommitTasks

/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
Expand All @@ -18,29 +16,86 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks
* specific language governing permissions and limitations
* under the License.
*/

import org.elasticsearch.gradle.precommit.PrecommitTasks
import org.gradle.api.XmlProvider
import org.gradle.api.publish.maven.MavenPublication

buildscript {
repositories {
maven {
url 'https://plugins.gradle.org/m2/'
}
}
dependencies {
classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
}
}

apply plugin: 'elasticsearch.build'
apply plugin: 'elasticsearch.rest-test'
apply plugin: 'nebula.maven-base-publish'
apply plugin: 'nebula.maven-scm'
apply plugin: 'com.github.johnrengelman.shadow'

group = 'org.elasticsearch.client'
archivesBaseName = 'elasticsearch-rest-high-level-client'

publishing {
publications {
nebula {
artifactId = archivesBaseName
publications {
nebula(MavenPublication) {
artifact shadowJar
artifactId = archivesBaseName
/*
* Configure the pom to include the "shadow" as compile dependencies
* because that is how we're using them but remove all other dependencies
* because they've been shaded into the jar.
*/
pom.withXml { XmlProvider xml ->
Node root = xml.asNode()
root.remove(root.dependencies)
Node dependenciesNode = root.appendNode('dependencies')
project.configurations.shadow.allDependencies.each {
if (false == it instanceof SelfResolvingDependency) {
Node dependencyNode = dependenciesNode.appendNode('dependency')
dependencyNode.appendNode('groupId', it.group)
dependencyNode.appendNode('artifactId', it.name)
dependencyNode.appendNode('version', it.version)
dependencyNode.appendNode('scope', 'compile')
}
}
}
}
}
}

/*
* We need somewhere to configure dependencies that we don't wish to shade
* into the high level REST client. The shadow plugin creates a "shadow"
* configuration which is *almost* exactly that. It is never bundled into
* the shaded jar but is used for main source compilation. Unfortunately,
* by default it is not used for *test* source compilation and isn't used
* in tests at all. This change makes it available for test compilation.
* A change below makes it available for testing.
*/
sourceSets {
test {
compileClasspath += configurations.shadow
}
}

dependencies {
compile "org.elasticsearch:elasticsearch:${version}"
compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
compile "org.elasticsearch.plugin:parent-join-client:${version}"
compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
compile "org.elasticsearch.plugin:rank-eval-client:${version}"
compile "org.elasticsearch.plugin:lang-mustache-client:${version}"
/*
* Everything in the "shadow" configuration is *not* copied into the
* shadowJar.
*/
shadow "org.elasticsearch:elasticsearch:${version}"
shadow "org.elasticsearch.client:elasticsearch-rest-client:${version}"
shadow "org.elasticsearch.plugin:parent-join-client:${version}"
shadow "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
shadow "org.elasticsearch.plugin:rank-eval-client:${version}"
shadow "org.elasticsearch.plugin:lang-mustache-client:${version}"
compile project(':x-pack:protocol')

testCompile "org.elasticsearch.client:test:${version}"
testCompile "org.elasticsearch.test:framework:${version}"
Expand All @@ -63,3 +118,48 @@ forbiddenApisMain {
signaturesURLs += [PrecommitTasks.getResource('/forbidden/http-signatures.txt')]
signaturesURLs += [file('src/main/resources/forbidden/rest-high-level-signatures.txt').toURI().toURL()]
}

shadowJar {
classifier = null
mergeServiceFiles()
}

// We don't need normal jar, we use shadow jar instead
jar.enabled = false
assemble.dependsOn shadowJar

javadoc {
/*
* Bundle all of the javadoc from all of the shaded projects into this one
* so we don't *have* to publish javadoc for all of the "client" jars.
*/
configurations.compile.dependencies.all { Dependency dep ->
Project p = dependencyToProject(dep)
if (p != null) {
evaluationDependsOn(p.path)
source += p.sourceSets.main.allJava
}
}
}

/*
* Use the jar for testing so we have tests of the bundled jar.
* Use the "shadow" configuration for testing because we need things
* in it.
*/
test {
classpath -= compileJava.outputs.files
classpath -= configurations.compile
classpath -= configurations.runtime
classpath += configurations.shadow
classpath += shadowJar.outputs.files
dependsOn shadowJar
}
integTestRunner {
classpath -= compileJava.outputs.files
classpath -= configurations.compile
classpath -= configurations.runtime
classpath += configurations.shadow
classpath += shadowJar.outputs.files
dependsOn shadowJar
}
37 changes: 37 additions & 0 deletions client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
View file
Expand Up @@ -39,6 +39,7 @@
import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsRequest;
import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
Expand Down Expand Up @@ -103,6 +104,7 @@
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.VersionType;
import org.elasticsearch.index.rankeval.RankEvalRequest;
import org.elasticsearch.protocol.xpack.XPackInfoRequest;
import org.elasticsearch.rest.action.search.RestSearchAction;
import org.elasticsearch.script.mustache.MultiSearchTemplateRequest;
import org.elasticsearch.script.mustache.SearchTemplateRequest;
Expand All @@ -114,8 +116,10 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.util.EnumSet;
import java.util.Locale;
import java.util.StringJoiner;
import java.util.stream.Collectors;

final class RequestConverters {
static final XContentType REQUEST_BODY_CONTENT_TYPE = XContentType.JSON;
Expand Down Expand Up @@ -950,6 +954,26 @@ static Request createSnapshot(CreateSnapshotRequest createSnapshotRequest) throw
return request;
}

static Request getSnapshots(GetSnapshotsRequest getSnapshotsRequest) {
EndpointBuilder endpointBuilder = new EndpointBuilder().addPathPartAsIs("_snapshot")
.addPathPart(getSnapshotsRequest.repository());
String endpoint;
if (getSnapshotsRequest.snapshots().length == 0) {
endpoint = endpointBuilder.addPathPart("_all").build();
} else {
endpoint = endpointBuilder.addCommaSeparatedPathParts(getSnapshotsRequest.snapshots()).build();
}

Request request = new Request(HttpGet.METHOD_NAME, endpoint);

Params parameters = new Params(request);
parameters.withMasterTimeout(getSnapshotsRequest.masterNodeTimeout());
parameters.putParam("ignore_unavailable", Boolean.toString(getSnapshotsRequest.ignoreUnavailable()));
parameters.putParam("verbose", Boolean.toString(getSnapshotsRequest.verbose()));

return request;
}

static Request deleteSnapshot(DeleteSnapshotRequest deleteSnapshotRequest) {
String endpoint = new EndpointBuilder().addPathPartAsIs("_snapshot")
.addPathPart(deleteSnapshotRequest.repository())
Expand Down Expand Up @@ -1055,6 +1079,19 @@ static Request deleteScript(DeleteStoredScriptRequest deleteStoredScriptRequest)
return request;
}

static Request xPackInfo(XPackInfoRequest infoRequest) {
Request request = new Request(HttpGet.METHOD_NAME, "/_xpack");
if (false == infoRequest.isVerbose()) {
request.addParameter("human", "false");
}
if (false == infoRequest.getCategories().equals(EnumSet.allOf(XPackInfoRequest.Category.class))) {
request.addParameter("categories", infoRequest.getCategories().stream()
.map(c -> c.toString().toLowerCase(Locale.ROOT))
.collect(Collectors.joining(",")));
}
return request;
}

private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef();
return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));
Expand Down
23 changes: 18 additions & 5 deletions client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
View file
Expand Up @@ -202,6 +202,7 @@ public class RestHighLevelClient implements Closeable {
private final IngestClient ingestClient = new IngestClient(this);
private final SnapshotClient snapshotClient = new SnapshotClient(this);
private final TasksClient tasksClient = new TasksClient(this);
private final XPackClient xPackClient = new XPackClient(this);

/**
* Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the
Expand Down Expand Up @@ -292,6 +293,19 @@ public final TasksClient tasks() {
return tasksClient;
}

/**
* A wrapper for the {@link RestHighLevelClient} that provides methods for
* accessing the Elastic Licensed X-Pack APIs that are shipped with the
* default distribution of Elasticsearch. All of these APIs will 404 if run
* against the OSS distribution of Elasticsearch.
* <p>
* See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/xpack-api.html">
* X-Pack APIs on elastic.co</a> for more information.
*/
public final XPackClient xpack() {
return xPackClient;
}

/**
* Executes a bulk request using the Bulk API.
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html">Bulk API on elastic.co</a>
Expand Down Expand Up @@ -979,7 +993,6 @@ public final void rankEvalAsync(RankEvalRequest rankEvalRequest, RequestOptions
emptySet());
}


/**
* Executes a request using the Multi Search Template API.
*
Expand All @@ -989,9 +1002,9 @@ public final void rankEvalAsync(RankEvalRequest rankEvalRequest, RequestOptions
public final MultiSearchTemplateResponse multiSearchTemplate(MultiSearchTemplateRequest multiSearchTemplateRequest,
RequestOptions options) throws IOException {
return performRequestAndParseEntity(multiSearchTemplateRequest, RequestConverters::multiSearchTemplate,
options, MultiSearchTemplateResponse::fromXContext, emptySet());
}
options, MultiSearchTemplateResponse::fromXContext, emptySet());
}

/**
* Asynchronously executes a request using the Multi Search Template API
*
Expand All @@ -1003,7 +1016,7 @@ public final void multiSearchTemplateAsync(MultiSearchTemplateRequest multiSearc
ActionListener<MultiSearchTemplateResponse> listener) {
performRequestAsyncAndParseEntity(multiSearchTemplateRequest, RequestConverters::multiSearchTemplate,
options, MultiSearchTemplateResponse::fromXContext, listener, emptySet());
}
}

/**
* Asynchronously executes a request using the Ranking Evaluation API.
Expand Down
31 changes: 31 additions & 0 deletions client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java
View file
Expand Up @@ -32,6 +32,8 @@
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse;
import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;

import java.io.IOException;

Expand Down Expand Up @@ -190,6 +192,35 @@ public void createSnapshotAsync(CreateSnapshotRequest createSnapshotRequest, Req
CreateSnapshotResponse::fromXContent, listener, emptySet());
}

/**
* Get snapshots.
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
* API on elastic.co</a>
*
* @param getSnapshotsRequest the request
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @return the response
* @throws IOException in case there is a problem sending the request or parsing back the response
*/
public GetSnapshotsResponse get(GetSnapshotsRequest getSnapshotsRequest, RequestOptions options) throws IOException {
return restHighLevelClient.performRequestAndParseEntity(getSnapshotsRequest, RequestConverters::getSnapshots, options,
GetSnapshotsResponse::fromXContent, emptySet());
}

/**
* Asynchronously get snapshots.
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
* API on elastic.co</a>
*
* @param getSnapshotsRequest the request
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @param listener the listener to be notified upon request completion
*/
public void getAsync(GetSnapshotsRequest getSnapshotsRequest, RequestOptions options, ActionListener<GetSnapshotsResponse> listener) {
restHighLevelClient.performRequestAsyncAndParseEntity(getSnapshotsRequest, RequestConverters::getSnapshots, options,
GetSnapshotsResponse::fromXContent, listener, emptySet());
}

/**
* Deletes a snapshot.
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
Expand Down

0 comments on commit 4363363

Please sign in to comment.