diff --git a/build.gradle b/build.gradle index c5bf39bc65563..7d6b760a19c57 100644 --- a/build.gradle +++ b/build.gradle @@ -62,7 +62,12 @@ configure(subprojects.findAll { it.projectDir.toPath().startsWith(rootPath) }) { } } -// introspect all versions of ES that may be tested agains for backwards compatibility +/* Introspect all versions of ES that may be tested agains for backwards + * compatibility. It is *super* important that this logic is the same as the + * logic in VersionUtils.java, modulo alphas, betas, and rcs which are ignored + * in gradle because they don't have any backwards compatibility guarantees + * but are not ignored in VersionUtils.java because the tests expect them not + * to be. */ Version currentVersion = Version.fromString(VersionProperties.elasticsearch.minus('-SNAPSHOT')) int prevMajor = currentVersion.major - 1 File versionFile = file('core/src/main/java/org/elasticsearch/Version.java') @@ -72,13 +77,14 @@ List versions = [] int prevMinorIndex = -1 // index in the versions list of the last minor from the prev major int lastPrevMinor = -1 // the minor version number from the prev major we most recently seen for (String line : versionLines) { - Matcher match = line =~ /\W+public static final Version V_(\d+)_(\d+)_(\d+)(_UNRELEASED)? .*/ + /* Note that this skips alphas and betas which is fine because they aren't + * compatible with anything. */ + Matcher match = line =~ /\W+public static final Version V_(\d+)_(\d+)_(\d+) .*/ if (match.matches()) { int major = Integer.parseInt(match.group(1)) int minor = Integer.parseInt(match.group(2)) int bugfix = Integer.parseInt(match.group(3)) - boolean unreleased = match.group(4) != null - Version foundVersion = new Version(major, minor, bugfix, false, unreleased) + Version foundVersion = new Version(major, minor, bugfix, false) if (currentVersion != foundVersion) { versions.add(foundVersion) } @@ -96,10 +102,13 @@ if (currentVersion.bugfix == 0) { // If on a release branch, after the initial release of that branch, the bugfix version will // be bumped, and will be != 0. On master and N.x branches, we want to test against the // unreleased version of closest branch. So for those cases, the version includes -SNAPSHOT, - // and the bwc-zip distribution will checkout and build that version. + // and the bwc distribution will checkout and build that version. Version last = versions[-1] - versions[-1] = new Version(last.major, last.minor, last.bugfix, - true, last.unreleased) + versions[-1] = new Version(last.major, last.minor, last.bugfix, true) + if (last.bugfix == 0) { + versions[-2] = new Version( + versions[-2].major, versions[-2].minor, versions[-2].bugfix, true) + } } // injecting groovy property variables into all projects @@ -114,6 +123,44 @@ allprojects { } } +task('verifyVersions') { + description 'Verifies that all released versions that are indexed compatible are listed in Version.java.' + group 'Verification' + enabled = false == gradle.startParameter.isOffline() + doLast { + // Read the list from maven central + Node xml + new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s -> + xml = new XmlParser().parse(s) + } + Set knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ }) + + // Limit the known versions to those that should be index compatible + knownVersions = knownVersions.findAll { Integer.parseInt(it.split('\\.')[0]) >= prevMajor } + + /* Limit the listed versions to those that have been marked as released. + * Versions not marked as released don't get the same testing and we want + * to make sure that we flip all unreleased versions to released as soon + * as possible after release. */ + Set actualVersions = new TreeSet<>( + indexCompatVersions + .findAll { false == it.snapshot } + .collect { it.toString() }) + + // TODO this is almost certainly going to fail on 5.4 when we release 5.5.0 + + // Finally, compare! + if (!knownVersions.equals(actualVersions)) { + throw new GradleException("out-of-date versions\nActual :" + + actualVersions + "\nExpected:" + knownVersions + + "; update Version.java") + } + } +} +task('precommit') { + dependsOn(verifyVersions) +} + subprojects { project.afterEvaluate { // include license and notice in jars @@ -173,8 +220,10 @@ subprojects { ] if (wireCompatVersions[-1].snapshot) { // if the most previous version is a snapshot, we need to connect that version to the - // bwc-zip project which will checkout and build that snapshot version - ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc-zip' + // bwc project which will checkout and build that snapshot version + ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc' + ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc' + ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc' } project.afterEvaluate { configurations.all { diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 64ebba7578c4f..87d5ec9ae5336 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -311,7 +311,13 @@ class BuildPlugin implements Plugin { /** * Returns a closure which can be used with a MavenPom for fixing problems with gradle generated poms. * - * The current fixup is to set compile time deps back to compile from runtime (known issue with maven-publish plugin). + *
    + *
  • Remove transitive dependencies. We currently exclude all artifacts explicitly instead of using wildcards + * as Ivy incorrectly translates POMs with * excludes to Ivy XML with * excludes which results in the main artifact + * being excluded as well (see https://issues.apache.org/jira/browse/IVY-1531). Note that Gradle 2.14+ automatically + * translates non-transitive dependencies to * excludes. We should revisit this when upgrading Gradle.
    • + *
  • Set compile time deps back to compile from runtime (known issue with maven-publish plugin)
    • + *
*/ private static Closure fixupDependencies(Project project) { return { XmlProvider xml -> @@ -321,15 +327,53 @@ class BuildPlugin implements Plugin { return } - // fix deps incorrectly marked as runtime back to compile time deps - // see https://discuss.gradle.org/t/maven-publish-plugin-generated-pom-making-dependency-scope-runtime/7494/4 + // check each dependency for any transitive deps for (Node depNode : depsNodes.get(0).children()) { + String groupId = depNode.get('groupId').get(0).text() + String artifactId = depNode.get('artifactId').get(0).text() + String version = depNode.get('version').get(0).text() + + // fix deps incorrectly marked as runtime back to compile time deps + // see https://discuss.gradle.org/t/maven-publish-plugin-generated-pom-making-dependency-scope-runtime/7494/4 boolean isCompileDep = project.configurations.compile.allDependencies.find { dep -> dep.name == depNode.artifactId.text() } if (depNode.scope.text() == 'runtime' && isCompileDep) { depNode.scope*.value = 'compile' } + + // remove any exclusions added by gradle, they contain wildcards and systems like ivy have bugs with wildcards + // see https://github.com/elastic/elasticsearch/issues/24490 + NodeList exclusionsNode = depNode.get('exclusions') + if (exclusionsNode.size() > 0) { + depNode.remove(exclusionsNode.get(0)) + } + + // collect the transitive deps now that we know what this dependency is + String depConfig = transitiveDepConfigName(groupId, artifactId, version) + Configuration configuration = project.configurations.findByName(depConfig) + if (configuration == null) { + continue // we did not make this dep non-transitive + } + Set artifacts = configuration.resolvedConfiguration.resolvedArtifacts + if (artifacts.size() <= 1) { + // this dep has no transitive deps (or the only artifact is itself) + continue + } + + // we now know we have something to exclude, so add exclusions for all artifacts except the main one + Node exclusions = depNode.appendNode('exclusions') + for (ResolvedArtifact artifact : artifacts) { + ModuleVersionIdentifier moduleVersionIdentifier = artifact.moduleVersion.id; + String depGroupId = moduleVersionIdentifier.group + String depArtifactId = moduleVersionIdentifier.name + // add exclusions for all artifacts except the main one + if (depGroupId != groupId || depArtifactId != artifactId) { + Node exclusion = exclusions.appendNode('exclusion') + exclusion.appendNode('groupId', depGroupId) + exclusion.appendNode('artifactId', depArtifactId) + } + } } } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy index 03fe4c3428162..1c236c6c44ca5 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy @@ -29,20 +29,14 @@ public class Version { final int bugfix final int id final boolean snapshot - /** - * Is the vesion listed as {@code _UNRELEASED} in Version.java. - */ - final boolean unreleased - public Version(int major, int minor, int bugfix, boolean snapshot, - boolean unreleased) { + public Version(int major, int minor, int bugfix, boolean snapshot) { this.major = major this.minor = minor this.bugfix = bugfix this.snapshot = snapshot this.id = major * 100000 + minor * 1000 + bugfix * 10 + (snapshot ? 1 : 0) - this.unreleased = unreleased } public static Version fromString(String s) { @@ -54,7 +48,7 @@ public class Version { bugfix = bugfix.split('-')[0] } return new Version(parts[0] as int, parts[1] as int, bugfix as int, - snapshot, false) + snapshot) } @Override diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index c9965caa96e7d..afb8b62182985 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -46,11 +46,11 @@ class ClusterConfiguration { int transportPort = 0 /** - * An override of the data directory. This may only be used with a single node. - * The value is lazily evaluated at runtime as a String path. + * An override of the data directory. Input is the node number and output + * is the override data directory. */ @Input - Object dataDir = null + Closure dataDir = null /** Optional override of the cluster name. */ @Input diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy index c8617d9a36022..46542708420f1 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy @@ -111,10 +111,7 @@ class NodeInfo { homeDir = homeDir(baseDir, config.distribution, nodeVersion) confDir = confDir(baseDir, config.distribution, nodeVersion) if (config.dataDir != null) { - if (config.numNodes != 1) { - throw new IllegalArgumentException("Cannot set data dir for integ test with more than one node") - } - dataDir = config.dataDir + dataDir = "${config.dataDir(nodeNum)}" } else { dataDir = new File(homeDir, "data") } diff --git a/core/build.gradle b/core/build.gradle index 516aeb1d0eb3e..141c647f4889d 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -277,43 +277,3 @@ if (isEclipse == false || project.path == ":core-tests") { check.dependsOn integTest integTest.mustRunAfter test } - -task('verifyVersions') { - description 'Verifies that all released versions that are indexed compatible are listed in Version.java.' - group 'Verification' - enabled = false == gradle.startParameter.isOffline() - doLast { - // Read the list from maven central - Node xml - new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s -> - xml = new XmlParser().parse(s) - } - Set knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ }) - - // Limit the known versions to those that should be wire compatible - String currentVersion = versions.elasticsearch.minus('-SNAPSHOT') - int prevMajor = Integer.parseInt(currentVersion.split('\\.')[0]) - 1 - if (prevMajor == 4) { - // 4 didn't exist, it was 2. - prevMajor = 2; - } - knownVersions = knownVersions.findAll { Integer.parseInt(it.split('\\.')[0]) >= prevMajor } - - /* Limit the listed versions to those that have been marked as released. - * Versions not marked as released don't get the same testing and we want - * to make sure that we flip all unreleased versions to released as soon - * as possible after release. */ - Set actualVersions = new TreeSet<>( - indexCompatVersions - .findAll { false == it.unreleased } - .collect { it.toString() }) - - // Finally, compare! - if (!knownVersions.equals(actualVersions)) { - throw new GradleException("out-of-date versions\nActual :" + - actualVersions + "\nExpected:" + knownVersions + - "; update Version.java") - } - } -} -check.dependsOn(verifyVersions) diff --git a/core/src/main/java/org/apache/lucene/analysis/miscellaneous/DeDuplicatingTokenFilter.java b/core/src/main/java/org/apache/lucene/analysis/miscellaneous/DeDuplicatingTokenFilter.java new file mode 100644 index 0000000000000..3265ab8addce8 --- /dev/null +++ b/core/src/main/java/org/apache/lucene/analysis/miscellaneous/DeDuplicatingTokenFilter.java @@ -0,0 +1,201 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lucene.analysis.miscellaneous; + +import org.apache.lucene.analysis.FilteringTokenFilter; +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.hash.MurmurHash3; + +import java.io.IOException; +import java.util.ArrayList; + +/** + * Inspects token streams for duplicate sequences of tokens. Token sequences + * have a minimum length - 6 is a good heuristic as it avoids filtering common + * idioms/phrases but detects longer sections that are typical of cut+paste + * copies of text. + * + *

+ * Internally each token is hashed/moduloed into a single byte (so 256 possible + * values for each token) and then recorded in a trie of seen byte sequences + * using a {@link DuplicateByteSequenceSpotter}. This trie is passed into the + * TokenFilter constructor so a single object can be reused across multiple + * documents. + * + *

+ * The emitDuplicates setting controls if duplicate tokens are filtered from + * results or are output (the {@link DuplicateSequenceAttribute} attribute can + * be used to inspect the number of prior sightings when emitDuplicates is true) + */ +public class DeDuplicatingTokenFilter extends FilteringTokenFilter { + private final DuplicateSequenceAttribute seqAtt = addAttribute(DuplicateSequenceAttribute.class); + private final boolean emitDuplicates; + static final MurmurHash3.Hash128 seed = new MurmurHash3.Hash128(); + + public DeDuplicatingTokenFilter(TokenStream in, DuplicateByteSequenceSpotter byteStreamDuplicateSpotter) { + this(in, byteStreamDuplicateSpotter, false); + } + + /** + * + * @param in + * The input token stream + * @param byteStreamDuplicateSpotter + * object which retains trie of token sequences + * @param emitDuplicates + * true if duplicate tokens are to be emitted (use + * {@link DuplicateSequenceAttribute} attribute to inspect number + * of prior sightings of tokens as part of a sequence). + */ + public DeDuplicatingTokenFilter(TokenStream in, DuplicateByteSequenceSpotter byteStreamDuplicateSpotter, boolean emitDuplicates) { + super(new DuplicateTaggingFilter(byteStreamDuplicateSpotter, in)); + this.emitDuplicates = emitDuplicates; + } + + @Override + protected boolean accept() throws IOException { + return emitDuplicates || seqAtt.getNumPriorUsesInASequence() < 1; + } + + private static class DuplicateTaggingFilter extends TokenFilter { + private final DuplicateSequenceAttribute seqAtt = addAttribute(DuplicateSequenceAttribute.class); + + TermToBytesRefAttribute termBytesAtt = addAttribute(TermToBytesRefAttribute.class); + private DuplicateByteSequenceSpotter byteStreamDuplicateSpotter; + private ArrayList allTokens; + int pos = 0; + private final int windowSize; + + protected DuplicateTaggingFilter(DuplicateByteSequenceSpotter byteStreamDuplicateSpotter, TokenStream input) { + super(input); + this.byteStreamDuplicateSpotter = byteStreamDuplicateSpotter; + this.windowSize = DuplicateByteSequenceSpotter.TREE_DEPTH; + } + + + @Override + public final boolean incrementToken() throws IOException { + if (allTokens == null) { + loadAllTokens(); + } + clearAttributes(); + if (pos < allTokens.size()) { + State earlierToken = allTokens.get(pos); + pos++; + restoreState(earlierToken); + return true; + } else { + return false; + } + } + + public void loadAllTokens() throws IOException { + // TODO consider changing this implementation to emit tokens as-we-go + // rather than buffering all. However this array is perhaps not the + // bulk of memory usage (in practice the dupSequenceSpotter requires + // ~5x the original content size in its internal tree ). + allTokens = new ArrayList(256); + + /* + * Given the bytes 123456123456 and a duplicate sequence size of 6 + * the byteStreamDuplicateSpotter will only flag the final byte as + * part of a duplicate sequence due to the byte-at-a-time streaming + * nature of its assessments. When this happens we retain a buffer + * of the last 6 tokens so that we can mark the states of prior + * tokens (bytes 7 to 11) as also being duplicates + */ + + pos = 0; + boolean isWrapped = false; + State priorStatesBuffer[] = new State[windowSize]; + short priorMaxNumSightings[] = new short[windowSize]; + int cursor = 0; + while (input.incrementToken()) { + BytesRef bytesRef = termBytesAtt.getBytesRef(); + long tokenHash = MurmurHash3.hash128(bytesRef.bytes, bytesRef.offset, bytesRef.length, 0, seed).h1; + byte tokenByte = (byte) (tokenHash & 0xFF); + short numSightings = byteStreamDuplicateSpotter.addByte(tokenByte); + priorStatesBuffer[cursor] = captureState(); + // Revise prior captured State objects if the latest + // token is marked as a duplicate + if (numSightings >= 1) { + int numLengthsToRecord = windowSize; + int pos = cursor; + while (numLengthsToRecord > 0) { + if (pos < 0) { + pos = windowSize - 1; + } + priorMaxNumSightings[pos] = (short) Math.max(priorMaxNumSightings[pos], numSightings); + numLengthsToRecord--; + pos--; + } + } + // Reposition cursor to next free slot + cursor++; + if (cursor >= windowSize) { + // wrap around the buffer + cursor = 0; + isWrapped = true; + } + // clean out the end of the tail that we may overwrite if the + // next iteration adds a new head + if (isWrapped) { + // tokenPos is now positioned on tail - emit any valid + // tokens we may about to overwrite in the next iteration + if (priorStatesBuffer[cursor] != null) { + recordLengthInfoState(priorMaxNumSightings, priorStatesBuffer, cursor); + } + } + } // end loop reading all tokens from stream + + // Flush the buffered tokens + int pos = isWrapped ? nextAfter(cursor) : 0; + while (pos != cursor) { + recordLengthInfoState(priorMaxNumSightings, priorStatesBuffer, pos); + pos = nextAfter(pos); + } + } + + private int nextAfter(int pos) { + pos++; + if (pos >= windowSize) { + pos = 0; + } + return pos; + } + + private void recordLengthInfoState(short[] maxNumSightings, State[] tokenStates, int cursor) { + if (maxNumSightings[cursor] > 0) { + // We need to patch in the max sequence length we recorded at + // this position into the token state + restoreState(tokenStates[cursor]); + seqAtt.setNumPriorUsesInASequence(maxNumSightings[cursor]); + maxNumSightings[cursor] = 0; + // record the patched state + tokenStates[cursor] = captureState(); + } + allTokens.add(tokenStates[cursor]); + } + + } +} \ No newline at end of file diff --git a/core/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateByteSequenceSpotter.java b/core/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateByteSequenceSpotter.java new file mode 100644 index 0000000000000..7a58eaa1375f1 --- /dev/null +++ b/core/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateByteSequenceSpotter.java @@ -0,0 +1,311 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lucene.analysis.miscellaneous; + +import org.apache.lucene.util.RamUsageEstimator; + +/** + * A Trie structure for analysing byte streams for duplicate sequences. Bytes + * from a stream are added one at a time using the addByte method and the number + * of times it has been seen as part of a sequence is returned. + * + * The minimum required length for a duplicate sequence detected is 6 bytes. + * + * The design goals are to maximize speed of lookup while minimizing the space + * required to do so. This has led to a hybrid solution for representing the + * bytes that make up a sequence in the trie. + * + * If we have 6 bytes in sequence e.g. abcdef then they are represented as + * object nodes in the tree as follows: + *

+ * (a)-(b)-(c)-(def as an int) + *

+ * + * + * {@link RootTreeNode} objects are used for the first two levels of the tree + * (representing bytes a and b in the example sequence). The combinations of + * objects at these 2 levels are few so internally these objects allocate an + * array of 256 child node objects to quickly address children by indexing + * directly into the densely packed array using a byte value. The third level in + * the tree holds {@link LightweightTreeNode} nodes that have few children + * (typically much less than 256) and so use a dynamically-grown array to hold + * child nodes as simple int primitives. These ints represent the final 3 bytes + * of a sequence and also hold a count of the number of times the entire sequence + * path has been visited (count is a single byte). + *

+ * The Trie grows indefinitely as more content is added and while theoretically + * it could be massive (a 6-depth tree could produce 256^6 nodes) non-random + * content e.g English text contains fewer variations. + *

+ * In future we may look at using one of these strategies when memory is tight: + *

    + *
  1. auto-pruning methods to remove less-visited parts of the tree + *
  2. auto-reset to wipe the whole tree and restart when a memory threshold is + * reached + *
  3. halting any growth of the tree + *
+ * + * Tests on real-world-text show that the size of the tree is a multiple of the + * input text where that multiplier varies between 10 and 5 times as the content + * size increased from 10 to 100 megabytes of content. + * + */ +public class DuplicateByteSequenceSpotter { + public static final int TREE_DEPTH = 6; + // The maximum number of repetitions that are counted + public static final int MAX_HIT_COUNT = 255; + private final TreeNode root; + private boolean sequenceBufferFilled = false; + private final byte[] sequenceBuffer = new byte[TREE_DEPTH]; + private int nextFreePos = 0; + + // ==Performance info + private final int[] nodesAllocatedByDepth; + private int nodesResizedByDepth; + // ==== RAM usage estimation settings ==== + private long bytesAllocated; + // Root node object plus inner-class reference to containing "this" + // (profiler suggested this was a cost) + static final long TREE_NODE_OBJECT_SIZE = RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_OBJECT_REF; + // A TreeNode specialization with an array ref (dynamically allocated and + // fixed-size) + static final long ROOT_TREE_NODE_OBJECT_SIZE = TREE_NODE_OBJECT_SIZE + RamUsageEstimator.NUM_BYTES_OBJECT_REF; + // A KeyedTreeNode specialization with an array ref (dynamically allocated + // and grown) + static final long LIGHTWEIGHT_TREE_NODE_OBJECT_SIZE = TREE_NODE_OBJECT_SIZE + RamUsageEstimator.NUM_BYTES_OBJECT_REF; + // A KeyedTreeNode specialization with a short-based hit count and a + // sequence of bytes encoded as an int + static final long LEAF_NODE_OBJECT_SIZE = TREE_NODE_OBJECT_SIZE + Short.BYTES + Integer.BYTES; + + public DuplicateByteSequenceSpotter() { + this.nodesAllocatedByDepth = new int[4]; + this.bytesAllocated = 0; + root = new RootTreeNode((byte) 1, null, 0); + } + + /** + * Reset the sequence detection logic to avoid any continuation of the + * immediately previous bytes. A minimum of dupSequenceSize bytes need to be + * added before any new duplicate sequences will be reported. + * Hit counts are not reset by calling this method. + */ + public void startNewSequence() { + sequenceBufferFilled = false; + nextFreePos = 0; + } + + /** + * Add a byte to the sequence. + * @param b + * the next byte in a sequence + * @return number of times this byte and the preceding 6 bytes have been + * seen before as a sequence (only counts up to 255) + * + */ + public short addByte(byte b) { + // Add latest byte to circular buffer + sequenceBuffer[nextFreePos] = b; + nextFreePos++; + if (nextFreePos >= sequenceBuffer.length) { + nextFreePos = 0; + sequenceBufferFilled = true; + } + if (sequenceBufferFilled == false) { + return 0; + } + TreeNode node = root; + // replay updated sequence of bytes represented in the circular + // buffer starting from the tail + int p = nextFreePos; + + // The first tier of nodes are addressed using individual bytes from the + // sequence + node = node.add(sequenceBuffer[p], 0); + p = nextBufferPos(p); + node = node.add(sequenceBuffer[p], 1); + p = nextBufferPos(p); + node = node.add(sequenceBuffer[p], 2); + + // The final 3 bytes in the sequence are represented in an int + // where the 4th byte will contain a hit count. + + + p = nextBufferPos(p); + int sequence = 0xFF & sequenceBuffer[p]; + p = nextBufferPos(p); + sequence = sequence << 8 | (0xFF & sequenceBuffer[p]); + p = nextBufferPos(p); + sequence = sequence << 8 | (0xFF & sequenceBuffer[p]); + return (short) (node.add(sequence << 8) - 1); + } + + private int nextBufferPos(int p) { + p++; + if (p >= sequenceBuffer.length) { + p = 0; + } + return p; + } + + /** + * Base class for nodes in the tree. Subclasses are optimised for use at + * different locations in the tree - speed-optimized nodes represent + * branches near the root while space-optimized nodes are used for deeper + * leaves/branches. + */ + abstract class TreeNode { + + TreeNode(byte key, TreeNode parentNode, int depth) { + nodesAllocatedByDepth[depth]++; + } + + public abstract TreeNode add(byte b, int depth); + + /** + * + * @param byteSequence + * a sequence of bytes encoded as an int + * @return the number of times the full sequence has been seen (counting + * up to a maximum of 32767). + */ + public abstract short add(int byteSequence); + } + + // Node implementation for use at the root of the tree that sacrifices space + // for speed. + class RootTreeNode extends TreeNode { + + // A null-or-256 sized array that can be indexed into using a byte for + // fast access. + // Being near the root of the tree it is expected that this is a + // non-sparse array. + TreeNode[] children; + + RootTreeNode(byte key, TreeNode parentNode, int depth) { + super(key, parentNode, depth); + bytesAllocated += ROOT_TREE_NODE_OBJECT_SIZE; + } + + public TreeNode add(byte b, int depth) { + if (children == null) { + children = new TreeNode[256]; + bytesAllocated += (RamUsageEstimator.NUM_BYTES_OBJECT_REF * 256); + } + int bIndex = 0xFF & b; + TreeNode node = children[bIndex]; + if (node == null) { + if (depth <= 1) { + // Depths 0 and 1 use RootTreeNode impl and create + // RootTreeNodeImpl children + node = new RootTreeNode(b, this, depth); + } else { + // Deeper-level nodes are less visited but more numerous + // so use a more space-friendly data structure + node = new LightweightTreeNode(b, this, depth); + } + children[bIndex] = node; + } + return node; + } + + @Override + public short add(int byteSequence) { + throw new UnsupportedOperationException("Root nodes do not support byte sequences encoded as integers"); + } + + } + + // Node implementation for use by the depth 3 branches of the tree that + // sacrifices speed for space. + final class LightweightTreeNode extends TreeNode { + + // An array dynamically resized but frequently only sized 1 as most + // sequences leading to end leaves are one-off paths. + // It is scanned for matches sequentially and benchmarks showed + // that sorting contents on insertion didn't improve performance. + int[] children = null; + + LightweightTreeNode(byte key, TreeNode parentNode, int depth) { + super(key, parentNode, depth); + bytesAllocated += LIGHTWEIGHT_TREE_NODE_OBJECT_SIZE; + + } + + @Override + public short add(int byteSequence) { + if (children == null) { + // Create array adding new child with the byte sequence combined with hitcount of 1. + // Most nodes at this level we expect to have only 1 child so we start with the + // smallest possible child array. + children = new int[1]; + bytesAllocated += RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + Integer.BYTES; + children[0] = byteSequence + 1; + return 1; + } + // Find existing child and if discovered increment count + for (int i = 0; i < children.length; i++) { + int child = children[i]; + if (byteSequence == (child & 0xFFFFFF00)) { + int hitCount = child & 0xFF; + if (hitCount < MAX_HIT_COUNT) { + children[i]++; + } + return (short) (hitCount + 1); + } + } + // Grow array adding new child + int[] newChildren = new int[children.length + 1]; + bytesAllocated += Integer.BYTES; + + System.arraycopy(children, 0, newChildren, 0, children.length); + children = newChildren; + // Combine the byte sequence with a hit count of 1 into an int. + children[newChildren.length - 1] = byteSequence + 1; + nodesResizedByDepth++; + return 1; + } + + @Override + public TreeNode add(byte b, int depth) { + throw new UnsupportedOperationException("Leaf nodes do not take byte sequences"); + } + + } + + public final long getEstimatedSizeInBytes() { + return bytesAllocated; + } + + /** + * @return Performance info - the number of nodes allocated at each depth + */ + public int[] getNodesAllocatedByDepth() { + return nodesAllocatedByDepth.clone(); + } + + /** + * @return Performance info - the number of resizing of children arrays, at + * each depth + */ + public int getNodesResizedByDepth() { + return nodesResizedByDepth; + } + +} diff --git a/core/src/main/java/org/elasticsearch/index/engine/IgnoreOnRecoveryEngineException.java b/core/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateSequenceAttribute.java similarity index 63% rename from core/src/main/java/org/elasticsearch/index/engine/IgnoreOnRecoveryEngineException.java rename to core/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateSequenceAttribute.java index dcb3e43fb975b..bd797823d6835 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/IgnoreOnRecoveryEngineException.java +++ b/core/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateSequenceAttribute.java @@ -17,10 +17,19 @@ * under the License. */ -package org.elasticsearch.index.engine; +package org.apache.lucene.analysis.miscellaneous; + +import org.apache.lucene.util.Attribute; /** - * Exceptions implementing this interface will be ignored during recovery. + * Provides statistics useful for detecting duplicate sections of text */ -public interface IgnoreOnRecoveryEngineException { +public interface DuplicateSequenceAttribute extends Attribute { + /** + * @return The number of times this token has been seen previously as part + * of a sequence (counts to a max of 255) + */ + short getNumPriorUsesInASequence(); + + void setNumPriorUsesInASequence(short len); } \ No newline at end of file diff --git a/core/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateSequenceAttributeImpl.java b/core/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateSequenceAttributeImpl.java new file mode 100644 index 0000000000000..4c989a5b3cc38 --- /dev/null +++ b/core/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateSequenceAttributeImpl.java @@ -0,0 +1,53 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lucene.analysis.miscellaneous; + +import org.apache.lucene.util.AttributeImpl; +import org.apache.lucene.util.AttributeReflector; + +public class DuplicateSequenceAttributeImpl extends AttributeImpl implements DuplicateSequenceAttribute { + protected short numPriorUsesInASequence = 0; + + @Override + public void clear() { + numPriorUsesInASequence = 0; + } + + @Override + public void copyTo(AttributeImpl target) { + DuplicateSequenceAttributeImpl t = (DuplicateSequenceAttributeImpl) target; + t.numPriorUsesInASequence = numPriorUsesInASequence; + } + + @Override + public short getNumPriorUsesInASequence() { + return numPriorUsesInASequence; + } + + @Override + public void setNumPriorUsesInASequence(short len) { + numPriorUsesInASequence = len; + } + + @Override + public void reflectWith(AttributeReflector reflector) { + reflector.reflect(DuplicateSequenceAttribute.class, "sequenceLength", numPriorUsesInASequence); + } +} diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchException.java b/core/src/main/java/org/elasticsearch/ElasticsearchException.java index ae575af61c8d7..b020dc6ea1f5b 100644 --- a/core/src/main/java/org/elasticsearch/ElasticsearchException.java +++ b/core/src/main/java/org/elasticsearch/ElasticsearchException.java @@ -877,8 +877,7 @@ private enum ElasticsearchExceptionHandle { org.elasticsearch.transport.ReceiveTimeoutTransportException::new, 83, UNKNOWN_VERSION_ADDED), NODE_DISCONNECTED_EXCEPTION(org.elasticsearch.transport.NodeDisconnectedException.class, org.elasticsearch.transport.NodeDisconnectedException::new, 84, UNKNOWN_VERSION_ADDED), - ALREADY_EXPIRED_EXCEPTION(org.elasticsearch.index.AlreadyExpiredException.class, - org.elasticsearch.index.AlreadyExpiredException::new, 85, UNKNOWN_VERSION_ADDED), + // 85 used to be for AlreadyExpiredException AGGREGATION_EXECUTION_EXCEPTION(org.elasticsearch.search.aggregations.AggregationExecutionException.class, org.elasticsearch.search.aggregations.AggregationExecutionException::new, 86, UNKNOWN_VERSION_ADDED), // 87 used to be for MergeMappingException diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java index 1e60a4ac83c46..c693975681639 100644 --- a/core/src/main/java/org/elasticsearch/Version.java +++ b/core/src/main/java/org/elasticsearch/Version.java @@ -74,15 +74,17 @@ public class Version implements Comparable { public static final Version V_5_3_2 = new Version(V_5_3_2_ID, org.apache.lucene.util.Version.LUCENE_6_4_2); public static final int V_5_4_0_ID = 5040099; public static final Version V_5_4_0 = new Version(V_5_4_0_ID, org.apache.lucene.util.Version.LUCENE_6_5_0); - public static final int V_5_5_0_ID_UNRELEASED = 5050099; - public static final Version V_5_5_0_UNRELEASED = new Version(V_5_5_0_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_5_0); - public static final int V_6_0_0_alpha1_ID_UNRELEASED = 6000001; - public static final Version V_6_0_0_alpha1_UNRELEASED = - new Version(V_6_0_0_alpha1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_7_0_0); - public static final int V_6_0_0_alpha2_ID_UNRELEASED = 6000002; - public static final Version V_6_0_0_alpha2_UNRELEASED = - new Version(V_6_0_0_alpha2_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_7_0_0); - public static final Version CURRENT = V_6_0_0_alpha2_UNRELEASED; + public static final int V_5_4_1_ID = 5040199; + public static final Version V_5_4_1 = new Version(V_5_4_1_ID, org.apache.lucene.util.Version.LUCENE_6_5_1); + public static final int V_5_5_0_ID = 5050099; + public static final Version V_5_5_0 = new Version(V_5_5_0_ID, org.apache.lucene.util.Version.LUCENE_6_5_1); + public static final int V_6_0_0_alpha1_ID = 6000001; + public static final Version V_6_0_0_alpha1 = + new Version(V_6_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_0_0); + public static final int V_6_0_0_alpha2_ID = 6000002; + public static final Version V_6_0_0_alpha2 = + new Version(V_6_0_0_alpha2_ID, org.apache.lucene.util.Version.LUCENE_7_0_0); + public static final Version CURRENT = V_6_0_0_alpha2; // unreleased versions must be added to the above list with the suffix _UNRELEASED (with the exception of CURRENT) @@ -97,12 +99,14 @@ public static Version readVersion(StreamInput in) throws IOException { public static Version fromId(int id) { switch (id) { - case V_6_0_0_alpha2_ID_UNRELEASED: - return V_6_0_0_alpha2_UNRELEASED; - case V_6_0_0_alpha1_ID_UNRELEASED: - return V_6_0_0_alpha1_UNRELEASED; - case V_5_5_0_ID_UNRELEASED: - return V_5_5_0_UNRELEASED; + case V_6_0_0_alpha2_ID: + return V_6_0_0_alpha2; + case V_6_0_0_alpha1_ID: + return V_6_0_0_alpha1; + case V_5_5_0_ID: + return V_5_5_0; + case V_5_4_1_ID: + return V_5_4_1; case V_5_4_0_ID: return V_5_4_0; case V_5_3_2_ID: diff --git a/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java b/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java index aacb0ff17e744..64f63025279c3 100644 --- a/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java +++ b/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java @@ -261,7 +261,7 @@ public void readFrom(StreamInput in) throws IOException { type = in.readString(); id = in.readString(); version = in.readZLong(); - if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { seqNo = in.readZLong(); primaryTerm = in.readVLong(); } else { @@ -279,7 +279,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(type); out.writeString(id); out.writeZLong(version); - if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { out.writeZLong(seqNo); out.writeVLong(primaryTerm); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java index 4fc1c9c96fa8d..cdc869e529d3b 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java @@ -79,7 +79,7 @@ public void readFrom(StreamInput in) throws IOException { super.readFrom(in); clusterName = new ClusterName(in); clusterState = ClusterState.readFrom(in, null); - if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { totalCompressedSize = new ByteSizeValue(in); } else { // in a mixed cluster, if a pre 6.0 node processes the get cluster state @@ -95,7 +95,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); clusterName.writeTo(out); clusterState.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { totalCompressedSize.writeTo(out); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index a294d6cb771e3..0139726903b7c 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -492,7 +492,7 @@ public void readFrom(StreamInput in) throws IOException { for (int i = 0; i < size; i++) { final String type = in.readString(); String source = in.readString(); - if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to 5.3.0 after backport + if (in.getVersion().before(Version.V_6_0_0_alpha1)) { // TODO change to 5.3.0 after backport // we do not know the content type that comes from earlier versions so we autodetect and convert source = XContentHelper.convertToJson(new BytesArray(source), false, false, XContentFactory.xContentType(source)); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java index dfd50cf8cfd44..d15e390532e78 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java @@ -308,7 +308,7 @@ public void readFrom(StreamInput in) throws IOException { indicesOptions = IndicesOptions.readIndicesOptions(in); type = in.readOptionalString(); source = in.readString(); - if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to V_5_3 once backported + if (in.getVersion().before(Version.V_5_3_0)) { // we do not know the format from earlier versions so convert if necessary source = XContentHelper.convertToJson(new BytesArray(source), false, false, XContentFactory.xContentType(source)); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java index 6ad389c13d99f..8cded12b03073 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; + import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; @@ -34,7 +35,6 @@ import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.shard.ShardStateMetaData; import java.io.IOException; import java.util.ArrayList; @@ -165,7 +165,7 @@ public static StoreStatus readStoreStatus(StreamInput in) throws IOException { @Override public void readFrom(StreamInput in) throws IOException { node = new DiscoveryNode(in); - if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().before(Version.V_6_0_0_alpha1)) { // legacy version in.readLong(); } @@ -179,7 +179,7 @@ public void readFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { node.writeTo(out); - if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().before(Version.V_6_0_0_alpha1)) { // legacy version out.writeLong(-1L); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java index 150b7c6a52bc5..3d1e567fa1cea 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java @@ -104,7 +104,7 @@ public void readFrom(StreamInput in) throws IOException { statePath = in.readString(); dataPath = in.readString(); isCustomDataPath = in.readBoolean(); - if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { seqNoStats = in.readOptionalWriteable(SeqNoStats::new); } } @@ -117,7 +117,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(statePath); out.writeString(dataPath); out.writeBoolean(isCustomDataPath); - if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { out.writeOptionalWriteable(seqNoStats); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index b954aab7a6778..99ad163f48dd2 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -475,7 +475,7 @@ public void readFrom(StreamInput in) throws IOException { cause = in.readString(); name = in.readString(); - if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { indexPatterns = in.readList(StreamInput::readString); } else { indexPatterns = Collections.singletonList(in.readString()); @@ -487,7 +487,7 @@ public void readFrom(StreamInput in) throws IOException { for (int i = 0; i < size; i++) { final String type = in.readString(); String mappingSource = in.readString(); - if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to V_5_3_0 once backported + if (in.getVersion().before(Version.V_5_3_0)) { // we do not know the incoming type so convert it if needed mappingSource = XContentHelper.convertToJson(new BytesArray(mappingSource), false, false, XContentFactory.xContentType(mappingSource)); @@ -512,7 +512,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(cause); out.writeString(name); - if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { out.writeStringList(indexPatterns); } else { out.writeString(indexPatterns.size() > 0 ? indexPatterns.get(0) : ""); diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java index 50da1476f49f3..39e03277c37bb 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java @@ -78,7 +78,7 @@ public void readFrom(StreamInput in) throws IOException { if (in.readBoolean()) { primaryResponse = BulkItemResponse.readBulkItem(in); } - if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO remove once backported + if (in.getVersion().before(Version.V_6_0_0_alpha1)) { // TODO remove once backported boolean ignoreOnReplica = in.readBoolean(); if (ignoreOnReplica == false && primaryResponse != null) { assert primaryResponse.isFailed() == false : "expected no failure on the primary response"; @@ -89,7 +89,7 @@ public void readFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { out.writeVInt(id); - if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO remove once backported + if (out.getVersion().before(Version.V_6_0_0_alpha1)) { // TODO remove once backported // old nodes expect updated version and version type on the request if (primaryResponse != null) { request.version(primaryResponse.getVersion()); @@ -102,7 +102,7 @@ public void writeTo(StreamOutput out) throws IOException { DocWriteRequest.writeDocumentRequest(out, request); } out.writeOptionalStreamable(primaryResponse); - if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO remove once backported + if (out.getVersion().before(Version.V_6_0_0_alpha1)) { // TODO remove once backported if (primaryResponse != null) { out.writeBoolean(primaryResponse.isFailed() || primaryResponse.getResponse().getResult() == DocWriteResponse.Result.NOOP); diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java index 0ddd01187a6c2..daf10521f9ed5 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java @@ -211,7 +211,7 @@ public Failure(StreamInput in) throws IOException { id = in.readOptionalString(); cause = in.readException(); status = ExceptionsHelper.status(cause); - if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { seqNo = in.readZLong(); } else { seqNo = SequenceNumbersService.UNASSIGNED_SEQ_NO; @@ -224,7 +224,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(getType()); out.writeOptionalString(getId()); out.writeException(getCause()); - if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { out.writeZLong(getSeqNo()); } } diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index f0aede639e6ac..597b27eae4bd2 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -446,6 +446,7 @@ public WriteReplicaResult shardOperationOnReplica(BulkShardReq public static Translog.Location performOnReplica(BulkShardRequest request, IndexShard replica) throws Exception { Translog.Location location = null; + final long primaryTerm = request.primaryTerm(); for (int i = 0; i < request.items().length; i++) { BulkItemRequest item = request.items()[i]; final Engine.Result operationResult; @@ -457,10 +458,12 @@ public static Translog.Location performOnReplica(BulkShardRequest request, Index switch (docWriteRequest.opType()) { case CREATE: case INDEX: - operationResult = executeIndexRequestOnReplica(primaryResponse, (IndexRequest) docWriteRequest, replica); + operationResult = + executeIndexRequestOnReplica(primaryResponse, (IndexRequest) docWriteRequest, primaryTerm, replica); break; case DELETE: - operationResult = executeDeleteRequestOnReplica(primaryResponse, (DeleteRequest) docWriteRequest, replica); + operationResult = + executeDeleteRequestOnReplica(primaryResponse, (DeleteRequest) docWriteRequest, primaryTerm, replica); break; default: throw new IllegalStateException("Unexpected request operation type on replica: " @@ -528,14 +531,12 @@ private static Translog.Location locationToSync(Translog.Location current, * Execute the given {@link IndexRequest} on a replica shard, throwing a * {@link RetryOnReplicaException} if the operation needs to be re-tried. */ - private static Engine.IndexResult executeIndexRequestOnReplica( - DocWriteResponse primaryResponse, - IndexRequest request, - IndexShard replica) throws IOException { + private static Engine.IndexResult executeIndexRequestOnReplica(DocWriteResponse primaryResponse, IndexRequest request, + long primaryTerm, IndexShard replica) throws IOException { final Engine.Index operation; try { - operation = prepareIndexOperationOnReplica(primaryResponse, request, replica); + operation = prepareIndexOperationOnReplica(primaryResponse, request, primaryTerm, replica); } catch (MapperParsingException e) { return new Engine.IndexResult(e, primaryResponse.getVersion(), primaryResponse.getSeqNo()); } @@ -553,6 +554,7 @@ private static Engine.IndexResult executeIndexRequestOnReplica( static Engine.Index prepareIndexOperationOnReplica( DocWriteResponse primaryResponse, IndexRequest request, + long primaryTerm, IndexShard replica) { final ShardId shardId = replica.shardId(); @@ -565,7 +567,7 @@ static Engine.Index prepareIndexOperationOnReplica( final VersionType versionType = request.versionType().versionTypeForReplicationAndRecovery(); assert versionType.validateVersionForWrites(version); - return replica.prepareIndexOnReplica(sourceToParse, seqNo, version, versionType, + return replica.prepareIndexOnReplica(sourceToParse, seqNo, primaryTerm, version, versionType, request.getAutoGeneratedTimestamp(), request.isRetry()); } @@ -647,7 +649,7 @@ private static Engine.DeleteResult executeDeleteRequestOnPrimary(DeleteRequest r } private static Engine.DeleteResult executeDeleteRequestOnReplica(DocWriteResponse primaryResponse, DeleteRequest request, - IndexShard replica) throws Exception { + final long primaryTerm, IndexShard replica) throws Exception { if (replica.indexSettings().isSingleType()) { // We need to wait for the replica to have the mappings Mapping update; @@ -667,7 +669,7 @@ private static Engine.DeleteResult executeDeleteRequestOnReplica(DocWriteRespons final long version = primaryResponse.getVersion(); assert versionType.validateVersionForWrites(version); final Engine.Delete delete = replica.prepareDeleteOnReplica(request.type(), request.id(), - primaryResponse.getSeqNo(), request.primaryTerm(), version, versionType); + primaryResponse.getSeqNo(), primaryTerm, version, versionType); return replica.delete(delete); } diff --git a/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java index 7a47405d92b95..b04f882076326 100644 --- a/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java +++ b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java @@ -78,7 +78,7 @@ void setMergeResults(boolean mergeResults) { public void readFrom(StreamInput in) throws IOException { super.readFrom(in); fields = in.readStringArray(); - if (in.getVersion().onOrAfter(Version.V_5_5_0_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_5_5_0)) { indices = in.readStringArray(); indicesOptions = IndicesOptions.readIndicesOptions(in); mergeResults = in.readBoolean(); @@ -91,7 +91,7 @@ public void readFrom(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeStringArray(fields); - if (out.getVersion().onOrAfter(Version.V_5_5_0_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_5_5_0)) { out.writeStringArray(indices); indicesOptions.writeIndicesOptions(out); out.writeBoolean(mergeResults); diff --git a/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java index 00c424e1fe70a..ae5db5835670a 100644 --- a/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java +++ b/core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java @@ -86,7 +86,7 @@ public void readFrom(StreamInput in) throws IOException { super.readFrom(in); this.responseMap = in.readMap(StreamInput::readString, FieldCapabilitiesResponse::readField); - if (in.getVersion().onOrAfter(Version.V_5_5_0_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_5_5_0)) { indexResponses = in.readList(FieldCapabilitiesIndexResponse::new); } else { indexResponses = Collections.emptyList(); @@ -101,7 +101,7 @@ private static Map readField(StreamInput in) throws I public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeMap(responseMap, StreamOutput::writeString, FieldCapabilitiesResponse::writeField); - if (out.getVersion().onOrAfter(Version.V_5_5_0_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_5_5_0)) { out.writeList(indexResponses); } diff --git a/core/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/core/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index 8fad95257a879..3f0fb77781bdd 100644 --- a/core/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/core/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -118,38 +118,45 @@ public void onFailure(Exception e) { for (Map.Entry remoteIndices : remoteClusterIndices.entrySet()) { String clusterAlias = remoteIndices.getKey(); OriginalIndices originalIndices = remoteIndices.getValue(); - Transport.Connection connection = remoteClusterService.getConnection(remoteIndices.getKey()); - FieldCapabilitiesRequest remoteRequest = new FieldCapabilitiesRequest(); - remoteRequest.setMergeResults(false); // we need to merge on this node - remoteRequest.indicesOptions(originalIndices.indicesOptions()); - remoteRequest.indices(originalIndices.indices()); - remoteRequest.fields(request.fields()); - transportService.sendRequest(connection, FieldCapabilitiesAction.NAME, remoteRequest, TransportRequestOptions.EMPTY, - new TransportResponseHandler() { - @Override - public FieldCapabilitiesResponse newInstance() { - return new FieldCapabilitiesResponse(); - } - - @Override - public void handleResponse(FieldCapabilitiesResponse response) { - for (FieldCapabilitiesIndexResponse res : response.getIndexResponses()) { - indexResponses.add(new FieldCapabilitiesIndexResponse(RemoteClusterAware.buildRemoteIndexName(clusterAlias, - res.getIndexName()), res.get())); - } - onResponse.run(); - } - - @Override - public void handleException(TransportException exp) { - onResponse.run(); - } - - @Override - public String executor() { - return ThreadPool.Names.SAME; - } - }); + // if we are connected this is basically a no-op, if we are not we try to connect in parallel in a non-blocking fashion + remoteClusterService.ensureConnected(clusterAlias, ActionListener.wrap(v -> { + Transport.Connection connection = remoteClusterService.getConnection(clusterAlias); + FieldCapabilitiesRequest remoteRequest = new FieldCapabilitiesRequest(); + remoteRequest.setMergeResults(false); // we need to merge on this node + remoteRequest.indicesOptions(originalIndices.indicesOptions()); + remoteRequest.indices(originalIndices.indices()); + remoteRequest.fields(request.fields()); + transportService.sendRequest(connection, FieldCapabilitiesAction.NAME, remoteRequest, TransportRequestOptions.EMPTY, + new TransportResponseHandler() { + + @Override + public FieldCapabilitiesResponse newInstance() { + return new FieldCapabilitiesResponse(); + } + + @Override + public void handleResponse(FieldCapabilitiesResponse response) { + try { + for (FieldCapabilitiesIndexResponse res : response.getIndexResponses()) { + indexResponses.add(new FieldCapabilitiesIndexResponse(RemoteClusterAware. + buildRemoteIndexName(clusterAlias, res.getIndexName()), res.get())); + } + } finally { + onResponse.run(); + } + } + + @Override + public void handleException(TransportException exp) { + onResponse.run(); + } + + @Override + public String executor() { + return ThreadPool.Names.SAME; + } + }); + }, e -> onResponse.run())); } } diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java index ee9a5b2dfe684..552780e73951e 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -523,7 +523,7 @@ public void readFrom(StreamInput in) throws IOException { id = in.readOptionalString(); routing = in.readOptionalString(); parent = in.readOptionalString(); - if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().before(Version.V_6_0_0_alpha1)) { in.readOptionalString(); // timestamp in.readOptionalWriteable(TimeValue::new); // ttl } @@ -548,7 +548,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(id); out.writeOptionalString(routing); out.writeOptionalString(parent); - if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().before(Version.V_6_0_0_alpha1)) { // Serialize a fake timestamp. 5.x expect this value to be set by the #process method so we can't use null. // On the other hand, indices created on 5.x do not index the timestamp field. Therefore passing a 0 (or any value) for // the transport layer OK as it will be ignored. diff --git a/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index a8c5bdeacf396..078bd6e0b4e6b 100644 --- a/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.function.Function; /** @@ -59,7 +60,7 @@ private boolean isCollapseRequest() { final SearchRequest searchRequest = context.getRequest(); return searchRequest.source() != null && searchRequest.source().collapse() != null && - searchRequest.source().collapse().getInnerHit() != null; + searchRequest.source().collapse().getInnerHits().isEmpty() == false; } @Override @@ -67,6 +68,7 @@ public void run() throws IOException { if (isCollapseRequest() && searchResponse.getHits().getHits().length > 0) { SearchRequest searchRequest = context.getRequest(); CollapseBuilder collapseBuilder = searchRequest.source().collapse(); + final List innerHitBuilders = collapseBuilder.getInnerHits(); MultiSearchRequest multiRequest = new MultiSearchRequest(); if (collapseBuilder.getMaxConcurrentGroupRequests() > 0) { multiRequest.maxConcurrentSearchRequests(collapseBuilder.getMaxConcurrentGroupRequests()); @@ -83,27 +85,31 @@ public void run() throws IOException { if (origQuery != null) { groupQuery.must(origQuery); } - SearchSourceBuilder sourceBuilder = buildExpandSearchSourceBuilder(collapseBuilder.getInnerHit()) - .query(groupQuery); - SearchRequest groupRequest = new SearchRequest(searchRequest.indices()) - .types(searchRequest.types()) - .source(sourceBuilder); - multiRequest.add(groupRequest); + for (InnerHitBuilder innerHitBuilder : innerHitBuilders) { + SearchSourceBuilder sourceBuilder = buildExpandSearchSourceBuilder(innerHitBuilder) + .query(groupQuery); + SearchRequest groupRequest = new SearchRequest(searchRequest.indices()) + .types(searchRequest.types()) + .source(sourceBuilder); + multiRequest.add(groupRequest); + } } context.getSearchTransport().sendExecuteMultiSearch(multiRequest, context.getTask(), ActionListener.wrap(response -> { Iterator it = response.iterator(); for (SearchHit hit : searchResponse.getHits()) { - MultiSearchResponse.Item item = it.next(); - if (item.isFailure()) { - context.onPhaseFailure(this, "failed to expand hits", item.getFailure()); - return; - } - SearchHits innerHits = item.getResponse().getHits(); - if (hit.getInnerHits() == null) { - hit.setInnerHits(new HashMap<>(1)); + for (InnerHitBuilder innerHitBuilder : innerHitBuilders) { + MultiSearchResponse.Item item = it.next(); + if (item.isFailure()) { + context.onPhaseFailure(this, "failed to expand hits", item.getFailure()); + return; + } + SearchHits innerHits = item.getResponse().getHits(); + if (hit.getInnerHits() == null) { + hit.setInnerHits(new HashMap<>(innerHitBuilders.size())); + } + hit.getInnerHits().put(innerHitBuilder.getName(), innerHits); } - hit.getInnerHits().put(collapseBuilder.getInnerHit().getName(), innerHits); } context.executeNextPhase(this, nextPhaseFactory.apply(searchResponse)); }, context::onFailure) diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index b3dcf6fe1ef4f..946692f182643 100644 --- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -1011,7 +1011,7 @@ public ReplicaResponse(String allocationId, long localCheckpoint) { @Override public void readFrom(StreamInput in) throws IOException { - if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { super.readFrom(in); localCheckpoint = in.readZLong(); allocationId = in.readString(); @@ -1022,7 +1022,7 @@ public void readFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { super.writeTo(out); out.writeZLong(localCheckpoint); out.writeString(allocationId); @@ -1191,7 +1191,7 @@ public ConcreteReplicaRequest(final R request, final String targetAllocationID, @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { globalCheckpoint = in.readZLong(); } } @@ -1199,7 +1199,7 @@ public void readFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { out.writeZLong(globalCheckpoint); } } diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java index c88b604e8c9f9..6d3098c5caa5b 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java @@ -44,10 +44,8 @@ import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.lookup.SourceLookup; @@ -301,8 +299,8 @@ Result prepareUpdateScriptRequest(ShardId shardId, UpdateRequest request, GetRes private Map executeScript(Script script, Map ctx) { try { if (scriptService != null) { - CompiledScript compiledScript = scriptService.compile(script, ScriptContext.UPDATE); - ExecutableScript executableScript = scriptService.executable(compiledScript, script.getParams()); + ExecutableScript.Factory factory = scriptService.compile(script, ExecutableScript.UPDATE_CONTEXT); + ExecutableScript executableScript = factory.newInstance(script.getParams()); executableScript.setNextVar(ContextFields.CTX, ctx); executableScript.run(); } diff --git a/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java b/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java index 133c20411fb95..2caaa0bddf3c9 100644 --- a/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java +++ b/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java @@ -138,7 +138,7 @@ public void readFrom(StreamInput in) throws IOException { retryable = in.readBoolean(); disableStatePersistence = in.readBoolean(); status = RestStatus.readFrom(in); - if (in.getVersion().onOrAfter(Version.V_5_5_0_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_5_5_0)) { allowReleaseResources = in.readBoolean(); } else { allowReleaseResources = false; @@ -156,7 +156,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(retryable); out.writeBoolean(disableStatePersistence); RestStatus.writeTo(out, status); - if (out.getVersion().onOrAfter(Version.V_5_5_0_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_5_5_0)) { out.writeBoolean(allowReleaseResources); } } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index 5bba34904d0c8..b22106d97107a 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -210,7 +210,7 @@ public int hashCode() { public static IndexTemplateMetaData readFrom(StreamInput in) throws IOException { Builder builder = new Builder(in.readString()); builder.order(in.readInt()); - if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { builder.patterns(in.readList(StreamInput::readString)); } else { builder.patterns(Collections.singletonList(in.readString())); @@ -245,7 +245,7 @@ public static Diff readDiffFrom(StreamInput in) throws IO public void writeTo(StreamOutput out) throws IOException { out.writeString(name); out.writeInt(order); - if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { out.writeStringList(patterns); } else { out.writeString(patterns.size() > 0 ? patterns.get(0) : ""); diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java index 08d5211e12d74..1f6aeb11220f7 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java @@ -196,7 +196,7 @@ public void writeTo(StreamOutput out) throws IOException { source().writeTo(out); // routing out.writeBoolean(routing().required()); - if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().before(Version.V_6_0_0_alpha1)) { // timestamp out.writeBoolean(false); // enabled out.writeString(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format()); @@ -233,7 +233,7 @@ public MappingMetaData(StreamInput in) throws IOException { source = CompressedXContent.readCompressedString(in); // routing routing = new Routing(in.readBoolean()); - if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().before(Version.V_6_0_0_alpha1)) { // timestamp boolean enabled = in.readBoolean(); if (enabled) { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java index 6068d61aaedfb..8a88ee1751a14 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java @@ -179,7 +179,7 @@ private ShardIterator preferenceActiveShardIterator(IndexShardRoutingTable index } // if not, then use it as the index int routingHash = Murmur3HashFunction.hash(preference); - if (nodes.getMinNodeVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (nodes.getMinNodeVersion().onOrAfter(Version.V_6_0_0_alpha1)) { // The AllocationService lists shards in a fixed order based on nodes // so earlier versions of this class would have a tendency to // select the same node across different shardIds. diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java index d3071eb58f077..b0a1f2ed09f59 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -344,7 +344,7 @@ public Boolean getAsBooleanLenientForPreEs6Indices( final String setting, final Boolean defaultValue, final DeprecationLogger deprecationLogger) { - if (indexVersion.before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (indexVersion.before(Version.V_6_0_0_alpha1)) { //Only emit a warning if the setting's value is not a proper boolean final String value = get(setting, "false"); if (Booleans.isBoolean(value) == false) { diff --git a/core/src/main/java/org/elasticsearch/common/transport/TransportAddress.java b/core/src/main/java/org/elasticsearch/common/transport/TransportAddress.java index 5ed7f40fad0ab..965811f42ac51 100644 --- a/core/src/main/java/org/elasticsearch/common/transport/TransportAddress.java +++ b/core/src/main/java/org/elasticsearch/common/transport/TransportAddress.java @@ -78,7 +78,7 @@ public TransportAddress(StreamInput in) throws IOException { * {@link Version#V_5_0_2} as the hostString was not serialized */ public TransportAddress(StreamInput in, @Nullable String hostString) throws IOException { - if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // bwc layer for 5.x where we had more than one transport address + if (in.getVersion().before(Version.V_6_0_0_alpha1)) { // bwc layer for 5.x where we had more than one transport address final short i = in.readShort(); if(i != 1) { // we fail hard to ensure nobody tries to use some custom transport address impl even if that is difficult to add throw new AssertionError("illegal transport ID from node of version: " + in.getVersion() + " got: " + i + " expected: 1"); @@ -101,7 +101,7 @@ public TransportAddress(StreamInput in, @Nullable String hostString) throws IOEx @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().before(Version.V_6_0_0_alpha1)) { out.writeShort((short)1); // this maps to InetSocketTransportAddress in 5.x } byte[] bytes = address.getAddress().getAddress(); // 4 bytes (IPv4) or 16 bytes (IPv6) diff --git a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java index 0dc9647c126d2..f2b046acd97e8 100644 --- a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java +++ b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java @@ -290,7 +290,7 @@ public Exception storeException() { @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().before(Version.V_6_0_0_alpha1)) { // legacy version in.readLong(); } @@ -304,7 +304,7 @@ public void readFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().before(Version.V_6_0_0_alpha1)) { // legacy version out.writeLong(-1L); } diff --git a/core/src/main/java/org/elasticsearch/index/AlreadyExpiredException.java b/core/src/main/java/org/elasticsearch/index/AlreadyExpiredException.java deleted file mode 100644 index 24e910b905e56..0000000000000 --- a/core/src/main/java/org/elasticsearch/index/AlreadyExpiredException.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.index.engine.IgnoreOnRecoveryEngineException; - -import java.io.IOException; - -public class AlreadyExpiredException extends ElasticsearchException implements IgnoreOnRecoveryEngineException { - private String index; - private String type; - private String id; - private final long timestamp; - private final long ttl; - private final long now; - - public AlreadyExpiredException(String index, String type, String id, long timestamp, long ttl, long now) { - super("already expired [" + index + "]/[" + type + "]/[" + id + "] due to expire at [" + (timestamp + ttl) + "] and was processed at [" + now + "]"); - this.setIndex(index); - this.type = type; - this.id = id; - this.timestamp = timestamp; - this.ttl = ttl; - this.now = now; - } - - public String index() { - return index; - } - - public String type() { - return type; - } - - public String id() { - return id; - } - - public long timestamp() { - return timestamp; - } - - public long ttl() { - return ttl; - } - - public long now() { - return now; - } - - public AlreadyExpiredException(StreamInput in) throws IOException{ - super(in); - index = in.readOptionalString(); - type = in.readOptionalString(); - id = in.readOptionalString(); - timestamp = in.readLong(); - ttl = in.readLong(); - now = in.readLong(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeOptionalString(index); - out.writeOptionalString(type); - out.writeOptionalString(id); - out.writeLong(timestamp); - out.writeLong(ttl); - out.writeLong(now); - } -} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/core/src/main/java/org/elasticsearch/index/IndexSortConfig.java index 1d3f5f0fc23ea..7ec5acbe3ab0e 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/core/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -120,13 +120,13 @@ public IndexSortConfig(IndexSettings indexSettings) { .map((name) -> new FieldSortSpec(name)) .toArray(FieldSortSpec[]::new); - if (sortSpecs.length > 0 && indexSettings.getIndexVersionCreated().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (sortSpecs.length > 0 && indexSettings.getIndexVersionCreated().before(Version.V_6_0_0_alpha1)) { /** * This index might be assigned to a node where the index sorting feature is not available * (ie. versions prior to {@link Version.V_6_0_0_alpha1_UNRELEASED}) so we must fail here rather than later. */ throw new IllegalArgumentException("unsupported index.version.created:" + indexSettings.getIndexVersionCreated() + - ", can't set index.sort on versions prior to " + Version.V_6_0_0_alpha1_UNRELEASED); + ", can't set index.sort on versions prior to " + Version.V_6_0_0_alpha1); } if (INDEX_SORT_ORDER_SETTING.exists(settings)) { diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 03007d96b9f23..ca653ff41249f 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -550,14 +550,14 @@ private boolean assertVersionType(final Engine.Operation operation) { } private boolean assertIncomingSequenceNumber(final Engine.Operation.Origin origin, final long seqNo) { - if (engineConfig.getIndexSettings().getIndexVersionCreated().before(Version.V_6_0_0_alpha1_UNRELEASED) && origin == Operation.Origin.LOCAL_TRANSLOG_RECOVERY) { + if (engineConfig.getIndexSettings().getIndexVersionCreated().before(Version.V_6_0_0_alpha1) && origin == Operation.Origin.LOCAL_TRANSLOG_RECOVERY) { // legacy support assert seqNo == SequenceNumbersService.UNASSIGNED_SEQ_NO : "old op recovering but it already has a seq no.;" + " index version: " + engineConfig.getIndexSettings().getIndexVersionCreated() + ", seqNo: " + seqNo; } else if (origin == Operation.Origin.PRIMARY) { // sequence number should not be set when operation origin is primary assert seqNo == SequenceNumbersService.UNASSIGNED_SEQ_NO : "primary ops should never have an assigned seq no.; seqNo: " + seqNo; - } else if (engineConfig.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + } else if (engineConfig.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1)) { // sequence number should be set when operation origin is not primary assert seqNo >= 0 : "recovery or replica ops should have an assigned seq no.; origin: " + origin; } @@ -565,7 +565,7 @@ private boolean assertIncomingSequenceNumber(final Engine.Operation.Origin origi } private boolean assertSequenceNumberBeforeIndexing(final Engine.Operation.Origin origin, final long seqNo) { - if (engineConfig.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED) || + if (engineConfig.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1) || origin == Operation.Origin.PRIMARY) { // sequence number should be set when operation origin is primary or when all shards are on new nodes assert seqNo >= 0 : "ops should have an assigned seq no.; origin: " + origin; @@ -679,7 +679,7 @@ private IndexingStrategy planIndexingAsNonPrimary(Index index) throws IOExceptio } else { // This can happen if the primary is still on an old node and send traffic without seq# or we recover from translog // created by an old version. - assert config().getIndexSettings().getIndexVersionCreated().before(Version.V_6_0_0_alpha1_UNRELEASED) : + assert config().getIndexSettings().getIndexVersionCreated().before(Version.V_6_0_0_alpha1) : "index is newly created but op has no sequence numbers. op: " + index; opVsLucene = compareOpToLuceneDocBasedOnVersions(index); } @@ -963,7 +963,7 @@ private DeletionStrategy planDeletionAsNonPrimary(Delete delete) throws IOExcept if (delete.seqNo() != SequenceNumbersService.UNASSIGNED_SEQ_NO) { opVsLucene = compareOpToLuceneDocBasedOnSeqNo(delete); } else { - assert config().getIndexSettings().getIndexVersionCreated().before(Version.V_6_0_0_alpha1_UNRELEASED) : + assert config().getIndexSettings().getIndexVersionCreated().before(Version.V_6_0_0_alpha1) : "index is newly created but op has no sequence numbers. op: " + delete; opVsLucene = compareOpToLuceneDocBasedOnVersions(delete); } diff --git a/core/src/main/java/org/elasticsearch/index/engine/Segment.java b/core/src/main/java/org/elasticsearch/index/engine/Segment.java index 4de85d0020d9f..b9df309970462 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/Segment.java +++ b/core/src/main/java/org/elasticsearch/index/engine/Segment.java @@ -168,7 +168,7 @@ public void readFrom(StreamInput in) throws IOException { // verbose mode ramTree = readRamTree(in); } - if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { segmentSort = readSegmentSort(in); } else { segmentSort = null; @@ -193,7 +193,7 @@ public void writeTo(StreamOutput out) throws IOException { if (verbose) { writeRamTree(out, ramTree); } - if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { writeSegmentSort(out, segmentSort); } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java index 85403ccc03b85..f3001db39260a 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java @@ -104,7 +104,7 @@ public static class TypeParser implements MetadataFieldMapper.TypeParser { public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { if (node.isEmpty() == false && - parserContext.indexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + parserContext.indexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1)) { throw new IllegalArgumentException("[_all] is disabled in 6.0. As a replacement, you can use an [copy_to] " + "on mapping fields to create your own catch all field."); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 7bd8c79ee670a..93c8759d1dc52 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -233,7 +233,7 @@ protected void parseCreateField(ParseContext context, List field value = fieldType().nullValue(); } } else { - if (indexCreatedVersion.onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (indexCreatedVersion.onOrAfter(Version.V_6_0_0_alpha1)) { value = context.parser().booleanValue(); } else { value = context.parser().booleanValueLenient(); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/core/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java index ca8fcd1ffd8ba..0570cd7325243 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java @@ -205,7 +205,7 @@ public static DynamicTemplate parse(String name, Map conf, try { xcontentFieldType = XContentFieldType.fromString(matchMappingType); } catch (IllegalArgumentException e) { - if (indexVersionCreated.onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (indexVersionCreated.onOrAfter(Version.V_6_0_0_alpha1)) { throw e; } else { DEPRECATION_LOGGER.deprecated("match_mapping_type [" + matchMappingType + "] is invalid and will be ignored: " diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 4d68deef3b7c0..50f424c268f7c 100755 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -102,7 +102,7 @@ public enum MergeReason { Function defValue = settings -> { boolean singleType = true; if (settings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null) != null) { - singleType = Version.indexCreated(settings).onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED); + singleType = Version.indexCreated(settings).onOrAfter(Version.V_6_0_0_alpha1); } return Boolean.valueOf(singleType).toString(); }; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java b/core/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java index 5b2ec6b2d9375..18928246772f6 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java @@ -55,7 +55,7 @@ public class TypeParsers { //TODO 22298: Remove this method and have all call-sites use XContentMapValues.nodeBooleanValue(node) directly. public static boolean nodeBooleanValue(String fieldName, String propertyName, Object node, Mapper.TypeParser.ParserContext parserContext) { - if (parserContext.indexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (parserContext.indexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1)) { return XContentMapValues.nodeBooleanValue(node, fieldName + "." + propertyName); } else { return nodeBooleanValueLenient(fieldName, propertyName, node); @@ -247,7 +247,7 @@ && parseNorms(builder, name, propName, propNode, parserContext)) { if (parserContext.isWithinMultiField()) { throw new MapperParsingException("include_in_all in multi fields is not allowed. Found the include_in_all in field [" + name + "] which is within a multi field."); - } else if (parserContext.indexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + } else if (parserContext.indexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1)) { throw new MapperParsingException("[include_in_all] is not allowed for indices created on or after version 6.0.0 as " + "[_all] is deprecated. As a replacement, you can use an [copy_to] on mapping fields to create your " + "own catch all field."); diff --git a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java index c02ff4175c92c..cab2aee910fe5 100644 --- a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java @@ -88,7 +88,7 @@ public BoolQueryBuilder(StreamInput in) throws IOException { shouldClauses.addAll(readQueries(in)); filterClauses.addAll(readQueries(in)); adjustPureNegative = in.readBoolean(); - if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().before(Version.V_6_0_0_alpha1)) { in.readBoolean(); // disable_coord } minimumShouldMatch = in.readOptionalString(); @@ -101,7 +101,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { writeQueries(out, shouldClauses); writeQueries(out, filterClauses); out.writeBoolean(adjustPureNegative); - if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().before(Version.V_6_0_0_alpha1)) { out.writeBoolean(true); // disable_coord } out.writeOptionalString(minimumShouldMatch); diff --git a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java index e5fff943cd39a..63b09540a187d 100644 --- a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java @@ -111,7 +111,7 @@ public CommonTermsQueryBuilder(StreamInput in) throws IOException { analyzer = in.readOptionalString(); lowFreqMinimumShouldMatch = in.readOptionalString(); highFreqMinimumShouldMatch = in.readOptionalString(); - if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().before(Version.V_6_0_0_alpha1)) { in.readBoolean(); // disable_coord } cutoffFrequency = in.readFloat(); @@ -126,7 +126,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeOptionalString(analyzer); out.writeOptionalString(lowFreqMinimumShouldMatch); out.writeOptionalString(highFreqMinimumShouldMatch); - if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().before(Version.V_6_0_0_alpha1)) { out.writeBoolean(true); // disable_coord } out.writeFloat(cutoffFrequency); diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java index fde98ee0d4117..7b331dd487b51 100644 --- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java @@ -126,7 +126,7 @@ public InnerHitBuilder(String name) { */ public InnerHitBuilder(StreamInput in) throws IOException { name = in.readOptionalString(); - if (in.getVersion().before(Version.V_5_5_0_UNRELEASED)) { + if (in.getVersion().before(Version.V_5_5_0)) { in.readOptionalString(); in.readOptionalString(); } @@ -156,7 +156,7 @@ public InnerHitBuilder(StreamInput in) throws IOException { } } highlightBuilder = in.readOptionalWriteable(HighlightBuilder::new); - if (in.getVersion().before(Version.V_5_5_0_UNRELEASED)) { + if (in.getVersion().before(Version.V_5_5_0)) { /** * this is needed for BWC with nodes pre 5.5 */ @@ -168,8 +168,8 @@ public InnerHitBuilder(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getVersion().before(Version.V_5_5_0_UNRELEASED)) { - throw new IOException("Invalid output version, must >= " + Version.V_5_5_0_UNRELEASED.toString()); + if (out.getVersion().before(Version.V_5_5_0)) { + throw new IOException("Invalid output version, must >= " + Version.V_5_5_0.toString()); } out.writeOptionalString(name); out.writeBoolean(ignoreUnmapped); @@ -207,8 +207,8 @@ public void writeTo(StreamOutput out) throws IOException { * Should only be used to send nested inner hits to nodes pre 5.5. */ protected void writeToNestedBWC(StreamOutput out, QueryBuilder query, String nestedPath) throws IOException { - assert out.getVersion().before(Version.V_5_5_0_UNRELEASED) : - "invalid output version, must be < " + Version.V_5_5_0_UNRELEASED.toString(); + assert out.getVersion().before(Version.V_5_5_0) : + "invalid output version, must be < " + Version.V_5_5_0.toString(); writeToBWC(out, query, nestedPath, null); } @@ -217,8 +217,8 @@ protected void writeToNestedBWC(StreamOutput out, QueryBuilder query, String nes * Should only be used to send collapsing inner hits to nodes pre 5.5. */ public void writeToCollapseBWC(StreamOutput out) throws IOException { - assert out.getVersion().before(Version.V_5_5_0_UNRELEASED) : - "invalid output version, must be < " + Version.V_5_5_0_UNRELEASED.toString(); + assert out.getVersion().before(Version.V_5_5_0) : + "invalid output version, must be < " + Version.V_5_5_0.toString(); writeToBWC(out, new MatchAllQueryBuilder(), null, null); } @@ -227,8 +227,8 @@ public void writeToCollapseBWC(StreamOutput out) throws IOException { * Should only be used to send hasParent or hasChild inner hits to nodes pre 5.5. */ public void writeToParentChildBWC(StreamOutput out, QueryBuilder query, String parentChildPath) throws IOException { - assert(out.getVersion().before(Version.V_5_5_0_UNRELEASED)) : - "invalid output version, must be < " + Version.V_5_5_0_UNRELEASED.toString(); + assert(out.getVersion().before(Version.V_5_5_0)) : + "invalid output version, must be < " + Version.V_5_5_0.toString(); writeToBWC(out, query, null, parentChildPath); } diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java index 5863d02f903cf..74c80554109e3 100644 --- a/core/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java @@ -75,7 +75,7 @@ protected void setupInnerHitsContext(QueryShardContext queryShardContext, if (innerHitBuilder.getScriptFields() != null) { for (SearchSourceBuilder.ScriptField field : innerHitBuilder.getScriptFields()) { SearchScript searchScript = innerHitsContext.getQueryShardContext().getSearchScript(field.script(), - ScriptContext.SEARCH); + SearchScript.CONTEXT); innerHitsContext.scriptFields().add(new org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField( field.fieldName(), searchScript, field.ignoreFailure())); } diff --git a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java index a62b95337f945..ae092443f4989 100644 --- a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java @@ -103,7 +103,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeString(path); out.writeVInt(scoreMode.ordinal()); out.writeNamedWriteable(query); - if (out.getVersion().before(Version.V_5_5_0_UNRELEASED)) { + if (out.getVersion().before(Version.V_5_5_0)) { final boolean hasInnerHit = innerHitBuilder != null; out.writeBoolean(hasInnerHit); if (hasInnerHit) { diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java index 37473710e2507..02da924f93165 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java @@ -20,12 +20,10 @@ import org.apache.lucene.index.IndexReader; import org.elasticsearch.client.Client; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; @@ -106,7 +104,7 @@ public long nowInMillis() { } public String getTemplateBytes(Script template) { - CompiledTemplate compiledTemplate = scriptService.compileTemplate(template, ScriptContext.SEARCH); + CompiledTemplate compiledTemplate = scriptService.compileTemplate(template, ExecutableScript.CONTEXT); return compiledTemplate.run(template.getParams()); } } diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 31b2a71846c18..1ab89a0fe01d9 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.index.Index; @@ -50,7 +49,6 @@ import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.support.NestedScope; import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; @@ -331,39 +329,43 @@ public final Index index() { * Compiles (or retrieves from cache) and binds the parameters to the * provided script */ - public final SearchScript getSearchScript(Script script, ScriptContext context) { + public final SearchScript getSearchScript(Script script, ScriptContext context) { failIfFrozen(); - CompiledScript compile = scriptService.compile(script, context); - return scriptService.search(lookup(), compile, script.getParams()); + SearchScript.Factory factory = scriptService.compile(script, context); + return factory.newInstance(script.getParams(), lookup()); } /** * Returns a lazily created {@link SearchScript} that is compiled immediately but can be pulled later once all * parameters are available. */ - public final Function, SearchScript> getLazySearchScript(Script script, ScriptContext context) { + public final Function, SearchScript> getLazySearchScript( + Script script, ScriptContext context) { + // TODO: this "lazy" binding can be removed once scripted metric aggs have their own contexts, which take _agg/_aggs as a parameter failIfFrozen(); - CompiledScript compile = scriptService.compile(script, context); - return (p) -> scriptService.search(lookup(), compile, p); + SearchScript.Factory factory = scriptService.compile(script, context); + return (p) -> factory.newInstance(p, lookup()); } /** * Compiles (or retrieves from cache) and binds the parameters to the * provided script */ - public final ExecutableScript getExecutableScript(Script script, ScriptContext context) { + public final ExecutableScript getExecutableScript(Script script, ScriptContext context) { failIfFrozen(); - CompiledScript compiledScript = scriptService.compile(script, context); - return scriptService.executable(compiledScript, script.getParams()); + ExecutableScript.Factory factory = scriptService.compile(script, context); + return factory.newInstance(script.getParams()); } /** * Returns a lazily created {@link ExecutableScript} that is compiled immediately but can be pulled later once all * parameters are available. */ - public final Function, ExecutableScript> getLazyExecutableScript(Script script, ScriptContext context) { + public final Function, ExecutableScript> getLazyExecutableScript( + Script script, ScriptContext context) { + // TODO: this "lazy" binding can be removed once scripted metric aggs have their own contexts, which take _agg/_aggs as a parameter failIfFrozen(); - CompiledScript executable = scriptService.compile(script, context); - return (p) -> scriptService.executable(executable, p); + ExecutableScript.Factory factory = scriptService.compile(script, context); + return factory::newInstance; } /** diff --git a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java index 140fc04c95562..88fde50eb1b87 100644 --- a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java @@ -131,7 +131,7 @@ public static ScriptQueryBuilder fromXContent(QueryParseContext parseContext) th @Override protected Query doToQuery(QueryShardContext context) throws IOException { - return new ScriptQuery(script, context.getSearchScript(script, ScriptContext.SEARCH)); + return new ScriptQuery(script, context.getSearchScript(script, SearchScript.CONTEXT)); } static class ScriptQuery extends Query { diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java index 3195380d21678..2af4c9dd7516b 100644 --- a/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java @@ -94,7 +94,7 @@ protected int doHashCode() { @Override protected ScoreFunction doToFunction(QueryShardContext context) { try { - SearchScript searchScript = context.getSearchScript(script, ScriptContext.SEARCH); + SearchScript searchScript = context.getSearchScript(script, SearchScript.CONTEXT); return new ScriptScoreFunction(script, searchScript); } catch (Exception e) { throw new QueryShardException(context, "script_score: the script could not be loaded", e); diff --git a/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java b/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java index 716667dbb4dac..37c7b13ec79c9 100644 --- a/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java +++ b/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java @@ -80,8 +80,10 @@ public void onFailedQueryPhase(SearchContext searchContext) { computeStats(searchContext, statsHolder -> { if (searchContext.hasOnlySuggest()) { statsHolder.suggestCurrent.dec(); + assert statsHolder.suggestCurrent.count() >= 0; } else { statsHolder.queryCurrent.dec(); + assert statsHolder.queryCurrent.count() >= 0; } }); } @@ -92,9 +94,11 @@ public void onQueryPhase(SearchContext searchContext, long tookInNanos) { if (searchContext.hasOnlySuggest()) { statsHolder.suggestMetric.inc(tookInNanos); statsHolder.suggestCurrent.dec(); + assert statsHolder.suggestCurrent.count() >= 0; } else { statsHolder.queryMetric.inc(tookInNanos); statsHolder.queryCurrent.dec(); + assert statsHolder.queryCurrent.count() >= 0; } }); } @@ -114,6 +118,7 @@ public void onFetchPhase(SearchContext searchContext, long tookInNanos) { computeStats(searchContext, statsHolder -> { statsHolder.fetchMetric.inc(tookInNanos); statsHolder.fetchCurrent.dec(); + assert statsHolder.fetchCurrent.count() >= 0; }); } @@ -174,6 +179,7 @@ public void onNewScrollContext(SearchContext context) { @Override public void onFreeScrollContext(SearchContext context) { totalStats.scrollCurrent.dec(); + assert totalStats.scrollCurrent.count() >= 0; totalStats.scrollMetric.inc(System.nanoTime() - context.getOriginNanoTime()); } diff --git a/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java b/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java index a55b920bb27a9..7baed972fcba8 100644 --- a/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java +++ b/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java @@ -86,7 +86,7 @@ protected void sendReplicaRequest( final ConcreteReplicaRequest replicaRequest, final DiscoveryNode node, final ActionListener listener) { - if (node.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (node.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { super.sendReplicaRequest(replicaRequest, node, listener); } else { listener.onResponse(new ReplicaResponse(replicaRequest.getTargetAllocationID(), SequenceNumbersService.UNASSIGNED_SEQ_NO)); diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index f294252456ac2..35ab1db14189a 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -522,11 +522,12 @@ public Engine.Index prepareIndexOnPrimary(SourceToParse source, long version, Ve } } - public Engine.Index prepareIndexOnReplica(SourceToParse source, long seqNo, long version, VersionType versionType, long autoGeneratedIdTimestamp, - boolean isRetry) { + public Engine.Index prepareIndexOnReplica(SourceToParse source, long opSeqNo, long opPrimaryTerm, long version, VersionType versionType, + long autoGeneratedIdTimestamp, boolean isRetry) { try { verifyReplicationTarget(); - return prepareIndex(docMapper(source.type()), source, seqNo, primaryTerm, version, versionType, + assert opPrimaryTerm <= this.primaryTerm : "op term [ " + opPrimaryTerm + " ] > shard term [" + this.primaryTerm + "]"; + return prepareIndex(docMapper(source.type()), source, opSeqNo, opPrimaryTerm, version, versionType, Engine.Operation.Origin.REPLICA, autoGeneratedIdTimestamp, isRetry); } catch (Exception e) { verifyNotClosed(e); @@ -594,11 +595,12 @@ public Engine.Delete prepareDeleteOnPrimary(String type, String id, long version versionType, Engine.Operation.Origin.PRIMARY); } - public Engine.Delete prepareDeleteOnReplica(String type, String id, long seqNo, long primaryTerm, + public Engine.Delete prepareDeleteOnReplica(String type, String id, long opSeqNo, long opPrimaryTerm, long version, VersionType versionType) { verifyReplicationTarget(); + assert opPrimaryTerm <= this.primaryTerm : "op term [ " + opPrimaryTerm + " ] > shard term [" + this.primaryTerm + "]"; final Term uid = extractUidForDelete(type, id); - return prepareDelete(type, id, uid, seqNo, primaryTerm, version, versionType, Engine.Operation.Origin.REPLICA); + return prepareDelete(type, id, uid, opSeqNo, opPrimaryTerm, version, versionType, Engine.Operation.Origin.REPLICA); } private static Engine.Delete prepareDelete(String type, String id, Term uid, long seqNo, long primaryTerm, long version, @@ -1090,7 +1092,7 @@ private void internalPerformTranslogRecovery(boolean skipTranslogRecovery, boole private boolean assertMaxUnsafeAutoIdInCommit() throws IOException { final Map userData = SegmentInfos.readLatestCommit(store.directory()).getUserData(); - if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_5_5_0_UNRELEASED) && + if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_5_5_0) && // TODO: LOCAL_SHARDS need to transfer this information recoveryState().getRecoverySource().getType() != RecoverySource.Type.LOCAL_SHARDS) { // as of 5.5.0, the engine stores the maxUnsafeAutoIdTimestamp in the commit point. @@ -1875,10 +1877,12 @@ public void acquireReplicaOperationPermit( if (operationPrimaryTerm > primaryTerm) { try { indexShardOperationPermits.blockOperations(30, TimeUnit.MINUTES, () -> { - assert operationPrimaryTerm > primaryTerm; + assert operationPrimaryTerm > primaryTerm : + "shard term already update. op term [" + operationPrimaryTerm + "], shardTerm [" + primaryTerm + "]"; primaryTerm = operationPrimaryTerm; + getEngine().getTranslog().rollGeneration(); }); - } catch (final InterruptedException | TimeoutException e) { + } catch (final Exception e) { onPermitAcquired.onFailure(e); return; } diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java index 016067259c11b..fea26168efa9d 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.action.support.ThreadedActionListener; +import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext; @@ -70,7 +71,8 @@ public void close() { * @throws TimeoutException if timed out waiting for in-flight operations to finish * @throws IndexShardClosedException if operation permit has been closed */ - public void blockOperations(long timeout, TimeUnit timeUnit, Runnable onBlocked) throws InterruptedException, TimeoutException { + public void blockOperations(long timeout, TimeUnit timeUnit, CheckedRunnable onBlocked) throws + InterruptedException, TimeoutException, E { if (closed) { throw new IndexShardClosedException(shardId); } @@ -109,9 +111,9 @@ public void blockOperations(long timeout, TimeUnit timeUnit, Runnable onBlocked) /** * Acquires a permit whenever permit acquisition is not blocked. If the permit is directly available, the provided - * {@link ActionListener} will be called on the calling thread. During calls of {@link #blockOperations(long, TimeUnit, Runnable)}, - * permit acquisition can be delayed. The provided ActionListener will then be called using the provided executor once operations are no - * longer blocked. + * {@link ActionListener} will be called on the calling thread. During calls of + * {@link #blockOperations(long, TimeUnit, CheckedRunnable)}, permit acquisition can be delayed. The provided ActionListener will + * then be called using the provided executor once operations are no longer blocked. * * @param onAcquired {@link ActionListener} that is invoked once acquisition is successful or failed * @param executorOnDelay executor to use for delayed call diff --git a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java index f4f611b733b5a..668e957ae52ec 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java +++ b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java @@ -24,12 +24,10 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.engine.IgnoreOnRecoveryEngineException; import org.elasticsearch.index.mapper.DocumentMapperForType; import org.elasticsearch.index.mapper.MapperException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; -import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.rest.RestStatus; @@ -149,59 +147,39 @@ private void maybeAddMappingUpdate(String type, Mapping update, String docId, bo * is encountered. */ private void performRecoveryOperation(Engine engine, Translog.Operation operation, boolean allowMappingUpdates, Engine.Operation.Origin origin) throws IOException { - - try { - switch (operation.opType()) { - case INDEX: - Translog.Index index = (Translog.Index) operation; - // we set canHaveDuplicates to true all the time such that we de-optimze the translog case and ensure that all - // autoGeneratedID docs that are coming from the primary are updated correctly. - Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()), - source(shardId.getIndexName(), index.type(), index.id(), index.source(), XContentFactory.xContentType(index.source())) - .routing(index.routing()).parent(index.parent()), index.seqNo(), index.primaryTerm(), - index.version(), index.versionType().versionTypeForReplicationAndRecovery(), origin, index.getAutoGeneratedIdTimestamp(), true); - maybeAddMappingUpdate(engineIndex.type(), engineIndex.parsedDoc().dynamicMappingsUpdate(), engineIndex.id(), allowMappingUpdates); - logger.trace("[translog] recover [index] op [({}, {})] of [{}][{}]", index.seqNo(), index.primaryTerm(), index.type(), index.id()); - index(engine, engineIndex); - break; - case DELETE: - Translog.Delete delete = (Translog.Delete) operation; - logger.trace("[translog] recover [delete] op [({}, {})] of [{}][{}]", delete.seqNo(), delete.primaryTerm(), delete.type(), delete.id()); - final Engine.Delete engineDelete = new Engine.Delete(delete.type(), delete.id(), delete.uid(), delete.seqNo(), - delete.primaryTerm(), delete.version(), delete.versionType().versionTypeForReplicationAndRecovery(), - origin, System.nanoTime()); - delete(engine, engineDelete); - break; - case NO_OP: - final Translog.NoOp noOp = (Translog.NoOp) operation; - final long seqNo = noOp.seqNo(); - final long primaryTerm = noOp.primaryTerm(); - final String reason = noOp.reason(); - logger.trace("[translog] recover [no_op] op [({}, {})] of [{}]", seqNo, primaryTerm, reason); - final Engine.NoOp engineNoOp = - new Engine.NoOp(seqNo, primaryTerm, origin, System.nanoTime(), reason); - noOp(engine, engineNoOp); - break; - default: - throw new IllegalStateException("No operation defined for [" + operation + "]"); - } - } catch (ElasticsearchException e) { - boolean hasIgnoreOnRecoveryException = false; - ElasticsearchException current = e; - while (true) { - if (current instanceof IgnoreOnRecoveryEngineException) { - hasIgnoreOnRecoveryException = true; - break; - } - if (current.getCause() instanceof ElasticsearchException) { - current = (ElasticsearchException) current.getCause(); - } else { - break; - } - } - if (!hasIgnoreOnRecoveryException) { - throw e; - } + switch (operation.opType()) { + case INDEX: + Translog.Index index = (Translog.Index) operation; + // we set canHaveDuplicates to true all the time such that we de-optimze the translog case and ensure that all + // autoGeneratedID docs that are coming from the primary are updated correctly. + Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()), + source(shardId.getIndexName(), index.type(), index.id(), index.source(), XContentFactory.xContentType(index.source())) + .routing(index.routing()).parent(index.parent()), index.seqNo(), index.primaryTerm(), + index.version(), index.versionType().versionTypeForReplicationAndRecovery(), origin, index.getAutoGeneratedIdTimestamp(), true); + maybeAddMappingUpdate(engineIndex.type(), engineIndex.parsedDoc().dynamicMappingsUpdate(), engineIndex.id(), allowMappingUpdates); + logger.trace("[translog] recover [index] op [({}, {})] of [{}][{}]", index.seqNo(), index.primaryTerm(), index.type(), index.id()); + index(engine, engineIndex); + break; + case DELETE: + Translog.Delete delete = (Translog.Delete) operation; + logger.trace("[translog] recover [delete] op [({}, {})] of [{}][{}]", delete.seqNo(), delete.primaryTerm(), delete.type(), delete.id()); + final Engine.Delete engineDelete = new Engine.Delete(delete.type(), delete.id(), delete.uid(), delete.seqNo(), + delete.primaryTerm(), delete.version(), delete.versionType().versionTypeForReplicationAndRecovery(), + origin, System.nanoTime()); + delete(engine, engineDelete); + break; + case NO_OP: + final Translog.NoOp noOp = (Translog.NoOp) operation; + final long seqNo = noOp.seqNo(); + final long primaryTerm = noOp.primaryTerm(); + final String reason = noOp.reason(); + logger.trace("[translog] recover [no_op] op [({}, {})] of [{}]", seqNo, primaryTerm, reason); + final Engine.NoOp engineNoOp = + new Engine.NoOp(seqNo, primaryTerm, origin, System.nanoTime(), reason); + noOp(engine, engineNoOp); + break; + default: + throw new IllegalStateException("No operation defined for [" + operation + "]"); } operationProcessed(); } diff --git a/core/src/main/java/org/elasticsearch/index/store/StoreStats.java b/core/src/main/java/org/elasticsearch/index/store/StoreStats.java index 422508d8237b1..1ed7d6d01e823 100644 --- a/core/src/main/java/org/elasticsearch/index/store/StoreStats.java +++ b/core/src/main/java/org/elasticsearch/index/store/StoreStats.java @@ -68,7 +68,7 @@ public ByteSizeValue getSize() { @Override public void readFrom(StreamInput in) throws IOException { sizeInBytes = in.readVLong(); - if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().before(Version.V_6_0_0_alpha1)) { in.readVLong(); // throttleTimeInNanos } } @@ -76,7 +76,7 @@ public void readFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(sizeInBytes); - if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().before(Version.V_6_0_0_alpha1)) { out.writeVLong(0L); // throttleTimeInNanos } } diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java index 8d2be4104135d..7b6922e786723 100644 --- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -910,11 +910,11 @@ public Index(Engine.Index index, Engine.IndexResult indexResult) { this.autoGeneratedIdTimestamp = index.getAutoGeneratedIdTimestamp(); } - public Index(String type, String id, byte[] source) { + public Index(String type, String id, long seqNo, byte[] source) { this.type = type; this.id = id; this.source = new BytesArray(source); - this.seqNo = 0; + this.seqNo = seqNo; version = Versions.MATCH_ANY; versionType = VersionType.INTERNAL; routing = null; @@ -1037,9 +1037,11 @@ public int hashCode() { @Override public String toString() { return "Index{" + - "id='" + id + '\'' + - ", type='" + type + '\'' + - '}'; + "id='" + id + '\'' + + ", type='" + type + '\'' + + ", seqNo=" + seqNo + + ", primaryTerm=" + primaryTerm + + '}'; } public long getAutoGeneratedIdTimestamp() { @@ -1079,8 +1081,8 @@ public Delete(Engine.Delete delete, Engine.DeleteResult deleteResult) { } /** utility for testing */ - public Delete(String type, String id, Term uid) { - this(type, id, uid, 0, 0, Versions.MATCH_ANY, VersionType.INTERNAL); + public Delete(String type, String id, long seqNo, Term uid) { + this(type, id, uid, seqNo, 0, Versions.MATCH_ANY, VersionType.INTERNAL); } public Delete(String type, String id, Term uid, long seqNo, long primaryTerm, long version, VersionType versionType) { @@ -1180,10 +1182,11 @@ public int hashCode() { @Override public String toString() { return "Delete{" + - "uid=" + uid + - '}'; + "uid=" + uid + + ", seqNo=" + seqNo + + ", primaryTerm=" + primaryTerm + + '}'; } - } public static class NoOp implements Operation { @@ -1260,9 +1263,16 @@ public int hashCode() { return 31 * 31 * 31 + 31 * 31 * Long.hashCode(seqNo) + 31 * Long.hashCode(primaryTerm) + reason().hashCode(); } + @Override + public String toString() { + return "NoOp{" + + "seqNo=" + seqNo + + ", primaryTerm=" + primaryTerm + + ", reason='" + reason + '\'' + + '}'; + } } - public enum Durability { /** diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java index daf9a44b6665e..4a98365e02fba 100644 --- a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java +++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java @@ -24,7 +24,10 @@ import org.apache.lucene.store.OutputStreamDataOutput; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; +import org.elasticsearch.Assertions; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.Channels; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.index.seqno.SequenceNumbers; @@ -39,6 +42,8 @@ import java.nio.channels.FileChannel; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.LongSupplier; @@ -71,6 +76,8 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable { // lock order synchronized(syncLock) -> synchronized(this) private final Object syncLock = new Object(); + private final Map> seenSequenceNumbers; + private TranslogWriter( final ChannelFactory channelFactory, final ShardId shardId, @@ -90,6 +97,7 @@ private TranslogWriter( assert initialCheckpoint.maxSeqNo == SequenceNumbersService.NO_OPS_PERFORMED : initialCheckpoint.maxSeqNo; this.maxSeqNo = initialCheckpoint.maxSeqNo; this.globalCheckpointSupplier = globalCheckpointSupplier; + this.seenSequenceNumbers = Assertions.ENABLED ? new HashMap<>() : null; } static int getHeaderLength(String translogUUID) { @@ -195,9 +203,30 @@ public synchronized Translog.Location add(final BytesReference data, final long operationCounter++; + assert assertNoSeqNumberConflict(seqNo, data); + return new Translog.Location(generation, offset, data.length()); } + private synchronized boolean assertNoSeqNumberConflict(long seqNo, BytesReference data) throws IOException { + if (seqNo == SequenceNumbersService.UNASSIGNED_SEQ_NO) { + // nothing to do + } else if (seenSequenceNumbers.containsKey(seqNo)) { + final Tuple previous = seenSequenceNumbers.get(seqNo); + if (previous.v1().equals(data) == false) { + Translog.Operation newOp = Translog.readOperation(new BufferedChecksumStreamInput(data.streamInput())); + Translog.Operation prvOp = Translog.readOperation(new BufferedChecksumStreamInput(previous.v1().streamInput())); + throw new AssertionError( + "seqNo [" + seqNo + "] was processed twice in generation [" + generation + "], with different data. " + + "prvOp [" + prvOp + "], newOp [" + newOp + "]", previous.v2()); + } + } else { + seenSequenceNumbers.put(seqNo, + new Tuple<>(new BytesArray(data.toBytesRef(), true), new RuntimeException("stack capture previous op"))); + } + return true; + } + /** * write all buffered ops to disk and fsync file. * diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index 147a952507ad9..d230785a23e4f 100644 --- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -591,7 +591,7 @@ private Set allocationIdsForShardsOnNodesThatUnderstandSeqNos( final DiscoveryNodes nodes) { return shardRoutings .stream() - .filter(sr -> nodes.get(sr.currentNodeId()).getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) + .filter(sr -> nodes.get(sr.currentNodeId()).getVersion().onOrAfter(Version.V_6_0_0_alpha1)) .map(ShardRouting::allocationId) .map(AllocationId::getId) .collect(Collectors.toSet()); diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryFinalizeRecoveryRequest.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryFinalizeRecoveryRequest.java index eaace66107790..2bdf45fede28c 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryFinalizeRecoveryRequest.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryFinalizeRecoveryRequest.java @@ -60,7 +60,7 @@ public void readFrom(StreamInput in) throws IOException { super.readFrom(in); recoveryId = in.readLong(); shardId = ShardId.readShardId(in); - if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { globalCheckpoint = in.readZLong(); } else { globalCheckpoint = SequenceNumbersService.UNASSIGNED_SEQ_NO; @@ -72,7 +72,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeLong(recoveryId); shardId.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { out.writeZLong(globalCheckpoint); } } diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryPrepareForTranslogOperationsRequest.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryPrepareForTranslogOperationsRequest.java index 155aa53e71a5d..61cd986a1aef4 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryPrepareForTranslogOperationsRequest.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryPrepareForTranslogOperationsRequest.java @@ -61,7 +61,7 @@ public void readFrom(StreamInput in) throws IOException { recoveryId = in.readLong(); shardId = ShardId.readShardId(in); totalTranslogOps = in.readVInt(); - if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().before(Version.V_6_0_0_alpha1)) { in.readLong(); // maxUnsafeAutoIdTimestamp } } @@ -72,7 +72,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(recoveryId); shardId.writeTo(out); out.writeVInt(totalTranslogOps); - if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().before(Version.V_6_0_0_alpha1)) { out.writeLong(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP); // maxUnsafeAutoIdTimestamp } } diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTranslogOperationsResponse.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTranslogOperationsResponse.java index 7427e631fb070..731eb28ed92c7 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTranslogOperationsResponse.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTranslogOperationsResponse.java @@ -44,7 +44,7 @@ public class RecoveryTranslogOperationsResponse extends TransportResponse { @Override public void writeTo(final StreamOutput out) throws IOException { // before 6.0.0 we responded with an empty response so we have to maintain that - if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { out.writeZLong(localCheckpoint); } } @@ -52,7 +52,7 @@ public void writeTo(final StreamOutput out) throws IOException { @Override public void readFrom(final StreamInput in) throws IOException { // before 6.0.0 we received an empty response so we have to maintain that - if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { localCheckpoint = in.readZLong(); } else { diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java b/core/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java index a2a578cc7229a..825fa8306bada 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java @@ -119,7 +119,7 @@ public void readFrom(StreamInput in) throws IOException { targetNode = new DiscoveryNode(in); metadataSnapshot = new Store.MetadataSnapshot(in); primaryRelocation = in.readBoolean(); - if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { startingSeqNo = in.readLong(); } else { startingSeqNo = SequenceNumbersService.UNASSIGNED_SEQ_NO; @@ -136,7 +136,7 @@ public void writeTo(StreamOutput out) throws IOException { targetNode.writeTo(out); metadataSnapshot.writeTo(out); out.writeBoolean(primaryRelocation); - if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { out.writeLong(startingSeqNo); } } diff --git a/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java b/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java index 34a85ab9ab1f7..fa5444102a8d8 100644 --- a/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java +++ b/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java @@ -24,6 +24,7 @@ import org.apache.logging.log4j.util.Supplier; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; @@ -44,7 +45,7 @@ public Template compile(String template) { int mustacheEnd = template.indexOf("}}"); if (mustacheStart != -1 && mustacheEnd != -1 && mustacheStart < mustacheEnd) { Script script = new Script(ScriptType.INLINE, "mustache", template, Collections.emptyMap()); - CompiledTemplate compiledTemplate = scriptService.compileTemplate(script, ScriptContext.INGEST); + CompiledTemplate compiledTemplate = scriptService.compileTemplate(script, ExecutableScript.INGEST_CONTEXT); return new Template() { @Override public String execute(Map model) { diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java index 6f5e9a52b4fd0..f2538365a7174 100644 --- a/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java +++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java @@ -70,7 +70,7 @@ public Path(StreamInput in) throws IOException { total = in.readLong(); free = in.readLong(); available = in.readLong(); - if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().before(Version.V_6_0_0_alpha1)) { in.readOptionalBoolean(); } } @@ -83,7 +83,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(total); out.writeLong(free); out.writeLong(available); - if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().before(Version.V_6_0_0_alpha1)) { out.writeOptionalBoolean(null); } } @@ -455,7 +455,7 @@ public FsInfo(StreamInput in) throws IOException { paths[i] = new Path(in); } this.total = total(); - if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { this.leastDiskEstimate = in.readOptionalWriteable(DiskUsage::new); this.mostDiskEstimate = in.readOptionalWriteable(DiskUsage::new); } else { @@ -472,7 +472,7 @@ public void writeTo(StreamOutput out) throws IOException { for (Path path : paths) { path.writeTo(out); } - if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { out.writeOptionalWriteable(this.leastDiskEstimate); out.writeOptionalWriteable(this.mostDiskEstimate); } diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java index 8e3cd53e74f1c..f88ddcf482530 100644 --- a/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java +++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java @@ -155,8 +155,8 @@ public static FsInfo.Path getFSInfo(NodePath nodePath) throws IOException { // since recomputing these once per second (default) could be costly, // and they should not change: fsPath.total = adjustForHugeFilesystems(nodePath.fileStore.getTotalSpace()); - fsPath.free = nodePath.fileStore.getUnallocatedSpace(); - fsPath.available = nodePath.fileStore.getUsableSpace(); + fsPath.free = adjustForHugeFilesystems(nodePath.fileStore.getUnallocatedSpace()); + fsPath.available = adjustForHugeFilesystems(nodePath.fileStore.getUsableSpace()); fsPath.type = nodePath.fileStore.type(); fsPath.mount = nodePath.fileStore.toString(); return fsPath; diff --git a/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java b/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java index 125b58a498d2c..88af291983af8 100644 --- a/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java +++ b/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.plugins; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -32,9 +33,11 @@ public interface ScriptPlugin { /** - * Returns a {@link ScriptEngine} instance or null if this plugin doesn't add a new script engine + * Returns a {@link ScriptEngine} instance or null if this plugin doesn't add a new script engine. + * @param settings Node settings + * @param contexts The contexts that {@link ScriptEngine#compile(String, String, ScriptContext, Map)} may be called with */ - default ScriptEngine getScriptEngine(Settings settings) { + default ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { return null; } diff --git a/core/src/main/java/org/elasticsearch/script/CompiledScript.java b/core/src/main/java/org/elasticsearch/script/CompiledScript.java deleted file mode 100644 index 818971f0f8990..0000000000000 --- a/core/src/main/java/org/elasticsearch/script/CompiledScript.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.script; - -/** - * CompiledScript holds all the parameters necessary to execute a previously compiled script. - */ -public class CompiledScript { - - private final ScriptType type; - private final String name; - private final String lang; - private final Object compiled; - - /** - * Constructor for CompiledScript. - * @param type The type of script to be executed. - * @param name The name of the script to be executed. - * @param lang The language of the script to be executed. - * @param compiled The compiled script Object that is executable. - */ - public CompiledScript(ScriptType type, String name, String lang, Object compiled) { - this.type = type; - this.name = name; - this.lang = lang; - this.compiled = compiled; - } - - /** - * Method to get the type of language. - * @return The type of language the script was compiled in. - */ - public ScriptType type() { - return type; - } - - /** - * Method to get the name of the script. - * @return The name of the script to be executed. - */ - public String name() { - return name; - } - - /** - * Method to get the language. - * @return The language of the script to be executed. - */ - public String lang() { - return lang; - } - - /** - * Method to get the compiled script object. - * @return The compiled script Object that is executable. - */ - public Object compiled() { - return compiled; - } - - /** - * @return A string composed of type, lang, and name to describe the CompiledScript. - */ - @Override - public String toString() { - return type + " script [" + name + "] using lang [" + lang + "]"; - } -} diff --git a/core/src/main/java/org/elasticsearch/script/ExecutableScript.java b/core/src/main/java/org/elasticsearch/script/ExecutableScript.java index e3f8eb4744f2c..e87b7cdf3890a 100644 --- a/core/src/main/java/org/elasticsearch/script/ExecutableScript.java +++ b/core/src/main/java/org/elasticsearch/script/ExecutableScript.java @@ -19,6 +19,8 @@ package org.elasticsearch.script; +import java.util.Map; + /** * An executable script, can't be used concurrently. */ @@ -38,4 +40,15 @@ public interface ExecutableScript { * Executes the script. */ Object run(); + + interface Factory { + ExecutableScript newInstance(Map params); + } + + ScriptContext CONTEXT = new ScriptContext<>("executable", Factory.class); + + // TODO: remove these once each has its own script interface + ScriptContext AGGS_CONTEXT = new ScriptContext<>("aggs_executable", Factory.class); + ScriptContext UPDATE_CONTEXT = new ScriptContext<>("update", Factory.class); + ScriptContext INGEST_CONTEXT = new ScriptContext<>("ingest", Factory.class); } diff --git a/core/src/main/java/org/elasticsearch/script/ScriptContext.java b/core/src/main/java/org/elasticsearch/script/ScriptContext.java index cd3bff33794ea..420257e0e5089 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptContext.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptContext.java @@ -19,35 +19,56 @@ package org.elasticsearch.script; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; +import java.lang.reflect.Method; /** - * A holder for information about a context in which a script is compiled and run. + * The information necessary to compile and run a script. + * + * A {@link ScriptContext} contains the information related to a single use case and the interfaces + * and methods necessary for a {@link ScriptEngine} to implement. + *

+ * There are two related classes which must be supplied to construct a {@link ScriptContext}. + *

+ * The FactoryType is a factory class for constructing instances of a script. The + * {@link ScriptService} returns an instance of FactoryType when compiling a script. This class + * must be stateless so it is cacheable by the {@link ScriptService}. It must have an abstract method + * named {@code newInstance} which {@link ScriptEngine} implementations will define. + *

+ * The InstanceType is a class returned by the {@code newInstance} method of the + * FactoryType. It is an instance of a script and may be stateful. Instances of + * the InstanceType may be executed multiple times by a caller with different arguments. This + * class must have an abstract method named {@code execute} which {@link ScriptEngine} implementations + * will define. */ -public final class ScriptContext { - - public static final ScriptContext AGGS = new ScriptContext("aggs"); - public static final ScriptContext SEARCH = new ScriptContext("search"); - public static final ScriptContext UPDATE = new ScriptContext("update"); - public static final ScriptContext INGEST = new ScriptContext("ingest"); - - public static final Map BUILTINS; - static { - Map builtins = new HashMap<>(); - builtins.put(AGGS.name, AGGS); - builtins.put(SEARCH.name, SEARCH); - builtins.put(UPDATE.name, UPDATE); - builtins.put(INGEST.name, INGEST); - BUILTINS = Collections.unmodifiableMap(builtins); - } +public final class ScriptContext { /** A unique identifier for this context. */ public final String name; - // pkg private ctor, only created by script module - public ScriptContext(String name) { + /** A factory class for constructing instances of a script. */ + public final Class factoryClazz; + + /** A class that is an instance of a script. */ + public final Class instanceClazz; + + /** Construct a context with the related instance and compiled classes. */ + public ScriptContext(String name, Class factoryClazz) { this.name = name; + this.factoryClazz = factoryClazz; + Method newInstanceMethod = null; + for (Method method : factoryClazz.getMethods()) { + if (method.getName().equals("newInstance")) { + if (newInstanceMethod != null) { + throw new IllegalArgumentException("Cannot have multiple newInstance methods on FactoryType class [" + + factoryClazz.getName() + "] for script context [" + name + "]"); + } + newInstanceMethod = method; + } + } + if (newInstanceMethod == null) { + throw new IllegalArgumentException("Could not find method newInstance on FactoryType class [" + + factoryClazz.getName() + "] for script context [" + name + "]"); + } + instanceClazz = newInstanceMethod.getReturnType(); } } diff --git a/core/src/main/java/org/elasticsearch/script/ScriptEngine.java b/core/src/main/java/org/elasticsearch/script/ScriptEngine.java index 9572e891b3c09..bd32cce0b3781 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptEngine.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptEngine.java @@ -19,10 +19,8 @@ package org.elasticsearch.script; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.search.lookup.SearchLookup; - import java.io.Closeable; +import java.io.IOException; import java.util.Map; /** @@ -39,12 +37,12 @@ public interface ScriptEngine extends Closeable { * Compiles a script. * @param name the name of the script. {@code null} if it is anonymous (inline). For a stored script, its the identifier. * @param code actual source of the script + * @param context the context this script will be used for * @param params compile-time parameters (such as flags to the compiler) - * @return an opaque compiled script which may be cached and later passed to + * @return A compiled script of the FactoryType from {@link ScriptContext} */ - Object compile(String name, String code, Map params); - - ExecutableScript executable(CompiledScript compiledScript, @Nullable Map vars); + FactoryType compile(String name, String code, ScriptContext context, Map params); - SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map vars); + @Override + default void close() throws IOException {} } diff --git a/core/src/main/java/org/elasticsearch/script/ScriptModule.java b/core/src/main/java/org/elasticsearch/script/ScriptModule.java index b29a199c3aba2..cc098c14359a5 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptModule.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptModule.java @@ -19,10 +19,14 @@ package org.elasticsearch.script; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; @@ -32,11 +36,24 @@ * Manages building {@link ScriptService}. */ public class ScriptModule { + + public static final Map> CORE_CONTEXTS; + static { + CORE_CONTEXTS = Stream.of( + SearchScript.CONTEXT, + SearchScript.AGGS_CONTEXT, + ExecutableScript.CONTEXT, + ExecutableScript.AGGS_CONTEXT, + ExecutableScript.UPDATE_CONTEXT, + ExecutableScript.INGEST_CONTEXT + ).collect(Collectors.toMap(c -> c.name, Function.identity())); + } + private final ScriptService scriptService; public ScriptModule(Settings settings, List scriptPlugins) { Map engines = new HashMap<>(); - Map contexts = new HashMap<>(ScriptContext.BUILTINS); + Map> contexts = new HashMap<>(CORE_CONTEXTS); for (ScriptPlugin plugin : scriptPlugins) { for (ScriptContext context : plugin.getContexts()) { ScriptContext oldContext = contexts.put(context.name, context); @@ -44,7 +61,9 @@ public ScriptModule(Settings settings, List scriptPlugins) { throw new IllegalArgumentException("Context name [" + context.name + "] defined twice"); } } - ScriptEngine engine = plugin.getScriptEngine(settings); + } + for (ScriptPlugin plugin : scriptPlugins) { + ScriptEngine engine = plugin.getScriptEngine(settings, contexts.values()); if (engine != null) { ScriptEngine existing = engines.put(engine.getType(), engine); if (existing != null) { diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java index e5892ef1dcd4b..2634cbda8dd78 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java @@ -45,7 +45,6 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.template.CompiledTemplate; import java.io.Closeable; @@ -82,9 +81,9 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust private final Set contextsAllowed; private final Map engines; - private final Map contexts; + private final Map> contexts; - private final Cache cache; + private final Cache cache; private final ScriptMetrics scriptMetrics = new ScriptMetrics(); @@ -95,7 +94,7 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust private double scriptsPerMinCounter; private double compilesAllowedPerNano; - public ScriptService(Settings settings, Map engines, Map contexts) { + public ScriptService(Settings settings, Map engines, Map> contexts) { super(settings); Objects.requireNonNull(settings); @@ -176,7 +175,7 @@ public ScriptService(Settings settings, Map engines, Map cacheBuilder = CacheBuilder.builder(); + CacheBuilder cacheBuilder = CacheBuilder.builder(); if (cacheMaxSize >= 0) { cacheBuilder.setMaximumWeight(cacheMaxSize); } @@ -218,9 +217,11 @@ void setMaxCompilationsPerMinute(Integer newMaxPerMinute) { } /** - * Checks if a script can be executed and compiles it if needed, or returns the previously compiled and cached script. + * Compiles a script using the given context. + * + * @return a compiled script which may be used to construct instances of a script for the given context */ - public CompiledScript compile(Script script, ScriptContext context) { + public FactoryType compile(Script script, ScriptContext context) { Objects.requireNonNull(script); Objects.requireNonNull(context); @@ -263,7 +264,7 @@ public CompiledScript compile(Script script, ScriptContext context) { // TODO: fix this through some API or something, that's wrong // special exception to prevent expressions from compiling as update or mapping scripts boolean expression = "expression".equals(script.getLang()); - boolean notSupported = context.name.equals(ScriptContext.UPDATE.name); + boolean notSupported = context.name.equals(ExecutableScript.UPDATE_CONTEXT.name); if (expression && notSupported) { throw new UnsupportedOperationException("scripts of type [" + script.getType() + "]," + " operation [" + context.name + "] and lang [" + lang + "] are not supported"); @@ -287,11 +288,11 @@ public CompiledScript compile(Script script, ScriptContext context) { logger.trace("compiling lang: [{}] type: [{}] script: {}", lang, type, idOrCode); } - CacheKey cacheKey = new CacheKey(lang, idOrCode, options); - CompiledScript compiledScript = cache.get(cacheKey); + CacheKey cacheKey = new CacheKey(lang, idOrCode, context.name, options); + Object compiledScript = cache.get(cacheKey); if (compiledScript != null) { - return compiledScript; + return context.factoryClazz.cast(compiledScript); } // Synchronize so we don't compile scripts many times during multiple shards all compiling a script @@ -311,7 +312,7 @@ public CompiledScript compile(Script script, ScriptContext context) { } // Check whether too many compilations have happened checkCompilationLimit(); - compiledScript = new CompiledScript(type, id, lang, scriptEngine.compile(id, idOrCode, options)); + compiledScript = scriptEngine.compile(id, idOrCode, context, options); } catch (ScriptException good) { // TODO: remove this try-catch completely, when all script engines have good exceptions! throw good; // its already good @@ -325,14 +326,14 @@ public CompiledScript compile(Script script, ScriptContext context) { cache.put(cacheKey, compiledScript); } - return compiledScript; + return context.factoryClazz.cast(compiledScript); } } /** Compiles a template. Note this will be moved to a separate TemplateService in the future. */ - public CompiledTemplate compileTemplate(Script script, ScriptContext scriptContext) { - CompiledScript compiledScript = compile(script, scriptContext); - return params -> (String)executable(compiledScript, params).run(); + public CompiledTemplate compileTemplate(Script script, ScriptContext scriptContext) { + ExecutableScript.Factory factory = compile(script, scriptContext); + return params -> (String) factory.newInstance(params).run(); } /** @@ -431,7 +432,8 @@ public void putStoredScript(ClusterService clusterService, PutStoredScriptReques throw new IllegalArgumentException( "cannot put [" + ScriptType.STORED + "] script, no script contexts are enabled"); } else { - Object compiled = scriptEngine.compile(request.id(), source.getCode(), Collections.emptyMap()); + // TODO: executable context here is just a placeholder, replace with optional context name passed into PUT stored script req + Object compiled = scriptEngine.compile(request.id(), source.getCode(), ExecutableScript.CONTEXT, Collections.emptyMap()); if (compiled == null) { throw new IllegalArgumentException("failed to parse/compile stored script [" + request.id() + "]" + @@ -498,21 +500,6 @@ public StoredScriptSource getStoredScript(ClusterState state, GetStoredScriptReq } } - /** - * Executes a previously compiled script provided as an argument - */ - public ExecutableScript executable(CompiledScript compiledScript, Map params) { - return getEngine(compiledScript.lang()).executable(compiledScript, params); - } - - /** - * Binds provided parameters to a compiled script returning a - * {@link SearchScript} ready for execution - */ - public SearchScript search(SearchLookup lookup, CompiledScript compiledScript, Map params) { - return getEngine(compiledScript.lang()).search(compiledScript, lookup, params); - } - public ScriptStats stats() { return scriptMetrics.stats(); } @@ -527,9 +514,9 @@ public void clusterChanged(ClusterChangedEvent event) { * {@code ScriptEngine}'s {@code scriptRemoved} method when the * script has been removed from the cache */ - private class ScriptCacheRemovalListener implements RemovalListener { + private class ScriptCacheRemovalListener implements RemovalListener { @Override - public void onRemoval(RemovalNotification notification) { + public void onRemoval(RemovalNotification notification) { if (logger.isDebugEnabled()) { logger.debug("removed {} from cache, reason: {}", notification.getValue(), notification.getRemovalReason()); } @@ -540,11 +527,13 @@ public void onRemoval(RemovalNotification notification private static final class CacheKey { final String lang; final String idOrCode; + final String context; final Map options; - private CacheKey(String lang, String idOrCode, Map options) { + private CacheKey(String lang, String idOrCode, String context, Map options) { this.lang = lang; this.idOrCode = idOrCode; + this.context = context; this.options = options; } @@ -552,21 +541,16 @@ private CacheKey(String lang, String idOrCode, Map options) { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - - CacheKey cacheKey = (CacheKey)o; - - if (lang != null ? !lang.equals(cacheKey.lang) : cacheKey.lang != null) return false; - if (!idOrCode.equals(cacheKey.idOrCode)) return false; - return options != null ? options.equals(cacheKey.options) : cacheKey.options == null; - + CacheKey cacheKey = (CacheKey) o; + return Objects.equals(lang, cacheKey.lang) && + Objects.equals(idOrCode, cacheKey.idOrCode) && + Objects.equals(context, cacheKey.context) && + Objects.equals(options, cacheKey.options); } @Override public int hashCode() { - int result = lang != null ? lang.hashCode() : 0; - result = 31 * result + idOrCode.hashCode(); - result = 31 * result + (options != null ? options.hashCode() : 0); - return result; + return Objects.hash(lang, idOrCode, context, options); } } } diff --git a/core/src/main/java/org/elasticsearch/script/SearchScript.java b/core/src/main/java/org/elasticsearch/script/SearchScript.java index 5d5f218763237..bbee2910c882e 100644 --- a/core/src/main/java/org/elasticsearch/script/SearchScript.java +++ b/core/src/main/java/org/elasticsearch/script/SearchScript.java @@ -19,8 +19,10 @@ package org.elasticsearch.script; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; +import java.util.Map; /** * A search script. @@ -36,4 +38,11 @@ public interface SearchScript { */ boolean needsScores(); + interface Factory { + SearchScript newInstance(Map params, SearchLookup lookup); + } + + ScriptContext CONTEXT = new ScriptContext<>("search", Factory.class); + // TODO: remove aggs context when it has its own interface + ScriptContext AGGS_CONTEXT = new ScriptContext<>("aggs", Factory.class); } \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java index 85308f0c249fb..fea834c02ac52 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java @@ -132,6 +132,7 @@ import org.elasticsearch.search.aggregations.bucket.significant.SignificantLongTerms; import org.elasticsearch.search.aggregations.bucket.significant.SignificantStringTerms; import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.significant.SignificantTextAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.significant.UnmappedSignificantTerms; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.ChiSquare; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.GND; @@ -377,6 +378,8 @@ private void registerAggregations(List plugins) { .addResultReader(SignificantStringTerms.NAME, SignificantStringTerms::new) .addResultReader(SignificantLongTerms.NAME, SignificantLongTerms::new) .addResultReader(UnmappedSignificantTerms.NAME, UnmappedSignificantTerms::new)); + registerAggregation(new AggregationSpec(SignificantTextAggregationBuilder.NAME, SignificantTextAggregationBuilder::new, + SignificantTextAggregationBuilder.getParser(significanceHeuristicParserRegistry))); registerAggregation(new AggregationSpec(RangeAggregationBuilder.NAME, RangeAggregationBuilder::new, RangeAggregationBuilder::parse).addResultReader(InternalRange::new)); registerAggregation(new AggregationSpec(DateRangeAggregationBuilder.NAME, DateRangeAggregationBuilder::new, diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java index a5ad038fd534f..fef20f44f5237 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchService.java +++ b/core/src/main/java/org/elasticsearch/search/SearchService.java @@ -48,8 +48,6 @@ import org.elasticsearch.index.shard.SearchOperationListener; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason; -import org.elasticsearch.script.CompiledScript; -import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.aggregations.AggregationInitializationException; @@ -253,6 +251,7 @@ public SearchPhaseResult executeQueryPhase(ShardSearchRequest request, SearchTas final SearchContext context = createAndPutContext(request); final SearchOperationListener operationListener = context.indexShard().getSearchOperationListener(); context.incRef(); + boolean queryPhaseSuccess = false; try { context.setTask(task); operationListener.onPreQueryPhase(context); @@ -267,6 +266,7 @@ public SearchPhaseResult executeQueryPhase(ShardSearchRequest request, SearchTas contextProcessedSuccessfully(context); } final long afterQueryTime = System.nanoTime(); + queryPhaseSuccess = true; operationListener.onQueryPhase(context, afterQueryTime - time); if (request.numberOfShards() == 1) { return executeFetchPhase(context, operationListener, afterQueryTime); @@ -278,7 +278,9 @@ public SearchPhaseResult executeQueryPhase(ShardSearchRequest request, SearchTas e = (e.getCause() == null || e.getCause() instanceof Exception) ? (Exception) e.getCause() : new ElasticsearchException(e.getCause()); } - operationListener.onFailedQueryPhase(context); + if (!queryPhaseSuccess) { + operationListener.onFailedQueryPhase(context); + } logger.trace("Query phase failed", e); processFailure(context, e); throw ExceptionsHelper.convertToRuntime(e); @@ -686,8 +688,8 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc } if (source.scriptFields() != null) { for (org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField field : source.scriptFields()) { - CompiledScript compile = scriptService.compile(field.script(), ScriptContext.SEARCH); - SearchScript searchScript = scriptService.search(context.lookup(), compile, field.script().getParams()); + SearchScript.Factory factory = scriptService.compile(field.script(), SearchScript.CONTEXT); + SearchScript searchScript = factory.newInstance(field.script().getParams(), context.lookup()); context.scriptFields().add(new ScriptField(field.fieldName(), searchScript, field.ignoreFailure())); } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java index 8b704ee8a69a2..a7a4a5f843a3e 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java @@ -51,6 +51,7 @@ import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms; import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.significant.SignificantTextAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.avg.Avg; @@ -246,6 +247,15 @@ public static SignificantTermsAggregationBuilder significantTerms(String name) { return new SignificantTermsAggregationBuilder(name, null); } + + /** + * Create a new {@link SignificantTextAggregationBuilder} aggregation with the given name and text field name + */ + public static SignificantTextAggregationBuilder significantText(String name, String fieldName) { + return new SignificantTextAggregationBuilder(name, fieldName); + } + + /** * Create a new {@link DateHistogramAggregationBuilder} aggregation with the given * name. diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java index 4a04fa645b11b..a9af38ef6e091 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java @@ -55,7 +55,7 @@ public abstract class AggregatorBase extends Aggregator { private DeferringBucketCollector recordingWrapper; private final List pipelineAggregators; private final CircuitBreakerService breakerService; - private boolean failed = false; + private long requestBytesUsed; /** * Constructs a new Aggregator. @@ -105,16 +105,31 @@ public boolean needsScores() { return false; // unreachable } }; + addRequestCircuitBreakerBytes(DEFAULT_WEIGHT); + } + + /** + * Increment the number of bytes that have been allocated to service this request + * and potentially trigger a {@link CircuitBreakingException}. The number of bytes + * allocated is automatically decremented with the circuit breaker service on + * closure of this aggregator. + * For performance reasons subclasses should not call this millions of times + * each with small increments and instead batch up into larger allocations. + * + * @param bytesAllocated the number of additional bytes allocated + * @return the cumulative size in bytes allocated by this aggregator to service this request + */ + protected long addRequestCircuitBreakerBytes(long bytesAllocated) { try { this.breakerService .getBreaker(CircuitBreaker.REQUEST) - .addEstimateBytesAndMaybeBreak(DEFAULT_WEIGHT, ""); + .addEstimateBytesAndMaybeBreak(bytesAllocated, ""); + this.requestBytesUsed += bytesAllocated; + return requestBytesUsed; } catch (CircuitBreakingException cbe) { - this.failed = true; throw cbe; - } + } } - /** * Most aggregators don't need scores, make sure to extend this method if * your aggregator needs them. @@ -265,9 +280,7 @@ public void close() { try { doClose(); } finally { - if (!this.failed) { - this.breakerService.getBreaker(CircuitBreaker.REQUEST).addWithoutBreaking(-DEFAULT_WEIGHT); - } + this.breakerService.getBreaker(CircuitBreaker.REQUEST).addWithoutBreaking(-this.requestBytesUsed); } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java index e81c0b1890bdc..31594e2804164 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java @@ -430,7 +430,7 @@ public static BucketOrder readOrder(StreamInput in) throws IOException { * @throws IOException on error reading from the stream. */ public static BucketOrder readHistogramOrder(StreamInput in, boolean bwcOrderFlag) throws IOException { - if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha2_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha2)) { return Streams.readOrder(in); } else { // backwards compat logic if (bwcOrderFlag == false || in.readBoolean()) { @@ -486,7 +486,7 @@ public static void writeOrder(BucketOrder order, StreamOutput out) throws IOExce * @throws IOException on error writing to the stream. */ public static void writeHistogramOrder(BucketOrder order, StreamOutput out, boolean bwcOrderFlag) throws IOException { - if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha2_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha2)) { order.writeTo(out); } else { // backwards compat logic if(bwcOrderFlag) { // need to add flag that determines if order exists diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregationBuilder.java new file mode 100644 index 0000000000000..c0fecd8be469e --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregationBuilder.java @@ -0,0 +1,386 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.significant; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ParseFieldRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationInitializationException; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic; +import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicParser; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds; +import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +public class SignificantTextAggregationBuilder extends AbstractAggregationBuilder { + public static final String NAME = "significant_text"; + + static final ParseField FIELD_NAME = new ParseField("field"); + static final ParseField SOURCE_FIELDS_NAME = new ParseField("source_fields"); + static final ParseField FILTER_DUPLICATE_TEXT_FIELD_NAME = new ParseField( + "filter_duplicate_text"); + + static final TermsAggregator.BucketCountThresholds DEFAULT_BUCKET_COUNT_THRESHOLDS = + SignificantTermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS; + static final SignificanceHeuristic DEFAULT_SIGNIFICANCE_HEURISTIC = SignificantTermsAggregationBuilder.DEFAULT_SIGNIFICANCE_HEURISTIC; + + private String fieldName = null; + private String [] sourceFieldNames = null; + private boolean filterDuplicateText = false; + private IncludeExclude includeExclude = null; + private QueryBuilder filterBuilder = null; + private TermsAggregator.BucketCountThresholds bucketCountThresholds = new BucketCountThresholds( + DEFAULT_BUCKET_COUNT_THRESHOLDS); + private SignificanceHeuristic significanceHeuristic = DEFAULT_SIGNIFICANCE_HEURISTIC; + + public static Aggregator.Parser getParser( + ParseFieldRegistry significanceHeuristicParserRegistry) { + ObjectParser parser = new ObjectParser<>( + SignificantTextAggregationBuilder.NAME); + + parser.declareInt(SignificantTextAggregationBuilder::shardSize, + TermsAggregationBuilder.SHARD_SIZE_FIELD_NAME); + + parser.declareLong(SignificantTextAggregationBuilder::minDocCount, + TermsAggregationBuilder.MIN_DOC_COUNT_FIELD_NAME); + + parser.declareLong(SignificantTextAggregationBuilder::shardMinDocCount, + TermsAggregationBuilder.SHARD_MIN_DOC_COUNT_FIELD_NAME); + + parser.declareInt(SignificantTextAggregationBuilder::size, + TermsAggregationBuilder.REQUIRED_SIZE_FIELD_NAME); + + parser.declareString(SignificantTextAggregationBuilder::fieldName, FIELD_NAME); + + parser.declareStringArray(SignificantTextAggregationBuilder::sourceFieldNames, SOURCE_FIELDS_NAME); + + + parser.declareBoolean(SignificantTextAggregationBuilder::filterDuplicateText, + FILTER_DUPLICATE_TEXT_FIELD_NAME); + + parser.declareObject(SignificantTextAggregationBuilder::backgroundFilter, + (p, context) -> context.parseInnerQueryBuilder(), + SignificantTermsAggregationBuilder.BACKGROUND_FILTER); + + parser.declareField((b, v) -> b.includeExclude(IncludeExclude.merge(v, b.includeExclude())), + IncludeExclude::parseInclude, IncludeExclude.INCLUDE_FIELD, + ObjectParser.ValueType.OBJECT_ARRAY_OR_STRING); + + parser.declareField((b, v) -> b.includeExclude(IncludeExclude.merge(b.includeExclude(), v)), + IncludeExclude::parseExclude, IncludeExclude.EXCLUDE_FIELD, + ObjectParser.ValueType.STRING_ARRAY); + + for (String name : significanceHeuristicParserRegistry.getNames()) { + parser.declareObject(SignificantTextAggregationBuilder::significanceHeuristic, + (p, context) -> { + SignificanceHeuristicParser significanceHeuristicParser = significanceHeuristicParserRegistry + .lookupReturningNullIfNotFound(name); + return significanceHeuristicParser.parse(context); + }, new ParseField(name)); + } + return new Aggregator.Parser() { + @Override + public AggregationBuilder parse(String aggregationName, QueryParseContext context) + throws IOException { + return parser.parse(context.parser(), + new SignificantTextAggregationBuilder(aggregationName, null), context); + } + }; + } + + protected TermsAggregator.BucketCountThresholds getBucketCountThresholds() { + return new TermsAggregator.BucketCountThresholds(bucketCountThresholds); + } + + public TermsAggregator.BucketCountThresholds bucketCountThresholds() { + return bucketCountThresholds; + } + + + @Override + public SignificantTextAggregationBuilder subAggregations(Builder subFactories) { + throw new AggregationInitializationException("Aggregator [" + name + "] of type [" + + getType() + "] cannot accept sub-aggregations"); + } + + @Override + public SignificantTextAggregationBuilder subAggregation(AggregationBuilder aggregation) { + throw new AggregationInitializationException("Aggregator [" + name + "] of type [" + + getType() + "] cannot accept sub-aggregations"); + } + + public SignificantTextAggregationBuilder bucketCountThresholds( + TermsAggregator.BucketCountThresholds bucketCountThresholds) { + if (bucketCountThresholds == null) { + throw new IllegalArgumentException( + "[bucketCountThresholds] must not be null: [" + name + "]"); + } + this.bucketCountThresholds = bucketCountThresholds; + return this; + } + + /** + * Sets the size - indicating how many term buckets should be returned + * (defaults to 10) + */ + public SignificantTextAggregationBuilder size(int size) { + if (size <= 0) { + throw new IllegalArgumentException( + "[size] must be greater than 0. Found [" + size + "] in [" + name + "]"); + } + bucketCountThresholds.setRequiredSize(size); + return this; + } + + /** + * Sets the shard_size - indicating the number of term buckets each shard + * will return to the coordinating node (the node that coordinates the + * search execution). The higher the shard size is, the more accurate the + * results are. + */ + public SignificantTextAggregationBuilder shardSize(int shardSize) { + if (shardSize <= 0) { + throw new IllegalArgumentException("[shardSize] must be greater than 0. Found [" + + shardSize + "] in [" + name + "]"); + } + bucketCountThresholds.setShardSize(shardSize); + return this; + } + + /** + * Sets the name of the text field that will be the subject of this + * aggregation. + */ + public SignificantTextAggregationBuilder fieldName(String fieldName) { + this.fieldName = fieldName; + return this; + } + + + /** + * Selects the fields to load from _source JSON and analyze. + * If none are specified, the indexed "fieldName" value is assumed + * to also be the name of the JSON field holding the value + */ + public SignificantTextAggregationBuilder sourceFieldNames(List names) { + this.sourceFieldNames = names.toArray(new String [names.size()]); + return this; + } + + + /** + * Control if duplicate paragraphs of text should try be filtered from the + * statistical text analysis. Can improve results but slows down analysis. + * Default is false. + */ + public SignificantTextAggregationBuilder filterDuplicateText(boolean filterDuplicateText) { + this.filterDuplicateText = filterDuplicateText; + return this; + } + + /** + * Set the minimum document count terms should have in order to appear in + * the response. + */ + public SignificantTextAggregationBuilder minDocCount(long minDocCount) { + if (minDocCount < 0) { + throw new IllegalArgumentException( + "[minDocCount] must be greater than or equal to 0. Found [" + minDocCount + + "] in [" + name + "]"); + } + bucketCountThresholds.setMinDocCount(minDocCount); + return this; + } + + /** + * Set the minimum document count terms should have on the shard in order to + * appear in the response. + */ + public SignificantTextAggregationBuilder shardMinDocCount(long shardMinDocCount) { + if (shardMinDocCount < 0) { + throw new IllegalArgumentException( + "[shardMinDocCount] must be greater than or equal to 0. Found [" + + shardMinDocCount + "] in [" + name + "]"); + } + bucketCountThresholds.setShardMinDocCount(shardMinDocCount); + return this; + } + + public SignificantTextAggregationBuilder backgroundFilter(QueryBuilder backgroundFilter) { + if (backgroundFilter == null) { + throw new IllegalArgumentException( + "[backgroundFilter] must not be null: [" + name + "]"); + } + this.filterBuilder = backgroundFilter; + return this; + } + + public QueryBuilder backgroundFilter() { + return filterBuilder; + } + + /** + * Set terms to include and exclude from the aggregation results + */ + public SignificantTextAggregationBuilder includeExclude(IncludeExclude includeExclude) { + this.includeExclude = includeExclude; + return this; + } + + /** + * Get terms to include and exclude from the aggregation results + */ + public IncludeExclude includeExclude() { + return includeExclude; + } + + public SignificantTextAggregationBuilder significanceHeuristic( + SignificanceHeuristic significanceHeuristic) { + if (significanceHeuristic == null) { + throw new IllegalArgumentException( + "[significanceHeuristic] must not be null: [" + name + "]"); + } + this.significanceHeuristic = significanceHeuristic; + return this; + } + + public SignificanceHeuristic significanceHeuristic() { + return significanceHeuristic; + } + + /** + * @param name + * the name of this aggregation + * @param fieldName + * the name of the text field that will be the subject of this + * aggregation + * + */ + public SignificantTextAggregationBuilder(String name, String fieldName) { + super(name); + this.fieldName = fieldName; + } + + /** + * Read from a stream. + */ + public SignificantTextAggregationBuilder(StreamInput in) throws IOException { + super(in); + fieldName = in.readString(); + filterDuplicateText = in.readBoolean(); + bucketCountThresholds = new BucketCountThresholds(in); + filterBuilder = in.readOptionalNamedWriteable(QueryBuilder.class); + includeExclude = in.readOptionalWriteable(IncludeExclude::new); + significanceHeuristic = in.readNamedWriteable(SignificanceHeuristic.class); + sourceFieldNames = in.readOptionalStringArray(); + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeString(fieldName); + out.writeBoolean(filterDuplicateText); + bucketCountThresholds.writeTo(out); + out.writeOptionalNamedWriteable(filterBuilder); + out.writeOptionalWriteable(includeExclude); + out.writeNamedWriteable(significanceHeuristic); + out.writeOptionalStringArray(sourceFieldNames); + } + + @Override + protected AggregatorFactory doBuild(SearchContext context, AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { + SignificanceHeuristic executionHeuristic = this.significanceHeuristic.rewrite(context); + String[] execFieldNames = sourceFieldNames; + if (execFieldNames == null) { + execFieldNames = new String[] { fieldName }; + } + return new SignificantTextAggregatorFactory(name, includeExclude, filterBuilder, + bucketCountThresholds, executionHeuristic, context, parent, subFactoriesBuilder, + fieldName, execFieldNames, filterDuplicateText, metaData); + } + + @Override + protected XContentBuilder internalXContent(XContentBuilder builder, Params params) + throws IOException { + builder.startObject(); + bucketCountThresholds.toXContent(builder, params); + if (fieldName != null) { + builder.field(FIELD_NAME.getPreferredName(), fieldName); + } + if (sourceFieldNames != null) { + builder.array(SOURCE_FIELDS_NAME.getPreferredName(), sourceFieldNames); + } + + if (filterDuplicateText) { + builder.field(FILTER_DUPLICATE_TEXT_FIELD_NAME.getPreferredName(), filterDuplicateText); + } + if (filterBuilder != null) { + builder.field(SignificantTermsAggregationBuilder.BACKGROUND_FILTER.getPreferredName(), + filterBuilder); + } + if (includeExclude != null) { + includeExclude.toXContent(builder, params); + } + significanceHeuristic.toXContent(builder, params); + + builder.endObject(); + return builder; + } + + @Override + protected int doHashCode() { + return Objects.hash(bucketCountThresholds, fieldName, filterDuplicateText, filterBuilder, + includeExclude, significanceHeuristic, Arrays.hashCode(sourceFieldNames)); + } + + @Override + protected boolean doEquals(Object obj) { + SignificantTextAggregationBuilder other = (SignificantTextAggregationBuilder) obj; + return Objects.equals(bucketCountThresholds, other.bucketCountThresholds) + && Objects.equals(fieldName, other.fieldName) + && Arrays.equals(sourceFieldNames, other.sourceFieldNames) + && filterDuplicateText == other.filterDuplicateText + && Objects.equals(filterBuilder, other.filterBuilder) + && Objects.equals(includeExclude, other.includeExclude) + && Objects.equals(significanceHeuristic, other.significanceHeuristic); + } + + @Override + public String getType() { + return NAME; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregator.java new file mode 100644 index 0000000000000..c7539a4ca0216 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregator.java @@ -0,0 +1,256 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.significant; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.miscellaneous.DeDuplicatingTokenFilter; +import org.apache.lucene.analysis.miscellaneous.DuplicateByteSequenceSpotter; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Terms; +import org.apache.lucene.search.highlight.TokenStreamFromTermVector; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.util.BytesRefHash; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; +import org.elasticsearch.search.aggregations.bucket.BucketsAggregator; +import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds; +import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; +import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude.StringFilter; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.internal.ContextIndexSearcher; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyList; + +public class SignificantTextAggregator extends BucketsAggregator { + + private final StringFilter includeExclude; + protected final BucketCountThresholds bucketCountThresholds; + protected long numCollectedDocs; + private final BytesRefHash bucketOrds; + private final SignificanceHeuristic significanceHeuristic; + private SignificantTextAggregatorFactory termsAggFactory; + private final DocValueFormat format = DocValueFormat.RAW; + private final String fieldName; + private final String[] sourceFieldNames; + private DuplicateByteSequenceSpotter dupSequenceSpotter = null ; + private long lastTrieSize; + private static final int MEMORY_GROWTH_REPORTING_INTERVAL_BYTES = 5000; + + + + public SignificantTextAggregator(String name, AggregatorFactories factories, + SearchContext context, Aggregator parent, List pipelineAggregators, + BucketCountThresholds bucketCountThresholds, IncludeExclude.StringFilter includeExclude, + SignificanceHeuristic significanceHeuristic, SignificantTextAggregatorFactory termsAggFactory, + String fieldName, String [] sourceFieldNames, boolean filterDuplicateText, + Map metaData) throws IOException { + super(name, factories, context, parent, pipelineAggregators, metaData); + this.bucketCountThresholds = bucketCountThresholds; + this.includeExclude = includeExclude; + this.significanceHeuristic = significanceHeuristic; + this.termsAggFactory = termsAggFactory; + this.fieldName = fieldName; + this.sourceFieldNames = sourceFieldNames; + bucketOrds = new BytesRefHash(1, context.bigArrays()); + if(filterDuplicateText){ + dupSequenceSpotter = new DuplicateByteSequenceSpotter(); + lastTrieSize = dupSequenceSpotter.getEstimatedSizeInBytes(); + } + } + + + + + @Override + public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, + final LeafBucketCollector sub) throws IOException { + final BytesRefBuilder previous = new BytesRefBuilder(); + return new LeafBucketCollectorBase(sub, null) { + + @Override + public void collect(int doc, long bucket) throws IOException { + collectFromSource(doc, bucket, fieldName, sourceFieldNames); + numCollectedDocs++; + if (dupSequenceSpotter != null) { + dupSequenceSpotter.startNewSequence(); + } + } + + private void processTokenStream(int doc, long bucket, TokenStream ts, String fieldText) throws IOException{ + if (dupSequenceSpotter != null) { + ts = new DeDuplicatingTokenFilter(ts, dupSequenceSpotter); + } + CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); + ts.reset(); + try { + //Assume tokens will average 5 bytes in length to size number of tokens + BytesRefHash inDocTerms = new BytesRefHash(1+(fieldText.length()/5), context.bigArrays()); + + try{ + while (ts.incrementToken()) { + if (dupSequenceSpotter != null) { + long newTrieSize = dupSequenceSpotter.getEstimatedSizeInBytes(); + long growth = newTrieSize - lastTrieSize; + // Only update the circuitbreaker after + if (growth > MEMORY_GROWTH_REPORTING_INTERVAL_BYTES) { + addRequestCircuitBreakerBytes(growth); + lastTrieSize = newTrieSize; + } + } + previous.clear(); + previous.copyChars(termAtt); + BytesRef bytes = previous.get(); + if (inDocTerms.add(bytes) >= 0) { + if (includeExclude == null || includeExclude.accept(bytes)) { + long bucketOrdinal = bucketOrds.add(bytes); + if (bucketOrdinal < 0) { // already seen + bucketOrdinal = -1 - bucketOrdinal; + collectExistingBucket(sub, doc, bucketOrdinal); + } else { + collectBucket(sub, doc, bucketOrdinal); + } + } + } + } + } finally{ + Releasables.close(inDocTerms); + } + } finally{ + ts.close(); + } + } + + private void collectFromSource(int doc, long bucket, String indexedFieldName, String[] sourceFieldNames) throws IOException { + MappedFieldType fieldType = context.getQueryShardContext().fieldMapper(indexedFieldName); + if(fieldType == null){ + throw new IllegalArgumentException("Aggregation [" + name + "] cannot process field ["+indexedFieldName + +"] since it is not present"); + } + + SourceLookup sourceLookup = context.lookup().source(); + sourceLookup.setSegmentAndDocument(ctx, doc); + + for (String sourceField : sourceFieldNames) { + List textsToHighlight = sourceLookup.extractRawValues(sourceField); + textsToHighlight = textsToHighlight.stream().map(obj -> { + if (obj instanceof BytesRef) { + return fieldType.valueForDisplay(obj).toString(); + } else { + return obj; + } + }).collect(Collectors.toList()); + + Analyzer analyzer = fieldType.indexAnalyzer(); + for (Object fieldValue : textsToHighlight) { + String fieldText = fieldValue.toString(); + TokenStream ts = analyzer.tokenStream(indexedFieldName, fieldText); + processTokenStream(doc, bucket, ts, fieldText); + } + } + } + }; + } + + @Override + public SignificantStringTerms buildAggregation(long owningBucketOrdinal) throws IOException { + assert owningBucketOrdinal == 0; + + final int size = (int) Math.min(bucketOrds.size(), bucketCountThresholds.getShardSize()); + long supersetSize = termsAggFactory.getSupersetNumDocs(); + long subsetSize = numCollectedDocs; + + BucketSignificancePriorityQueue ordered = new BucketSignificancePriorityQueue<>(size); + SignificantStringTerms.Bucket spare = null; + for (int i = 0; i < bucketOrds.size(); i++) { + final int docCount = bucketDocCount(i); + if (docCount < bucketCountThresholds.getShardMinDocCount()) { + continue; + } + + if (spare == null) { + spare = new SignificantStringTerms.Bucket(new BytesRef(), 0, 0, 0, 0, null, format); + } + + bucketOrds.get(i, spare.termBytes); + spare.subsetDf = docCount; + spare.subsetSize = subsetSize; + spare.supersetDf = termsAggFactory.getBackgroundFrequency(spare.termBytes); + spare.supersetSize = supersetSize; + // During shard-local down-selection we use subset/superset stats + // that are for this shard only + // Back at the central reducer these properties will be updated with + // global stats + spare.updateScore(significanceHeuristic); + + spare.bucketOrd = i; + spare = ordered.insertWithOverflow(spare); + } + + final SignificantStringTerms.Bucket[] list = new SignificantStringTerms.Bucket[ordered.size()]; + for (int i = ordered.size() - 1; i >= 0; i--) { + final SignificantStringTerms.Bucket bucket = ordered.pop(); + // the terms are owned by the BytesRefHash, we need to pull a copy since the BytesRef hash data may be recycled at some point + bucket.termBytes = BytesRef.deepCopyOf(bucket.termBytes); + bucket.aggregations = bucketAggregations(bucket.bucketOrd); + list[i] = bucket; + } + + return new SignificantStringTerms( name, bucketCountThresholds.getRequiredSize(), + bucketCountThresholds.getMinDocCount(), pipelineAggregators(), + metaData(), format, subsetSize, supersetSize, significanceHeuristic, Arrays.asList(list)); + } + + + @Override + public SignificantStringTerms buildEmptyAggregation() { + // We need to account for the significance of a miss in our global stats - provide corpus size as context + ContextIndexSearcher searcher = context.searcher(); + IndexReader topReader = searcher.getIndexReader(); + int supersetSize = topReader.numDocs(); + return new SignificantStringTerms(name, bucketCountThresholds.getRequiredSize(), bucketCountThresholds.getMinDocCount(), + pipelineAggregators(), metaData(), format, 0, supersetSize, significanceHeuristic, emptyList()); + } + + @Override + public void doClose() { + Releasables.close(bucketOrds, termsAggFactory); + } + +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java new file mode 100644 index 0000000000000..629f6989941d0 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java @@ -0,0 +1,187 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.significant; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.lucene.index.FilterableTermsEnum; +import org.elasticsearch.common.lucene.index.FreqTermsEnum; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.bucket.BucketUtils; +import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds; +import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class SignificantTextAggregatorFactory extends AggregatorFactory + implements Releasable { + + private final IncludeExclude includeExclude; + private String indexedFieldName; + private MappedFieldType fieldType; + private final String[] sourceFieldNames; + private FilterableTermsEnum termsEnum; + private int numberOfAggregatorsCreated; + private final Query filter; + private final int supersetNumDocs; + private final TermsAggregator.BucketCountThresholds bucketCountThresholds; + private final SignificanceHeuristic significanceHeuristic; + private final DocValueFormat format = DocValueFormat.RAW; + private final boolean filterDuplicateText; + + public SignificantTextAggregatorFactory(String name, IncludeExclude includeExclude, + QueryBuilder filterBuilder, TermsAggregator.BucketCountThresholds bucketCountThresholds, + SignificanceHeuristic significanceHeuristic, SearchContext context, AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, String fieldName, String [] sourceFieldNames, + boolean filterDuplicateText, Map metaData) throws IOException { + super(name, context, parent, subFactoriesBuilder, metaData); + this.includeExclude = includeExclude; + this.filter = filterBuilder == null + ? null + : filterBuilder.toQuery(context.getQueryShardContext()); + this.indexedFieldName = fieldName; + this.sourceFieldNames = sourceFieldNames; + this.filterDuplicateText = filterDuplicateText; + IndexSearcher searcher = context.searcher(); + // Important - need to use the doc count that includes deleted docs + // or we have this issue: https://github.com/elastic/elasticsearch/issues/7951 + this.supersetNumDocs = filter == null + ? searcher.getIndexReader().maxDoc() + : searcher.count(filter); + this.bucketCountThresholds = bucketCountThresholds; + this.significanceHeuristic = significanceHeuristic; + fieldType = context.getQueryShardContext().fieldMapper(indexedFieldName); + + } + + + /** + * Get the number of docs in the superset. + */ + public long getSupersetNumDocs() { + return supersetNumDocs; + } + + private FilterableTermsEnum getTermsEnum(String field) throws IOException { + if (termsEnum != null) { + return termsEnum; + } + IndexReader reader = context.searcher().getIndexReader(); + if (numberOfAggregatorsCreated > 1) { + termsEnum = new FreqTermsEnum(reader, field, true, false, filter, context.bigArrays()); + } else { + termsEnum = new FilterableTermsEnum(reader, indexedFieldName, PostingsEnum.NONE, filter); + } + return termsEnum; + } + + private long getBackgroundFrequency(String value) throws IOException { + Query query = fieldType.termQuery(value, context.getQueryShardContext()); + if (query instanceof TermQuery) { + // for types that use the inverted index, we prefer using a caching terms + // enum that will do a better job at reusing index inputs + Term term = ((TermQuery) query).getTerm(); + FilterableTermsEnum termsEnum = getTermsEnum(term.field()); + if (termsEnum.seekExact(term.bytes())) { + return termsEnum.docFreq(); + } else { + return 0; + } + } + // otherwise do it the naive way + if (filter != null) { + query = new BooleanQuery.Builder() + .add(query, Occur.FILTER) + .add(filter, Occur.FILTER) + .build(); + } + return context.searcher().count(query); + } + + public long getBackgroundFrequency(BytesRef termBytes) throws IOException { + String value = format.format(termBytes); + return getBackgroundFrequency(value); + } + + + @Override + public void close() { + try { + if (termsEnum instanceof Releasable) { + ((Releasable) termsEnum).close(); + } + } finally { + termsEnum = null; + } + } + + @Override + protected Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBucket, + List pipelineAggregators, Map metaData) + throws IOException { + if (collectsFromSingleBucket == false) { + return asMultiBucketAggregator(this, context, parent); + } + + numberOfAggregatorsCreated++; + BucketCountThresholds bucketCountThresholds = new BucketCountThresholds(this.bucketCountThresholds); + if (bucketCountThresholds.getShardSize() == SignificantTextAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize()) { + // The user has not made a shardSize selection. + // Use default heuristic to avoid any wrong-ranking caused by + // distributed counting but request double the usual amount. + // We typically need more than the number of "top" terms requested + // by other aggregations as the significance algorithm is in less + // of a position to down-select at shard-level - some of the things + // we want to find have only one occurrence on each shard and as + // such are impossible to differentiate from non-significant terms + // at that early stage. + bucketCountThresholds.setShardSize(2 * BucketUtils.suggestShardSideQueueSize(bucketCountThresholds.getRequiredSize(), + context.numberOfShards())); + } + +// TODO - need to check with mapping that this is indeed a text field.... + + IncludeExclude.StringFilter incExcFilter = includeExclude == null ? null: + includeExclude.convertToStringFilter(DocValueFormat.RAW); + + return new SignificantTextAggregator(name, factories, context, parent, pipelineAggregators, bucketCountThresholds, + incExcFilter, significanceHeuristic, this, indexedFieldName, sourceFieldNames, filterDuplicateText, metaData); + + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java index b3df54247b961..e1fb72288bb94 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java @@ -28,10 +28,8 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryShardException; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptContext; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.internal.SearchContext; @@ -93,13 +91,14 @@ public void writeTo(StreamOutput out) throws IOException { @Override public SignificanceHeuristic rewrite(InternalAggregation.ReduceContext context) { - CompiledScript compiledScript = context.scriptService().compile(script, ScriptContext.AGGS); - return new ExecutableScriptHeuristic(script, context.scriptService().executable(compiledScript, script.getParams())); + ExecutableScript.Factory factory = context.scriptService().compile(script, ExecutableScript.AGGS_CONTEXT); + return new ExecutableScriptHeuristic(script, factory.newInstance(script.getParams())); } @Override public SignificanceHeuristic rewrite(SearchContext context) { - return new ExecutableScriptHeuristic(script, context.getQueryShardContext().getExecutableScript(script, ScriptContext.AGGS)); + return new ExecutableScriptHeuristic(script, + context.getQueryShardContext().getExecutableScript(script, ExecutableScript.AGGS_CONTEXT)); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java index 2bedd16ebff79..49cb6dcf9673e 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java @@ -22,10 +22,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptContext; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; @@ -96,9 +94,9 @@ public InternalAggregation doReduce(List aggregations, Redu if (firstAggregation.reduceScript.getParams() != null) { vars.putAll(firstAggregation.reduceScript.getParams()); } - CompiledScript compiledScript = reduceContext.scriptService().compile( - firstAggregation.reduceScript, ScriptContext.AGGS); - ExecutableScript script = reduceContext.scriptService().executable(compiledScript, vars); + ExecutableScript.Factory factory = reduceContext.scriptService().compile( + firstAggregation.reduceScript, ExecutableScript.AGGS_CONTEXT); + ExecutableScript script = factory.newInstance(vars); aggregation = Collections.singletonList(script.run()); } else if (reduceContext.isFinalReduce()) { aggregation = Collections.singletonList(aggregationObjects); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java index 6d8c297eebd67..34c06ec3c5598 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java @@ -186,15 +186,15 @@ protected ScriptedMetricAggregatorFactory doBuild(SearchContext context, Aggrega QueryShardContext queryShardContext = context.getQueryShardContext(); Function, ExecutableScript> executableInitScript; if (initScript != null) { - executableInitScript = queryShardContext.getLazyExecutableScript(initScript, ScriptContext.AGGS); + executableInitScript = queryShardContext.getLazyExecutableScript(initScript, ExecutableScript.AGGS_CONTEXT); } else { executableInitScript = (p) -> null; } - Function, SearchScript> searchMapScript = queryShardContext.getLazySearchScript(mapScript, - ScriptContext.AGGS); + Function, SearchScript> searchMapScript = + queryShardContext.getLazySearchScript(mapScript, SearchScript.AGGS_CONTEXT); Function, ExecutableScript> executableCombineScript; if (combineScript != null) { - executableCombineScript = queryShardContext.getLazyExecutableScript(combineScript, ScriptContext.AGGS); + executableCombineScript = queryShardContext.getLazyExecutableScript(combineScript, ExecutableScript.AGGS_CONTEXT); } else { executableCombineScript = (p) -> null; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java index 0ff94e9aa9873..94467add5114b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java @@ -534,7 +534,7 @@ protected TopHitsAggregatorFactory doBuild(SearchContext context, AggregatorFact if (scriptFields != null) { for (ScriptField field : scriptFields) { SearchScript searchScript = context.getQueryShardContext().getSearchScript(field.script(), - ScriptContext.SEARCH); + SearchScript.CONTEXT); fields.add(new org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField( field.fieldName(), searchScript, field.ignoreFailure())); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java index 46f4fbb2375a2..0a56ae2c1cbfa 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java @@ -21,10 +21,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.InternalAggregation; @@ -91,7 +89,7 @@ public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext (InternalMultiBucketAggregation) aggregation; List buckets = originalAgg.getBuckets(); - CompiledScript compiledScript = reduceContext.scriptService().compile(script, ScriptContext.AGGS); + ExecutableScript.Factory factory = reduceContext.scriptService().compile(script, ExecutableScript.AGGS_CONTEXT); List newBuckets = new ArrayList<>(); for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) { Map vars = new HashMap<>(); @@ -112,7 +110,7 @@ public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext if (skipBucket) { newBuckets.add(bucket); } else { - ExecutableScript executableScript = reduceContext.scriptService().executable(compiledScript, vars); + ExecutableScript executableScript = factory.newInstance(vars); Object returned = executableScript.run(); if (returned == null) { newBuckets.add(bucket); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java index f5bc5cc4adec1..a54ad0ec21f39 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java @@ -22,10 +22,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptContext; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext; import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; @@ -84,7 +82,7 @@ public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext (InternalMultiBucketAggregation) aggregation; List buckets = originalAgg.getBuckets(); - CompiledScript compiledScript = reduceContext.scriptService().compile(script, ScriptContext.AGGS); + ExecutableScript.Factory factory = reduceContext.scriptService().compile(script, ExecutableScript.AGGS_CONTEXT); List newBuckets = new ArrayList<>(); for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) { Map vars = new HashMap<>(); @@ -97,7 +95,8 @@ public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext Double value = resolveBucketValue(originalAgg, bucket, bucketsPath, gapPolicy); vars.put(varName, value); } - ExecutableScript executableScript = reduceContext.scriptService().executable(compiledScript, vars); + // TODO: can we use one instance of the script for all buckets? it should be stateless? + ExecutableScript executableScript = factory.newInstance(vars); Object scriptReturnValue = executableScript.run(); final boolean keepBucket; // TODO: WTF!!!!! diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java index ea2afdb7e809d..6404cae0e4cef 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java @@ -120,7 +120,7 @@ private static SearchScript createScript(Script script, QueryShardContext contex if (script == null) { return null; } else { - return context.getSearchScript(script, ScriptContext.AGGS); + return context.getSearchScript(script, SearchScript.AGGS_CONTEXT); } } diff --git a/core/src/main/java/org/elasticsearch/search/collapse/CollapseBuilder.java b/core/src/main/java/org/elasticsearch/search/collapse/CollapseBuilder.java index 9ae8bd2b3833c..93f4b8bf41c9a 100644 --- a/core/src/main/java/org/elasticsearch/search/collapse/CollapseBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/collapse/CollapseBuilder.java @@ -20,15 +20,17 @@ import org.apache.lucene.index.IndexOptions; import org.elasticsearch.Version; -import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; @@ -38,12 +40,15 @@ import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Objects; /** * A builder that enables field collapsing on search request. */ -public class CollapseBuilder extends ToXContentToBytes implements Writeable { +public class CollapseBuilder implements Writeable, ToXContentObject { public static final ParseField FIELD_FIELD = new ParseField("field"); public static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits"); public static final ParseField MAX_CONCURRENT_GROUP_REQUESTS_FIELD = new ParseField("max_concurrent_group_searches"); @@ -53,12 +58,27 @@ public class CollapseBuilder extends ToXContentToBytes implements Writeable { static { PARSER.declareString(CollapseBuilder::setField, FIELD_FIELD); PARSER.declareInt(CollapseBuilder::setMaxConcurrentGroupRequests, MAX_CONCURRENT_GROUP_REQUESTS_FIELD); - PARSER.declareObject(CollapseBuilder::setInnerHits, - (p, c) -> InnerHitBuilder.fromXContent(c), INNER_HITS_FIELD); + PARSER.declareField((parser, builder, context) -> { + XContentParser.Token currentToken = parser.currentToken(); + if (currentToken == XContentParser.Token.START_OBJECT) { + builder.setInnerHits(InnerHitBuilder.fromXContent(context)); + } else if (currentToken == XContentParser.Token.START_ARRAY) { + List innerHitBuilders = new ArrayList<>(); + for (currentToken = parser.nextToken(); currentToken != XContentParser.Token.END_ARRAY; currentToken = parser.nextToken()) { + if (currentToken == XContentParser.Token.START_OBJECT) { + innerHitBuilders.add(InnerHitBuilder.fromXContent(context)); + } else { + throw new ParsingException(parser.getTokenLocation(), "Invalid token in inner_hits array"); + } + } + + builder.setInnerHits(innerHitBuilders); + } + }, INNER_HITS_FIELD, ObjectParser.ValueType.OBJECT_ARRAY); } private String field; - private InnerHitBuilder innerHit; + private List innerHits = Collections.emptyList(); private int maxConcurrentGroupRequests = 0; private CollapseBuilder() {} @@ -75,22 +95,31 @@ public CollapseBuilder(String field) { public CollapseBuilder(StreamInput in) throws IOException { this.field = in.readString(); this.maxConcurrentGroupRequests = in.readVInt(); - this.innerHit = in.readOptionalWriteable(InnerHitBuilder::new); + if (in.getVersion().onOrAfter(Version.V_5_5_0)) { + this.innerHits = in.readList(InnerHitBuilder::new); + } else { + InnerHitBuilder innerHitBuilder = in.readOptionalWriteable(InnerHitBuilder::new); + if (innerHitBuilder != null) { + this.innerHits = Collections.singletonList(innerHitBuilder); + } else { + this.innerHits = Collections.emptyList(); + } + } } @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(field); out.writeVInt(maxConcurrentGroupRequests); - if (out.getVersion().before(Version.V_5_5_0_UNRELEASED)) { - final boolean hasInnerHit = innerHit != null; + if (out.getVersion().onOrAfter(Version.V_5_5_0)) { + out.writeList(innerHits); + } else { + boolean hasInnerHit = innerHits.isEmpty() == false; out.writeBoolean(hasInnerHit); if (hasInnerHit) { - innerHit.writeToCollapseBWC(out); + innerHits.get(0).writeToCollapseBWC(out); } - } else { - out.writeOptionalWriteable(innerHit); - } + } } public static CollapseBuilder fromXContent(QueryParseContext context) throws IOException { @@ -108,7 +137,12 @@ private CollapseBuilder setField(String field) { } public CollapseBuilder setInnerHits(InnerHitBuilder innerHit) { - this.innerHit = innerHit; + this.innerHits = Collections.singletonList(innerHit); + return this; + } + + public CollapseBuilder setInnerHits(List innerHits) { + this.innerHits = innerHits; return this; } @@ -130,8 +164,8 @@ public String getField() { /** * The inner hit options to expand the collapsed results */ - public InnerHitBuilder getInnerHit() { - return this.innerHit; + public List getInnerHits() { + return this.innerHits; } /** @@ -154,8 +188,16 @@ private void innerToXContent(XContentBuilder builder) throws IOException { if (maxConcurrentGroupRequests > 0) { builder.field(MAX_CONCURRENT_GROUP_REQUESTS_FIELD.getPreferredName(), maxConcurrentGroupRequests); } - if (innerHit != null) { - builder.field(INNER_HITS_FIELD.getPreferredName(), innerHit); + if (innerHits.isEmpty() == false) { + if (innerHits.size() == 1) { + builder.field(INNER_HITS_FIELD.getPreferredName(), innerHits.get(0)); + } else { + builder.startArray(INNER_HITS_FIELD.getPreferredName()); + for (InnerHitBuilder innerHit : innerHits) { + innerHit.toXContent(builder, ToXContent.EMPTY_PARAMS); + } + builder.endArray(); + } } } @@ -168,14 +210,12 @@ public boolean equals(Object o) { if (maxConcurrentGroupRequests != that.maxConcurrentGroupRequests) return false; if (!field.equals(that.field)) return false; - return innerHit != null ? innerHit.equals(that.innerHit) : that.innerHit == null; - + return Objects.equals(innerHits, that.innerHits); } @Override public int hashCode() { - int result = field.hashCode(); - result = 31 * result + (innerHit != null ? innerHit.hashCode() : 0); + int result = Objects.hash(field, innerHits); result = 31 * result + maxConcurrentGroupRequests; return result; } @@ -204,10 +244,11 @@ public CollapseContext build(SearchContext context) { if (fieldType.hasDocValues() == false) { throw new SearchContextException(context, "cannot collapse on field `" + field + "` without `doc_values`"); } - if (fieldType.indexOptions() == IndexOptions.NONE && innerHit != null) { + if (fieldType.indexOptions() == IndexOptions.NONE && (innerHits != null && !innerHits.isEmpty())) { throw new SearchContextException(context, "cannot expand `inner_hits` for collapse field `" + field + "`, " + "only indexed field can retrieve `inner_hits`"); } - return new CollapseContext(fieldType, innerHit); + + return new CollapseContext(fieldType, innerHits); } } diff --git a/core/src/main/java/org/elasticsearch/search/collapse/CollapseContext.java b/core/src/main/java/org/elasticsearch/search/collapse/CollapseContext.java index d0ea2154ab3e5..cb1587cd7d9e9 100644 --- a/core/src/main/java/org/elasticsearch/search/collapse/CollapseContext.java +++ b/core/src/main/java/org/elasticsearch/search/collapse/CollapseContext.java @@ -26,17 +26,24 @@ import org.elasticsearch.index.query.InnerHitBuilder; import java.io.IOException; +import java.util.Collections; +import java.util.List; /** * Context used for field collapsing */ public class CollapseContext { private final MappedFieldType fieldType; - private final InnerHitBuilder innerHit; + private final List innerHits; public CollapseContext(MappedFieldType fieldType, InnerHitBuilder innerHit) { this.fieldType = fieldType; - this.innerHit = innerHit; + this.innerHits = Collections.singletonList(innerHit); + } + + public CollapseContext(MappedFieldType fieldType, List innerHits) { + this.fieldType = fieldType; + this.innerHits = innerHits; } /** The field type used for collapsing **/ @@ -44,10 +51,9 @@ public MappedFieldType getFieldType() { return fieldType; } - /** The inner hit options to expand the collapsed results **/ - public InnerHitBuilder getInnerHit() { - return innerHit; + public List getInnerHit() { + return innerHits; } public CollapsingTopDocsCollector createTopDocs(Sort sort, int topN, boolean trackMaxScore) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/search/internal/ScrollContext.java b/core/src/main/java/org/elasticsearch/search/internal/ScrollContext.java index 1b7bcfb93c788..163dbcc73d924 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/ScrollContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/ScrollContext.java @@ -22,11 +22,35 @@ import org.apache.lucene.search.ScoreDoc; import org.elasticsearch.search.Scroll; +import java.util.HashMap; +import java.util.Map; + /** Wrapper around information that needs to stay around when scrolling. */ -public class ScrollContext { +public final class ScrollContext { + + private Map context = null; public int totalHits = -1; public float maxScore; public ScoreDoc lastEmittedDoc; public Scroll scroll; + + /** + * Returns the object or null if the given key does not have a + * value in the context + */ + @SuppressWarnings("unchecked") // (T)object + public T getFromContext(String key) { + return context != null ? (T) context.get(key) : null; + } + + /** + * Puts the object into the context + */ + public void putInContext(String key, Object value) { + if (context == null) { + context = new HashMap<>(); + } + context.put(key, value); + } } diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java index c775cc814991c..269021fd4671a 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java @@ -242,7 +242,7 @@ public static ScriptSortBuilder fromXContent(QueryParseContext context, String e @Override public SortFieldAndFormat build(QueryShardContext context) throws IOException { - final SearchScript searchScript = context.getSearchScript(script, ScriptContext.SEARCH); + final SearchScript searchScript = context.getSearchScript(script, SearchScript.CONTEXT); MultiValueMode valueMode = null; if (sortMode != null) { diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java index f5a6eca35b7f9..1194488e50616 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java @@ -631,7 +631,7 @@ public SuggestionContext build(QueryShardContext context) throws IOException { if (this.collateQuery != null) { Function, ExecutableScript> compiledScript = context.getLazyExecutableScript(this.collateQuery, - ScriptContext.SEARCH); + ExecutableScript.CONTEXT); suggestionContext.setCollateQueryScript(compiledScript); if (this.collateParams != null) { suggestionContext.setCollateScriptParams(this.collateParams); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java index 99e2e18496b61..31dcc22bec013 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java @@ -23,7 +23,6 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.search.suggest.DirectSpellcheckerSettings; import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 37cce0ac60174..a54e72159f8ae 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -70,7 +70,7 @@ public final class SnapshotInfo implements Comparable, ToXContent, private static final String SUCCESSFUL_SHARDS = "successful_shards"; private static final Version VERSION_INCOMPATIBLE_INTRODUCED = Version.V_5_2_0; - public static final Version VERBOSE_INTRODUCED = Version.V_5_5_0_UNRELEASED; + public static final Version VERBOSE_INTRODUCED = Version.V_5_5_0; private static final Comparator COMPARATOR = Comparator.comparing(SnapshotInfo::startTime).thenComparing(SnapshotInfo::snapshotId); diff --git a/core/src/main/java/org/elasticsearch/template/CompiledTemplate.java b/core/src/main/java/org/elasticsearch/template/CompiledTemplate.java index 380c36a590197..d03993583c43a 100644 --- a/core/src/main/java/org/elasticsearch/template/CompiledTemplate.java +++ b/core/src/main/java/org/elasticsearch/template/CompiledTemplate.java @@ -21,10 +21,6 @@ import java.util.Map; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.script.CompiledScript; -import org.elasticsearch.script.ScriptType; - /** * A template that may be executed. */ diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index 7b0c4eb752bdd..b61da9c27f126 100644 --- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -612,7 +612,7 @@ public Info(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { out.writeString(name); if (type == ThreadPoolType.FIXED_AUTO_QUEUE_SIZE && - out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + out.getVersion().before(Version.V_6_0_0_alpha1)) { // 5.x doesn't know about the "fixed_auto_queue_size" thread pool type, just write fixed. out.writeString(ThreadPoolType.FIXED.getType()); } else { diff --git a/core/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/core/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java index e4f0d0d4af766..2b16c26931b86 100644 --- a/core/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java +++ b/core/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java @@ -160,12 +160,24 @@ public void fetchSearchShards(ClusterSearchShardsRequest searchRequest, // we can't proceed with a search on a cluster level. // in the future we might want to just skip the remote nodes in such a case but that can already be implemented on the caller // end since they provide the listener. - connectHandler.connect(ActionListener.wrap((x) -> fetchShardsInternal(searchRequest, listener), listener::onFailure)); + ensureConnected(ActionListener.wrap((x) -> fetchShardsInternal(searchRequest, listener), listener::onFailure)); } else { fetchShardsInternal(searchRequest, listener); } } + /** + * Ensures that this cluster is connected. If the cluster is connected this operation + * will invoke the listener immediately. + */ + public void ensureConnected(ActionListener voidActionListener) { + if (connectedNodes.isEmpty()) { + connectHandler.connect(voidActionListener); + } else { + voidActionListener.onResponse(null); + } + } + private void fetchShardsInternal(ClusterSearchShardsRequest searchShardsRequest, final ActionListener listener) { final DiscoveryNode node = nodeSupplier.get(); diff --git a/core/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/core/src/main/java/org/elasticsearch/transport/RemoteClusterService.java index 91a5cebbcd2a2..621713c8ab11e 100644 --- a/core/src/main/java/org/elasticsearch/transport/RemoteClusterService.java +++ b/core/src/main/java/org/elasticsearch/transport/RemoteClusterService.java @@ -22,6 +22,7 @@ import org.apache.lucene.util.IOUtils; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest; @@ -46,6 +47,7 @@ import java.io.Closeable; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -265,6 +267,18 @@ public Transport.Connection getConnection(DiscoveryNode node, String cluster) { return connection.getConnection(node); } + /** + * Ensures that the given cluster alias is connected. If the cluster is connected this operation + * will invoke the listener immediately. + */ + public void ensureConnected(String clusterAlias, ActionListener listener) { + RemoteClusterConnection remoteClusterConnection = remoteClusters.get(clusterAlias); + if (remoteClusterConnection == null) { + throw new IllegalArgumentException("no such remote cluster: " + clusterAlias); + } + remoteClusterConnection.ensureConnected(listener); + } + public Transport.Connection getConnection(String cluster) { RemoteClusterConnection connection = remoteClusters.get(cluster); if (connection == null) { diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java index 5713cc27c09c3..de083ead10e36 100644 --- a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java +++ b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java @@ -1324,16 +1324,8 @@ public final void messageReceived(BytesReference reference, Channel channel, Str } streamIn = compressor.streamInput(streamIn); } - // for handshakes we are compatible with N-2 since otherwise we can't figure out our initial version - // since we are compatible with N-1 and N+1 so we always send our minCompatVersion as the initial version in the - // handshake. This looks odd but it's required to establish the connection correctly we check for real compatibility - // once the connection is established - final Version compatibilityVersion = TransportStatus.isHandshake(status) ? getCurrentVersion().minimumCompatibilityVersion() - : getCurrentVersion(); - if (version.isCompatible(compatibilityVersion) == false) { - throw new IllegalStateException("Received message from unsupported version: [" + version - + "] minimal compatible version is: [" + compatibilityVersion.minimumCompatibilityVersion() + "]"); - } + final boolean isHandshake = TransportStatus.isHandshake(status); + ensureVersionCompatibility(version, getCurrentVersion(), isHandshake); streamIn = new NamedWriteableAwareStreamInput(streamIn, namedWriteableRegistry); streamIn.setVersion(version); threadPool.getThreadContext().readHeaders(streamIn); @@ -1341,7 +1333,7 @@ public final void messageReceived(BytesReference reference, Channel channel, Str handleRequest(channel, profileName, streamIn, requestId, messageLengthBytes, version, remoteAddress, status); } else { final TransportResponseHandler handler; - if (TransportStatus.isHandshake(status)) { + if (isHandshake) { handler = pendingHandshakes.remove(requestId); } else { TransportResponseHandler theHandler = transportServiceAdapter.onResponseReceived(requestId); @@ -1377,6 +1369,19 @@ public final void messageReceived(BytesReference reference, Channel channel, Str } } + static void ensureVersionCompatibility(Version version, Version currentVersion, boolean isHandshake) { + // for handshakes we are compatible with N-2 since otherwise we can't figure out our initial version + // since we are compatible with N-1 and N+1 so we always send our minCompatVersion as the initial version in the + // handshake. This looks odd but it's required to establish the connection correctly we check for real compatibility + // once the connection is established + final Version compatibilityVersion = isHandshake ? currentVersion.minimumCompatibilityVersion() : currentVersion; + if (version.isCompatible(compatibilityVersion) == false) { + final Version minCompatibilityVersion = isHandshake ? compatibilityVersion : compatibilityVersion.minimumCompatibilityVersion(); + String msg = "Received " + (isHandshake? "handshake " : "") + "message from unsupported version: ["; + throw new IllegalStateException(msg + version + "] minimal compatible version is: [" + minCompatibilityVersion + "]"); + } + } + private void handleResponse(InetSocketAddress remoteAddress, final StreamInput stream, final TransportResponseHandler handler) { final TransportResponse response = handler.newInstance(); response.remoteAddress(new TransportAddress(remoteAddress)); diff --git a/core/src/test/java/org/apache/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java b/core/src/test/java/org/apache/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java new file mode 100644 index 0000000000000..0154800edc886 --- /dev/null +++ b/core/src/test/java/org/apache/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java @@ -0,0 +1,127 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lucene.analysis.miscellaneous; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class DeDuplicatingTokenFilterTests extends ESTestCase { + public void testSimple() throws IOException { + DuplicateByteSequenceSpotter bytesDeDuper = new DuplicateByteSequenceSpotter(); + Analyzer analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer t = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(t, new DeDuplicatingTokenFilter(t, bytesDeDuper)); + } + }; + + String input = "a b c 1 2 3 4 5 6 7 a b c d 1 2 3 4 5 6 7 e f 1 2 3 4 5 6 7"; + String expectedOutput = "a b c 1 2 3 4 5 6 7 a b c d e f"; + TokenStream test = analyzer.tokenStream("test", input); + CharTermAttribute termAttribute = test.addAttribute(CharTermAttribute.class); + + test.reset(); + + StringBuilder sb = new StringBuilder(); + while (test.incrementToken()) { + sb.append(termAttribute.toString()); + sb.append(" "); + } + String output = sb.toString().trim(); + assertThat(output, equalTo(expectedOutput)); + + } + + public void testHitCountLimits() throws IOException { + DuplicateByteSequenceSpotter bytesDeDuper = new DuplicateByteSequenceSpotter(); + long peakMemoryUsed = 0; + for (int i = 0; i < DuplicateByteSequenceSpotter.MAX_HIT_COUNT * 2; i++) { + Analyzer analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer t = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(t, new DeDuplicatingTokenFilter(t, bytesDeDuper, true)); + } + }; + try { + String input = "1 2 3 4 5 6"; + bytesDeDuper.startNewSequence(); + + TokenStream test = analyzer.tokenStream("test", input); + DuplicateSequenceAttribute dsa = test.addAttribute(DuplicateSequenceAttribute.class); + + test.reset(); + + while (test.incrementToken()) { + assertEquals(Math.min(DuplicateByteSequenceSpotter.MAX_HIT_COUNT, i), dsa.getNumPriorUsesInASequence()); + } + + if (i == 0) { + peakMemoryUsed = bytesDeDuper.getEstimatedSizeInBytes(); + } else { + // Given we are feeding the same content repeatedly the + // actual memory + // used by bytesDeDuper should not grow + assertEquals(peakMemoryUsed, bytesDeDuper.getEstimatedSizeInBytes()); + } + + } finally { + analyzer.close(); + } + } + } + + public void testTaggedFrequencies() throws IOException { + DuplicateByteSequenceSpotter bytesDeDuper = new DuplicateByteSequenceSpotter(); + Analyzer analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer t = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(t, new DeDuplicatingTokenFilter(t, bytesDeDuper, true)); + } + }; + try { + String input = "a b c 1 2 3 4 5 6 7 a b c d 1 2 3 4 5 6 7 e f 1 2 3 4 5 6 7"; + short[] expectedFrequencies = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 2, 2, 2, 2, 2, 2, 2 }; + TokenStream test = analyzer.tokenStream("test", input); + DuplicateSequenceAttribute seqAtt = test.addAttribute(DuplicateSequenceAttribute.class); + + test.reset(); + + for (int i = 0; i < expectedFrequencies.length; i++) { + assertThat(test.incrementToken(), equalTo(true)); + assertThat(seqAtt.getNumPriorUsesInASequence(), equalTo(expectedFrequencies[i])); + } + assertThat(test.incrementToken(), equalTo(false)); + } finally { + analyzer.close(); + } + + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java index 106c24982a87a..764a6d3b3512d 100644 --- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java +++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java @@ -56,7 +56,6 @@ import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.env.ShardLockObtainFailedException; -import org.elasticsearch.index.AlreadyExpiredException; import org.elasticsearch.index.Index; import org.elasticsearch.index.engine.RecoveryEngineException; import org.elasticsearch.index.query.QueryShardException; @@ -296,24 +295,6 @@ public void testSearchException() throws IOException { assertTrue(ex.getCause() instanceof NullPointerException); } - public void testAlreadyExpiredException() throws IOException { - AlreadyExpiredException alreadyExpiredException = serialize(new AlreadyExpiredException("index", "type", "id", 1, 2, 3)); - assertEquals("index", alreadyExpiredException.getIndex().getName()); - assertEquals("type", alreadyExpiredException.type()); - assertEquals("id", alreadyExpiredException.id()); - assertEquals(2, alreadyExpiredException.ttl()); - assertEquals(1, alreadyExpiredException.timestamp()); - assertEquals(3, alreadyExpiredException.now()); - - alreadyExpiredException = serialize(new AlreadyExpiredException(null, null, null, -1, -2, -3)); - assertNull(alreadyExpiredException.getIndex()); - assertNull(alreadyExpiredException.type()); - assertNull(alreadyExpiredException.id()); - assertEquals(-2, alreadyExpiredException.ttl()); - assertEquals(-1, alreadyExpiredException.timestamp()); - assertEquals(-3, alreadyExpiredException.now()); - } - public void testActionNotFoundTransportException() throws IOException { ActionNotFoundTransportException ex = serialize(new ActionNotFoundTransportException("AACCCTION")); assertEquals("AACCCTION", ex.action()); @@ -780,7 +761,7 @@ public void testIds() { ids.put(82, org.elasticsearch.repositories.RepositoryException.class); ids.put(83, org.elasticsearch.transport.ReceiveTimeoutTransportException.class); ids.put(84, org.elasticsearch.transport.NodeDisconnectedException.class); - ids.put(85, org.elasticsearch.index.AlreadyExpiredException.class); + ids.put(85, null); ids.put(86, org.elasticsearch.search.aggregations.AggregationExecutionException.class); ids.put(88, org.elasticsearch.indices.InvalidIndexTemplateException.class); ids.put(90, org.elasticsearch.index.engine.RefreshFailedEngineException.class); diff --git a/core/src/test/java/org/elasticsearch/VersionTests.java b/core/src/test/java/org/elasticsearch/VersionTests.java index 27782511de725..3bdec11472916 100644 --- a/core/src/test/java/org/elasticsearch/VersionTests.java +++ b/core/src/test/java/org/elasticsearch/VersionTests.java @@ -34,7 +34,7 @@ import java.util.Set; import static org.elasticsearch.Version.V_5_3_0; -import static org.elasticsearch.Version.V_6_0_0_alpha2_UNRELEASED; +import static org.elasticsearch.Version.V_6_0_0_alpha2; import static org.elasticsearch.test.VersionUtils.randomVersion; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.containsString; @@ -46,30 +46,30 @@ public class VersionTests extends ESTestCase { public void testVersionComparison() throws Exception { - assertThat(V_5_3_0.before(V_6_0_0_alpha2_UNRELEASED), is(true)); + assertThat(V_5_3_0.before(V_6_0_0_alpha2), is(true)); assertThat(V_5_3_0.before(V_5_3_0), is(false)); - assertThat(V_6_0_0_alpha2_UNRELEASED.before(V_5_3_0), is(false)); + assertThat(V_6_0_0_alpha2.before(V_5_3_0), is(false)); - assertThat(V_5_3_0.onOrBefore(V_6_0_0_alpha2_UNRELEASED), is(true)); + assertThat(V_5_3_0.onOrBefore(V_6_0_0_alpha2), is(true)); assertThat(V_5_3_0.onOrBefore(V_5_3_0), is(true)); - assertThat(V_6_0_0_alpha2_UNRELEASED.onOrBefore(V_5_3_0), is(false)); + assertThat(V_6_0_0_alpha2.onOrBefore(V_5_3_0), is(false)); - assertThat(V_5_3_0.after(V_6_0_0_alpha2_UNRELEASED), is(false)); + assertThat(V_5_3_0.after(V_6_0_0_alpha2), is(false)); assertThat(V_5_3_0.after(V_5_3_0), is(false)); - assertThat(V_6_0_0_alpha2_UNRELEASED.after(V_5_3_0), is(true)); + assertThat(V_6_0_0_alpha2.after(V_5_3_0), is(true)); - assertThat(V_5_3_0.onOrAfter(V_6_0_0_alpha2_UNRELEASED), is(false)); + assertThat(V_5_3_0.onOrAfter(V_6_0_0_alpha2), is(false)); assertThat(V_5_3_0.onOrAfter(V_5_3_0), is(true)); - assertThat(V_6_0_0_alpha2_UNRELEASED.onOrAfter(V_5_3_0), is(true)); + assertThat(V_6_0_0_alpha2.onOrAfter(V_5_3_0), is(true)); assertTrue(Version.fromString("5.0.0-alpha2").onOrAfter(Version.fromString("5.0.0-alpha1"))); assertTrue(Version.fromString("5.0.0").onOrAfter(Version.fromString("5.0.0-beta2"))); assertTrue(Version.fromString("5.0.0-rc1").onOrAfter(Version.fromString("5.0.0-beta24"))); assertTrue(Version.fromString("5.0.0-alpha24").before(Version.fromString("5.0.0-beta0"))); - assertThat(V_5_3_0, is(lessThan(V_6_0_0_alpha2_UNRELEASED))); + assertThat(V_5_3_0, is(lessThan(V_6_0_0_alpha2))); assertThat(V_5_3_0.compareTo(V_5_3_0), is(0)); - assertThat(V_6_0_0_alpha2_UNRELEASED, is(greaterThan(V_5_3_0))); + assertThat(V_6_0_0_alpha2, is(greaterThan(V_5_3_0))); } public void testMin() { @@ -97,7 +97,7 @@ public void testMax() { } public void testMinimumIndexCompatibilityVersion() { - assertEquals(Version.V_5_0_0, Version.V_6_0_0_alpha2_UNRELEASED.minimumIndexCompatibilityVersion()); + assertEquals(Version.V_5_0_0, Version.V_6_0_0_alpha2.minimumIndexCompatibilityVersion()); assertEquals(Version.fromId(2000099), Version.V_5_0_0.minimumIndexCompatibilityVersion()); assertEquals(Version.fromId(2000099), Version.V_5_1_1.minimumIndexCompatibilityVersion()); @@ -157,7 +157,7 @@ public void testVersionNoPresentInSettings() { public void testIndexCreatedVersion() { // an actual index has a IndexMetaData.SETTING_INDEX_UUID final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_2, - Version.V_5_2_0, Version.V_6_0_0_alpha2_UNRELEASED); + Version.V_5_2_0, Version.V_6_0_0_alpha2); assertEquals(version, Version.indexCreated(Settings.builder().put(IndexMetaData.SETTING_INDEX_UUID, "foo").put(IndexMetaData.SETTING_VERSION_CREATED, version).build())); } @@ -170,11 +170,11 @@ public void testMinCompatVersion() { assertThat(Version.fromString("2.3.0").minimumCompatibilityVersion(), equalTo(major)); // from 6.0 on we are supporting the latest minor of the previous major... this might fail once we add a new version ie. 5.x is // released since we need to bump the supported minor in Version#minimumCompatibilityVersion() - Version lastVersion = VersionUtils.getPreviousVersion(Version.V_6_0_0_alpha2_UNRELEASED); - assertEquals(lastVersion.major, Version.V_6_0_0_alpha2_UNRELEASED.minimumCompatibilityVersion().major); + Version lastVersion = VersionUtils.getPreviousVersion(Version.V_6_0_0_alpha1); + assertEquals(lastVersion.major, Version.V_6_0_0_alpha2.minimumCompatibilityVersion().major); assertEquals("did you miss to bump the minor in Version#minimumCompatibilityVersion()", - lastVersion.minor, Version.V_6_0_0_alpha2_UNRELEASED.minimumCompatibilityVersion().minor); - assertEquals(0, Version.V_6_0_0_alpha2_UNRELEASED.minimumCompatibilityVersion().revision); + lastVersion.minor, Version.V_6_0_0_alpha2.minimumCompatibilityVersion().minor); + assertEquals(0, Version.V_6_0_0_alpha2.minimumCompatibilityVersion().revision); } public void testToString() { @@ -325,8 +325,8 @@ public static void assertUnknownVersion(Version version) { public void testIsCompatible() { assertTrue(isCompatible(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion())); - assertTrue(isCompatible(Version.V_5_5_0_UNRELEASED, Version.V_6_0_0_alpha2_UNRELEASED)); - assertFalse(isCompatible(Version.fromId(2000099), Version.V_6_0_0_alpha2_UNRELEASED)); + assertTrue(isCompatible(Version.V_5_5_0, Version.V_6_0_0_alpha2)); + assertFalse(isCompatible(Version.fromId(2000099), Version.V_6_0_0_alpha2)); assertFalse(isCompatible(Version.fromId(2000099), Version.V_5_0_0)); assertTrue(isCompatible(Version.fromString("6.0.0"), Version.fromString("7.0.0"))); if (Version.CURRENT.isRelease()) { diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java index 2870b04fdb747..69b82e232d001 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java @@ -81,14 +81,19 @@ public void testPutMappingRequestSerialization() throws IOException { request.source(mapping, XContentType.YAML); assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.source()); - BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - request.writeTo(bytesStreamOutput); - StreamInput in = StreamInput.wrap(bytesStreamOutput.bytes().toBytesRef().bytes); - PutMappingRequest serialized = new PutMappingRequest(); - serialized.readFrom(in); + final Version version = randomFrom(Version.CURRENT, Version.V_5_3_0, Version.V_5_3_1, Version.V_5_3_2, Version.V_5_4_0); + try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) { + bytesStreamOutput.setVersion(version); + request.writeTo(bytesStreamOutput); + try (StreamInput in = StreamInput.wrap(bytesStreamOutput.bytes().toBytesRef().bytes)) { + in.setVersion(version); + PutMappingRequest serialized = new PutMappingRequest(); + serialized.readFrom(in); - String source = serialized.source(); - assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), source); + String source = serialized.source(); + assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), source); + } + } } public void testSerializationBwc() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java index 453efb2a6059f..2137b33eb089d 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java @@ -77,13 +77,19 @@ public void testPutIndexTemplateRequestSerializationXContent() throws IOExceptio assertNotEquals(mapping, request.mappings().get("bar")); assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.mappings().get("bar")); - BytesStreamOutput out = new BytesStreamOutput(); - request.writeTo(out); + final Version version = randomFrom(Version.CURRENT, Version.V_5_3_0, Version.V_5_3_1, Version.V_5_3_2, Version.V_5_4_0); + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + request.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes); - PutIndexTemplateRequest serialized = new PutIndexTemplateRequest(); - serialized.readFrom(in); - assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), serialized.mappings().get("bar")); + try (StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes)) { + in.setVersion(version); + PutIndexTemplateRequest serialized = new PutIndexTemplateRequest(); + serialized.readFrom(in); + assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), + serialized.mappings().get("bar")); + } + } } public void testPutIndexTemplateRequestSerializationXContentBwc() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java b/core/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java index 20406acc9b6e7..39a4bb2feca3f 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java @@ -631,16 +631,17 @@ public void testPrepareIndexOpOnReplica() throws Exception { IndexMetaData metaData = indexMetaData(); IndexShard shard = newStartedShard(false); - DocWriteResponse primaryResponse = new IndexResponse(shardId, "index", "id", 1, 17, 1, randomBoolean()); + DocWriteResponse primaryResponse = new IndexResponse(shardId, "index", "id", 17, 0, 1, randomBoolean()); IndexRequest request = new IndexRequest("index", "type", "id") .source(Requests.INDEX_CONTENT_TYPE, "field", "value"); Engine.Index op = TransportShardBulkAction.prepareIndexOperationOnReplica( - primaryResponse, request, shard); + primaryResponse, request, shard.getPrimaryTerm(), shard); assertThat(op.version(), equalTo(primaryResponse.getVersion())); assertThat(op.seqNo(), equalTo(primaryResponse.getSeqNo())); assertThat(op.versionType(), equalTo(VersionType.EXTERNAL)); + assertThat(op.primaryTerm(), equalTo(shard.getPrimaryTerm())); closeShards(shard); } diff --git a/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index 255025302c78d..a85f4892933a7 100644 --- a/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -36,25 +36,38 @@ import org.hamcrest.Matchers; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.IntStream; public class ExpandSearchPhaseTests extends ESTestCase { public void testCollapseSingleHit() throws IOException { final int iters = randomIntBetween(5, 10); for (int i = 0; i < iters; i++) { - SearchHits collapsedHits = new SearchHits(new SearchHit[]{new SearchHit(2, "ID", new Text("type"), - Collections.emptyMap()), new SearchHit(3, "ID", new Text("type"), - Collections.emptyMap())}, 1, 1.0F); + final int numInnerHits = randomIntBetween(1, 5); + List collapsedHits = new ArrayList<>(numInnerHits); + for (int innerHitNum = 0; innerHitNum < numInnerHits; innerHitNum++) { + SearchHits hits = new SearchHits(new SearchHit[]{new SearchHit(innerHitNum, "ID", new Text("type"), + Collections.emptyMap()), new SearchHit(innerHitNum + 1, "ID", new Text("type"), + Collections.emptyMap())}, 2, 1.0F); + collapsedHits.add(hits); + } + AtomicBoolean executedMultiSearch = new AtomicBoolean(false); QueryBuilder originalQuery = randomBoolean() ? null : QueryBuilders.termQuery("foo", "bar"); - MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1); + final MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1); String collapseValue = randomBoolean() ? null : "boom"; + mockSearchPhaseContext.getRequest().source(new SearchSourceBuilder() - .collapse(new CollapseBuilder("someField").setInnerHits(new InnerHitBuilder().setName("foobarbaz")))); + .collapse(new CollapseBuilder("someField") + .setInnerHits(IntStream.range(0, numInnerHits).mapToObj(hitNum -> new InnerHitBuilder().setName("innerHit" + hitNum)) + .collect(Collectors.toList())))); mockSearchPhaseContext.getRequest().source().query(originalQuery); mockSearchPhaseContext.searchTransport = new SearchTransportService( Settings.builder().put("search.remote.connect", false).build(), null) { @@ -62,9 +75,10 @@ public void testCollapseSingleHit() throws IOException { @Override void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener listener) { assertTrue(executedMultiSearch.compareAndSet(false, true)); - assertEquals(1, request.requests().size()); + assertEquals(numInnerHits, request.requests().size()); SearchRequest searchRequest = request.requests().get(0); assertTrue(searchRequest.source().query() instanceof BoolQueryBuilder); + BoolQueryBuilder groupBuilder = (BoolQueryBuilder) searchRequest.source().query(); if (collapseValue == null) { assertThat(groupBuilder.mustNot(), Matchers.contains(QueryBuilders.existsQuery("someField"))); @@ -78,13 +92,15 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL assertArrayEquals(mockSearchPhaseContext.getRequest().types(), searchRequest.types()); - InternalSearchResponse internalSearchResponse = new InternalSearchResponse(collapsedHits, - null, null, null, false, null, 1); - SearchResponse response = mockSearchPhaseContext.buildSearchResponse(internalSearchResponse, null); - listener.onResponse(new MultiSearchResponse(new MultiSearchResponse.Item[]{ - new MultiSearchResponse.Item(response, null) - })); + List mSearchResponses = new ArrayList<>(numInnerHits); + for (int innerHitNum = 0; innerHitNum < numInnerHits; innerHitNum++) { + InternalSearchResponse internalSearchResponse = new InternalSearchResponse(collapsedHits.get(innerHitNum), + null, null, null, false, null, 1); + SearchResponse response = mockSearchPhaseContext.buildSearchResponse(internalSearchResponse, null); + mSearchResponses.add(new MultiSearchResponse.Item(response, null)); + } + listener.onResponse(new MultiSearchResponse(mSearchResponses.toArray(new MultiSearchResponse.Item[0]))); } }; @@ -108,8 +124,12 @@ public void run() throws IOException { assertNotNull(reference.get()); SearchResponse theResponse = reference.get(); assertSame(theResponse, response); - assertEquals(1, theResponse.getHits().getHits()[0].getInnerHits().size()); - assertSame(theResponse.getHits().getHits()[0].getInnerHits().get("foobarbaz"), collapsedHits); + assertEquals(numInnerHits, theResponse.getHits().getHits()[0].getInnerHits().size()); + + for (int innerHitNum = 0; innerHitNum < numInnerHits; innerHitNum++) { + assertSame(theResponse.getHits().getHits()[0].getInnerHits().get("innerHit" + innerHitNum), collapsedHits.get(innerHitNum)); + } + assertTrue(executedMultiSearch.get()); assertEquals(1, mockSearchPhaseContext.phasesExecuted.get()); } diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index 89026d9d1dbd3..f91fab381d313 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -21,6 +21,7 @@ import org.apache.lucene.store.AlreadyClosedException; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.UnavailableShardsException; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; @@ -49,10 +50,14 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexService; @@ -63,12 +68,16 @@ import org.elasticsearch.index.shard.ShardNotFoundException; import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.indices.cluster.ClusterStateChanges; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.CapturingTransport; +import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.MockTcpTransport; +import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportRequest; @@ -82,10 +91,13 @@ import org.junit.BeforeClass; import java.io.IOException; +import java.net.UnknownHostException; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -918,6 +930,61 @@ protected ReplicaResult shardOperationOnReplica(Request request, IndexShard repl assertConcreteShardRequest(capturedRequest.request, request, replica.allocationId()); } + public void testRetryOnReplicaWithRealTransport() throws Exception { + final ShardId shardId = new ShardId("test", "_na_", 0); + final ClusterState initialState = state(shardId.getIndexName(), true, ShardRoutingState.STARTED, ShardRoutingState.STARTED); + final ShardRouting replica = initialState.getRoutingTable().shardRoutingTable(shardId).replicaShards().get(0); + // simulate execution of the node holding the replica + final ClusterState stateWithNodes = ClusterState.builder(initialState) + .nodes(DiscoveryNodes.builder(initialState.nodes()).localNodeId(replica.currentNodeId())).build(); + setState(clusterService, stateWithNodes); + AtomicBoolean throwException = new AtomicBoolean(true); + final ReplicationTask task = maybeTask(); + NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList()); + final Transport transport = new MockTcpTransport(Settings.EMPTY, threadPool, BigArrays.NON_RECYCLING_INSTANCE, + new NoneCircuitBreakerService(), namedWriteableRegistry, new NetworkService(Settings.EMPTY, Collections.emptyList()), + Version.CURRENT); + transportService = new MockTransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, + x -> clusterService.localNode(),null); + transportService.start(); + transportService.acceptIncomingRequests(); + + AtomicBoolean calledSuccessfully = new AtomicBoolean(false); + TestAction action = new TestAction(Settings.EMPTY, "testActionWithExceptions", transportService, clusterService, shardStateAction, + threadPool) { + @Override + protected ReplicaResult shardOperationOnReplica(Request request, IndexShard replica) { + assertPhase(task, "replica"); + if (throwException.get()) { + throw new RetryOnReplicaException(shardId, "simulation"); + } + calledSuccessfully.set(true); + return new ReplicaResult(); + } + }; + final TestAction.ReplicaOperationTransportHandler replicaOperationTransportHandler = action.new ReplicaOperationTransportHandler(); + final PlainActionFuture listener = new PlainActionFuture<>(); + final Request request = new Request().setShardId(shardId); + final long checkpoint = randomNonNegativeLong(); + request.primaryTerm(stateWithNodes.metaData().getIndexSafe(shardId.getIndex()).primaryTerm(shardId.id())); + replicaOperationTransportHandler.messageReceived( + new TransportReplicationAction.ConcreteReplicaRequest<>(request, replica.allocationId().getId(), checkpoint), + createTransportChannel(listener), task); + if (listener.isDone()) { + listener.get(); // fail with the exception if there + fail("listener shouldn't be done"); + } + + // release the waiting + throwException.set(false); + // publish a new state (same as the old state with the version incremented) + setState(clusterService, stateWithNodes); + + // Assert that the request was retried, this time successfull + assertTrue("action should have been successfully called on retry but was not", calledSuccessfully.get()); + transportService.stop(); + } + private void assertConcreteShardRequest(TransportRequest capturedRequest, Request expectedRequest, AllocationId expectedAllocationId) { final TransportReplicationAction.ConcreteShardRequest concreteShardRequest = (TransportReplicationAction.ConcreteShardRequest) capturedRequest; diff --git a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java index 0159ffe8994c8..e872d3d854ecc 100644 --- a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java @@ -42,8 +42,8 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptType; import org.elasticsearch.test.ESTestCase; @@ -136,7 +136,7 @@ public void setUp() throws Exception { scripts.put("return", vars -> null); final MockScriptEngine engine = new MockScriptEngine("mock", scripts); Map engines = Collections.singletonMap(engine.getType(), engine); - ScriptService scriptService = new ScriptService(baseSettings, engines, ScriptContext.BUILTINS); + ScriptService scriptService = new ScriptService(baseSettings, engines, ScriptModule.CORE_CONTEXTS); final Settings settings = settings(Version.CURRENT).build(); updateHelper = new UpdateHelper(settings, scriptService); diff --git a/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java index 11d42516331ab..eb68fe17d9a70 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java @@ -24,12 +24,6 @@ import org.apache.lucene.util.TestUtil; import org.elasticsearch.Version; import org.elasticsearch.VersionTests; -import org.elasticsearch.action.admin.indices.get.GetIndexResponse; -import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; -import org.elasticsearch.action.admin.indices.segments.IndexSegments; -import org.elasticsearch.action.admin.indices.segments.IndexShardSegments; -import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse; -import org.elasticsearch.action.admin.indices.segments.ShardSegments; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchRequestBuilder; @@ -38,7 +32,6 @@ import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.Settings; @@ -51,9 +44,7 @@ import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.gateway.MetaDataStateFormat; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.engine.Segment; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.node.Node; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHit; @@ -238,7 +229,6 @@ void assertOldIndexWorks(String index) throws Exception { // node startup upgradeIndexFolder(); importIndex(indexName); - assertIndexSanity(indexName, version); assertBasicSearchWorks(indexName); assertAllSearchWorks(indexName); assertBasicAggregationWorks(indexName); @@ -251,54 +241,6 @@ void assertOldIndexWorks(String index) throws Exception { unloadIndex(indexName); } - void assertIndexSanity(String indexName, Version indexCreated) { - GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().addIndices(indexName).get(); - assertEquals(1, getIndexResponse.indices().length); - assertEquals(indexName, getIndexResponse.indices()[0]); - Version actualVersionCreated = Version.indexCreated(getIndexResponse.getSettings().get(indexName)); - assertEquals(indexCreated, actualVersionCreated); - ensureYellow(indexName); - RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(indexName) - .setDetailed(true).setActiveOnly(false).get(); - boolean foundTranslog = false; - for (List states : recoveryResponse.shardRecoveryStates().values()) { - for (RecoveryState state : states) { - if (state.getStage() == RecoveryState.Stage.DONE - && state.getPrimary() - && state.getRecoverySource().getType() == RecoverySource.Type.EXISTING_STORE) { - assertFalse("more than one primary recoverd?", foundTranslog); - assertNotEquals(0, state.getTranslog().recoveredOperations()); - foundTranslog = true; - } - } - } - assertTrue("expected translog but nothing was recovered", foundTranslog); - IndicesSegmentResponse segmentsResponse = client().admin().indices().prepareSegments(indexName).get(); - IndexSegments segments = segmentsResponse.getIndices().get(indexName); - int numCurrent = 0; - int numBWC = 0; - for (IndexShardSegments indexShardSegments : segments) { - for (ShardSegments shardSegments : indexShardSegments) { - for (Segment segment : shardSegments) { - if (indexCreated.luceneVersion.equals(segment.version)) { - numBWC++; - if (Version.CURRENT.luceneVersion.equals(segment.version)) { - numCurrent++; - } - } else if (Version.CURRENT.luceneVersion.equals(segment.version)) { - numCurrent++; - } else { - fail("unexpected version " + segment.version); - } - } - } - } - assertNotEquals("expected at least 1 current segment after translog recovery", 0, numCurrent); - assertNotEquals("expected at least 1 old segment", 0, numBWC); - SearchResponse test = client().prepareSearch(indexName).get(); - assertThat(test.getHits().getTotalHits(), greaterThanOrEqualTo(1L)); - } - void assertBasicSearchWorks(String indexName) { logger.info("--> testing basic search"); SearchRequestBuilder searchReq = client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()); diff --git a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java index 2d2a4cec7e3b0..35d46879639ae 100644 --- a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java @@ -43,13 +43,11 @@ import java.util.Collections; -import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertExists; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.lessThan; @ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false) public class NoMasterNodeIT extends ESIntegTestCase { @@ -70,7 +68,7 @@ public void testNoMasterActions() throws Exception { .put(DiscoverySettings.NO_MASTER_BLOCK_SETTING.getKey(), "all") .build(); - TimeValue timeout = TimeValue.timeValueMillis(200); + final TimeValue timeout = TimeValue.timeValueMillis(10); internalCluster().startNode(settings); // start a second node, create an index, and then shut it down so we have no master block @@ -78,12 +76,9 @@ public void testNoMasterActions() throws Exception { createIndex("test"); client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet(); internalCluster().stopRandomDataNode(); - assertBusy(new Runnable() { - @Override - public void run() { - ClusterState state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); - assertTrue(state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID)); - } + assertBusy(() -> { + ClusterState state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); + assertTrue(state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID)); }); assertThrows(client().prepareGet("test", "type1", "1"), @@ -130,25 +125,23 @@ public void run() { ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "test script", Collections.emptyMap())).setTimeout(timeout)); - checkWriteAction(false, timeout, - client().prepareIndex("test", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()).setTimeout(timeout)); + checkWriteAction( + client().prepareIndex("test", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()).setTimeout(timeout)); - checkWriteAction(true, timeout, - client().prepareIndex("no_index", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()).setTimeout(timeout)); + checkWriteAction( + client().prepareIndex("no_index", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()).setTimeout(timeout)); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); bulkRequestBuilder.add(client().prepareIndex("test", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject())); bulkRequestBuilder.add(client().prepareIndex("test", "type1", "2").setSource(XContentFactory.jsonBuilder().startObject().endObject())); - // the request should fail very quickly - use a large timeout and make sure it didn't pass... - timeout = new TimeValue(5000); bulkRequestBuilder.setTimeout(timeout); - checkWriteAction(false, timeout, bulkRequestBuilder); + checkWriteAction(bulkRequestBuilder); bulkRequestBuilder = client().prepareBulk(); bulkRequestBuilder.add(client().prepareIndex("no_index", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject())); bulkRequestBuilder.add(client().prepareIndex("no_index", "type1", "2").setSource(XContentFactory.jsonBuilder().startObject().endObject())); bulkRequestBuilder.setTimeout(timeout); - checkWriteAction(true, timeValueSeconds(5), bulkRequestBuilder); + checkWriteAction(bulkRequestBuilder); internalCluster().startNode(settings); client().admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForNodes("2").execute().actionGet(); @@ -156,7 +149,6 @@ public void run() { void checkUpdateAction(boolean autoCreateIndex, TimeValue timeout, ActionRequestBuilder builder) { // we clean the metadata when loosing a master, therefore all operations on indices will auto create it, if allowed - long now = System.currentTimeMillis(); try { builder.get(); fail("expected ClusterBlockException or MasterNotDiscoveredException"); @@ -166,26 +158,16 @@ void checkUpdateAction(boolean autoCreateIndex, TimeValue timeout, ActionRequest } else { assertFalse(autoCreateIndex); } - // verify we waited before giving up... assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE)); - assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50)); } } - void checkWriteAction(boolean indexShouldBeAutoCreated, TimeValue timeout, ActionRequestBuilder builder) { - long now = System.currentTimeMillis(); + void checkWriteAction(ActionRequestBuilder builder) { try { builder.get(); fail("Expected ClusterBlockException"); } catch (ClusterBlockException e) { - if (indexShouldBeAutoCreated) { - // timeout is 200 - assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50)); - assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE)); - } else { - // timeout is 5000 - assertThat(System.currentTimeMillis() - now, lessThan(timeout.millis() + 300)); - } + assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE)); } } @@ -244,12 +226,10 @@ public void testNoMasterActionsWriteMasterBlock() throws Exception { assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE)); } - now = System.currentTimeMillis(); try { client().prepareIndex("test1", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()).setTimeout(timeout).get(); fail("Expected ClusterBlockException"); } catch (ClusterBlockException e) { - assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50)); assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE)); } diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java index 6eec34a90e981..80ca8cc275a41 100644 --- a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java +++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java @@ -155,7 +155,7 @@ public void testLenientBooleanForPreEs6Index() throws IOException { // time to say goodbye? assertTrue( "It's time to implement #22298. Please delete this test and Settings#getAsBooleanLenientForPreEs6Indices().", - Version.CURRENT.minimumCompatibilityVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)); + Version.CURRENT.minimumCompatibilityVersion().before(Version.V_6_0_0_alpha1)); String falsy = randomFrom("false", "off", "no", "0"); diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index e72f68e1d2b4c..8bf948240b5d4 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -2578,7 +2578,7 @@ public void testUpgradeOldIndex() throws IOException { for (int i = 0; i < numExtraDocs; i++) { ParsedDocument doc = testParsedDocument("extra" + Integer.toString(i), null, testDocument(), new BytesArray("{}"), null); Term uid; - if (indexMetaData.getCreationVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (indexMetaData.getCreationVersion().onOrAfter(Version.V_6_0_0_alpha1)) { uid = new Term(IdFieldMapper.NAME, doc.id()); } else { uid = new Term(UidFieldMapper.NAME, Uid.createUid(doc.type(), doc.id())); @@ -2730,7 +2730,7 @@ public void testRecoverFromForeignTranslog() throws IOException { new TranslogConfig(shardId, createTempDir(), INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE), null, () -> SequenceNumbersService.UNASSIGNED_SEQ_NO); - translog.add(new Translog.Index("test", "SomeBogusId", "{}".getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "SomeBogusId", 0, "{}".getBytes(Charset.forName("UTF-8")))); assertEquals(generation.translogFileGeneration, translog.currentFileGeneration()); translog.close(); @@ -3015,8 +3015,8 @@ public void testDoubleDeliveryPrimary() throws IOException { TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), 10); assertEquals(1, topDocs.totalHits); } - operation = randomAppendOnly(doc, false, 1); - retry = randomAppendOnly(doc, true, 1); + operation = appendOnlyPrimary(doc, false, 1); + retry = appendOnlyPrimary(doc, true, 1); if (randomBoolean()) { Engine.IndexResult indexResult = engine.index(operation); assertNotNull(indexResult.getTranslogLocation()); @@ -3328,10 +3328,11 @@ public void testAppendConcurrently() throws InterruptedException, IOException { int numDocs = randomIntBetween(1000, 10000); assertEquals(0, engine.getNumVersionLookups()); assertEquals(0, engine.getNumIndexVersionsLookups()); + boolean primary = randomBoolean(); List docs = new ArrayList<>(); for (int i = 0; i < numDocs; i++) { final ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null); - Engine.Index index = randomAppendOnly(doc, false, i); + Engine.Index index = primary ? appendOnlyPrimary(doc, false, i) : appendOnlyReplica(doc, false, i, i); docs.add(index); } Collections.shuffle(docs, random()); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java index ffc921d013afa..7ed6efe516ab0 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java @@ -56,7 +56,7 @@ public void testParseUnknownMatchType() { templateDef2.put("mapping", Collections.singletonMap("store", true)); // if a wrong match type is specified, we ignore the template IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> DynamicTemplate.parse("my_template", templateDef2, Version.V_6_0_0_alpha1_UNRELEASED)); + () -> DynamicTemplate.parse("my_template", templateDef2, Version.V_6_0_0_alpha1)); assertEquals("No field type matched on [text], possible values are [object, string, long, double, boolean, date, binary]", e.getMessage()); } diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index daaff4b1fc434..a113132351b78 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -122,8 +122,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; -import java.util.function.BiFunction; -import java.util.function.Function; import java.util.function.LongFunction; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -338,11 +336,12 @@ private Releasable acquireReplicaOperationPermitBlockingly(IndexShard indexShard public void testOperationPermitOnReplicaShards() throws InterruptedException, ExecutionException, IOException, BrokenBarrierException { final ShardId shardId = new ShardId("test", "_na_", 0); final IndexShard indexShard; - + final boolean engineClosed; switch (randomInt(2)) { case 0: // started replica indexShard = newStartedShard(false); + engineClosed = false; break; case 1: { // initializing replica / primary @@ -353,6 +352,7 @@ public void testOperationPermitOnReplicaShards() throws InterruptedException, Ex ShardRoutingState.INITIALIZING, relocating ? AllocationId.newRelocation(AllocationId.newInitializing()) : AllocationId.newInitializing()); indexShard = newShard(routing); + engineClosed = true; break; } case 2: { @@ -363,6 +363,7 @@ public void testOperationPermitOnReplicaShards() throws InterruptedException, Ex true, ShardRoutingState.RELOCATING, AllocationId.newRelocation(routing.allocationId())); indexShard.updateRoutingEntry(routing); indexShard.relocated("test"); + engineClosed = false; break; } default: @@ -380,6 +381,7 @@ public void testOperationPermitOnReplicaShards() throws InterruptedException, Ex } final long primaryTerm = indexShard.getPrimaryTerm(); + final long translogGen = engineClosed ? -1 : indexShard.getTranslog().getGeneration().translogFileGeneration; final Releasable operation1 = acquireReplicaOperationPermitBlockingly(indexShard, primaryTerm); assertEquals(1, indexShard.getActiveOperationsCount()); @@ -414,8 +416,9 @@ public void onFailure(Exception e) { { final AtomicBoolean onResponse = new AtomicBoolean(); - final AtomicBoolean onFailure = new AtomicBoolean(); + final AtomicReference onFailure = new AtomicReference<>(); final CyclicBarrier barrier = new CyclicBarrier(2); + final long newPrimaryTerm = primaryTerm + 1 + randomInt(20); // but you can not increment with a new primary term until the operations on the older primary term complete final Thread thread = new Thread(() -> { try { @@ -424,23 +427,29 @@ public void onFailure(Exception e) { throw new RuntimeException(e); } indexShard.acquireReplicaOperationPermit( - primaryTerm + 1 + randomInt(20), + newPrimaryTerm, new ActionListener() { @Override public void onResponse(Releasable releasable) { + assertThat(indexShard.getPrimaryTerm(), equalTo(newPrimaryTerm)); onResponse.set(true); releasable.close(); + finish(); + } + + @Override + public void onFailure(Exception e) { + onFailure.set(e); + finish(); + } + + private void finish() { try { barrier.await(); } catch (final BrokenBarrierException | InterruptedException e) { throw new RuntimeException(e); } } - - @Override - public void onFailure(Exception e) { - onFailure.set(true); - } }, ThreadPool.Names.SAME); }); @@ -448,16 +457,25 @@ public void onFailure(Exception e) { barrier.await(); // our operation should be blocked until the previous operations complete assertFalse(onResponse.get()); - assertFalse(onFailure.get()); + assertNull(onFailure.get()); + assertThat(indexShard.getPrimaryTerm(), equalTo(primaryTerm)); Releasables.close(operation1); // our operation should still be blocked assertFalse(onResponse.get()); - assertFalse(onFailure.get()); + assertNull(onFailure.get()); + assertThat(indexShard.getPrimaryTerm(), equalTo(primaryTerm)); Releasables.close(operation2); barrier.await(); // now lock acquisition should have succeeded - assertTrue(onResponse.get()); - assertFalse(onFailure.get()); + assertThat(indexShard.getPrimaryTerm(), equalTo(newPrimaryTerm)); + if (engineClosed) { + assertFalse(onResponse.get()); + assertThat(onFailure.get(), instanceOf(AlreadyClosedException.class)); + } else { + assertTrue(onResponse.get()); + assertNull(onFailure.get()); + assertThat(indexShard.getTranslog().getGeneration().translogFileGeneration, equalTo(translogGen + 1)); + } thread.join(); assertEquals(0, indexShard.getActiveOperationsCount()); } @@ -1046,7 +1064,7 @@ public void testRecoverFromStoreWithNoOps() throws IOException { test = otherShard.prepareIndexOnReplica( SourceToParse.source(shard.shardId().getIndexName(), test.type(), test.id(), test.source(), XContentType.JSON), - 1, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false); + 1, 1, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false); otherShard.index(test); final ShardRouting primaryShardRouting = shard.routingEntry(); diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 26e8aa2ea189f..0021139adb92d 100644 --- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -248,12 +248,12 @@ public void testSimpleOperations() throws IOException { Translog.Snapshot snapshot = translog.newSnapshot(); assertThat(snapshot, SnapshotMatchers.size(0)); - addToTranslogAndList(translog, ops, new Translog.Index("test", "1", new byte[]{1})); + addToTranslogAndList(translog, ops, new Translog.Index("test", "1", 0, new byte[]{1})); snapshot = translog.newSnapshot(); assertThat(snapshot, SnapshotMatchers.equalsTo(ops)); assertThat(snapshot.totalOperations(), equalTo(ops.size())); - addToTranslogAndList(translog, ops, new Translog.Delete("test", "2", newUid("2"))); + addToTranslogAndList(translog, ops, new Translog.Delete("test", "2", 1, newUid("2"))); snapshot = translog.newSnapshot(); assertThat(snapshot, SnapshotMatchers.equalsTo(ops)); assertThat(snapshot.totalOperations(), equalTo(ops.size())); @@ -316,7 +316,7 @@ public void testStats() throws IOException { assertThat(stats.estimatedNumberOfOperations(), equalTo(0L)); } assertThat((int) firstOperationPosition, greaterThan(CodecUtil.headerLength(TranslogWriter.TRANSLOG_CODEC))); - translog.add(new Translog.Index("test", "1", new byte[]{1})); + translog.add(new Translog.Index("test", "1", 0, new byte[]{1})); { final TranslogStats stats = stats(); @@ -324,23 +324,21 @@ public void testStats() throws IOException { assertThat(stats.getTranslogSizeInBytes(), equalTo(97L)); } - translog.add(new Translog.Delete("test", "2", newUid("2"))); + translog.add(new Translog.Delete("test", "2", 1, newUid("2"))); { final TranslogStats stats = stats(); assertThat(stats.estimatedNumberOfOperations(), equalTo(2L)); assertThat(stats.getTranslogSizeInBytes(), equalTo(139L)); } - translog.add(new Translog.Delete("test", "3", newUid("3"))); + translog.add(new Translog.Delete("test", "3", 2, newUid("3"))); { final TranslogStats stats = stats(); assertThat(stats.estimatedNumberOfOperations(), equalTo(3L)); assertThat(stats.getTranslogSizeInBytes(), equalTo(181L)); } - final long seqNo = 1; - final long primaryTerm = 1; - translog.add(new Translog.NoOp(seqNo, primaryTerm, randomAlphaOfLength(16))); + translog.add(new Translog.NoOp(3, 1, randomAlphaOfLength(16))); { final TranslogStats stats = stats(); assertThat(stats.estimatedNumberOfOperations(), equalTo(4L)); @@ -416,7 +414,7 @@ public void testSnapshot() throws IOException { Translog.Snapshot snapshot = translog.newSnapshot(); assertThat(snapshot, SnapshotMatchers.size(0)); - addToTranslogAndList(translog, ops, new Translog.Index("test", "1", new byte[]{1})); + addToTranslogAndList(translog, ops, new Translog.Index("test", "1", 0, new byte[]{1})); snapshot = translog.newSnapshot(); assertThat(snapshot, SnapshotMatchers.equalsTo(ops)); @@ -436,15 +434,15 @@ public void testSnapshotWithNewTranslog() throws IOException { Translog.Snapshot snapshot = translog.newSnapshot(); assertThat(snapshot, SnapshotMatchers.size(0)); - addToTranslogAndList(translog, ops, new Translog.Index("test", "1", new byte[]{1})); + addToTranslogAndList(translog, ops, new Translog.Index("test", "1", 0, new byte[]{1})); Translog.Snapshot snapshot1 = translog.newSnapshot(); - addToTranslogAndList(translog, ops, new Translog.Index("test", "2", new byte[]{2})); + addToTranslogAndList(translog, ops, new Translog.Index("test", "2", 1, new byte[]{2})); assertThat(snapshot1, SnapshotMatchers.equalsTo(ops.get(0))); translog.prepareCommit(); - addToTranslogAndList(translog, ops, new Translog.Index("test", "3", new byte[]{3})); + addToTranslogAndList(translog, ops, new Translog.Index("test", "3", 2, new byte[]{3})); try (Translog.View view = translog.newView()) { Translog.Snapshot snapshot2 = translog.newSnapshot(); @@ -456,7 +454,7 @@ public void testSnapshotWithNewTranslog() throws IOException { public void testSnapshotOnClosedTranslog() throws IOException { assertTrue(Files.exists(translogDir.resolve(Translog.getFilename(1)))); - translog.add(new Translog.Index("test", "1", new byte[]{1})); + translog.add(new Translog.Index("test", "1", 0, new byte[]{1})); translog.close(); try { Translog.Snapshot snapshot = translog.newSnapshot(); @@ -501,10 +499,11 @@ public void testConcurrentWritesWithVaryingSize() throws Throwable { Thread[] threads = new Thread[threadCount]; final Exception[] threadExceptions = new Exception[threadCount]; + final AtomicLong seqNoGenerator = new AtomicLong(); final CountDownLatch downLatch = new CountDownLatch(1); for (int i = 0; i < threadCount; i++) { final int threadId = i; - threads[i] = new TranslogThread(translog, downLatch, opsPerThread, threadId, writtenOperations, threadExceptions); + threads[i] = new TranslogThread(translog, downLatch, opsPerThread, threadId, writtenOperations, seqNoGenerator, threadExceptions); threads[i].setDaemon(true); threads[i].start(); } @@ -566,7 +565,7 @@ public void testTranslogChecksums() throws Exception { int translogOperations = randomIntBetween(10, 100); for (int op = 0; op < translogOperations; op++) { String ascii = randomAlphaOfLengthBetween(1, 50); - locations.add(translog.add(new Translog.Index("test", "" + op, ascii.getBytes("UTF-8")))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, ascii.getBytes("UTF-8")))); } translog.sync(); @@ -592,7 +591,7 @@ public void testTruncatedTranslogs() throws Exception { int translogOperations = randomIntBetween(10, 100); for (int op = 0; op < translogOperations; op++) { String ascii = randomAlphaOfLengthBetween(1, 50); - locations.add(translog.add(new Translog.Index("test", "" + op, ascii.getBytes("UTF-8")))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, ascii.getBytes("UTF-8")))); } translog.sync(); @@ -655,7 +654,7 @@ private Term newUid(String uid) { public void testVerifyTranslogIsNotDeleted() throws IOException { assertFileIsPresent(translog, 1); - translog.add(new Translog.Index("test", "1", new byte[]{1})); + translog.add(new Translog.Index("test", "1", 0, new byte[]{1})); Translog.Snapshot snapshot = translog.newSnapshot(); assertThat(snapshot, SnapshotMatchers.size(1)); assertFileIsPresent(translog, 1); @@ -702,13 +701,13 @@ public void doRun() throws BrokenBarrierException, InterruptedException, IOExcep switch (type) { case CREATE: case INDEX: - op = new Translog.Index("type", "" + id, new byte[]{(byte) id}); + op = new Translog.Index("type", "" + id, id, new byte[]{(byte) id}); break; case DELETE: - op = new Translog.Delete("test", Long.toString(id), newUid(Long.toString(id))); + op = new Translog.Delete("test", Long.toString(id), id, newUid(Long.toString(id))); break; case NO_OP: - op = new Translog.NoOp(id, id, Long.toString(id)); + op = new Translog.NoOp(id, 1, Long.toString(id)); break; default: throw new AssertionError("unsupported operation type [" + type + "]"); @@ -853,12 +852,15 @@ public void testSyncUpTo() throws IOException { int translogOperations = randomIntBetween(10, 100); int count = 0; for (int op = 0; op < translogOperations; op++) { - final Translog.Location location = translog.add(new Translog.Index("test", "" + op, Integer.toString(++count).getBytes(Charset.forName("UTF-8")))); + int seqNo = ++count; + final Translog.Location location = + translog.add(new Translog.Index("test", "" + op, seqNo, Integer.toString(seqNo).getBytes(Charset.forName("UTF-8")))); if (randomBoolean()) { assertTrue("at least one operation pending", translog.syncNeeded()); assertTrue("this operation has not been synced", translog.ensureSynced(location)); assertFalse("the last call to ensureSycned synced all previous ops", translog.syncNeeded()); // we are the last location so everything should be synced - translog.add(new Translog.Index("test", "" + op, Integer.toString(++count).getBytes(Charset.forName("UTF-8")))); + seqNo = ++count; + translog.add(new Translog.Index("test", "" + op, seqNo, Integer.toString(seqNo).getBytes(Charset.forName("UTF-8")))); assertTrue("one pending operation", translog.syncNeeded()); assertFalse("this op has been synced before", translog.ensureSynced(location)); // not syncing now assertTrue("we only synced a previous operation yet", translog.syncNeeded()); @@ -886,7 +888,8 @@ public void testSyncUpToStream() throws IOException { if (rarely()) { translog.commit(translog.currentFileGeneration()); // do this first so that there is at least one pending tlog entry } - final Translog.Location location = translog.add(new Translog.Index("test", "" + op, Integer.toString(++count).getBytes(Charset.forName("UTF-8")))); + final Translog.Location location = + translog.add(new Translog.Index("test", "" + op, op, Integer.toString(++count).getBytes(Charset.forName("UTF-8")))); locations.add(location); } Collections.shuffle(locations, random()); @@ -913,7 +916,8 @@ public void testLocationComparison() throws IOException { int translogOperations = randomIntBetween(10, 100); int count = 0; for (int op = 0; op < translogOperations; op++) { - locations.add(translog.add(new Translog.Index("test", "" + op, Integer.toString(++count).getBytes(Charset.forName("UTF-8"))))); + locations.add( + translog.add(new Translog.Index("test", "" + op, op, Integer.toString(++count).getBytes(Charset.forName("UTF-8"))))); if (rarely() && translogOperations > op + 1) { translog.commit(translog.currentFileGeneration()); } @@ -949,7 +953,7 @@ public void testBasicCheckpoint() throws IOException { int lastSynced = -1; long lastSyncedGlobalCheckpoint = globalCheckpoint.get(); for (int op = 0; op < translogOperations; op++) { - locations.add(translog.add(new Translog.Index("test", "" + op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); if (randomBoolean()) { globalCheckpoint.set(globalCheckpoint.get() + randomIntBetween(1, 16)); } @@ -960,7 +964,8 @@ public void testBasicCheckpoint() throws IOException { } } assertEquals(translogOperations, translog.totalOperations()); - translog.add(new Translog.Index("test", "" + translogOperations, Integer.toString(translogOperations).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index( + "test", "" + translogOperations, translogOperations, Integer.toString(translogOperations).getBytes(Charset.forName("UTF-8")))); final Checkpoint checkpoint = Checkpoint.read(translog.location().resolve(Translog.CHECKPOINT_FILE_NAME)); try (TranslogReader reader = translog.openReader(translog.location().resolve(Translog.getFilename(translog.currentFileGeneration())), checkpoint)) { @@ -1077,7 +1082,7 @@ public void testBasicRecovery() throws IOException { int minUncommittedOp = -1; final boolean commitOften = randomBoolean(); for (int op = 0; op < translogOperations; op++) { - locations.add(translog.add(new Translog.Index("test", "" + op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); final boolean commit = commitOften ? frequently() : rarely(); if (commit && op < translogOperations - 1) { translog.commit(translog.currentFileGeneration()); @@ -1116,7 +1121,7 @@ public void testRecoveryUncommitted() throws IOException { Translog.TranslogGeneration translogGeneration = null; final boolean sync = randomBoolean(); for (int op = 0; op < translogOperations; op++) { - locations.add(translog.add(new Translog.Index("test", "" + op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); if (op == prepareOp) { translogGeneration = translog.getGeneration(); translog.prepareCommit(); @@ -1139,7 +1144,7 @@ public void testRecoveryUncommitted() throws IOException { for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null synced: " + sync, next); - assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); + assertEquals("payload mismatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); } } if (randomBoolean()) { // recover twice @@ -1152,7 +1157,7 @@ public void testRecoveryUncommitted() throws IOException { for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null synced: " + sync, next); - assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); + assertEquals("payload mismatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); } } } @@ -1166,7 +1171,7 @@ public void testRecoveryUncommittedFileExists() throws IOException { Translog.TranslogGeneration translogGeneration = null; final boolean sync = randomBoolean(); for (int op = 0; op < translogOperations; op++) { - locations.add(translog.add(new Translog.Index("test", "" + op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); if (op == prepareOp) { translogGeneration = translog.getGeneration(); translog.prepareCommit(); @@ -1193,7 +1198,7 @@ public void testRecoveryUncommittedFileExists() throws IOException { for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null synced: " + sync, next); - assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); + assertEquals("payload mismatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); } } @@ -1208,7 +1213,7 @@ public void testRecoveryUncommittedFileExists() throws IOException { for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null synced: " + sync, next); - assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); + assertEquals("payload mismatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); } } } @@ -1221,7 +1226,7 @@ public void testRecoveryUncommittedCorruptedCheckpoint() throws IOException { Translog.TranslogGeneration translogGeneration = null; final boolean sync = randomBoolean(); for (int op = 0; op < translogOperations; op++) { - locations.add(translog.add(new Translog.Index("test", "" + op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); if (op == prepareOp) { translogGeneration = translog.getGeneration(); translog.prepareCommit(); @@ -1240,7 +1245,9 @@ public void testRecoveryUncommittedCorruptedCheckpoint() throws IOException { try (Translog ignored = new Translog(config, translogGeneration, () -> SequenceNumbersService.UNASSIGNED_SEQ_NO)) { fail("corrupted"); } catch (IllegalStateException ex) { - assertEquals("Checkpoint file translog-2.ckp already exists but has corrupted content expected: Checkpoint{offset=3123, numOps=55, generation=2, minSeqNo=0, maxSeqNo=0, globalCheckpoint=-2} but got: Checkpoint{offset=0, numOps=0, generation=0, minSeqNo=-1, maxSeqNo=-1, globalCheckpoint=-2}", ex.getMessage()); + assertEquals("Checkpoint file translog-2.ckp already exists but has corrupted content expected: Checkpoint{offset=3123, " + + "numOps=55, generation=2, minSeqNo=45, maxSeqNo=99, globalCheckpoint=-2} but got: Checkpoint{offset=0, numOps=0, " + + "generation=0, minSeqNo=-1, maxSeqNo=-1, globalCheckpoint=-2}", ex.getMessage()); } Checkpoint.write(FileChannel::open, config.getTranslogPath().resolve(Translog.getCommitCheckpointFileName(read.generation)), read, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); try (Translog translog = new Translog(config, translogGeneration, () -> SequenceNumbersService.UNASSIGNED_SEQ_NO)) { @@ -1252,7 +1259,7 @@ public void testRecoveryUncommittedCorruptedCheckpoint() throws IOException { for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null synced: " + sync, next); - assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); + assertEquals("payload mismatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); } } } @@ -1262,7 +1269,7 @@ public void testSnapshotFromStreamInput() throws IOException { List ops = new ArrayList<>(); int translogOperations = randomIntBetween(10, 100); for (int op = 0; op < translogOperations; op++) { - Translog.Index test = new Translog.Index("test", "" + op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))); + Translog.Index test = new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))); ops.add(test); } Translog.writeOperations(out, ops); @@ -1277,8 +1284,8 @@ public void testLocationHashCodeEquals() throws IOException { int translogOperations = randomIntBetween(10, 100); try (Translog translog2 = create(createTempDir())) { for (int op = 0; op < translogOperations; op++) { - locations.add(translog.add(new Translog.Index("test", "" + op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); - locations2.add(translog2.add(new Translog.Index("test", "" + op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + locations2.add(translog2.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); } int iters = randomIntBetween(10, 100); for (int i = 0; i < iters; i++) { @@ -1304,7 +1311,7 @@ public void testOpenForeignTranslog() throws IOException { int translogOperations = randomIntBetween(1, 10); int firstUncommitted = 0; for (int op = 0; op < translogOperations; op++) { - locations.add(translog.add(new Translog.Index("test", "" + op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); if (randomBoolean()) { translog.commit(translog.currentFileGeneration()); firstUncommitted = op + 1; @@ -1333,13 +1340,13 @@ public void testOpenForeignTranslog() throws IOException { } public void testFailOnClosedWrite() throws IOException { - translog.add(new Translog.Index("test", "1", Integer.toString(1).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "1", 0, Integer.toString(1).getBytes(Charset.forName("UTF-8")))); translog.close(); try { - translog.add(new Translog.Index("test", "1", Integer.toString(1).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "1", 0, Integer.toString(1).getBytes(Charset.forName("UTF-8")))); fail("closed"); } catch (AlreadyClosedException ex) { - // all is welll + // all is well } } @@ -1353,9 +1360,10 @@ public void testCloseConcurrently() throws Throwable { Thread[] threads = new Thread[threadCount]; final Exception[] threadExceptions = new Exception[threadCount]; final CountDownLatch downLatch = new CountDownLatch(1); + final AtomicLong seqNoGenerator = new AtomicLong(); for (int i = 0; i < threadCount; i++) { final int threadId = i; - threads[i] = new TranslogThread(translog, downLatch, opsPerThread, threadId, writtenOperations, threadExceptions); + threads[i] = new TranslogThread(translog, downLatch, opsPerThread, threadId, writtenOperations, seqNoGenerator, threadExceptions); threads[i].setDaemon(true); threads[i].start(); } @@ -1380,13 +1388,16 @@ private static class TranslogThread extends Thread { private final Collection writtenOperations; private final Exception[] threadExceptions; private final Translog translog; + private final AtomicLong seqNoGenerator; - TranslogThread(Translog translog, CountDownLatch downLatch, int opsPerThread, int threadId, Collection writtenOperations, Exception[] threadExceptions) { + TranslogThread(Translog translog, CountDownLatch downLatch, int opsPerThread, int threadId, + Collection writtenOperations, AtomicLong seqNoGenerator, Exception[] threadExceptions) { this.translog = translog; this.downLatch = downLatch; this.opsPerThread = opsPerThread; this.threadId = threadId; this.writtenOperations = writtenOperations; + this.seqNoGenerator = seqNoGenerator; this.threadExceptions = threadExceptions; } @@ -1400,20 +1411,20 @@ public void run() { switch (type) { case CREATE: case INDEX: - op = new Translog.Index("test", threadId + "_" + opCount, + op = new Translog.Index("test", threadId + "_" + opCount, seqNoGenerator.getAndIncrement(), randomUnicodeOfLengthBetween(1, 20 * 1024).getBytes("UTF-8")); break; case DELETE: op = new Translog.Delete( "test", threadId + "_" + opCount, new Term("_uid", threadId + "_" + opCount), - opCount, + seqNoGenerator.getAndIncrement(), 0, 1 + randomInt(100000), randomFrom(VersionType.values())); break; case NO_OP: - op = new Translog.NoOp(randomNonNegativeLong(), randomNonNegativeLong(), randomAlphaOfLength(16)); + op = new Translog.NoOp(seqNoGenerator.getAndIncrement(), randomNonNegativeLong(), randomAlphaOfLength(16)); break; default: throw new AssertionError("unsupported operation type [" + type + "]"); @@ -1447,7 +1458,8 @@ public void testFailFlush() throws IOException { boolean failed = false; while (failed == false) { try { - locations.add(translog.add(new Translog.Index("test", "" + opsSynced, Integer.toString(opsSynced).getBytes(Charset.forName("UTF-8"))))); + locations.add(translog.add( + new Translog.Index("test", "" + opsSynced, opsSynced, Integer.toString(opsSynced).getBytes(Charset.forName("UTF-8"))))); translog.sync(); opsSynced++; } catch (MockDirectoryWrapper.FakeIOException ex) { @@ -1467,7 +1479,8 @@ public void testFailFlush() throws IOException { fail.failNever(); if (randomBoolean()) { try { - locations.add(translog.add(new Translog.Index("test", "" + opsSynced, Integer.toString(opsSynced).getBytes(Charset.forName("UTF-8"))))); + locations.add(translog.add( + new Translog.Index("test", "" + opsSynced, opsSynced, Integer.toString(opsSynced).getBytes(Charset.forName("UTF-8"))))); fail("we are already closed"); } catch (AlreadyClosedException ex) { assertNotNull(ex.getCause()); @@ -1517,9 +1530,10 @@ public void testFailFlush() throws IOException { public void testTranslogOpsCountIsCorrect() throws IOException { List locations = new ArrayList<>(); int numOps = randomIntBetween(100, 200); - LineFileDocs lineFileDocs = new LineFileDocs(random()); // writes pretty big docs so we cross buffer boarders regularly + LineFileDocs lineFileDocs = new LineFileDocs(random()); // writes pretty big docs so we cross buffer borders regularly for (int opsAdded = 0; opsAdded < numOps; opsAdded++) { - locations.add(translog.add(new Translog.Index("test", "" + opsAdded, lineFileDocs.nextDoc().toString().getBytes(Charset.forName("UTF-8"))))); + locations.add(translog.add( + new Translog.Index("test", "" + opsAdded, opsAdded, lineFileDocs.nextDoc().toString().getBytes(Charset.forName("UTF-8"))))); Translog.Snapshot snapshot = this.translog.newSnapshot(); assertEquals(opsAdded + 1, snapshot.totalOperations()); for (int i = 0; i < opsAdded; i++) { @@ -1536,10 +1550,11 @@ public void testTragicEventCanBeAnyException() throws IOException { TranslogConfig config = getTranslogConfig(tempDir); Translog translog = getFailableTranslog(fail, config, false, true, null); LineFileDocs lineFileDocs = new LineFileDocs(random()); // writes pretty big docs so we cross buffer boarders regularly - translog.add(new Translog.Index("test", "1", lineFileDocs.nextDoc().toString().getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "1", 0, lineFileDocs.nextDoc().toString().getBytes(Charset.forName("UTF-8")))); fail.failAlways(); try { - Translog.Location location = translog.add(new Translog.Index("test", "2", lineFileDocs.nextDoc().toString().getBytes(Charset.forName("UTF-8")))); + Translog.Location location = translog.add( + new Translog.Index("test", "2", 1, lineFileDocs.nextDoc().toString().getBytes(Charset.forName("UTF-8")))); if (randomBoolean()) { translog.ensureSynced(location); } else { @@ -1568,10 +1583,11 @@ public void testFatalIOExceptionsWhileWritingConcurrently() throws IOException, final Exception[] threadExceptions = new Exception[threadCount]; final CountDownLatch downLatch = new CountDownLatch(1); final CountDownLatch added = new CountDownLatch(randomIntBetween(10, 100)); + final AtomicLong seqNoGenerator = new AtomicLong(); List writtenOperations = Collections.synchronizedList(new ArrayList<>()); for (int i = 0; i < threadCount; i++) { final int threadId = i; - threads[i] = new TranslogThread(translog, downLatch, 200, threadId, writtenOperations, threadExceptions) { + threads[i] = new TranslogThread(translog, downLatch, 200, threadId, writtenOperations, seqNoGenerator, threadExceptions) { @Override protected Translog.Location add(Translog.Operation op) throws IOException { Translog.Location add = super.add(op); @@ -1794,7 +1810,7 @@ public void testFailWhileCreateWriteWithRecoveredTLogs() throws IOException { Path tempDir = createTempDir(); TranslogConfig config = getTranslogConfig(tempDir); Translog translog = new Translog(config, null, () -> SequenceNumbersService.UNASSIGNED_SEQ_NO); - translog.add(new Translog.Index("test", "boom", "boom".getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "boom", 0, "boom".getBytes(Charset.forName("UTF-8")))); Translog.TranslogGeneration generation = translog.getGeneration(); translog.close(); try { @@ -1812,7 +1828,7 @@ protected TranslogWriter createWriter(long fileGeneration) throws IOException { } public void testRecoverWithUnbackedNextGen() throws IOException { - translog.add(new Translog.Index("test", "" + 0, Integer.toString(0).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "" + 0, 0, Integer.toString(0).getBytes(Charset.forName("UTF-8")))); Translog.TranslogGeneration translogGeneration = translog.getGeneration(); translog.close(); TranslogConfig config = translog.getConfig(); @@ -1830,7 +1846,7 @@ public void testRecoverWithUnbackedNextGen() throws IOException { assertNotNull("operation " + i + " must be non-null", next); assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.utf8ToString())); } - tlog.add(new Translog.Index("test", "" + 1, Integer.toString(1).getBytes(Charset.forName("UTF-8")))); + tlog.add(new Translog.Index("test", "" + 1, 1, Integer.toString(1).getBytes(Charset.forName("UTF-8")))); } try (Translog tlog = new Translog(config, translogGeneration, () -> SequenceNumbersService.UNASSIGNED_SEQ_NO)) { assertNotNull(translogGeneration); @@ -1845,7 +1861,7 @@ public void testRecoverWithUnbackedNextGen() throws IOException { } public void testRecoverWithUnbackedNextGenInIllegalState() throws IOException { - translog.add(new Translog.Index("test", "" + 0, Integer.toString(0).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "" + 0, 0, Integer.toString(0).getBytes(Charset.forName("UTF-8")))); Translog.TranslogGeneration translogGeneration = translog.getGeneration(); translog.close(); TranslogConfig config = translog.getConfig(); @@ -1865,7 +1881,7 @@ public void testRecoverWithUnbackedNextGenInIllegalState() throws IOException { } public void testRecoverWithUnbackedNextGenAndFutureFile() throws IOException { - translog.add(new Translog.Index("test", "" + 0, Integer.toString(0).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "" + 0, 0, Integer.toString(0).getBytes(Charset.forName("UTF-8")))); Translog.TranslogGeneration translogGeneration = translog.getGeneration(); translog.close(); TranslogConfig config = translog.getConfig(); @@ -1885,7 +1901,7 @@ public void testRecoverWithUnbackedNextGenAndFutureFile() throws IOException { assertNotNull("operation " + i + " must be non-null", next); assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.utf8ToString())); } - tlog.add(new Translog.Index("test", "" + 1, Integer.toString(1).getBytes(Charset.forName("UTF-8")))); + tlog.add(new Translog.Index("test", "" + 1, 1, Integer.toString(1).getBytes(Charset.forName("UTF-8")))); } try { @@ -1923,7 +1939,7 @@ public void testWithRandomException() throws IOException { LineFileDocs lineFileDocs = new LineFileDocs(random()); //writes pretty big docs so we cross buffer boarders regularly for (int opsAdded = 0; opsAdded < numOps; opsAdded++) { String doc = lineFileDocs.nextDoc().toString(); - failableTLog.add(new Translog.Index("test", "" + opsAdded, doc.getBytes(Charset.forName("UTF-8")))); + failableTLog.add(new Translog.Index("test", "" + opsAdded, opsAdded, doc.getBytes(Charset.forName("UTF-8")))); unsynced.add(doc); if (randomBoolean()) { failableTLog.sync(); @@ -2034,16 +2050,16 @@ public void testCheckpointOnDiskFull() throws IOException { * Tests that closing views after the translog is fine and we can reopen the translog */ public void testPendingDelete() throws IOException { - translog.add(new Translog.Index("test", "1", new byte[]{1})); + translog.add(new Translog.Index("test", "1", 0, new byte[]{1})); translog.prepareCommit(); Translog.TranslogGeneration generation = translog.getGeneration(); TranslogConfig config = translog.getConfig(); translog.close(); translog = new Translog(config, generation, () -> SequenceNumbersService.UNASSIGNED_SEQ_NO); - translog.add(new Translog.Index("test", "2", new byte[]{2})); + translog.add(new Translog.Index("test", "2", 1, new byte[]{2})); translog.prepareCommit(); Translog.View view = translog.newView(); - translog.add(new Translog.Index("test", "3", new byte[]{3})); + translog.add(new Translog.Index("test", "3", 2, new byte[]{3})); translog.close(); IOUtils.close(view); translog = new Translog(config, generation, () -> SequenceNumbersService.UNASSIGNED_SEQ_NO); @@ -2197,7 +2213,7 @@ public void testMinGenerationForSeqNo() throws IOException { for (final Long seqNo : shuffledSeqNos) { seqNos.add(Tuple.tuple(seqNo, terms.computeIfAbsent(seqNo, k -> 0L))); Long repeatingTermSeqNo = randomFrom(seqNos.stream().map(Tuple::v1).collect(Collectors.toList())); - seqNos.add(Tuple.tuple(repeatingTermSeqNo, terms.computeIfPresent(repeatingTermSeqNo, (s, t) -> t + 1))); + seqNos.add(Tuple.tuple(repeatingTermSeqNo, terms.get(repeatingTermSeqNo))); } for (final Tuple tuple : seqNos) { diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java index 1c588caadcd45..4f0fec4c85e52 100644 --- a/core/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java +++ b/core/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java @@ -61,7 +61,7 @@ Path translogLocation() { for (int i = 0; i < docs; i++) { Engine.Index indexOp = replica.prepareIndexOnReplica( SourceToParse.source(index, "type", "doc_" + i, new BytesArray("{}"), XContentType.JSON), - seqNo++, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false); + seqNo++, replica.getPrimaryTerm(), 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false); replica.index(indexOp); if (rarely()) { // insert a gap diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/StartRecoveryRequestTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/StartRecoveryRequestTests.java index 6981ebdae4d22..58c2fa6277c33 100644 --- a/core/src/test/java/org/elasticsearch/indices/recovery/StartRecoveryRequestTests.java +++ b/core/src/test/java/org/elasticsearch/indices/recovery/StartRecoveryRequestTests.java @@ -71,7 +71,7 @@ public void testSerialization() throws Exception { assertThat(outRequest.metadataSnapshot().asMap(), equalTo(inRequest.metadataSnapshot().asMap())); assertThat(outRequest.isPrimaryRelocation(), equalTo(inRequest.isPrimaryRelocation())); assertThat(outRequest.recoveryId(), equalTo(inRequest.recoveryId())); - if (targetNodeVersion.onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (targetNodeVersion.onOrAfter(Version.V_6_0_0_alpha1)) { assertThat(outRequest.startingSeqNo(), equalTo(inRequest.startingSeqNo())); } else { assertThat(SequenceNumbersService.UNASSIGNED_SEQ_NO, equalTo(inRequest.startingSeqNo())); diff --git a/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java b/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java index 0db1709e92c15..234524f16f454 100644 --- a/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java +++ b/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java @@ -246,6 +246,8 @@ List readProcDiskStats() throws IOException { public void testAdjustForHugeFilesystems() throws Exception { NodePath np = new FakeNodePath(createTempDir()); assertThat(FsProbe.getFSInfo(np).total, greaterThanOrEqualTo(0L)); + assertThat(FsProbe.getFSInfo(np).free, greaterThanOrEqualTo(0L)); + assertThat(FsProbe.getFSInfo(np).available, greaterThanOrEqualTo(0L)); } static class FakeNodePath extends NodeEnvironment.NodePath { @@ -284,12 +286,12 @@ public long getTotalSpace() throws IOException { @Override public long getUsableSpace() throws IOException { - return 10; + return randomIntBetween(-1000, 1000); } @Override public long getUnallocatedSpace() throws IOException { - return 10; + return randomIntBetween(-1000, 1000); } @Override diff --git a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java index 7010034bc4f26..8b37d8a7e4dbb 100644 --- a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java +++ b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java @@ -449,7 +449,12 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO } } - @TestLogging("org.elasticsearch.action.bulk:TRACE,org.elasticsearch.action.search:TRACE") + @TestLogging( + "org.elasticsearch.action.bulk:TRACE," + + "org.elasticsearch.action.search:TRACE," + + "org.elasticsearch.cluster.service:TRACE," + + "org.elasticsearch.index.seqno:TRACE") + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/24599") public void testIndexAndRelocateConcurrently() throws ExecutionException, InterruptedException { int halfNodes = randomIntBetween(1, 3); Settings[] nodeSettings = Stream.concat( diff --git a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java new file mode 100644 index 0000000000000..be49714973378 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.test.ESTestCase; + +public class ScriptContextTests extends ESTestCase { + + public interface TwoNewInstance { + String newInstance(int foo, int bar); + String newInstance(int foo); + } + + public interface MissingNewInstance { + String typoNewInstanceMethod(int foo); + } + + public interface DummyScript { + int execute(int foo); + + interface Factory { + DummyScript newInstance(); + } + } + + public void testTwoNewInstanceMethods() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> + new ScriptContext<>("test", TwoNewInstance.class)); + assertEquals("Cannot have multiple newInstance methods on FactoryType class [" + + TwoNewInstance.class.getName() + "] for script context [test]", e.getMessage()); + } + + public void testMissingNewInstanceMethod() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> + new ScriptContext<>("test", MissingNewInstance.class)); + assertEquals("Could not find method newInstance on FactoryType class [" + + MissingNewInstance.class.getName() + "] for script context [test]", e.getMessage()); + } + + public void testInstanceTypeReflection() { + ScriptContext context = new ScriptContext<>("test", DummyScript.Factory.class); + assertEquals("test", context.name); + assertEquals(DummyScript.class, context.instanceClazz); + assertEquals(DummyScript.Factory.class, context.factoryClazz); + } +} diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java index edc224da0db7a..af1b0dc3d01c3 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java @@ -30,7 +30,6 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -38,8 +37,6 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; -import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -51,7 +48,7 @@ public class ScriptServiceTests extends ESTestCase { private ScriptEngine scriptEngine; private Map engines; - private Map contexts; + private Map> contexts; private ScriptService scriptService; private Settings baseSettings; @@ -69,9 +66,8 @@ public void setup() throws IOException { } scripts.put("script", p -> null); scriptEngine = new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, scripts); - //prevent duplicates using map - contexts = new HashMap<>(ScriptContext.BUILTINS); + contexts = new HashMap<>(ScriptModule.CORE_CONTEXTS); engines = new HashMap<>(); engines.put(scriptEngine.getType(), scriptEngine); engines.put("test", new MockScriptEngine("test", scripts)); @@ -124,27 +120,26 @@ public void testNotSupportedDisableDynamicSetting() throws IOException { public void testInlineScriptCompiledOnceCache() throws IOException { buildScriptService(Settings.EMPTY); - CompiledScript compiledScript1 = scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), - randomFrom(contexts.values())); - CompiledScript compiledScript2 = scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), - randomFrom(contexts.values())); - assertThat(compiledScript1.compiled(), sameInstance(compiledScript2.compiled())); + Script script = new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()); + SearchScript.Factory factoryScript1 = scriptService.compile(script, SearchScript.CONTEXT); + SearchScript.Factory factoryScript2 = scriptService.compile(script, SearchScript.CONTEXT); + assertThat(factoryScript1, sameInstance(factoryScript2)); } public void testAllowAllScriptTypeSettings() throws IOException { buildScriptService(Settings.EMPTY); - assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.SEARCH); - assertCompileAccepted("painless", "script", ScriptType.STORED, ScriptContext.SEARCH); + assertCompileAccepted("painless", "script", ScriptType.INLINE, SearchScript.CONTEXT); + assertCompileAccepted("painless", "script", ScriptType.STORED, SearchScript.CONTEXT); } public void testAllowAllScriptContextSettings() throws IOException { buildScriptService(Settings.EMPTY); - assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.SEARCH); - assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.AGGS); - assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.UPDATE); - assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.INGEST); + assertCompileAccepted("painless", "script", ScriptType.INLINE, SearchScript.CONTEXT); + assertCompileAccepted("painless", "script", ScriptType.INLINE, SearchScript.AGGS_CONTEXT); + assertCompileAccepted("painless", "script", ScriptType.INLINE, ExecutableScript.UPDATE_CONTEXT); + assertCompileAccepted("painless", "script", ScriptType.INLINE, ExecutableScript.INGEST_CONTEXT); } public void testAllowSomeScriptTypeSettings() throws IOException { @@ -152,8 +147,8 @@ public void testAllowSomeScriptTypeSettings() throws IOException { builder.put("script.allowed_types", "inline"); buildScriptService(builder.build()); - assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.SEARCH); - assertCompileRejected("painless", "script", ScriptType.STORED, ScriptContext.SEARCH); + assertCompileAccepted("painless", "script", ScriptType.INLINE, SearchScript.CONTEXT); + assertCompileRejected("painless", "script", ScriptType.STORED, SearchScript.CONTEXT); } public void testAllowSomeScriptContextSettings() throws IOException { @@ -161,9 +156,9 @@ public void testAllowSomeScriptContextSettings() throws IOException { builder.put("script.allowed_contexts", "search, aggs"); buildScriptService(builder.build()); - assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.SEARCH); - assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.AGGS); - assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.UPDATE); + assertCompileAccepted("painless", "script", ScriptType.INLINE, SearchScript.CONTEXT); + assertCompileAccepted("painless", "script", ScriptType.INLINE, SearchScript.AGGS_CONTEXT); + assertCompileRejected("painless", "script", ScriptType.INLINE, ExecutableScript.UPDATE_CONTEXT); } public void testAllowNoScriptTypeSettings() throws IOException { @@ -171,8 +166,8 @@ public void testAllowNoScriptTypeSettings() throws IOException { builder.put("script.allowed_types", "none"); buildScriptService(builder.build()); - assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.SEARCH); - assertCompileRejected("painless", "script", ScriptType.STORED, ScriptContext.SEARCH); + assertCompileRejected("painless", "script", ScriptType.INLINE, SearchScript.CONTEXT); + assertCompileRejected("painless", "script", ScriptType.STORED, SearchScript.CONTEXT); } public void testAllowNoScriptContextSettings() throws IOException { @@ -180,18 +175,18 @@ public void testAllowNoScriptContextSettings() throws IOException { builder.put("script.allowed_contexts", "none"); buildScriptService(builder.build()); - assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.SEARCH); - assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.AGGS); + assertCompileRejected("painless", "script", ScriptType.INLINE, SearchScript.CONTEXT); + assertCompileRejected("painless", "script", ScriptType.INLINE, SearchScript.AGGS_CONTEXT); } public void testCompileNonRegisteredContext() throws IOException { - contexts.remove(ScriptContext.INGEST.name); + contexts.remove(ExecutableScript.INGEST_CONTEXT.name); buildScriptService(Settings.EMPTY); String type = scriptEngine.getType(); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> - scriptService.compile(new Script(randomFrom(ScriptType.values()), type, "test", Collections.emptyMap()), ScriptContext.INGEST)); - assertThat(e.getMessage(), containsString("script context [" + ScriptContext.INGEST.name + "] not supported")); + scriptService.compile(new Script(randomFrom(ScriptType.values()), type, "test", Collections.emptyMap()), ExecutableScript.INGEST_CONTEXT)); + assertThat(e.getMessage(), containsString("script context [" + ExecutableScript.INGEST_CONTEXT.name + "] not supported")); } public void testCompileCountedInCompilationStats() throws IOException { @@ -200,22 +195,6 @@ public void testCompileCountedInCompilationStats() throws IOException { assertEquals(1L, scriptService.stats().getCompilations()); } - public void testExecutableCountedInCompilationStats() throws IOException { - buildScriptService(Settings.EMPTY); - Script script = new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()); - CompiledScript compiledScript = scriptService.compile(script, randomFrom(contexts.values())); - scriptService.executable(compiledScript, script.getParams()); - assertEquals(1L, scriptService.stats().getCompilations()); - } - - public void testSearchCountedInCompilationStats() throws IOException { - buildScriptService(Settings.EMPTY); - Script script = new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()); - CompiledScript compile = scriptService.compile(script, randomFrom(contexts.values())); - scriptService.search(null, compile, script.getParams()); - assertEquals(1L, scriptService.stats().getCompilations()); - } - public void testMultipleCompilationsCountedInCompilationStats() throws IOException { buildScriptService(Settings.EMPTY); int numberOfCompilations = randomIntBetween(1, 20); @@ -231,8 +210,9 @@ public void testCompilationStatsOnCacheHit() throws IOException { builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), 1); buildScriptService(builder.build()); Script script = new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()); - scriptService.compile(script, randomFrom(contexts.values())); - scriptService.compile(script, randomFrom(contexts.values())); + ScriptContext context = randomFrom(contexts.values()); + scriptService.compile(script, context); + scriptService.compile(script, context); assertEquals(1L, scriptService.stats().getCompilations()); } @@ -252,14 +232,6 @@ public void testCacheEvictionCountedInCacheEvictionsStats() throws IOException { assertEquals(1L, scriptService.stats().getCacheEvictions()); } - public void testDefaultLanguage() throws IOException { - Settings.Builder builder = Settings.builder(); - buildScriptService(builder.build()); - CompiledScript script = scriptService.compile( - new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "1+1", Collections.emptyMap()), randomFrom(contexts.values())); - assertEquals(script.lang(), Script.DEFAULT_SCRIPT_LANG); - } - public void testStoreScript() throws Exception { BytesReference script = XContentFactory.jsonBuilder().startObject() .field("script", "abc") @@ -316,5 +288,4 @@ private void assertCompileAccepted(String lang, String script, ScriptType script notNullValue() ); } - } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalOrderTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalOrderTests.java index 43d10af99fbe9..d26eeca387184 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalOrderTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalOrderTests.java @@ -110,7 +110,7 @@ public void testHistogramOrderBwc() throws IOException { for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) { BucketOrder order = createTestInstance(); Version bwcVersion = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), - VersionUtils.getPreviousVersion(Version.V_6_0_0_alpha2_UNRELEASED)); + VersionUtils.getPreviousVersion(Version.V_6_0_0_alpha2)); boolean bwcOrderFlag = randomBoolean(); try (BytesStreamOutput out = new BytesStreamOutput()) { out.setVersion(bwcVersion); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java index ecdc7ebcc9a37..394870a2a9d2d 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.bucket; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -68,6 +69,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.search.aggregations.AggregationBuilders.filter; import static org.elasticsearch.search.aggregations.AggregationBuilders.significantTerms; +import static org.elasticsearch.search.aggregations.AggregationBuilders.significantText; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; @@ -102,18 +104,34 @@ public void testPlugin() throws Exception { String type = randomBoolean() ? "text" : "long"; String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}"; SharedSignificantTermsTestMethods.index01Docs(type, settings, this); - SearchResponse response = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) - .addAggregation( - terms("class") - .field(CLASS_FIELD) - .subAggregation((significantTerms("sig_terms")) - .field(TEXT_FIELD) - .significanceHeuristic(new SimpleHeuristic()) - .minDocCount(1) - ) - ) - .execute() - .actionGet(); + SearchRequestBuilder request; + if ("text".equals(type) && randomBoolean()) { + // Use significant_text on text fields but occasionally run with alternative of + // significant_terms on legacy fieldData=true too. + request = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) + .addAggregation( + terms("class") + .field(CLASS_FIELD) + .subAggregation((significantText("sig_terms", TEXT_FIELD)) + .significanceHeuristic(new SimpleHeuristic()) + .minDocCount(1) + ) + ); + }else + { + request = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) + .addAggregation( + terms("class") + .field(CLASS_FIELD) + .subAggregation((significantTerms("sig_terms")) + .field(TEXT_FIELD) + .significanceHeuristic(new SimpleHeuristic()) + .minDocCount(1) + ) + ); + } + + SearchResponse response = request.execute().actionGet(); assertSearchResponse(response); StringTerms classes = response.getAggregations().get("class"); assertThat(classes.getBuckets().size(), equalTo(2)); @@ -135,18 +153,7 @@ public void testPlugin() throws Exception { // we run the same test again but this time we do not call assertSearchResponse() before the assertions // the reason is that this would trigger toXContent and we would like to check that this has no potential side effects - response = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) - .addAggregation( - terms("class") - .field(CLASS_FIELD) - .subAggregation((significantTerms("sig_terms")) - .field(TEXT_FIELD) - .significanceHeuristic(new SimpleHeuristic()) - .minDocCount(1) - ) - ) - .execute() - .actionGet(); + response = request.execute().actionGet(); classes = (StringTerms) response.getAggregations().get("class"); assertThat(classes.getBuckets().size(), equalTo(2)); @@ -261,10 +268,23 @@ public void testXContentResponse() throws Exception { String type = randomBoolean() ? "text" : "long"; String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}"; SharedSignificantTermsTestMethods.index01Docs(type, settings, this); - SearchResponse response = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) - .addAggregation(terms("class").field(CLASS_FIELD).subAggregation(significantTerms("sig_terms").field(TEXT_FIELD))) - .execute() - .actionGet(); + + SearchRequestBuilder request; + if ("text".equals(type) && randomBoolean() ) { + // Use significant_text on text fields but occasionally run with alternative of + // significant_terms on legacy fieldData=true too. + request = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) + .addAggregation(terms("class").field(CLASS_FIELD) + .subAggregation(significantText("sig_terms", TEXT_FIELD))); + } else { + request = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) + .addAggregation(terms("class").field(CLASS_FIELD) + .subAggregation(significantTerms("sig_terms").field(TEXT_FIELD))); + } + + SearchResponse response = request.execute().actionGet(); + + assertSearchResponse(response); StringTerms classes = response.getAggregations().get("class"); assertThat(classes.getBuckets().size(), equalTo(2)); @@ -346,26 +366,40 @@ public void testDeletesIssue7951() throws Exception { indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "1").setSource(TEXT_FIELD, text, CLASS_FIELD, "1")); } indexRandom(true, false, indexRequestBuilderList); - - client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) + + + SearchRequestBuilder request; + if (randomBoolean() ) { + request = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) .addAggregation( terms("class") .field(CLASS_FIELD) .subAggregation( significantTerms("sig_terms") .field(TEXT_FIELD) - .minDocCount(1))) - .execute() - .actionGet(); + .minDocCount(1))); + }else + { + request = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) + .addAggregation( + terms("class") + .field(CLASS_FIELD) + .subAggregation( + significantText("sig_terms", TEXT_FIELD) + .minDocCount(1))); + } + + request.execute().actionGet(); + } public void testBackgroundVsSeparateSet() throws Exception { String type = randomBoolean() ? "text" : "long"; String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}"; SharedSignificantTermsTestMethods.index01Docs(type, settings, this); - testBackgroundVsSeparateSet(new MutualInformation(true, true), new MutualInformation(true, false)); - testBackgroundVsSeparateSet(new ChiSquare(true, true), new ChiSquare(true, false)); - testBackgroundVsSeparateSet(new GND(true), new GND(false)); + testBackgroundVsSeparateSet(new MutualInformation(true, true), new MutualInformation(true, false), type); + testBackgroundVsSeparateSet(new ChiSquare(true, true), new ChiSquare(true, false), type); + testBackgroundVsSeparateSet(new GND(true), new GND(false), type); } // compute significance score by @@ -373,35 +407,67 @@ public void testBackgroundVsSeparateSet() throws Exception { // 2. filter buckets and set the background to the other class and set is_background false // both should yield exact same result public void testBackgroundVsSeparateSet(SignificanceHeuristic significanceHeuristicExpectingSuperset, - SignificanceHeuristic significanceHeuristicExpectingSeparateSets) throws Exception { - - SearchResponse response1 = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) - .addAggregation(terms("class") - .field(CLASS_FIELD) - .subAggregation( - significantTerms("sig_terms") - .field(TEXT_FIELD) - .minDocCount(1) - .significanceHeuristic( - significanceHeuristicExpectingSuperset))) - .execute() - .actionGet(); + SignificanceHeuristic significanceHeuristicExpectingSeparateSets, + String type) throws Exception { + + final boolean useSigText = randomBoolean() && type.equals("text"); + SearchRequestBuilder request1; + if (useSigText) { + request1 = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) + .addAggregation(terms("class") + .field(CLASS_FIELD) + .subAggregation( + significantText("sig_terms", TEXT_FIELD) + .minDocCount(1) + .significanceHeuristic( + significanceHeuristicExpectingSuperset))); + }else + { + request1 = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) + .addAggregation(terms("class") + .field(CLASS_FIELD) + .subAggregation( + significantTerms("sig_terms") + .field(TEXT_FIELD) + .minDocCount(1) + .significanceHeuristic( + significanceHeuristicExpectingSuperset))); + } + + SearchResponse response1 = request1.execute().actionGet(); assertSearchResponse(response1); - SearchResponse response2 = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) - .addAggregation(filter("0", QueryBuilders.termQuery(CLASS_FIELD, "0")) - .subAggregation(significantTerms("sig_terms") - .field(TEXT_FIELD) - .minDocCount(1) - .backgroundFilter(QueryBuilders.termQuery(CLASS_FIELD, "1")) - .significanceHeuristic(significanceHeuristicExpectingSeparateSets))) - .addAggregation(filter("1", QueryBuilders.termQuery(CLASS_FIELD, "1")) - .subAggregation(significantTerms("sig_terms") - .field(TEXT_FIELD) - .minDocCount(1) - .backgroundFilter(QueryBuilders.termQuery(CLASS_FIELD, "0")) - .significanceHeuristic(significanceHeuristicExpectingSeparateSets))) - .execute() - .actionGet(); + + SearchRequestBuilder request2; + if (useSigText) { + request2 = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) + .addAggregation(filter("0", QueryBuilders.termQuery(CLASS_FIELD, "0")) + .subAggregation(significantText("sig_terms", TEXT_FIELD) + .minDocCount(1) + .backgroundFilter(QueryBuilders.termQuery(CLASS_FIELD, "1")) + .significanceHeuristic(significanceHeuristicExpectingSeparateSets))) + .addAggregation(filter("1", QueryBuilders.termQuery(CLASS_FIELD, "1")) + .subAggregation(significantText("sig_terms", TEXT_FIELD) + .minDocCount(1) + .backgroundFilter(QueryBuilders.termQuery(CLASS_FIELD, "0")) + .significanceHeuristic(significanceHeuristicExpectingSeparateSets))); + }else + { + request2 = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) + .addAggregation(filter("0", QueryBuilders.termQuery(CLASS_FIELD, "0")) + .subAggregation(significantTerms("sig_terms") + .field(TEXT_FIELD) + .minDocCount(1) + .backgroundFilter(QueryBuilders.termQuery(CLASS_FIELD, "1")) + .significanceHeuristic(significanceHeuristicExpectingSeparateSets))) + .addAggregation(filter("1", QueryBuilders.termQuery(CLASS_FIELD, "1")) + .subAggregation(significantTerms("sig_terms") + .field(TEXT_FIELD) + .minDocCount(1) + .backgroundFilter(QueryBuilders.termQuery(CLASS_FIELD, "0")) + .significanceHeuristic(significanceHeuristicExpectingSeparateSets))); + } + + SearchResponse response2 = request2.execute().actionGet(); StringTerms classes = response1.getAggregations().get("class"); @@ -438,14 +504,24 @@ public void testScoresEqualForPositiveAndNegative() throws Exception { public void testScoresEqualForPositiveAndNegative(SignificanceHeuristic heuristic) throws Exception { //check that results for both classes are the same with exclude negatives = false and classes are routing ids - SearchResponse response = client().prepareSearch("test") - .addAggregation(terms("class").field("class").subAggregation(significantTerms("mySignificantTerms") - .field("text") - .executionHint(randomExecutionHint()) - .significanceHeuristic(heuristic) - .minDocCount(1).shardSize(1000).size(1000))) - .execute() - .actionGet(); + SearchRequestBuilder request; + if (randomBoolean()) { + request = client().prepareSearch("test") + .addAggregation(terms("class").field("class").subAggregation(significantTerms("mySignificantTerms") + .field("text") + .executionHint(randomExecutionHint()) + .significanceHeuristic(heuristic) + .minDocCount(1).shardSize(1000).size(1000))); + }else + { + request = client().prepareSearch("test") + .addAggregation(terms("class").field("class").subAggregation(significantText("mySignificantTerms", "text") + .significanceHeuristic(heuristic) + .minDocCount(1).shardSize(1000).size(1000))); + } + SearchResponse response = request.execute().actionGet(); + assertSearchResponse(response); + assertSearchResponse(response); StringTerms classes = response.getAggregations().get("class"); assertThat(classes.getBuckets().size(), equalTo(2)); @@ -499,18 +575,29 @@ private void indexEqualTestData() throws ExecutionException, InterruptedExceptio } public void testScriptScore() throws ExecutionException, InterruptedException, IOException { - indexRandomFrequencies01(randomBoolean() ? "text" : "long"); + String type = randomBoolean() ? "text" : "long"; + indexRandomFrequencies01(type); ScriptHeuristic scriptHeuristic = getScriptSignificanceHeuristic(); - SearchResponse response = client().prepareSearch(INDEX_NAME) - .addAggregation(terms("class").field(CLASS_FIELD) - .subAggregation(significantTerms("mySignificantTerms") - .field(TEXT_FIELD) - .executionHint(randomExecutionHint()) - .significanceHeuristic(scriptHeuristic) - .minDocCount(1).shardSize(2).size(2))) - .execute() - .actionGet(); - assertSearchResponse(response); + + SearchRequestBuilder request; + if ("text".equals(type) && randomBoolean()) { + request = client().prepareSearch(INDEX_NAME) + .addAggregation(terms("class").field(CLASS_FIELD) + .subAggregation(significantText("mySignificantTerms", TEXT_FIELD) + .significanceHeuristic(scriptHeuristic) + .minDocCount(1).shardSize(2).size(2))); + }else + { + request = client().prepareSearch(INDEX_NAME) + .addAggregation(terms("class").field(CLASS_FIELD) + .subAggregation(significantTerms("mySignificantTerms") + .field(TEXT_FIELD) + .executionHint(randomExecutionHint()) + .significanceHeuristic(scriptHeuristic) + .minDocCount(1).shardSize(2).size(2))); + } + SearchResponse response = request.execute().actionGet(); + assertSearchResponse(response); for (Terms.Bucket classBucket : ((Terms) response.getAggregations().get("class")).getBuckets()) { SignificantTerms sigTerms = classBucket.getAggregations().get("mySignificantTerms"); for (SignificantTerms.Bucket bucket : sigTerms.getBuckets()) { @@ -577,8 +664,15 @@ public void testDontCacheScripts() throws Exception { // Test that a request using a script does not get cached ScriptHeuristic scriptHeuristic = getScriptSignificanceHeuristic(); - SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0) - .addAggregation(significantTerms("foo").field("s").significanceHeuristic(scriptHeuristic)).get(); + boolean useSigText = randomBoolean(); + SearchResponse r; + if (useSigText) { + r = client().prepareSearch("cache_test_idx").setSize(0) + .addAggregation(significantText("foo", "s").significanceHeuristic(scriptHeuristic)).get(); + } else { + r = client().prepareSearch("cache_test_idx").setSize(0) + .addAggregation(significantTerms("foo").field("s").significanceHeuristic(scriptHeuristic)).get(); + } assertSearchResponse(r); assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() @@ -588,7 +682,11 @@ public void testDontCacheScripts() throws Exception { // To make sure that the cache is working test that a request not using // a script is cached - r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(significantTerms("foo").field("s")).get(); + if (useSigText) { + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(significantText("foo", "s")).get(); + } else { + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(significantTerms("foo").field("s")).get(); + } assertSearchResponse(r); assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() @@ -596,5 +694,7 @@ public void testDontCacheScripts() throws Exception { assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() .getMissCount(), equalTo(1L)); } + + } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsTests.java index 9fe1c0ea479e0..a09dd9b4dcc04 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsTests.java @@ -103,77 +103,11 @@ protected SignificantTermsAggregationBuilder createTestAggregatorBuilder() { factory.format("###.##"); } if (randomBoolean()) { - IncludeExclude incExc = null; - switch (randomInt(5)) { - case 0: - incExc = new IncludeExclude(new RegExp("foobar"), null); - break; - case 1: - incExc = new IncludeExclude(null, new RegExp("foobaz")); - break; - case 2: - incExc = new IncludeExclude(new RegExp("foobar"), new RegExp("foobaz")); - break; - case 3: - SortedSet includeValues = new TreeSet<>(); - int numIncs = randomIntBetween(1, 20); - for (int i = 0; i < numIncs; i++) { - includeValues.add(new BytesRef(randomAlphaOfLengthBetween(1, 30))); - } - SortedSet excludeValues = null; - incExc = new IncludeExclude(includeValues, excludeValues); - break; - case 4: - SortedSet includeValues2 = null; - SortedSet excludeValues2 = new TreeSet<>(); - int numExcs2 = randomIntBetween(1, 20); - for (int i = 0; i < numExcs2; i++) { - excludeValues2.add(new BytesRef(randomAlphaOfLengthBetween(1, 30))); - } - incExc = new IncludeExclude(includeValues2, excludeValues2); - break; - case 5: - SortedSet includeValues3 = new TreeSet<>(); - int numIncs3 = randomIntBetween(1, 20); - for (int i = 0; i < numIncs3; i++) { - includeValues3.add(new BytesRef(randomAlphaOfLengthBetween(1, 30))); - } - SortedSet excludeValues3 = new TreeSet<>(); - int numExcs3 = randomIntBetween(1, 20); - for (int i = 0; i < numExcs3; i++) { - excludeValues3.add(new BytesRef(randomAlphaOfLengthBetween(1, 30))); - } - incExc = new IncludeExclude(includeValues3, excludeValues3); - break; - default: - fail(); - } + IncludeExclude incExc = getIncludeExclude(); factory.includeExclude(incExc); } if (randomBoolean()) { - SignificanceHeuristic significanceHeuristic = null; - switch (randomInt(5)) { - case 0: - significanceHeuristic = new PercentageScore(); - break; - case 1: - significanceHeuristic = new ChiSquare(randomBoolean(), randomBoolean()); - break; - case 2: - significanceHeuristic = new GND(randomBoolean()); - break; - case 3: - significanceHeuristic = new MutualInformation(randomBoolean(), randomBoolean()); - break; - case 4: - significanceHeuristic = new ScriptHeuristic(mockScript("foo")); - break; - case 5: - significanceHeuristic = new JLHScore(); - break; - default: - fail(); - } + SignificanceHeuristic significanceHeuristic = getSignificanceHeuristic(); factory.significanceHeuristic(significanceHeuristic); } if (randomBoolean()) { @@ -182,4 +116,80 @@ protected SignificantTermsAggregationBuilder createTestAggregatorBuilder() { return factory; } + static SignificanceHeuristic getSignificanceHeuristic() { + SignificanceHeuristic significanceHeuristic = null; + switch (randomInt(5)) { + case 0: + significanceHeuristic = new PercentageScore(); + break; + case 1: + significanceHeuristic = new ChiSquare(randomBoolean(), randomBoolean()); + break; + case 2: + significanceHeuristic = new GND(randomBoolean()); + break; + case 3: + significanceHeuristic = new MutualInformation(randomBoolean(), randomBoolean()); + break; + case 4: + significanceHeuristic = new ScriptHeuristic(mockScript("foo")); + break; + case 5: + significanceHeuristic = new JLHScore(); + break; + default: + fail(); + } + return significanceHeuristic; + } + + static IncludeExclude getIncludeExclude() { + IncludeExclude incExc = null; + switch (randomInt(5)) { + case 0: + incExc = new IncludeExclude(new RegExp("foobar"), null); + break; + case 1: + incExc = new IncludeExclude(null, new RegExp("foobaz")); + break; + case 2: + incExc = new IncludeExclude(new RegExp("foobar"), new RegExp("foobaz")); + break; + case 3: + SortedSet includeValues = new TreeSet<>(); + int numIncs = randomIntBetween(1, 20); + for (int i = 0; i < numIncs; i++) { + includeValues.add(new BytesRef(randomAlphaOfLengthBetween(1, 30))); + } + SortedSet excludeValues = null; + incExc = new IncludeExclude(includeValues, excludeValues); + break; + case 4: + SortedSet includeValues2 = null; + SortedSet excludeValues2 = new TreeSet<>(); + int numExcs2 = randomIntBetween(1, 20); + for (int i = 0; i < numExcs2; i++) { + excludeValues2.add(new BytesRef(randomAlphaOfLengthBetween(1, 30))); + } + incExc = new IncludeExclude(includeValues2, excludeValues2); + break; + case 5: + SortedSet includeValues3 = new TreeSet<>(); + int numIncs3 = randomIntBetween(1, 20); + for (int i = 0; i < numIncs3; i++) { + includeValues3.add(new BytesRef(randomAlphaOfLengthBetween(1, 30))); + } + SortedSet excludeValues3 = new TreeSet<>(); + int numExcs3 = randomIntBetween(1, 20); + for (int i = 0; i < numExcs3; i++) { + excludeValues3.add(new BytesRef(randomAlphaOfLengthBetween(1, 30))); + } + incExc = new IncludeExclude(includeValues3, excludeValues3); + break; + default: + fail(); + } + return incExc; + } + } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTextTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTextTests.java new file mode 100644 index 0000000000000..03f09ef7d5e5e --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTextTests.java @@ -0,0 +1,94 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket; + +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.aggregations.BaseAggregationTestCase; +import org.elasticsearch.search.aggregations.bucket.significant.SignificantTextAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic; +import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; + +import java.util.Arrays; + +public class SignificantTextTests extends BaseAggregationTestCase { + + @Override + protected SignificantTextAggregationBuilder createTestAggregatorBuilder() { + String name = randomAlphaOfLengthBetween(3, 20); + String field = randomAlphaOfLengthBetween(3, 20); + SignificantTextAggregationBuilder factory = new SignificantTextAggregationBuilder(name, field); + if (randomBoolean()) { + factory.bucketCountThresholds().setRequiredSize(randomIntBetween(1, Integer.MAX_VALUE)); + } + if (randomBoolean()) { + factory.sourceFieldNames(Arrays.asList(new String []{"foo", "bar"})); + } + + if (randomBoolean()) { + factory.bucketCountThresholds().setShardSize(randomIntBetween(1, Integer.MAX_VALUE)); + } + if (randomBoolean()) { + int minDocCount = randomInt(4); + switch (minDocCount) { + case 0: + break; + case 1: + case 2: + case 3: + case 4: + minDocCount = randomIntBetween(0, Integer.MAX_VALUE); + break; + } + factory.bucketCountThresholds().setMinDocCount(minDocCount); + } + if (randomBoolean()) { + int shardMinDocCount = randomInt(4); + switch (shardMinDocCount) { + case 0: + break; + case 1: + case 2: + case 3: + case 4: + shardMinDocCount = randomIntBetween(0, Integer.MAX_VALUE); + break; + default: + fail(); + } + factory.bucketCountThresholds().setShardMinDocCount(shardMinDocCount); + } + + factory.filterDuplicateText(randomBoolean()); + + if (randomBoolean()) { + IncludeExclude incExc = SignificantTermsTests.getIncludeExclude(); + factory.includeExclude(incExc); + } + if (randomBoolean()) { + SignificanceHeuristic significanceHeuristic = SignificantTermsTests.getSignificanceHeuristic(); + factory.significanceHeuristic(significanceHeuristic); + } + if (randomBoolean()) { + factory.backgroundFilter(QueryBuilders.termsQuery("foo", "bar")); + } + return factory; + } + +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java new file mode 100644 index 0000000000000..c8d8b6d59798a --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java @@ -0,0 +1,126 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.significant; + +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.analysis.AnalyzerScope; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.bucket.sampler.Sampler; +import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms; +import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms.Bucket; +import org.elasticsearch.search.aggregations.bucket.significant.SignificantTextAggregationBuilder; + +import java.io.IOException; +import java.util.Arrays; + +public class SignificantTextAggregatorTests extends AggregatorTestCase { + + + /** + * Uses the significant text aggregation to find the keywords in text fields + */ + public void testSignificance() throws IOException { + TextFieldType textFieldType = new TextFieldType(); + textFieldType.setName("text"); + textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer())); + + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + indexWriterConfig.setMaxBufferedDocs(100); + indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a + // single segment with + // predictable docIds + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + StringBuilder text = new StringBuilder("common "); + if (i % 2 == 0) { + text.append("odd "); + } else { + text.append("even separator" + i + " duplicate duplicate duplicate duplicate duplicate duplicate "); + } + + doc.add(new Field("text", text.toString(), textFieldType)); + String json ="{ \"text\" : \"" + text.toString() + "\","+ + " \"json_only_field\" : \"" + text.toString() + "\"" + + " }"; + doc.add(new StoredField("_source", new BytesRef(json))); + w.addDocument(doc); + } + + SignificantTextAggregationBuilder sigAgg = new SignificantTextAggregationBuilder("sig_text", "text").filterDuplicateText(true); + if(randomBoolean()){ + sigAgg.sourceFieldNames(Arrays.asList(new String [] {"json_only_field"})); + } + SamplerAggregationBuilder aggBuilder = new SamplerAggregationBuilder("sampler") + .subAggregation(sigAgg); + + try (IndexReader reader = DirectoryReader.open(w)) { + assertEquals("test expects a single segment", 1, reader.leaves().size()); + IndexSearcher searcher = new IndexSearcher(reader); + + // Search "odd" which should have no duplication + Sampler sampler = searchAndReduce(searcher, new TermQuery(new Term("text", "odd")), aggBuilder, textFieldType); + SignificantTerms terms = sampler.getAggregations().get("sig_text"); + + assertNull(terms.getBucketByKey("even")); + assertNull(terms.getBucketByKey("duplicate")); + assertNull(terms.getBucketByKey("common")); + assertNotNull(terms.getBucketByKey("odd")); + + // Search "even" which will have duplication + sampler = searchAndReduce(searcher, new TermQuery(new Term("text", "even")), aggBuilder, textFieldType); + terms = sampler.getAggregations().get("sig_text"); + + assertNull(terms.getBucketByKey("odd")); + assertNull(terms.getBucketByKey("duplicate")); + assertNull(terms.getBucketByKey("common")); + assertNull(terms.getBucketByKey("separator2")); + assertNull(terms.getBucketByKey("separator4")); + assertNull(terms.getBucketByKey("separator6")); + + assertNotNull(terms.getBucketByKey("even")); + + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java index 7511074bfa432..78df637e0683b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java @@ -25,6 +25,8 @@ import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.aggregations.ParsedAggregation; @@ -113,7 +115,8 @@ protected ScriptService mockScriptService() { @SuppressWarnings("unchecked") MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap(REDUCE_SCRIPT_NAME, script -> ((List) script.get("_aggs")).size())); - return new ScriptService(Settings.EMPTY, Collections.singletonMap(scriptEngine.getType(), scriptEngine), ScriptContext.BUILTINS); + Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); } @Override diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java index a336e759b4931..8357a31d7fed4 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.aggregations.AggregatorTestCase; @@ -197,7 +198,7 @@ protected QueryShardContext queryShardContextMock(MapperService mapperService, f CircuitBreakerService circuitBreakerService) { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, SCRIPTS); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptContext.BUILTINS); + ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); return new QueryShardContext(0, mapperService.getIndexSettings(), null, null, mapperService, null, scriptService, xContentRegistry(), null, null, System::currentTimeMillis); } diff --git a/core/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java b/core/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java index a0533a396a821..07eef1a2f3818 100644 --- a/core/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java @@ -26,30 +26,38 @@ import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.InnerHitBuilderTests; +import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.SearchContextException; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.internal.SearchContext; -import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.test.AbstractSerializingTestCase; import org.junit.AfterClass; import org.junit.BeforeClass; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import static java.util.Collections.emptyList; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class CollapseBuilderTests extends AbstractWireSerializingTestCase { +public class CollapseBuilderTests extends AbstractSerializingTestCase { private static NamedWriteableRegistry namedWriteableRegistry; private static NamedXContentRegistry xContentRegistry; @@ -67,17 +75,30 @@ public static void afterClass() throws Exception { } public static CollapseBuilder randomCollapseBuilder() { + return randomCollapseBuilder(true); + } + + public static CollapseBuilder randomCollapseBuilder(boolean multiInnerHits) { CollapseBuilder builder = new CollapseBuilder(randomAlphaOfLength(10)); builder.setMaxConcurrentGroupRequests(randomIntBetween(1, 48)); - if (randomBoolean()) { + int numInnerHits = randomIntBetween(0, multiInnerHits ? 5 : 1); + if (numInnerHits == 1) { InnerHitBuilder innerHit = InnerHitBuilderTests.randomInnerHits(); builder.setInnerHits(innerHit); + } else if (numInnerHits > 1) { + List innerHits = new ArrayList<>(numInnerHits); + for (int i = 0; i < numInnerHits; i++) { + innerHits.add(InnerHitBuilderTests.randomInnerHits()); + } + + builder.setInnerHits(innerHits); } + return builder; } @Override - protected Writeable createTestInstance() { + protected CollapseBuilder createTestInstance() { return randomCollapseBuilder(); } @@ -177,4 +198,26 @@ public Query termQuery(Object value, QueryShardContext context) { assertEquals(exc.getMessage(), "unknown type for collapse field `field`, only keywords and numbers are accepted"); } } + + @Override + protected CollapseBuilder doParseInstance(XContentParser parser) throws IOException { + return CollapseBuilder.fromXContent(new QueryParseContext(parser)); + } + + /** + * Rewrite this test to disable xcontent shuffling on the highlight builder + */ + public void testFromXContent() throws IOException { + for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) { + CollapseBuilder testInstance = createTestInstance(); + XContentType xContentType = randomFrom(XContentType.values()); + XContentBuilder builder = toXContent(testInstance, xContentType); + XContentBuilder shuffled = shuffleXContent(builder, "fields"); + assertParsedInstance(xContentType, shuffled.bytes(), testInstance); + for (Map.Entry alternateVersion : getAlternateVersions().entrySet()) { + String instanceAsString = alternateVersion.getKey(); + assertParsedInstance(XContentType.JSON, new BytesArray(instanceAsString), alternateVersion.getValue()); + } + } + } } diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java index 40418048e5a9f..d290bd6c3e010 100644 --- a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java @@ -24,24 +24,21 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; -import org.elasticsearch.script.CompiledScript; -import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ExplainableSearchScript; import org.elasticsearch.script.LeafSearchScript; import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.script.ScriptType; import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.lookup.LeafDocLookup; -import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; @@ -70,7 +67,7 @@ public class ExplainableScriptIT extends ESIntegTestCase { public static class ExplainableScriptPlugin extends Plugin implements ScriptPlugin { @Override - public ScriptEngine getScriptEngine(Settings settings) { + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { return new ScriptEngine() { @Override public String getType() { @@ -78,19 +75,10 @@ public String getType() { } @Override - public Object compile(String scriptName, String scriptSource, Map params) { + public T compile(String scriptName, String scriptSource, ScriptContext context, Map params) { assert scriptSource.equals("explainable_script"); - return null; - } - - @Override - public ExecutableScript executable(CompiledScript compiledScript, @Nullable Map vars) { - throw new UnsupportedOperationException(); - } - - @Override - public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map vars) { - return new SearchScript() { + assert context == SearchScript.CONTEXT; + SearchScript.Factory factory = (p, lookup) -> new SearchScript() { @Override public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException { return new MyScript(lookup.doc().getLeafDocLookup(context)); @@ -100,10 +88,8 @@ public boolean needsScores() { return false; } }; + return context.factoryClazz.cast(factory); } - - @Override - public void close() {} }; } } diff --git a/core/src/test/java/org/elasticsearch/search/internal/ScrollContextTests.java b/core/src/test/java/org/elasticsearch/search/internal/ScrollContextTests.java new file mode 100644 index 0000000000000..de4863dd92a08 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/internal/ScrollContextTests.java @@ -0,0 +1,36 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.internal; + +import org.elasticsearch.test.ESTestCase; + +public class ScrollContextTests extends ESTestCase { + + public void testStoringObjectsInScrollContext() { + final ScrollContext scrollContext = new ScrollContext(); + final String key = randomAlphaOfLengthBetween(1, 16); + assertNull(scrollContext.getFromContext(key)); + + final String value = randomAlphaOfLength(6); + scrollContext.putInContext(key, value); + + assertEquals(value, scrollContext.getFromContext(key)); + } +} diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 811401960bd01..a9d81c72f4a03 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -53,6 +53,7 @@ import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchModule; @@ -87,7 +88,7 @@ public static void init() throws IOException { .build(); Map, Object>> scripts = Collections.singletonMap("dummy", p -> null); ScriptEngine engine = new MockScriptEngine(MockScriptEngine.NAME, scripts); - scriptService = new ScriptService(baseSettings, Collections.singletonMap(engine.getType(), engine), ScriptContext.BUILTINS); + scriptService = new ScriptService(baseSettings, Collections.singletonMap(engine.getType(), engine), ScriptModule.CORE_CONTEXTS); SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList()); namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables()); diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java index b08eaee3daf65..50611a1cb9554 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java @@ -25,17 +25,14 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; -import org.elasticsearch.script.SearchScript; -import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.suggest.phrase.DirectCandidateGeneratorBuilder; import org.elasticsearch.search.suggest.phrase.Laplace; import org.elasticsearch.search.suggest.phrase.LinearInterpolation; @@ -1010,7 +1007,7 @@ protected Collection> nodePlugins() { public static class DummyTemplatePlugin extends Plugin implements ScriptPlugin { @Override - public ScriptEngine getScriptEngine(Settings settings) { + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { return new DummyTemplateScriptEngine(); } } @@ -1021,43 +1018,35 @@ public static class DummyTemplateScriptEngine implements ScriptEngine { // which makes the collate code thinks mustache is evaluating the query. public static final String NAME = "mustache"; - @Override - public void close() throws IOException { - } - @Override public String getType() { return NAME; } @Override - public Object compile(String scriptName, String scriptSource, Map params) { - return scriptSource; - } - - @Override - public ExecutableScript executable(CompiledScript compiledScript, Map params) { - String script = (String) compiledScript.compiled(); - for (Entry entry : params.entrySet()) { - script = script.replace("{{" + entry.getKey() + "}}", String.valueOf(entry.getValue())); + public T compile(String scriptName, String scriptSource, ScriptContext context, Map params) { + if (context.instanceClazz != ExecutableScript.class) { + throw new UnsupportedOperationException(); } - String result = script; - return new ExecutableScript() { - @Override - public void setNextVar(String name, Object value) { - throw new UnsupportedOperationException("setNextVar not supported"); - } - - @Override - public Object run() { - return result; + ExecutableScript.Factory factory = p -> { + String script = scriptSource; + for (Entry entry : p.entrySet()) { + script = script.replace("{{" + entry.getKey() + "}}", String.valueOf(entry.getValue())); } + String result = script; + return new ExecutableScript() { + @Override + public void setNextVar(String name, Object value) { + throw new UnsupportedOperationException("setNextVar not supported"); + } + + @Override + public Object run() { + return result; + } + }; }; - } - - @Override - public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, Map vars) { - throw new UnsupportedOperationException("search script not supported"); + return context.factoryClazz.cast(factory); } } diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 381755f87388c..4d00b1a91eaf9 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -808,6 +808,7 @@ public void testDataFileFailureDuringRestore() throws Exception { logger.info("--> total number of simulated failures during restore: [{}]", getFailureCount("test-repo")); } + @TestLogging("org.elasticsearch.cluster.routing:TRACE,org.elasticsearch.snapshots:TRACE") public void testDataFileCorruptionDuringRestore() throws Exception { Path repositoryLocation = randomRepoPath(); Client client = client(); @@ -826,9 +827,11 @@ public void testDataFileCorruptionDuringRestore() throws Exception { assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits(), equalTo(100L)); logger.info("--> snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get(); + CreateSnapshotResponse createSnapshotResponse = client.admin().cluster() + .prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get(); assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); - assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(createSnapshotResponse.getSnapshotInfo().successfulShards())); + assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), + equalTo(createSnapshotResponse.getSnapshotInfo().successfulShards())); logger.info("--> update repository with mock version"); assertAcked(client.admin().cluster().preparePutRepository("test-repo") @@ -844,14 +847,17 @@ public void testDataFileCorruptionDuringRestore() throws Exception { logger.info("--> delete index"); cluster().wipeIndices("test-idx"); logger.info("--> restore corrupt index"); - RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet(); + RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster() + .prepareRestoreSnapshot("test-repo", "test-snap").setMasterNodeTimeout("30s") + .setWaitForCompletion(true).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(restoreSnapshotResponse.getRestoreInfo().totalShards())); + assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), + equalTo(restoreSnapshotResponse.getRestoreInfo().totalShards())); // we have to delete the index here manually, otherwise the cluster will keep // trying to allocate the shards for the index, even though the restore operation // is completed and marked as failed, which can lead to nodes having pending // cluster states to process in their queue when the test is finished - client.admin().indices().prepareDelete("test-idx").get(); + cluster().wipeIndices("test-idx"); } public void testDeletionOfFailingToRecoverIndexShouldStopRestore() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/core/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 38e73c209ae6e..3c1181b68258d 100644 --- a/core/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/core/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.Build; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; @@ -730,4 +731,58 @@ public void onFailure(Exception e) { } return statsRef.get(); } + + public void testEnsureConnected() throws IOException, InterruptedException { + List knownNodes = new CopyOnWriteArrayList<>(); + try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); + MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { + DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); + DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); + knownNodes.add(seedTransport.getLocalDiscoNode()); + knownNodes.add(discoverableTransport.getLocalDiscoNode()); + Collections.shuffle(knownNodes, random()); + + try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { + service.start(); + service.acceptIncomingRequests(); + try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", + Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) { + assertFalse(service.nodeConnected(seedNode)); + assertFalse(service.nodeConnected(discoverableNode)); + assertTrue(connection.assertNoRunningConnections()); + CountDownLatch latch = new CountDownLatch(1); + connection.ensureConnected(new LatchedActionListener<>(new ActionListener() { + @Override + public void onResponse(Void aVoid) { + } + + @Override + public void onFailure(Exception e) { + throw new AssertionError(e); + } + }, latch)); + latch.await(); + assertTrue(service.nodeConnected(seedNode)); + assertTrue(service.nodeConnected(discoverableNode)); + assertTrue(connection.assertNoRunningConnections()); + + // exec again we are already connected + connection.ensureConnected(new LatchedActionListener<>(new ActionListener() { + @Override + public void onResponse(Void aVoid) { + } + + @Override + public void onFailure(Exception e) { + throw new AssertionError(e); + } + }, latch)); + latch.await(); + assertTrue(service.nodeConnected(seedNode)); + assertTrue(service.nodeConnected(discoverableNode)); + assertTrue(connection.assertNoRunningConnections()); + } + } + } + } } diff --git a/core/src/test/java/org/elasticsearch/transport/TCPTransportTests.java b/core/src/test/java/org/elasticsearch/transport/TCPTransportTests.java index eb9e6496521c0..d8e35bd6f1a1f 100644 --- a/core/src/test/java/org/elasticsearch/transport/TCPTransportTests.java +++ b/core/src/test/java/org/elasticsearch/transport/TCPTransportTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -150,6 +151,26 @@ public void testAddressLimit() throws Exception { assertEquals(102, addresses[2].getPort()); } + public void testEnsureVersionCompatibility() { + TcpTransport.ensureVersionCompatibility(VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), + Version.CURRENT), Version.CURRENT, randomBoolean()); + + TcpTransport.ensureVersionCompatibility(Version.fromString("5.0.0"), Version.fromString("6.0.0"), true); + IllegalStateException ise = expectThrows(IllegalStateException.class, () -> + TcpTransport.ensureVersionCompatibility(Version.fromString("5.0.0"), Version.fromString("6.0.0"), false)); + assertEquals("Received message from unsupported version: [5.0.0] minimal compatible version is: [5.4.0]", ise.getMessage()); + + ise = expectThrows(IllegalStateException.class, () -> + TcpTransport.ensureVersionCompatibility(Version.fromString("2.3.0"), Version.fromString("6.0.0"), true)); + assertEquals("Received handshake message from unsupported version: [2.3.0] minimal compatible version is: [5.4.0]", + ise.getMessage()); + + ise = expectThrows(IllegalStateException.class, () -> + TcpTransport.ensureVersionCompatibility(Version.fromString("2.3.0"), Version.fromString("6.0.0"), false)); + assertEquals("Received message from unsupported version: [2.3.0] minimal compatible version is: [5.4.0]", + ise.getMessage()); + } + public void testCompressRequest() throws IOException { final boolean compressed = randomBoolean(); final AtomicBoolean called = new AtomicBoolean(false); diff --git a/core/src/test/resources/org/apache/lucene/search/postingshighlight/CambridgeMA.utf8 b/core/src/test/resources/org/apache/lucene/search/postingshighlight/CambridgeMA.utf8 index d60b6fa15d8a2..f47f671ab4a18 100644 --- a/core/src/test/resources/org/apache/lucene/search/postingshighlight/CambridgeMA.utf8 +++ b/core/src/test/resources/org/apache/lucene/search/postingshighlight/CambridgeMA.utf8 @@ -1 +1 @@ -{{Distinguish|Cambridge, England}} {{primary sources|date=June 2012}} {{Use mdy dates|date=January 2011}} {{Infobox settlement |official_name = Cambridge, Massachusetts |nickname = |motto = "Boston's Left Bank"{{cite web|url= http://www.epodunk.com/cgi-bin/genInfo.php?locIndex=2894|title=Profile for Cambridge, Massachusetts, MA|publisher= ePodunk |accessdate= November 1, 2012}} |image_skyline = CambridgeMACityHall2.jpg |imagesize = 175px |image_caption = Cambridge City Hall |image_seal = |image_flag = |image_map = Cambridge ma highlight.png |mapsize = 250px |map_caption = Location in Middlesex County in Massachusetts |image_map1 = |mapsize1 = |map_caption1 = |coordinates_region = US-MA |subdivision_type = Country |subdivision_name = United States |subdivision_type1 = State |subdivision_name1 = [[Massachusetts]] |subdivision_type2 = [[List of counties in Massachusetts|County]] |subdivision_name2 = [[Middlesex County, Massachusetts|Middlesex]] |established_title = Settled |established_date = 1630 |established_title2 = Incorporated |established_date2 = 1636 |established_title3 = |established_date3 = |government_type = [[Council-manager government|Council-City Manager]] |leader_title = Mayor |leader_name = Henrietta Davis |leader_title1 = [[City manager|City Manager]] |leader_name1 = [[Robert W. Healy]] |area_magnitude = |area_total_km2 = 18.47 |area_total_sq_mi = 7.13 |area_land_km2 = 16.65 |area_land_sq_mi = 6.43 |area_water_km2 = 1.81 |area_water_sq_mi = 0.70 |population_as_of = 2010 |population_blank2_title = [[Demonym]] |population_blank2 = [[Cantabrigian]] |settlement_type = City |population_total = 105,162 |population_density_km2 = 6,341.98 |population_density_sq_mi = 16,422.08 |elevation_m = 12 |elevation_ft = 40 |timezone = [[Eastern Time Zone|Eastern]] |utc_offset = -5 |timezone_DST = [[Eastern Time Zone|Eastern]] |utc_offset_DST = -4 |coordinates_display = display=inline,title |latd = 42 |latm = 22 |lats = 25 |latNS = N |longd = 71 |longm = 06 |longs = 38 |longEW = W |website = [http://www.cambridgema.gov/ www.cambridgema.gov] |postal_code_type = ZIP code |postal_code = 02138, 02139, 02140, 02141, 02142 |area_code = [[Area code 617|617]] / [[Area code 857|857]] |blank_name = [[Federal Information Processing Standard|FIPS code]] |blank_info = 25-11000 |blank1_name = [[Geographic Names Information System|GNIS]] feature ID |blank1_info = 0617365 |footnotes = }} '''Cambridge''' is a city in [[Middlesex County, Massachusetts|Middlesex County]], [[Massachusetts]], [[United States]], in the [[Greater Boston]] area. It was named in honor of the [[University of Cambridge]] in [[England]], an important center of the [[Puritan]] theology embraced by the town's founders.{{cite book|last=Degler|first=Carl Neumann|title=Out of Our Pasts: The Forces That Shaped Modern America|publisher=HarperCollins|location=New York|year=1984|url=http://books.google.com/books?id=NebLe1ueuGQC&pg=PA18&lpg=PA18&dq=cambridge+university+puritans+newtowne#v=onepage&q=&f=false|accessdate=September 9, 2009 | isbn=978-0-06-131985-3}} Cambridge is home to two of the world's most prominent universities, [[Harvard University]] and the [[Massachusetts Institute of Technology]]. According to the [[2010 United States Census]], the city's population was 105,162.{{cite web|url=http://2010.census.gov/news/releases/operations/cb11-cn104.html |title=Census 2010 News | U.S. Census Bureau Delivers Massachusetts' 2010 Census Population Totals, Including First Look at Race and Hispanic Origin Data for Legislative Redistricting |publisher=2010.census.gov |date=2011-03-22 |accessdate=2012-04-28}} It is the fifth most populous city in the state, behind [[Boston]], [[Worcester, MA|Worcester]], [[Springfield, MA|Springfield]], and [[Lowell, Massachusetts|Lowell]]. Cambridge was one of the two [[county seat]]s of Middlesex County prior to the abolition of county government in 1997; [[Lowell, Massachusetts|Lowell]] was the other. ==History== {{See also|Timeline of Cambridge, Massachusetts history}} [[File:Formation of Massachusetts towns.svg|thumb|A map showing the original boundaries of Cambridge]] The site for what would become Cambridge was chosen in December 1630, because it was located safely upriver from Boston Harbor, which made it easily defensible from attacks by enemy ships. Also, the water from the local spring was so good that the local Native Americans believed it had medicinal properties.{{Citation needed|date=November 2009}} [[Thomas Dudley]], his daughter [[Anne Bradstreet]] and her husband Simon were among the first settlers of the town. The first houses were built in the spring of 1631. The settlement was initially referred to as "the newe towne".{{cite book|last=Drake|first=Samuel Adams|title=History of Middlesex County, Massachusetts|publisher=Estes and Lauriat|location=Boston|year=1880|volume=1|pages=305–16|url=http://books.google.com/books?id=QGolOAyd9RMC&pg=PA316&lpg=PA305&dq=newetowne&ct=result#PPA305,M1|accessdate=December 26, 2008}} Official Massachusetts records show the name capitalized as '''Newe Towne''' by 1632.{{cite book|title=Report on the Custody and Condition of the Public Records of Parishes|publisher=Massachusetts Secretary of the Commonwealth|url=http://books.google.com/books?id=IyYWAAAAYAAJ&pg=RA1-PA298&lpg=RA1-PA298&dq=%22Ordered+That+Newtowne+shall+henceforward+be+called%22|location=Boston|year=1889|page=298|accessdate=December 24, 2008}} Located at the first convenient [[Charles River]] crossing west of [[Boston]], Newe Towne was one of a number of towns (including Boston, [[Dorchester, Massachusetts|Dorchester]], [[Watertown, Massachusetts|Watertown]], and [[Weymouth, Massachusetts|Weymouth]]) founded by the 700 original [[Puritan]] colonists of the [[Massachusetts Bay Colony]] under governor [[John Winthrop]]. The original village site is in the heart of today's [[Harvard Square]]. The marketplace where farmers brought in crops from surrounding towns to sell survives today as the small park at the corner of John F. Kennedy (J.F.K.) and Winthrop Streets, then at the edge of a salt marsh, since filled. The town included a much larger area than the present city, with various outlying parts becoming independent towns over the years: [[Newton, Massachusetts|Newton (originally Cambridge Village, then Newtown)]] in 1688,{{cite book |last= Ritter |first= Priscilla R. |coauthors= Thelma Fleishman |title= Newton, Massachusetts 1679–1779: A Biographical Directory |year= 1982 |publisher= New England Historic Genealogical Society }} [[Lexington, Massachusetts|Lexington (Cambridge Farms)]] in 1712, and both [[Arlington, Massachusetts|West Cambridge (originally Menotomy)]] and [[Brighton, Massachusetts|Brighton (Little Cambridge)]] in 1807.{{cite web |url=http://www.brightonbot.com/history.php |title=A Short History of Allston-Brighton |first=Marchione |last=William P. |author= |authorlink= |coauthors= |date= |month= |year=2011 |work=Brighton-Allston Historical Society |publisher=Brighton Board of Trade |location= |page= |pages= |at= |language= |trans_title= |arxiv= |asin= |bibcode= |doi= |doibroken= |isbn= |issn= |jfm= |jstor= |lccn= |mr= |oclc= |ol= |osti= |pmc = |pmid= |rfc= |ssrn= |zbl= |id= |archiveurl= |archivedate= |deadurl= |accessdate=December 21, 2011 |quote= |ref= |separator= |postscript=}} Part of West Cambridge joined the new town of [[Belmont, Massachusetts|Belmont]] in 1859, and the rest of West Cambridge was renamed Arlington in 1867; Brighton was annexed by Boston in 1874. In the late 19th century, various schemes for annexing Cambridge itself to the City of Boston were pursued and rejected.{{cite news |title=ANNEXATION AND ITS FRUITS |author=Staff writer |first= |last= |authorlink= |url=http://query.nytimes.com/gst/abstract.html?res=9901E4DC173BEF34BC4D52DFB766838F669FDE |agency= |newspaper=[[The New York Times]] |publisher= |isbn= |issn= |pmid= |pmd= |bibcode= |doi= |date=January 15, 1874, Wednesday |page= 4 |pages= |accessdate=|archiveurl=http://query.nytimes.com/mem/archive-free/pdf?res=9901E4DC173BEF34BC4D52DFB766838F669FDE |archivedate=January 15, 1874 |ref= }}{{cite news |title=BOSTON'S ANNEXATION SCHEMES.; PROPOSAL TO ABSORB CAMBRIDGE AND OTHER NEAR-BY TOWNS |author=Staff writer |first= |last= |authorlink= |url=http://query.nytimes.com/gst/abstract.html?res=9C05E1DC1F39E233A25754C2A9659C94639ED7CF |agency= |newspaper=[[The New York Times]] |publisher= |isbn= |issn= |pmid= |pmd= |bibcode= |doi= |date=March 26, 1892, Wednesday |page= 11 |pages= |accessdate=August 21, 2010|archiveurl=http://query.nytimes.com/mem/archive-free/pdf?res=9C05E1DC1F39E233A25754C2A9659C94639ED7CF |archivedate=March 27, 1892 |ref= }} In 1636, [[Harvard College]] was founded by the colony to train [[minister (religion)|ministers]] and the new town was chosen for its site by [[Thomas Dudley]]. By 1638, the name "Newe Towne" had "compacted by usage into 'Newtowne'." In May 1638{{cite book|title=The Cambridge of Eighteen Hundred and Ninety-six|editor=Arthur Gilman, ed.|publisher=Committee on the Memorial Volume|location=Cambridge|year=1896|page=8}}{{cite web|author=Harvard News Office |url=http://news.harvard.edu/gazette/2002/05.02/02-history.html |title=''Harvard Gazette'' historical calendar giving May 12, 1638 as date of name change; certain other sources say May 2, 1638 or late 1637 |publisher=News.harvard.edu |date=2002-05-02 |accessdate=2012-04-28}} the name was changed to '''Cambridge''' in honor of the [[University of Cambridge|university]] in [[Cambridge, England]].{{cite book |last= Hannah Winthrop Chapter, D.A.R. |title= Historic Guide to Cambridge |edition= Second |year= 1907 |publisher= Hannah Winthrop Chapter, D.A.R. |location= Cambridge, Mass. |pages= 20–21 |quote= On October 15, 1637, the Great and General Court passed a vote that: "The college is ordered to bee at Newetowne." In this same year the name of Newetowne was changed to Cambridge, ("It is ordered that Newetowne shall henceforward be called Cambridge") in honor of the university in Cambridge, England, where many of the early settlers were educated. }} The first president ([[Henry Dunster]]), the first benefactor ([[John Harvard (clergyman)|John Harvard]]), and the first schoolmaster ([[Nathaniel Eaton]]) of Harvard were all Cambridge University alumni, as was the then ruling (and first) governor of the [[Massachusetts Bay Colony]], John Winthrop. In 1629, Winthrop had led the signing of the founding document of the city of Boston, which was known as the [[Cambridge Agreement]], after the university.{{cite web|url=http://www.winthropsociety.org/doc_cambr.php|publisher=The Winthrop Society|title=Descendants of the Great Migration|accessdate=September 8, 2008}} It was Governor Thomas Dudley who, in 1650, signed the charter creating the corporation which still governs Harvard College.{{cite web|url=http://hul.harvard.edu/huarc/charter.html |title=Harvard Charter of 1650, Harvard University Archives, Harvard University, harvard.edu |publisher=Hul.harvard.edu |date= |accessdate=2012-04-28}}{{cite book |last1= |first1= |authorlink1= |editor1-first= |editor1-last= |editor1-link= |others= |title=Constitution of the Commonwealth of Massachusetts|url=http://www.mass.gov/legis/const.htm |accessdate=December 13, 2009 |edition= |series= |volume= |date=September 1, 1779 |publisher=The General Court of Massachusetts |location= |isbn= |oclc= |doi= |page= |pages=|chapter=Chapter V: The University at Cambridge, and encouragement of literature, etc. |chapterurl= |ref= |bibcode= }} [[Image:Washington taking command of the American Army at Cambridge, 1775 - NARA - 532874.tif|thumb|right|George Washington in Cambridge, 1775]] Cambridge grew slowly as an agricultural village eight miles (13 km) by road from Boston, the capital of the colony. By the [[American Revolution]], most residents lived near the [[Cambridge Common|Common]] and Harvard College, with farms and estates comprising most of the town. Most of the inhabitants were descendants of the original Puritan colonists, but there was also a small elite of [[Anglicans|Anglican]] "worthies" who were not involved in village life, who made their livings from estates, investments, and trade, and lived in mansions along "the Road to Watertown" (today's [[Brattle Street (Cambridge, Massachusetts)|Brattle Street]], still known as [[Tory Row]]). In 1775, [[George Washington]] came up from [[Virginia]] to take command of fledgling volunteer American soldiers camped on the [[Cambridge Common]]—today called the birthplace of the [[U.S. Army]]. (The name of today's nearby Sheraton Commander Hotel refers to that event.) Most of the Tory estates were confiscated after the Revolution. On January 24, 1776, [[Henry Knox]] arrived with artillery captured from [[Fort Ticonderoga]], which enabled Washington to drive the British army out of Boston. [[File:Cambridge 1873 WardMap.jpg|thumb|300px|left|A map of Cambridge from 1873]] Between 1790 and 1840, Cambridge began to grow rapidly, with the construction of the [[West Boston Bridge]] in 1792, that connected Cambridge directly to Boston, making it no longer necessary to travel eight miles (13 km) through the [[Boston Neck]], [[Roxbury, Massachusetts|Roxbury]], and [[Brookline, Massachusetts|Brookline]] to cross the [[Charles River]]. A second bridge, the Canal Bridge, opened in 1809 alongside the new [[Middlesex Canal]]. The new bridges and roads made what were formerly estates and [[marsh]]land into prime industrial and residential districts. In the mid-19th century, Cambridge was the center of a literary revolution when it gave the country a new identity through poetry and literature. Cambridge was home to the famous Fireside Poets—so called because their poems would often be read aloud by families in front of their evening fires. In their day, the [[Fireside Poets]]—[[Henry Wadsworth Longfellow]], [[James Russell Lowell]], and [[Oliver Wendell Holmes, Sr.|Oliver Wendell Holmes]]—were as popular and influential as rock stars are today.{{Citation needed|date=November 2009}} Soon after, [[Toll road|turnpikes]] were built: the [[Cambridge and Concord Turnpike]] (today's Broadway and Concord Ave.), the [[Middlesex Turnpike (Massachusetts)|Middlesex Turnpike]] (Hampshire St. and [[Massachusetts Avenue (Boston)|Massachusetts Ave.]] northwest of [[Porter Square]]), and what are today's Cambridge, Main, and Harvard Streets were roads to connect various areas of Cambridge to the bridges. In addition, railroads crisscrossed the town during the same era, leading to the development of Porter Square as well as the creation of neighboring town [[Somerville, Massachusetts|Somerville]] from the formerly rural parts of [[Charlestown, Massachusetts|Charlestown]]. [[File:Middlesex Canal (Massachusetts) map, 1852.jpg|thumb|1852 Map of Boston area showing Cambridge and rail lines.]] Cambridge was incorporated as a city in 1846. This was despite noticeable tensions between East Cambridge, Cambridgeport, and Old Cambridge that stemmed from differences in in each area's culture, sources of income, and the national origins of the residents.Cambridge Considered: A Very Brief History of Cambridge, 1800-1900, Part I. http://cambridgeconsidered.blogspot.com/2011/01/very-brief-history-of-cambridge-1800.html The city's commercial center began to shift from Harvard Square to Central Square, which became the downtown of the city around this time. Between 1850 and 1900, Cambridge took on much of its present character—[[streetcar suburb]]an development along the turnpikes, with working-class and industrial neighborhoods focused on East Cambridge, comfortable middle-class housing being built on old estates in Cambridgeport and Mid-Cambridge, and upper-class enclaves near Harvard University and on the minor hills of the city. The coming of the railroad to North Cambridge and Northwest Cambridge then led to three major changes in the city: the development of massive brickyards and brickworks between Massachusetts Ave., Concord Ave. and [[Alewife Brook]]; the ice-cutting industry launched by [[Frederic Tudor]] on [[Fresh Pond, Cambridge, Massachusetts|Fresh Pond]]; and the carving up of the last estates into residential subdivisions to provide housing to the thousands of immigrants that arrived to work in the new industries. For many years, the city's largest employer was the [[New England Glass Company]], founded in 1818. By the middle of the 19th century it was the largest and most modern glassworks in the world. In 1888, all production was moved, by [[Edward Libbey|Edward Drummond Libbey]], to [[Toledo, Ohio]], where it continues today under the name Owens Illinois. Flint glassware with heavy lead content, produced by that company, is prized by antique glass collectors. There is none on public display in Cambridge, but there is a large collection in the [[Toledo Museum of Art]]. Among the largest businesses located in Cambridge was the firm of [[Carter's Ink Company]], whose neon sign long adorned the [[Charles River]] and which was for many years the largest manufacturer of ink in the world. By 1920, Cambridge was one of the main industrial cities of [[New England]], with nearly 120,000 residents. As industry in New England began to decline during the [[Great Depression]] and after World War II, Cambridge lost much of its industrial base. It also began the transition to being an intellectual, rather than an industrial, center. Harvard University had always been important in the city (both as a landowner and as an institution), but it began to play a more dominant role in the city's life and culture. Also, the move of the [[Massachusetts Institute of Technology]] from Boston in 1916 ensured Cambridge's status as an intellectual center of the United States. After the 1950s, the city's population began to decline slowly, as families tended to be replaced by single people and young couples. The 1980s brought a wave of high-technology startups, creating software such as [[Visicalc]] and [[Lotus 1-2-3]], and advanced computers, but many of these companies fell into decline with the fall of the minicomputer and [[DOS]]-based systems. However, the city continues to be home to many startups as well as a thriving biotech industry. By the end of the 20th century, Cambridge had one of the most expensive housing markets in the Northeastern United States. While maintaining much diversity in class, race, and age, it became harder and harder for those who grew up in the city to be able to afford to stay. The end of [[rent control]] in 1994 prompted many Cambridge renters to move to housing that was more affordable, in Somerville and other communities. In 2005, a reassessment of residential property values resulted in a disproportionate number of houses owned by non-affluent people jumping in value relative to other houses, with hundreds having their property tax increased by over 100%; this forced many homeowners in Cambridge to move elsewhere.Cambridge Chronicle, October 6, 13, 20, 27, 2005 As of 2012, Cambridge's mix of amenities and proximity to Boston has kept housing prices relatively stable. ==Geography== [[File:Charles River Cambridge USA.jpg|thumb|upright|A view from Boston of Harvard's [[Weld Boathouse]] and Cambridge in winter. The [[Charles River]] is in the foreground.]] According to the [[United States Census Bureau]], Cambridge has a total area of {{convert|7.1|sqmi|km2}}, of which {{convert|6.4|sqmi|km2}} of it is land and {{convert|0.7|sqmi|km2}} of it (9.82%) is water. ===Adjacent municipalities=== Cambridge is located in eastern Massachusetts, bordered by: *the city of [[Boston]] to the south (across the [[Charles River]]) and east *the city of [[Somerville, Massachusetts|Somerville]] to the north *the town of [[Arlington, Massachusetts|Arlington]] to the northwest *the town of [[Belmont, Massachusetts|Belmont]] and *the city of [[Watertown, Massachusetts|Watertown]] to the west The border between Cambridge and the neighboring city of [[Somerville, Massachusetts|Somerville]] passes through densely populated neighborhoods which are connected by the [[Red Line (MBTA)|MBTA Red Line]]. Some of the main squares, [[Inman Square|Inman]], [[Porter Square|Porter]], and to a lesser extent, [[Harvard Square|Harvard]], are very close to the city line, as are Somerville's [[Union Square (Somerville)|Union]] and [[Davis Square]]s. ===Neighborhoods=== ====Squares==== [[File:Centralsquarecambridgemass.jpg|thumb|[[Central Square (Cambridge)|Central Square]]]] [[File:Harvard square 2009j.JPG|thumb|[[Harvard Square]]]] [[File:Cambridge MA Inman Square.jpg|thumb|[[Inman Square]]]] Cambridge has been called the "City of Squares" by some,{{cite web|author=No Writer Attributed |url=http://www.thecrimson.com/article/1969/9/18/cambridge-a-city-of-squares-pcambridge/ |title="Cambridge: A City of Squares" Harvard Crimson, Sept. 18, 1969 |publisher=Thecrimson.com |date=1969-09-18 |accessdate=2012-04-28}}{{cite web|url=http://www.travelwritersmagazine.com/RonBernthal/Cambridge.html |title=Cambridge Journal: Massachusetts City No Longer in Boston's Shadow |publisher=Travelwritersmagazine.com |date= |accessdate=2012-04-28}} as most of its commercial districts are major street intersections known as [[Town square|squares]]. Each of the squares acts as a neighborhood center. These include: * [[Kendall Square]], formed by the junction of Broadway, Main Street, and Third Street, is also known as '''Technology Square''', a name shared with an office and laboratory building cluster in the neighborhood. Just over the [[Longfellow Bridge]] from Boston, at the eastern end of the [[Massachusetts Institute of Technology|MIT]] campus, it is served by the [[Kendall (MBTA station)|Kendall/MIT]] station on the [[Massachusetts Bay Transportation Authority|MBTA]] [[Red Line (MBTA)|Red Line]] subway. Most of Cambridge's large office towers are located here, giving the area somewhat of an office park feel. A flourishing [[biotech]] industry has grown up around this area. The "One Kendall Square" complex is nearby, but—confusingly—not actually in Kendall Square. Also, the "Cambridge Center" office complex is located here, and not at the actual center of Cambridge. * [[Central Square (Cambridge)|Central Square]], formed by the junction of Massachusetts Avenue, Prospect Street, and Western Avenue, is well known for its wide variety of ethnic restaurants. As recently as the late 1990s it was rather run-down; it underwent a controversial [[gentrification]] in recent years (in conjunction with the development of the nearby [[University Park at MIT]]), and continues to grow more expensive. It is served by the [[Central (MBTA station)|Central Station]] stop on the MBTA Red Line subway. '''Lafayette Square''', formed by the junction of Massachusetts Avenue, Columbia Street, Sidney Street, and Main Street, is considered part of the Central Square area. [[Cambridgeport]] is south of Central Square along Magazine Street and Brookline Street. * [[Harvard Square]], formed by the junction of Massachusetts Avenue, Brattle Street, and JFK Street. This is the primary site of [[Harvard University]], and is a major Cambridge shopping area. It is served by a [[Harvard (MBTA station)|Red Line station]]. Harvard Square was originally the northwestern terminus of the Red Line and a major transfer point to streetcars that also operated in a short [[Harvard Bus Tunnel|tunnel]]—which is still a major bus terminal, although the area under the Square was reconfigured dramatically in the 1980s when the Red Line was extended. The Harvard Square area includes '''Brattle Square''' and '''Eliot Square'''. A short distance away from the square lies the [[Cambridge Common]], while the neighborhood north of Harvard and east of Massachusetts Avenue is known as Agassiz in honor of the famed scientist [[Louis Agassiz]]. * [[Porter Square]], about a mile north on Massachusetts Avenue from Harvard Square, is formed by the junction of Massachusetts and Somerville Avenues, and includes part of the city of [[Somerville, Massachusetts|Somerville]]. It is served by the [[Porter (MBTA station)|Porter Square Station]], a complex housing a [[Red Line (MBTA)|Red Line]] stop and a [[Fitchburg Line]] [[MBTA commuter rail|commuter rail]] stop. [[Lesley University]]'s University Hall and Porter campus are located at Porter Square. * [[Inman Square]], at the junction of Cambridge and Hampshire streets in Mid-Cambridge. Inman Square is home to many diverse restaurants, bars, music venues and boutiques. The funky street scene still holds some urban flair, but was dressed up recently with Victorian streetlights, benches and bus stops. A new community park was installed and is a favorite place to enjoy some takeout food from the nearby restaurants and ice cream parlor. * [[Lechmere Square]], at the junction of Cambridge and First streets, adjacent to the CambridgeSide Galleria shopping mall. Perhaps best known as the northern terminus of the [[Massachusetts Bay Transportation Authority|MBTA]] [[Green Line (MBTA)|Green Line]] subway, at [[Lechmere (MBTA station)|Lechmere Station]]. ====Other neighborhoods==== The residential neighborhoods ([http://www.cambridgema.gov/CPD/publications/neighborhoods.cfm map]) in Cambridge border, but are not defined by the squares. These include: * [[East Cambridge, Massachusetts|East Cambridge]] (Area 1) is bordered on the north by the [[Somerville, Massachusetts|Somerville]] border, on the east by the Charles River, on the south by Broadway and Main Street, and on the west by the [[Grand Junction Railroad]] tracks. It includes the [[NorthPoint (Cambridge, Massachusetts)|NorthPoint]] development. * [[Massachusetts Institute of Technology|MIT]] Campus ([[MIT Campus (Area 2), Cambridge|Area 2]]) is bordered on the north by Broadway, on the south and east by the Charles River, and on the west by the Grand Junction Railroad tracks. * [[Wellington-Harrington]] (Area 3) is bordered on the north by the [[Somerville, Massachusetts|Somerville]] border, on the south and west by Hampshire Street, and on the east by the Grand Junction Railroad tracks. Referred to as "Mid-Block".{{clarify|What is? By whom? A full sentence would help.|date=September 2011}} * [[Area 4, Cambridge|Area 4]] is bordered on the north by Hampshire Street, on the south by Massachusetts Avenue, on the west by Prospect Street, and on the east by the Grand Junction Railroad tracks. Residents of Area 4 often refer to their neighborhood simply as "The Port", and refer to the area of Cambridgeport and Riverside as "The Coast". * [[Cambridgeport]] (Area 5) is bordered on the north by Massachusetts Avenue, on the south by the Charles River, on the west by River Street, and on the east by the Grand Junction Railroad tracks. * [[Mid-Cambridge]] (Area 6) is bordered on the north by Kirkland and Hampshire Streets and the [[Somerville, Massachusetts|Somerville]] border, on the south by Massachusetts Avenue, on the west by Peabody Street, and on the east by Prospect Street. * [[Riverside, Cambridge|Riverside]] (Area 7), an area sometimes referred to as "The Coast," is bordered on the north by Massachusetts Avenue, on the south by the Charles River, on the west by JFK Street, and on the east by River Street. * [[Agassiz, Cambridge, Massachusetts|Agassiz (Harvard North)]] (Area 8) is bordered on the north by the [[Somerville, Massachusetts|Somerville]] border, on the south and east by Kirkland Street, and on the west by Massachusetts Avenue. * [[Peabody, Cambridge, Massachusetts|Peabody]] (Area 9) is bordered on the north by railroad tracks, on the south by Concord Avenue, on the west by railroad tracks, and on the east by Massachusetts Avenue. The Avon Hill sub-neighborhood consists of the higher elevations bounded by Upland Road, Raymond Street, Linnaean Street and Massachusetts Avenue. * Brattle area/[[West Cambridge (neighborhood)|West Cambridge]] (Area 10) is bordered on the north by Concord Avenue and Garden Street, on the south by the Charles River and the [[Watertown, Massachusetts|Watertown]] border, on the west by Fresh Pond and the Collins Branch Library, and on the east by JFK Street. It includes the sub-neighborhoods of Brattle Street (formerly known as [[Tory Row]]) and Huron Village. * [[North Cambridge, Massachusetts|North Cambridge]] (Area 11) is bordered on the north by the [[Arlington, Massachusetts|Arlington]] and [[Somerville, Massachusetts|Somerville]] borders, on the south by railroad tracks, on the west by the [[Belmont, Massachusetts|Belmont]] border, and on the east by the [[Somerville, Massachusetts|Somerville]] border. * [[Cambridge Highlands]] (Area 12) is bordered on the north and east by railroad tracks, on the south by Fresh Pond, and on the west by the [[Belmont, Massachusetts|Belmont]] border. * [[Strawberry Hill, Cambridge|Strawberry Hill]] (Area 13) is bordered on the north by Fresh Pond, on the south by the [[Watertown, Massachusetts|Watertown]] border, on the west by the [[Belmont, Massachusetts|Belmont]] border, and on the east by railroad tracks. ===Parks and outdoors=== [[File:Alewife Brook Reservation.jpg|thumb|Alewife Brook Reservation]] Consisting largely of densely built residential space, Cambridge lacks significant tracts of public parkland. This is partly compensated for, however, by the presence of easily accessible open space on the university campuses, including [[Harvard Yard]] and MIT's Great Lawn, as well as the considerable open space of [[Mount Auburn Cemetery]]. At the western edge of Cambridge, the cemetery is well known as the first garden cemetery, for its distinguished inhabitants, for its superb landscaping (the oldest planned landscape in the country), and as a first-rate [[arboretum]]. Although known as a Cambridge landmark, much of the cemetery lies within the bounds of Watertown.http://www2.cambridgema.gov/CityOfCambridge_Content/documents/CambridgeStreetMap18x24_032007.pdf It is also a significant [[Important Bird Area]] (IBA) in the Greater Boston area. Public parkland includes the esplanade along the Charles River, which mirrors its [[Charles River Esplanade|Boston counterpart]], [[Cambridge Common]], a busy and historic public park immediately adjacent to the Harvard campus, and the [[Alewife Brook Reservation]] and [[Fresh Pond, Cambridge, Massachusetts|Fresh Pond]] in the western part of the city. ==Demographics== {{Historical populations | type=USA | align=right | 1790|2115 | 1800|2453 | 1810|2323 | 1820|3295 | 1830|6072 | 1840|8409 | 1850|15215 | 1860|26060 | 1870|39634 | 1880|52669 | 1890|70028 | 1900|91886 | 1910|104839 | 1920|109694 | 1930|113643 | 1940|110879 | 1950|120740 | 1960|107716 | 1970|100361 | 1980|95322 | 1990|95802 | 2000|101355 | 2010|105162 | footnote= {{Historical populations/Massachusetts municipalities references}}{{cite journal | title=1950 Census of Population | volume=1: Number of Inhabitants | at=Section 6, Pages 21-7 through 21-09, Massachusetts Table 4. Population of Urban Places of 10,000 or more from Earliest Census to 1920 | publisher=Bureau of the Census | accessdate=July 12, 2011 | year=1952 | url=http://www2.census.gov/prod2/decennial/documents/23761117v1ch06.pdf}} }} As of the census{{GR|2}} of 2010, there were 105,162 people, 44,032 households, and 17,420 families residing in the city. The population density was 16,422.08 people per square mile (6,341.98/km²), making Cambridge the fifth most densely populated city in the USCounty and City Data Book: 2000. Washington, DC: US Department of Commerce, Bureau of the Census. Table C-1. and the second most densely populated city in [[Massachusetts]] behind neighboring [[Somerville, Massachusetts|Somerville]].[http://www.boston.com/realestate/news/articles/2008/07/13/highest_population_density/ Highest Population Density, The Boston Globe] There were 47,291 housing units at an average density of 7,354.7 per square mile (2,840.3/km²). The racial makeup of the city was 66.60% [[White (U.S. Census)|White]], 11.70% [[Black (people)|Black]] or [[Race (United States Census)|African American]], 0.20% [[Native American (U.S. Census)|Native American]], 15.10% [[Asian (U.S. Census)|Asian]], 0.01% [[Pacific Islander (U.S. Census)|Pacific Islander]], 2.10% from [[Race (United States Census)|other races]], and 4.30% from two or more races. 7.60% of the population were [[Hispanics in the United States|Hispanic]] or [[Latino (U.S. Census)|Latino]] of any race. [[Non-Hispanic Whites]] were 62.1% of the population in 2010,{{cite web |url=http://quickfacts.census.gov/qfd/states/25/2511000.html |title=Cambridge (city), Massachusetts |work=State & County QuickFacts |publisher=U.S. Census Bureau}} down from 89.7% in 1970.{{cite web|title=Massachusetts - Race and Hispanic Origin for Selected Cities and Other Places: Earliest Census to 1990|publisher=U.S. Census Bureau|url=http://www.census.gov/population/www/documentation/twps0076/twps0076.html}} This rather closely parallels the average [[racial demographics of the United States]] as a whole, although Cambridge has significantly more Asians than the average, and fewer Hispanics and Caucasians. 11.0% were of [[irish people|Irish]], 7.2% English, 6.9% [[italians|Italian]], 5.5% [[West Indian]] and 5.3% [[germans|German]] ancestry according to [[Census 2000]]. 69.4% spoke English, 6.9% Spanish, 3.2% [[Standard Mandarin|Chinese]] or [[Standard Mandarin|Mandarin]], 3.0% [[portuguese language|Portuguese]], 2.9% [[French-based creole languages|French Creole]], 2.3% French, 1.5% [[korean language|Korean]], and 1.0% [[italian language|Italian]] as their first language. There were 44,032 households out of which 16.9% had children under the age of 18 living with them, 28.9% were married couples living together, 8.4% had a female householder with no husband present, and 60.4% were non-families. 40.7% of all households were made up of individuals and 9.6% had someone living alone who was 65 years of age or older. The average household size was 2.00 and the average family size was 2.76. In the city the population was spread out with 13.3% under the age of 18, 21.2% from 18 to 24, 38.6% from 25 to 44, 17.8% from 45 to 64, and 9.2% who were 65 years of age or older. The median age was 30.5 years. For every 100 females, there were 96.1 males. For every 100 females age 18 and over, there were 94.7 males. The median income for a household in the city was $47,979, and the median income for a family was $59,423 (these figures had risen to $58,457 and $79,533 respectively {{as of|2007|alt=as of a 2007 estimate}}{{cite web|url=http://factfinder.census.gov/servlet/ACSSAFFFacts?_event=Search&geo_id=16000US2418750&_geoContext=01000US%7C04000US24%7C16000US2418750&_street=&_county=cambridge&_cityTown=cambridge&_state=04000US25&_zip=&_lang=en&_sse=on&ActiveGeoDiv=geoSelect&_useEV=&pctxt=fph&pgsl=160&_submenuId=factsheet_1&ds_name=ACS_2007_3YR_SAFF&_ci_nbr=null&qr_name=null®=null%3Anull&_keyword=&_industry= |title=U.S. Census, 2000 |publisher=Factfinder.census.gov |date= |accessdate=2012-04-28}}). Males had a median income of $43,825 versus $38,489 for females. The per capita income for the city was $31,156. About 8.7% of families and 12.9% of the population were below the poverty line, including 15.1% of those under age 18 and 12.9% of those age 65 or over. Cambridge was ranked as one of the most liberal cities in America.{{cite web|author=Aug 16, 2005 12:00 AM |url=http://www.govpro.com/News/Article/31439/ |title=Study Ranks America’s Most Liberal and Conservative Cities |publisher=Govpro.com |date=2005-08-16 |accessdate=2012-04-28}} Locals living in and near the city jokingly refer to it as "The People's Republic of Cambridge."[http://www.universalhub.com/glossary/peoples_republic_the.html Wicked Good Guide to Boston English] Accessed February 2, 2009 For 2012, the residential property tax rate in Cambridge is $8.48 per $1,000.{{cite web|url=http://www.cambridgema.gov/finance/propertytaxinformation/fy12propertytaxinformation.aspx |title=FY12 Property Tax Information - City of Cambridge, Massachusetts |publisher=Cambridgema.gov |date= |accessdate=2012-04-28}} Cambridge enjoys the highest possible [[bond credit rating]], AAA, with all three Wall Street rating agencies.http://www.cambridgema.gov/CityOfCambridge_Content/documents/Understanding_Your_Taxes_2007.pdf Cambridge is noted for its diverse population, both racially and economically. Residents, known as ''Cantabrigians'', include affluent [[MIT]] and Harvard professors. The first legal applications in America for same-sex marriage licenses were issued at Cambridge's City Hall.{{cite web|url=http://www.boston.com/news/local/articles/2004/05/17/free_to_marry/ |title=Free to Marry |work=[[The Boston Globe]] |date=2004-05-17 |accessdate=2012-07-18}} Cambridge is also the birthplace of [[Thailand|Thai]] king [[Bhumibol Adulyadej|Bhumibol Adulyadej (Rama IX)]], who is the world's longest reigning monarch at age 82 (2010), as well as the longest reigning monarch in Thai history. He is also the first king of a foreign country to be born in the United States. ==Government== ===Federal and state representation=== {| class=wikitable ! colspan = 6 | Voter registration and party enrollment {{as of|lc=y|df=US|2008|10|15}}{{cite web|title = 2008 State Party Election Party Enrollment Statistics | publisher = Massachusetts Elections Division | format = PDF | accessdate = July 7, 2010 | url = http://www.sec.state.ma.us/ele/elepdf/st_county_town_enroll_breakdown_08.pdf}} |- ! colspan = 2 | Party ! Number of voters ! Percentage {{American politics/party colors/Democratic/row}} | [[Democratic Party (United States)|Democratic]] | style="text-align:center;"| 37,822 | style="text-align:center;"| 58.43% {{American politics/party colors/Republican/row}} | [[Republican Party (United States)|Republican]] | style="text-align:center;"| 3,280 | style="text-align:center;"| 5.07% {{American politics/party colors/Independent/row}} | Unaffiliated | style="text-align:center;"| 22,935 | style="text-align:center;"| 35.43% {{American politics/party colors/Libertarian/row}} | Minor Parties | style="text-align:center;"| 690 | style="text-align:center;"| 1.07% |- ! colspan = 2 | Total ! style="text-align:center;"| 64,727 ! style="text-align:center;"| 100% |} Cambridge is part of [[Massachusetts's 8th congressional district]], represented by Democrat [[Mike Capuano]], elected in 1998. The state's senior member of the [[United States Senate]] is Democrat [[John Kerry]], elected in 1984. The state's junior member is Republican [[Scott Brown]], [[United States Senate special election in Massachusetts, 2010|elected in 2010]] to fill the vacancy caused by the death of long-time Democratic Senator [[Ted Kennedy]]. The Governor of Massachusetts is Democrat [[Deval Patrick]], elected in 2006 and re-elected in 2010. On the state level, Cambridge is represented in six districts in the [[Massachusetts House of Representatives]]: the 24th Middlesex (which includes parts of Belmont and Arlington), the 25th and 26th Middlesex (the latter which includes a portion of Somerville), the 29th Middlesex (which includes a small part of Watertown), and the Eighth and Ninth Suffolk (both including parts of the City of Boston). The city is represented in the [[Massachusetts Senate]] as a part of the "First Suffolk and Middlesex" district (this contains parts of Boston, Revere and Winthrop each in Suffolk County); the "Middlesex, Suffolk and Essex" district, which includes Everett and Somerville, with Boston, Chelsea, and Revere of Suffolk, and Saugus in Essex; and the "Second Suffolk and Middlesex" district, containing parts of the City of Boston in Suffolk county, and Cambridge, Belmont and Watertown in Middlesex county.{{cite web|url=http://www.malegislature.gov/ |title=Index of Legislative Representation by City and Town, from |publisher=Mass.gov |date= |accessdate=2012-04-28}} In addition to the [[Cambridge Police Department (Massachusetts)|Cambridge Police Department]], the city is patrolled by the Fifth (Brighton) Barracks of Troop H of the [[Massachusetts State Police]].[http://www.mass.gov/?pageID=eopsterminal&L=5&L0=Home&L1=Law+Enforcement+%26+Criminal+Justice&L2=Law+Enforcement&L3=State+Police+Troops&L4=Troop+H&sid=Eeops&b=terminalcontent&f=msp_divisions_field_services_troops_troop_h_msp_field_troop_h_station_h5&csid=Eeops Station H-5, SP Brighton]{{dead link|date=April 2012}} Due, however, to close proximity, the city also practices functional cooperation with the Fourth (Boston) Barracks of Troop H, as well.[http://www.mass.gov/?pageID=eopsterminal&L=5&L0=Home&L1=Law+Enforcement+%26+Criminal+Justice&L2=Law+Enforcement&L3=State+Police+Troops&L4=Troop+H&sid=Eeops&b=terminalcontent&f=msp_divisions_field_services_troops_troop_h_msp_field_troop_h_station_h4&csid=Eeops Station H-4, SP Boston]{{dead link|date=April 2012}} ===City government=== [[File:CambridgeMACityHall1.jpg|thumb|right|[[Cambridge, Massachusetts City Hall|Cambridge City Hall]] in the 1980s]] Cambridge has a city government led by a [[List of mayors of Cambridge, Massachusetts|Mayor]] and nine-member City Council. There is also a six-member School Committee which functions alongside the Superintendent of public schools. The councilors and school committee members are elected every two years using the [[single transferable vote]] (STV) system.{{cite web|url=http://www.cambridgema.gov/election/Proportional_Representation.cfm |title=Proportional Representation Voting in Cambridge |publisher=Cambridgema.gov |date= |accessdate=2012-04-28}} Once a laborious process that took several days to complete by hand, ballot sorting and calculations to determine the outcome of elections are now quickly performed by computer, after the ballots have been [[Optical scan voting system|optically scanned]]. The mayor is elected by the city councilors from amongst themselves, and serves as the chair of City Council meetings. The mayor also sits on the School Committee. However, the Mayor is not the Chief Executive of the City. Rather, the City Manager, who is appointed by the City Council, serves in that capacity. Under the City's Plan E form of government the city council does not have the power to appoint or remove city officials who are under direction of the city manager. The city council and its individual members are also forbidden from giving orders to any subordinate of the city manager.http://www.cambridgema.gov/CityOfCambridge_Content/documents/planE.pdf [[Robert W. Healy]] is the City Manager; he has served in the position since 1981. In recent history, the media has highlighted the salary of the City Manager as being one of the highest in the State of Massachusetts.{{cite news |title=Cambridge city manager's salary almost as much as Obama's pay |url=http://www.wickedlocal.com/cambridge/features/x1837730973/Cambridge-city-managers-salary-almost-as-much-as-Obamas |agency= |newspaper=Wicked Local: Cambridge |publisher= |date=August 11, 2011 |accessdate=December 30, 2011 |quote= |archiveurl= |archivedate= |deadurl= |ref=}} The city council consists of:{{cite web|url=http://www.cambridgema.gov/ccouncil/citycouncilmembers.aspx |title=City of Cambridge – City Council Members |publisher=Cambridgema.gov |date= |accessdate=2012-04-28}}{{Refbegin|3}} *[[Leland Cheung]] (Jan. 2010–present) *Henrietta Davis (Jan. 1996–present)* *Marjorie C. Decker (Jan. 2000–present){{cite web |url= http://www.wickedlocal.com/cambridge/news/x738245499/Marjorie-Decker-announces-she-will-run-for-Alice-Wolfs-Cambridge-State-Representative-seat |title= Marjorie Decker announces she will run for Alice Wolf's Cambridge State Representative seat |date= 22 March 2012 |work= Wicked Local Cambridge |publisher= GateHouse Media, Inc. |accessdate= 4 April 2012 }} *Craig A. Kelley (Jan. 2006–present) *David Maher (Jan. 2000-Jan. 2006, Sept. 2007–present{{cite web|author=By ewelin, on September 5th, 2007 |url=http://www.cambridgehighlands.com/2007/09/david-p-maher-elected-to-fill-michael-sullivans-vacated-city-council-seat |title=David P. Maher Elected to fill Michael Sullivan’s Vacated City Council Seat • Cambridge Highlands Neighborhood Association |publisher=Cambridgehighlands.com |date=2007-09-05 |accessdate=2012-04-28}})** *[[Kenneth Reeves]] (Jan. 1990–present)** *[[E. Denise Simmons]] (Jan. 2002–present)** *[[Timothy J. Toomey, Jr.]] (Jan. 1990–present) *Minka vanBeuzekom (Jan. 2012–present){{Refend}} ''* = Current Mayor''
''** = former Mayor'' ===Fire Department=== The city of Cambridge is protected full-time by the 274 professional firefighters of the Cambridge Fire Department. The current Chief of Department is Gerald R. Reardon. The Cambridge Fire Department operates out of eight fire stations, located throughout the city, under the command of two divisions. The CFD also maintains and operates a front-line fire apparatus fleet of eight engines, four ladders, two Non-Transport Paramedic EMS units, a Haz-Mat unit, a Tactical Rescue unit, a Dive Rescue unit, two Marine units, and numerous special, support, and reserve units. John J. Gelinas, Chief of Operations, is in charge of day to day operation of the department.{{cite web|url=http://www2.cambridgema.gov/cfd/ |title=City of Cambridge Fire Department |publisher=.cambridgema.gov |date=2005-03-13 |accessdate=2012-06-26}} The CFD is rated as a Class 1 fire department by the [[Insurance Services Office]] (ISO), and is one of only 32 fire departments so rated, out of 37,000 departments in the United States. The other class 1 departments in New England are in [[Hartford, Connecticut]] and [[Milford, Connecticut]]. Class 1 signifies the highest level of fire protection according to various criteria.{{cite web|url=http://www2.cambridgema.gov/CFD/Class1FD.cfm |title=Class 1 Fire Department |publisher=.cambridgema.gov |date=1999-07-01 |accessdate=2012-06-26}} The CFD responds to approximately 15,000 emergency calls annually. {| class=wikitable |- valign=bottom ! Engine Company ! Ladder Company ! Special Unit ! Division ! Address ! Neighborhood |- | Engine 1 || Ladder 1 || || || 491 Broadway || Harvard Square |- | Engine 2 || Ladder 3 || Squad 2 || || 378 Massachusetts Ave. || Lafayette Square |- | Engine 3 || Ladder 2 || || || 175 Cambridge St. || East Cambridge |- | Engine 4 || || Squad 4 || || 2029 Massachusetts Ave. || Porter Square |- | Engine 5 || || || Division 1 || 1384 Cambridge St. || Inman Square |- | Engine 6 || || || || 176 River St. || Cambridgeport |- | Engine 8 || Ladder 4 || || Division 2 || 113 Garden St. || Taylor Square |- | Engine 9 || || || || 167 Lexington Ave || West Cambridge |- | Maintenance Facility || || || || 100 Smith Pl. || |} ===Water Department=== Cambridge is unusual among cities inside Route 128 in having a non-[[MWRA]] water supply. City water is obtained from [[Hobbs Brook]] (in [[Lincoln, Massachusetts|Lincoln]] and [[Waltham, Massachusetts|Waltham]]), [[Stony Brook (Boston)|Stony Brook]] (Waltham and [[Weston, Massachusetts|Weston]]), and [[Fresh Pond (Cambridge, Massachusetts)|Fresh Pond]] (Cambridge). The city owns over 1200 acres of land in other towns that includes these reservoirs and portions of their watershed.{{cite web|url=http://www2.cambridgema.gov/CWD/wat_lands.cfm |title=Cambridge Watershed Lands & Facilities |publisher=.cambridgema.gov |date= |accessdate=2012-04-28}} Water is treated at Fresh Pond, then pumped uphill to an elevation of {{convert|176|ft|m}} [[above sea level]] at the Payson Park Reservoir ([[Belmont, Massachusetts|Belmont]]); From there, the water is redistributed downhill via gravity to individual users in the city.{{cite web|url=http://www.cambridgema.gov/CityOfCambridge_Content/documents/CWD_March_2010.pdf |title=Water supply system |format=PDF |date= |accessdate=2012-04-28}}[http://www.cambridgema.gov/CWD/fpfaqs.cfm Is Fresh Pond really used for drinking water?], Cambridge Water Department ===County government=== Cambridge is a [[county seat]] of [[Middlesex County, Massachusetts]], along with [[Lowell, Massachusetts|Lowell]]. Though the county government was abolished in 1997, the county still exists as a geographical and political region. The employees of Middlesex County courts, jails, registries, and other county agencies now work directly for the state. At present, the county's registrars of [[Deed]]s and Probate remain in Cambridge; however, the Superior Court and District Attorney have had their base of operations transferred to [[Woburn, Massachusetts|Woburn]]. Third District court has shifted operations to [[Medford, Massachusetts|Medford]], and the Sheriff's office for the county is still awaiting a near-term relocation.{{cite news | url=http://www.boston.com/news/local/massachusetts/articles/2008/02/14/court_move_a_hassle_for_commuters/ |title=Court move a hassle for commuters |accessdate=July 25, 2009 |first=Eric |last=Moskowitz |authorlink= |coauthors= |date=February 14, 2008 |work=[[Boston Globe|The Boston Globe]] |pages= |archiveurl= |archivedate= |quote=In a little more than a month, Middlesex Superior Court will open in Woburn after nearly four decades at the Edward J. Sullivan Courthouse in Cambridge. With it, the court will bring the roughly 500 people who pass through its doors each day – the clerical staff, lawyers, judges, jurors, plaintiffs, defendants, and others who use or work in the system.}}{{cite news | url=http://www.wickedlocal.com/cambridge/homepage/x135741754/Cambridges-Middlesex-Jail-courts-may-be-shuttered-for-good |title=Cambridge's Middlesex Jail, courts may be shuttered for good |accessdate=July 25, 2009 |first=Charlie |last=Breitrose |authorlink= |coauthors= |date=July 7, 2009 |work=Wicked Local News: Cambridge |pages= |archiveurl= |archivedate= |quote=The courts moved out of the building to allow workers to remove asbestos. Superior Court moved to Woburn in March 2008, and in February, the Third District Court moved to Medford.}} ==Education== [[File:MIT Main Campus Aerial.jpg|thumb|Aerial view of part of [[MIT]]'s main campus]] [[File:Dunster House.jpg|thumb|[[Dunster House]], Harvard]] ===Higher education=== Cambridge is perhaps best known as an academic and intellectual center, owing to its colleges and universities, which include: *[[Cambridge College]] *[[Cambridge School of Culinary Arts]] *[[Episcopal Divinity School]] *[[Harvard University]] *[[Hult International Business School]] *[[Lesley University]] *[[Longy School of Music]] *[[Massachusetts Institute of Technology]] *[[Le Cordon Bleu College of Culinary Arts in Boston]] [[Nobel laureates by university affiliation|At least 129]] of the world's total 780 [[Nobel Prize]] winners have been, at some point in their careers, affiliated with universities in Cambridge. The [[American Academy of Arts and Sciences]] is also based in Cambridge. ===Primary and secondary public education=== The Cambridge Public School District encompasses 12 elementary schools that follow a variety of different educational systems and philosophies. All but one of the elementary schools extend up to the [[middle school]] grades as well. The 12 elementary schools are: *[[Amigos School]] *Baldwin School *Cambridgeport School *Fletcher-Maynard Academy *Graham and Parks Alternative School *Haggerty School *Kennedy-Longfellow School *King Open School *Martin Luther King, Jr. School *Morse School (a [[Core Knowledge Foundation|Core Knowledge]] school) *Peabody School *Tobin School (a [[Montessori school]]) There are three public high schools serving Cambridge students, including the [[Cambridge Rindge and Latin School]].{{cite web|url=http://www.cpsd.us/Web/PubInfo/SchoolsAtAGlance06-07.pdf|title=Cambridge Public Schools at a Glance|format=PDF}}{{dead link|date=June 2012}} and Community Charter School of Cambridge (www.ccscambridge.org) In 2003, the CRLS, also known as Rindge, came close to losing its educational accreditation when it was placed on probation by the [[New England Association of Schools and Colleges]].{{cite web|url=http://www.thecrimson.com/article.aspx?ref=512061|title=School Fights Achievement Gap|publisher=The Harvard Crimson|accessdate=May 14, 2009}} The school has improved under Principal Chris Saheed, graduation rates hover around 98%, and 70% of students gain college admission. Community Charter School of Cambridge serves 350 students, primarily from Boston and Cambridge, and is a tuition free public charter school with a college preparatory curriculum. All students from the class of 2009 and 2010 gained admission to college. Outside of the main public schools are public charter schools including: [[Benjamin Banneker Charter School]], which serves students in grades K-6,{{cite web|url=http://www.banneker.org/ |title=The Benjamin Banneker Charter Public School |publisher=Banneker.org |date=2012-03-01 |accessdate=2012-04-28}} [[Community Charter School of Cambridge]],{{cite web|url=http://www.ccscambridge.org/ |title=Community Charter School of Cambridge |publisher=Ccscambridge.org |date= |accessdate=2012-04-28}} which is located in Kendall Square and serves students in grades 7–12, and [[Prospect Hill Academy]], a [[charter school]] whose upper school is in [[Central Square (Cambridge)|Central Square]], though it is not a part of the Cambridge Public School District. ===Primary and secondary private education=== [[File:Cambridge Public Library, Cambridge, Massachusetts.JPG|thumb|right|[[Cambridge Public Library]] original building, part of an expanded facility]] There are also many private schools in the city including: *[[Boston Archdiocesan Choir School]] (BACS) *[[Buckingham Browne & Nichols]] (BB&N) *[[Cambridge montessori school|Cambridge Montessori School]] (CMS) *Cambridge [[Religious Society of Friends|Friends]] School. Thomas Waring served as founding headmaster of the school. *Fayerweather Street School (FSS)[http://www.fayerweather.org/ ] *[[International School of Boston]] (ISB, formerly École Bilingue) *[[Matignon High School]] *[[North Cambridge Catholic High School]] (re-branded as Cristo Rey Boston and relocated to Dorchester, MA in 2010) *[[Shady Hill School]] *St. Peter School ==Economy== [[File:Cambridge Skyline.jpg|thumb|Buildings of [[Kendall Square]], center of Cambridge's [[biotech]] economy, seen from the [[Charles River]]]] Manufacturing was an important part of the economy in the late 19th and early 20th century, but educational institutions are the city's biggest employers today. Harvard and [[Massachusetts Institute of Technology|MIT]] together employ about 20,000.[http://www2.cambridgema.gov/cdd/data/labor/top25/top25_2008.html Top 25 Cambridge Employers: 2008], City of Cambridge As a cradle of technological innovation, Cambridge was home to technology firms [[Analog Devices]], [[Akamai Technologies|Akamai]], [[BBN Technologies|Bolt, Beranek, and Newman (BBN Technologies)]] (now part of Raytheon), [[General Radio|General Radio (later GenRad)]], [[Lotus Development Corporation]] (now part of [[IBM]]), [[Polaroid Corporation|Polaroid]], [[Symbolics]], and [[Thinking Machines]]. In 1996, [[Polaroid Corporation|Polaroid]], [[Arthur D. Little]], and [[Lotus Development Corporation|Lotus]] were top employers with over 1,000 employees in Cambridge, but faded out a few years later. Health care and biotechnology firms such as [[Genzyme]], [[Biogen Idec]], [[Millennium Pharmaceuticals]], [[Sanofi]], [[Pfizer]] and [[Novartis]]{{cite news |title=Novartis doubles plan for Cambridge |author=Casey Ross and Robert Weisman |first= |last= |authorlink= |authorlink2= |url=http://articles.boston.com/2010-10-27/business/29323650_1_french-drug-maker-astrazeneca-plc-research-operations |agency= |newspaper=[[The Boston Globe]] |publisher= |isbn= |issn= |pmid= |pmd= |bibcode= |doi= |date=October 27, 2010 |page= |pages= |accessdate=April 12, 2011|quote=Already Cambridge’s largest corporate employer, the Swiss firm expects to hire an additional 200 to 300 employees over the next five years, bringing its total workforce in the city to around 2,300. Novartis’s global research operations are headquartered in Cambridge, across Massachusetts Avenue from the site of the new four-acre campus. |archiveurl= |archivedate= |ref=}} have significant presences in the city. Though headquartered in Switzerland, Novartis continues to expand its operations in Cambridge. Other major biotech and pharmaceutical firms expanding their presence in Cambridge include [[GlaxoSmithKline]], [[AstraZeneca]], [[Shire plc|Shire]], and [[Pfizer]].{{cite news|title=Novartis Doubles Plan for Cambridge|url=http://www.boston.com/business/healthcare/articles/2010/10/27/novartis_doubles_plan_for_cambridge/|accessdate=23 February 2012 | work=The Boston Globe|first1=Casey|last1=Ross|first2=Robert|last2=Weisman|date=October 27, 2010}} Most Biotech firms in Cambridge are located around [[Kendall Square]] and [[East Cambridge, Massachusetts|East Cambridge]], which decades ago were the city's center of manufacturing. A number of biotechnology companies are also located in [[University Park at MIT]], a new development in another former manufacturing area. None of the high technology firms that once dominated the economy was among the 25 largest employers in 2005, but by 2008 high tech companies [[Akamai Technologies|Akamai]] and [[ITA Software]] had grown to be among the largest 25 employers. [[Google]],{{cite web|url=http://www.google.com/corporate/address.html |title=Google Offices |publisher=Google.com |date= |accessdate=2012-07-18}} [[IBM Research]], and [[Microsoft Research]] maintain offices in Cambridge. In late January 2012—less than a year after acquiring [[Billerica, Massachusetts|Billerica]]-based analytic database management company, [[Vertica]]—[[Hewlett-Packard]] announced it would also be opening its first offices in Cambridge.{{cite web|last=Huang|first=Gregory|title=Hewlett-Packard Expands to Cambridge via Vertica’s "Big Data" Center|url=http://www.xconomy.com/boston/2012/01/23/hewlett-packard-expands-to-cambridge-via-verticas-big-data-center/?single_page=true}} Around this same time, e-commerce giants [[Staples Inc.|Staples]]{{cite web|title=Staples to bring e-commerce office to Cambridge's Kendall Square Read more: Staples to bring e-commerce office to Cambridge's Kendall Square - Cambridge, Massachusetts - Cambridge Chronicle http://www.wickedlocal.com/cambridge/news/x690035936/Staples-to-bring-E-commerce-office-to-Cambridges-Kendall-Square#ixzz1nDY39Who|url=http://www.wickedlocal.com/cambridge/news/x690035936/Staples-to-bring-E-commerce-office-to-Cambridges-Kendall-Square#axzz1kg3no7Zg}} and [[Amazon.com]]{{cite web|title=Amazon Seeks Brick-And-Mortar Presence In Boston Area|url=http://www.wbur.org/2011/12/22/amazon-boston}} said they would be opening research and innovation centers in Kendall Square. Video game developer [[Harmonix Music Systems]] is based in [[Central Square (Cambridge)|Central Square]]. The proximity of Cambridge's universities has also made the city a center for nonprofit groups and think tanks, including the [[National Bureau of Economic Research]], the [[Smithsonian Astrophysical Observatory]], the [[Lincoln Institute of Land Policy]], [[Cultural Survival]], and [[One Laptop per Child]]. In September 2011, an initiative by the City of Cambridge called the "[[Entrepreneur Walk of Fame]]" was launched. It seeks to highlight individuals who have made contributions to innovation in the global business community.{{cite news |title=Stars of invention |author= |first=Kathleen |last=Pierce |url=http://articles.boston.com/2011-09-16/business/30165912_1_gates-and-jobs-microsoft-granite-stars |agency= |newspaper=The Boston Globe|date=September 16, 2011 |page= |pages= |at= |accessdate=October 1, 2011}} ===Top employers=== The top ten employers in the city are:{{cite web|url=http://cambridgema.gov/citynewsandpublications/news/2012/01/fy11comprehensiveannualfinancialreportnowavailable.aspx |title=City of Cambridge, Massachusetts Comprehensive Annual Financial Report July 1, 2010—June 30, 2011 |publisher=Cambridgema.gov |date=2011-06-30 |accessdate=2012-04-28}} {| class="wikitable" |- ! # ! Employer ! # of employees |- | 1 |[[Harvard University]] |10,718 |- |2 |[[Massachusetts Institute of Technology]] |7,604 |- |3 |City of Cambridge |2,922 |- |4 |[[Novartis]] Institutes for BioMedical Research |2,095 |- |5 |[[Mount Auburn Hospital]] |1,665 |- |6 |[[Vertex Pharmaceuticals]] |1,600 |- |7 |[[Genzyme]] |1,504 |- |8 |[[Biogen Idec]] |1,350 |- |9 |[[Federal government of the United States|Federal Government]] |1,316 |- |10 |[[Pfizer]] |1,300 |} ==Transportation== {{See also|Boston transportation}} ===Road=== [[File:Harvard Square at Peabody Street and Mass Avenue.jpg|thumb|[[Massachusetts Avenue (Boston)|Massachusetts Avenue]] in [[Harvard Square]]]] Several major roads lead to Cambridge, including [[Massachusetts State Highway 2|Route 2]], [[Massachusetts State Highway 16|Route 16]] and the [[Massachusetts State Highway 28|McGrath Highway (Route 28)]]. The [[Massachusetts Turnpike]] does not pass through Cambridge, but provides access by an exit in nearby [[Allston, Massachusetts|Allston]]. Both [[U.S. Route 1]] and [[I-93 (MA)]] also provide additional access on the eastern end of Cambridge at Leverett Circle in [[Boston]]. [[Massachusetts State Highway 2A|Route 2A]] runs the length of the city, chiefly along Massachusetts Avenue. The Charles River forms the southern border of Cambridge and is crossed by 11 bridges connecting Cambridge to Boston, including the [[Longfellow Bridge]] and the [[Harvard Bridge]], eight of which are open to motorized road traffic. Cambridge has an irregular street network because many of the roads date from the colonial era. Contrary to popular belief, the road system did not evolve from longstanding cow-paths. Roads connected various village settlements with each other and nearby towns, and were shaped by geographic features, most notably streams, hills, and swampy areas. Today, the major "squares" are typically connected by long, mostly straight roads, such as Massachusetts Avenue between [[Harvard Square]] and [[Central Square (Cambridge)|Central Square]], or Hampshire Street between [[Kendall Square]] and [[Inman Square]]. ===Mass transit=== [[File:Central MBTA station.jpg|thumb|[[Central (MBTA)|Central station on the MBTA Red Line]]]] Cambridge is well served by the [[MBTA]], including the [[Porter (MBTA station)|Porter Square stop]] on the regional [[MBTA Commuter Rail|Commuter Rail]], the [[Lechmere (MBTA station)|Lechmere stop]] on the [[Green Line (MBTA)|Green Line]], and five stops on the [[Red Line (MBTA)|Red Line]] ([[Alewife Station (MBTA)|Alewife]], [[Porter (MBTA)|Porter Square]], [[Harvard (MBTA station)|Harvard Square]], [[Central (MBTA station)|Central Square]], and [[Kendall/MIT (MBTA station)|Kendall Square/MIT]]). Alewife Station, the current terminus of the Red Line, has a large multi-story parking garage (at a rate of $7 per day {{as of|lc=y|2009}}).{{cite web|url=http://www.mbta.com/schedules_and_maps/subway/lines/stations/?stopId=10029 |title=> Schedules & Maps > Subway > Alewife Station |publisher=MBTA |date= |accessdate=2012-04-28}} The [[Harvard Bus Tunnel]], under Harvard Square, reduces traffic congestion on the surface, and connects to the Red Line underground. This tunnel was originally opened for streetcars in 1912, and served trackless trolleys and buses as the routes were converted. The tunnel was partially reconfigured when the Red Line was extended to Alewife in the early 1980s. Outside of the state-owned transit agency, the city is also served by the Charles River Transportation Management Agency (CRTMA) shuttles which are supported by some of the largest companies operating in city, in addition to the municipal government itself.{{cite web |url=http://www.charlesrivertma.org/members.htm |title=Charles River TMA Members |author=Staff writer |date=(As of) January 1, 2013 |work=CRTMA |publisher= |language= |trans_title= |type= |archiveurl= |archivedate= |deadurl= |accessdate=January 1, 2013 |quote= |ref= |separator= |postscript=}} ===Cycling=== Cambridge has several [[bike path]]s, including one along the Charles River,{{cite web|url=http://www.mass.gov/dcr/parks/metroboston/maps/bikepaths_dudley.gif |title=Dr. Paul Dudley White Bikepath |date= |accessdate=2012-04-28}} and the [[Cambridge Linear Park|Linear Park]] connecting the [[Minuteman Bikeway]] at Alewife with the [[Somerville Community Path]]. Bike parking is common and there are bike lanes on many streets, although concerns have been expressed regarding the suitability of many of the lanes. On several central MIT streets, bike lanes transfer onto the sidewalk. Cambridge bans cycling on certain sections of sidewalk where pedestrian traffic is heavy.{{cite web|url=http://www.cambridgema.gov/cdd/et/bike/bike_ban.html |title=Sidewalk Bicycling Banned Areas – Cambridge Massachusetts |publisher=Cambridgema.gov |date= |accessdate=2012-04-28}}{{cite web|url=http://www.cambridgema.gov/cdd/et/bike/bike_reg.html |title=Traffic Regulations for Cyclists – Cambridge Massachusetts |publisher=Cambridgema.gov |date=1997-05-01 |accessdate=2012-04-28}} While ''[[Bicycling Magazine]]'' has rated Boston as one of the worst cities in the nation for bicycling (In their words, for "lousy roads, scarce and unconnected bike lanes and bike-friendly gestures from City Hall that go nowhere—such as hiring a bike coordinator in 2001, only to cut the position two years later"),[http://www.bicycling.com/article/1,6610,s1-2-16-14593-11,00.html Urban Treasures – bicycling.com]{{dead link|date=April 2012}} it has listed Cambridge as an honorable mention as one of the best[http://www.bicycling.com/article/1,6610,s1-2-16-14593-9,00.html Urban Treasures – bicycling.com]{{dead link|date=April 2012}} and was called by the magazine "Boston's Great Hope." Cambridge has an active, official bicycle committee. ===Walking=== [[File:Weeks Footbridge Cambridge, MA.jpg|thumb|The [[John W. Weeks Bridge|Weeks Bridge]] provides a pedestrian-only connection between Boston's Allston-Brighton neighborhood and Cambridge over the Charles River]] Walking is a popular activity in Cambridge. Per year 2000 data, of the communities in the U.S. with more than 100,000 residents, Cambridge has the highest percentage of commuters who walk to work.{{cite web|url=http://www.bikesatwork.com/carfree/census-lookup.php?state_select=ALL_STATES&lower_pop=100000&upper_pop=99999999&sort_num=2&show_rows=25&first_row=0 |title=The Carfree Census Database: Result of search for communities in any state with population over 100,000, sorted in descending order by % Pedestrian Commuters |publisher=Bikesatwork.com |date= |accessdate=2012-04-28}} Cambridge receives a "Walk Score" of 100 out of 100 possible points.[http://www.walkscore.com/get-score.php?street=cambridge%2C+ma&go=Go Walk Score site] Accessed July 28, 2009 Cambridge's major historic squares have been recently changed into a modern walking landscape, which has sparked a traffic calming program based on the needs of pedestrians rather than of motorists. ===Intercity=== The Boston intercity bus and train stations at [[South Station]], Boston, and [[Logan International Airport]] in [[East Boston]], are accessible by [[Red Line (MBTA)|subway]]. The [[Fitchburg Line]] rail service from [[Porter (MBTA station)|Porter Square]] connects to some western suburbs. Since October 2010, there has also been intercity bus service between [[Alewife (MBTA station)|Alewife Station]] (Cambridge) and [[New York City]].{{cite web|last=Thomas |first=Sarah |url=http://www.boston.com/yourtown/news/cambridge/2010/10/warren_mbta_welcome_world_wide.html |title=NYC-bound buses will roll from Newton, Cambridge |publisher=Boston.com |date=2010-10-19 |accessdate=2012-04-28}} ==Media== ===Newspapers=== Cambridge is served by several weekly newspapers. The most prominent is the ''[[Cambridge Chronicle]]'', which is also the oldest surviving weekly paper in the United States. ===Radio=== Cambridge is home to the following commercially licensed and student-run radio stations: {| class=wikitable |- ! [[Callsign]] !! Frequency !! City/town !! Licensee !! Format |- | [[WHRB]] || align=right | 95.3 FM || Cambridge (Harvard) || Harvard Radio Broadcasting Co., Inc. || [[Variety (US radio)|Musical variety]] |- | [[WJIB]] || align=right | 740 AM || Cambridge || Bob Bittner Broadcasting || [[Adult Standards]]/Pop |- | [[WMBR]] || align=right | 88.1 FM || Cambridge (MIT) || Technology Broadcasting Corporation || [[College radio]] |} ===Television=== Cambridge Community Television (CCTV) has served the Cambridge community since its inception in 1988. CCTV operates Cambridge's public access television facility and programs three television channels, 8, 9, and 96 on the Cambridge cable system (Comcast). ===Social media=== As of 2011, a growing number of social media efforts provide means for participatory engagement with the locality of Cambridge, such as Localocracy"Localocracy is an online town common where registered voters using real names can weigh in on local issues." [http://cambridge.localocracy.com/ Localocracy Cambridge, Massachusetts]. Accessed 2011-10-01 and [[foursquare (website)|Foursquare]]. ==Culture, art and architecture== [[File:Fogg.jpg|thumb|[[Fogg Museum]], Harvard]] ===Museums=== * [[Harvard Art Museum]], including the [[Busch-Reisinger Museum]], a collection of Germanic art the [[Fogg Art Museum]], a comprehensive collection of Western art, and the [[Arthur M. Sackler Museum]], a collection of Middle East and Asian art * [[Harvard Museum of Natural History]], including the [[Glass Flowers]] collection * [[Peabody Museum of Archaeology and Ethnology]], Harvard *[[Semitic Museum]], Harvard * [[MIT Museum]] * [[List Visual Arts Center]], MIT ===Public art=== Cambridge has a large and varied collection of permanent public art, both on city property (managed by the Cambridge Arts Council),{{cite web|url=http://www.cambridgema.gov/CAC/Public/overview.cfm |title=CAC Public Art Program |publisher=Cambridgema.gov |date=2007-03-13 |accessdate=2012-04-28}} and on the campuses of Harvard{{cite web|url=http://ofa.fas.harvard.edu/visualarts/pubart.php |title=Office for the Arts at Harvard: Public Art |publisher=Ofa.fas.harvard.edu |date= |accessdate=2012-04-28}} and MIT.{{cite web|url=http://listart.mit.edu/map |title=MIT Public Art Collection Map |publisher=Listart.mit.edu |date= |accessdate=2012-04-28}} Temporary public artworks are displayed as part of the annual Cambridge River Festival on the banks of the Charles River, during winter celebrations in Harvard and Central Squares, and at university campus sites. Experimental forms of public artistic and cultural expression include the Central Square World's Fair, the Somerville-based annual Honk! Festival,{{cite web|url=http://honkfest.org/ |title= Honk Fest}} and [[If This House Could Talk]],{{cite web|url=http://cambridgehistory.org/discover/ifthishousecouldtalk/index.html |title=The Cambridge Historical Society}} a neighborhood art and history event. {{or|date=April 2012}} {{Citation needed|date=April 2012}} An active tradition of street musicians and other performers in Harvard Square entertains an audience of tourists and local residents during the warmer months of the year. The performances are coordinated through a public process that has been developed collaboratively by the performers,{{cite web|url=http://www.buskersadvocates.org/ | title= Street Arts & Buskers Advocates}} city administrators, private organizations and business groups.{{cite web|url=http://harvardsquare.com/Home/Arts-and-Entertainment/Street-Arts-and-Buskers-Advocates.aspx |title=Street Arts and Buskers Advocates |publisher=Harvardsquare.com |date= |accessdate=2012-04-28}} [[File:Longfellow National Historic Site, Cambridge, Massachusetts.JPG|thumb|right|The [[Longfellow National Historic Site]]]] [[File:Wfm stata center.jpg|thumb|[[Stata Center]], MIT]] [[File:Simmons Hall, MIT, Cambridge, Massachusetts.JPG|thumb|[[List of MIT undergraduate dormitories|Simmons Hall]], MIT]] ===Architecture=== Despite intensive urbanization during the late 19th century and 20th century, Cambridge has preserved an unusual number of historic buildings, including some dating to the 17th century. The city also contains an abundance of innovative contemporary architecture, largely built by Harvard and MIT. ;Notable historic buildings in the city include: * The [[Asa Gray House]] (1810) * [[Austin Hall, Harvard University]] (1882–84) * [[Cambridge, Massachusetts City Hall|Cambridge City Hall]] (1888–89) * [[Cambridge Public Library]] (1888) * [[Christ Church, Cambridge]] (1761) * [[Cooper-Frost-Austin House]] (1689–1817) * [[Elmwood (Cambridge, Massachusetts)|Elmwood House]] (1767), residence of the [[President of Harvard University]] * [[First Church of Christ, Scientist (Cambridge, Massachusetts)|First Church of Christ, Scientist]] (1924–30) * [[The First Parish in Cambridge]] (1833) * [[Harvard-Epworth United Methodist Church]] (1891–93) * [[Harvard Lampoon Building]] (1909) * The [[Hooper-Lee-Nichols House]] (1685–1850) * [[Longfellow National Historic Site]] (1759), former home of poet [[Henry Wadsworth Longfellow]] * [[The Memorial Church of Harvard University]] (1932) * [[Memorial Hall, Harvard University]] (1870–77) * [[Middlesex County Courthouse (Massachusetts)|Middlesex County Courthouse]] (1814–48) * [[Urban Rowhouse (40-48 Pearl Street, Cambridge, Massachusetts)|Urban Rowhouse]] (1875) * [[spite house|O'Reilly Spite House]] (1908), built to spite a neighbor who would not sell his adjacent landBloom, Jonathan. (February 2, 2003) [[Boston Globe]] ''[http://nl.newsbank.com/nl-search/we/Archives?p_product=BG&p_theme=bg&p_action=search&p_maxdocs=200&p_topdoc=1&p_text_direct-0=0F907F2342522B5D&p_field_direct-0=document_id&p_perpage=10&p_sort=YMD_date:D Existing by the Thinnest of Margins. A Concord Avenue Landmark Gives New Meaning to Cozy.]'' Section: City Weekly; Page 11. Location: 260 Concord Ave, Cambridge, MA 02138. {{See also|List of Registered Historic Places in Cambridge, Massachusetts}} ;Contemporary architecture: * [[List of MIT undergraduate dormitories#Baker House|Baker House]] dormitory, MIT, by Finnish architect [[Alvar Aalto]], one of only two buildings by Aalto in the US * Harvard Graduate Center/Harkness Commons, by [[The Architects Collaborative]] (TAC, with [[Walter Gropius]]) * [[Carpenter Center for the Visual Arts]], Harvard, the only building in North America by [[Le Corbusier]] * [[Kresge Auditorium]], MIT, by [[Eero Saarinen]] * [[MIT Chapel]], by [[Eero Saarinen]] * [[Design Research Building]], by [[Benjamin Thompson and Associates]] * [[American Academy of Arts and Sciences]], by [[Kallmann McKinnell and Wood]], also architects of Boston City Hall * [[Arthur M. Sackler Museum]], Harvard, one of the few buildings in the U.S. by [[James Stirling (architect)|James Stirling]], winner of the [[Pritzker Prize]] * [[Stata Center]], MIT, by [[Frank Gehry]] * [[List of MIT undergraduate dormitories#Simmons Hall|Simmons Hall]], MIT, by [[Steven Holl]] ===Music=== The city has an active music scene from classical performances to the latest popular bands. ==Sister cities== Cambridge has 8 active, official [[Twin towns and sister cities|sister cities]], and an unofficial relationship with [[Cambridge]], England:"A message from the Peace Commission" [http://www.cambridgema.gov/peace/newsandpublications/news/detail.aspx?path=%2fsitecore%2fcontent%2fhome%2fpeace%2fnewsandpublications%2fnews%2f2008%2f02%2finformationoncambridgessistercities]. *{{Flagicon|PRT}} [[Coimbra]], [[Portugal]] *{{Flagicon|CUB}} [[Cienfuegos]], [[Cuba]] *{{Flagicon|ITA}} [[Gaeta]], [[Italy]] *{{Flagicon|IRL}} [[Galway]], [[Republic of Ireland|Ireland]] *{{Flagicon|ARM}} [[Yerevan]], [[Armenia]]{{cite web|url=http://www.cysca.org/ |title=Cambridge-Yerevan Sister City Association |publisher=Cysca.org |date= |accessdate=2012-04-28}} *{{Flagicon|SLV}} [[San José Las Flores, Chalatenango|San José Las Flores]], [[El Salvador]] *{{Flagicon|JPN}} [[Tsukuba, Ibaraki|Tsukuba Science City]], Japan *{{Flagicon|POL}} [[Kraków]], [[Poland]] *{{Flagicon|CHN}} [[Haidian District]], [[China]] Ten other official sister city relationships are inactive: [[Dublin]], Ireland; [[Ischia]], [[Catania]], and [[Florence]], Italy; [[Kraków]], Poland; [[Santo Domingo Oeste]], Dominican Republic; [[Southwark]], London, England; [[Yuseong]], Daejeon, Korea; and [[Haidian District|Haidian]], Beijing, China. There has also been an unofficial relationship with: *{{Flagicon|GBR}} [[Cambridge]], England, UK{{cite web|url=http://www.cambridgema.gov/peace/newsandpublications/news/detail.aspx?path=%2fsitecore%2fcontent%2fhome%2fpeace%2fnewsandpublications%2fnews%2f2008%2f02%2finformationoncambridgessistercities |title="Sister Cities", Cambridge Peace Commission |publisher=Cambridgema.gov |date=2008-02-15 |accessdate=2012-07-18}} ==Zip codes== *02138—Harvard Square/West Cambridge *02139—Central Square/Inman Square/MIT *02140—Porter Square/North Cambridge *02141—East Cambridge *02142—Kendall Square ==References== {{reflist|30em}} ==General references== * ''History of Middlesex County, Massachusetts'', [http://books.google.com/books?id=QGolOAyd9RMC&dq=intitle:History+intitle:of+intitle:Middlesex+intitle:County+intitle:Massachusetts&lr=&num=50&as_brr=0&source=gbs_other_versions_sidebar_s&cad=5 Volume 1 (A-H)], [http://books.google.com/books?id=hNaAnwRMedUC&pg=PA506&dq=intitle:History+intitle:of+intitle:Middlesex+intitle:County+intitle:Massachusetts&lr=&num=50&as_brr=0#PPA3,M1 Volume 2 (L-W)] compiled by Samuel Adams Drake, published 1879–1880. ** [http://books.google.com/books?id=QGolOAyd9RMC&printsec=titlepage#PPA305,M1 Cambridge article] by Rev. Edward Abbott in volume 1, pages 305–358. *Eliot, Samuel Atkins. ''A History of Cambridge, Massachusetts: 1630–1913''. Cambridge: The Cambridge Tribune, 1913. *Hiestand, Emily. "Watershed: An Excursion in Four Parts" The Georgia Review Spring 1998 pages 7–28 *[[Lucius Robinson Paige|Paige, Lucius]]. ''History of Cambridge, Massachusetts: 1630–1877''. Cambridge: The Riverside Press, 1877. *Survey of Architectural History in Cambridge: Mid Cambridge, 1967, Cambridge Historical Commission, Cambridge, Mass.{{ISBN missing}} *Survey of Architectural History in Cambridge: Cambridgeport, 1971 ISBN 0-262-53013-9, Cambridge Historical Commission, Cambridge, Mass. *Survey of Architectural History in Cambridge: Old Cambridge, 1973 ISBN 0-262-53014-7, Cambridge Historical Commission, Cambridge, Mass. *Survey of Architectural History in Cambridge: Northwest Cambridge, 1977 ISBN 0-262-53032-5, Cambridge Historical Commission, Cambridge, Mass. *Survey of Architectural History in Cambridge: East Cambridge, 1988 (revised) ISBN 0-262-53078-3, Cambridge Historical Commission, Cambridge, Mass. *{{cite book|last=Sinclair|first=Jill|title=Fresh Pond: The History of a Cambridge Landscape|publisher=MIT Press|location=Cambridge, Mass.|date=April 2009|isbn=978-0-262-19591-1 }} *{{cite book|last=Seaburg|first=Alan|title=Cambridge on the Charles|url=http://books.google.com/books?id=c7_oCS782-8C|publisher=Anne Miniver Press|location=Billerica, Mass.|year=2001|author=Seaburg, A. and Dahill, T. and Rose, C.H.|isbn=978-0-9625794-9-3}} ==External links== {{Commons category}} {{Wikivoyage|Cambridge (Massachusetts)}} {{Portal|Boston}} {{Commons category|Cambridge, Massachusetts}} *{{Official website|http://www.cambridgema.gov/}} *[http://www.cambridge-usa.org/ Cambridge Office for Tourism] *[http://www.city-data.com/city/Cambridge-Massachusetts.html City-Data.com] *[http://www.epodunk.com/cgi-bin/genInfo.php?locIndex=2894 ePodunk: Profile for Cambridge, Massachusetts] *{{dmoz|Regional/North_America/United_States/Massachusetts/Localities/C/Cambridge}}
===Maps=== *[http://www.cambridgema.gov/GIS/FindMapAtlas.cfm Cambridge Maps] *[http://www.cambridgema.gov/GIS City of Cambridge Geographic Information System (GIS)] *[http://www.salemdeeds.com/atlases_results.asp?ImageType=index&atlastype=MassWorld&atlastown=&atlas=MASSACHUSETTS+1871&atlas_desc=MASSACHUSETTS+1871 ''1871 Atlas of Massachusetts''.] by Wall & Gray. [http://www.salemdeeds.com/atlases_pages.asp?ImageName=PAGE_0010_0011.jpg&atlastype=MassWorld&atlastown=&atlas=MASSACHUSETTS+1871&atlas_desc=MASSACHUSETTS+1871&pageprefix= Map of Massachusetts.] [http://www.salemdeeds.com/atlases_pages.asp?ImageName=PAGE_0044_0045.jpg&atlastype=MassWorld&atlastown=&atlas=MASSACHUSETTS+1871&atlas_desc=MASSACHUSETTS+1871&pageprefix= Map of Middlesex County.] *Dutton, E.P. [http://maps.bpl.org/details_10717/?srch_query=Dutton%2C+E.P.&srch_fields=all&srch_author=on&srch_style=exact&srch_fa=save&srch_ok=Go+Search Chart of Boston Harbor and Massachusetts Bay with Map of Adjacent Country.] Published 1867. A good map of roads and rail lines around Cambridge. *[http://www.citymap.com/cambridge/index.htm Cambridge Citymap – Community, Business, and Visitor Map.] *[http://docs.unh.edu/towns/CambridgeMassachusettsMapList.htm Old USGS maps of Cambridge area.] {{Greater Boston}} {{Middlesex County, Massachusetts}} {{Massachusetts}} {{New England}} {{Massachusetts cities and mayors of 100,000 population}} [[Category:Cambridge, Massachusetts| ]] [[Category:University towns in the United States]] [[Category:County seats in Massachusetts]] [[Category:Populated places established in 1630]] [[Category:Charles River]] [[Category:Place names of English origin in the United States]] [[af:Cambridge, Massachusetts]] [[ar:كامبريدج، ماساتشوستس]] [[zh-min-nan:Cambridge, Massachusetts]] [[be:Горад Кембрыдж, Масачусетс]] [[be-x-old:Кембрыдж (Масачусэтс)]] [[bg:Кеймбридж (Масачузетс)]] [[br:Cambridge (Massachusetts)]] [[ca:Cambridge (Massachusetts)]] [[cs:Cambridge (Massachusetts)]] [[cy:Cambridge, Massachusetts]] [[da:Cambridge (Massachusetts)]] [[de:Cambridge (Massachusetts)]] [[et:Cambridge (Massachusetts)]] [[es:Cambridge (Massachusetts)]] [[eo:Kembriĝo (Masaĉuseco)]] [[eu:Cambridge (Massachusetts)]] [[fa:کمبریج (ماساچوست)]] [[fr:Cambridge (Massachusetts)]] [[gd:Cambridge (MA)]] [[ko:케임브리지 (매사추세츠 주)]] [[hy:Քեմբրիջ (Մասաչուսեթս)]] [[id:Cambridge, Massachusetts]] [[it:Cambridge (Massachusetts)]] [[he:קיימברידג' (מסצ'וסטס)]] [[jv:Cambridge, Massachusetts]] [[kk:Кэмбридж (Массачусетс)]] [[kw:Cambridge, Massachusetts]] [[sw:Cambridge, Massachusetts]] [[ht:Cambridge, Massachusetts]] [[la:Cantabrigia (Massachusetta)]] [[lv:Keimbridža]] [[lb:Cambridge (Massachusetts)]] [[hu:Cambridge (Massachusetts)]] [[mr:केंब्रिज, मॅसेच्युसेट्स]] [[ms:Cambridge, Massachusetts]] [[nl:Cambridge (Massachusetts)]] [[ja:ケンブリッジ (マサチューセッツ州)]] [[no:Cambridge (Massachusetts)]] [[pl:Cambridge (Massachusetts)]] [[pt:Cambridge (Massachusetts)]] [[ro:Cambridge, Massachusetts]] [[ru:Кембридж (Массачусетс)]] [[scn:Cambridge (Massachusetts), USA]] [[simple:Cambridge, Massachusetts]] [[sk:Cambridge (Massachusetts)]] [[sl:Cambridge, Massachusetts]] [[sr:Кембриџ (Масачусетс)]] [[fi:Cambridge (Massachusetts)]] [[sv:Cambridge, Massachusetts]] [[tl:Cambridge, Massachusetts]] [[ta:கேம்பிரிஜ், மாசசூசெட்ஸ்]] [[th:เคมบริดจ์ (รัฐแมสซาชูเซตส์)]] [[tg:Кембриҷ (Массачусетс)]] [[tr:Cambridge, Massachusetts]] [[uk:Кембридж (Массачусетс)]] [[vi:Cambridge, Massachusetts]] [[vo:Cambridge (Massachusetts)]] [[war:Cambridge, Massachusetts]] [[yi:קעמברידזש, מאסאטשוסעטס]] [[zh:剑桥 (马萨诸塞州)]] \ No newline at end of file +{{Distinguish|Cambridge, England}} {{primary sources|date=June 2012}} {{Use mdy dates|date=January 2011}} {{Infobox settlement |official_name = Cambridge, Massachusetts |nickname = |motto = "Boston's Left Bank"{{cite web|url= http://www.epodunk.com/cgi-bin/genInfo.php?locIndex=2894|title=Profile for Cambridge, Massachusetts, MA|publisher= ePodunk |accessdate= November 1, 2012}} |image_skyline = CambridgeMACityHall2.jpg |imagesize = 175px |image_caption = Cambridge City Hall |image_seal = |image_flag = |image_map = Cambridge ma highlight.png |mapsize = 250px |map_caption = Location in Middlesex County in Massachusetts |image_map1 = |mapsize1 = |map_caption1 = |coordinates_region = US-MA |subdivision_type = Country |subdivision_name = United States |subdivision_type1 = State |subdivision_name1 = [[Massachusetts]] |subdivision_type2 = [[List of counties in Massachusetts|County]] |subdivision_name2 = [[Middlesex County, Massachusetts|Middlesex]] |established_title = Settled |established_date = 1630 |established_title2 = Incorporated |established_date2 = 1636 |established_title3 = |established_date3 = |government_type = [[Council-manager government|Council-City Manager]] |leader_title = Mayor |leader_name = Henrietta Davis |leader_title1 = [[City manager|City Manager]] |leader_name1 = [[Robert W. Healy]] |area_magnitude = |area_total_km2 = 18.47 |area_total_sq_mi = 7.13 |area_land_km2 = 16.65 |area_land_sq_mi = 6.43 |area_water_km2 = 1.81 |area_water_sq_mi = 0.70 |population_as_of = 2010 |population_blank2_title = [[Demonym]] |population_blank2 = [[Cantabrigian]] |settlement_type = City |population_total = 105,162 |population_density_km2 = 6,341.98 |population_density_sq_mi = 16,422.08 |elevation_m = 12 |elevation_ft = 40 |timezone = [[Eastern Time Zone|Eastern]] |utc_offset = -5 |timezone_DST = [[Eastern Time Zone|Eastern]] |utc_offset_DST = -4 |coordinates_display = display=inline,title |latd = 42 |latm = 22 |lats = 25 |latNS = N |longd = 71 |longm = 06 |longs = 38 |longEW = W |website = [http://www.cambridgema.gov/ www.cambridgema.gov] |postal_code_type = ZIP code |postal_code = 02138, 02139, 02140, 02141, 02142 |area_code = [[Area code 617|617]] / [[Area code 857|857]] |blank_name = [[Federal Information Processing Standard|FIPS code]] |blank_info = 25-11000 |blank1_name = [[Geographic Names Information System|GNIS]] feature ID |blank1_info = 0617365 |footnotes = }} '''Cambridge''' is a city in [[Middlesex County, Massachusetts|Middlesex County]], [[Massachusetts]], [[United States]], in the [[Greater Boston]] area. It was named in honor of the [[University of Cambridge]] in [[England]], an important center of the [[Puritan]] theology embraced by the town's founders.{{cite book|last=Degler|first=Carl Neumann|title=Out of Our Pasts: The Forces That Shaped Modern America|publisher=HarperCollins|location=New York|year=1984|url=http://books.google.com/books?id=NebLe1ueuGQC&pg=PA18&lpg=PA18&dq=cambridge+university+puritans+newtowne#v=onepage&q=&f=false|accessdate=September 9, 2009 | isbn=978-0-06-131985-3}} Cambridge is home to two of the world's most prominent universities, [[Harvard University]] and the [[Massachusetts Institute of Technology]]. According to the [[2010 United States Census]], the city's population was 105,162.{{cite web|url=http://2010.census.gov/news/releases/operations/cb11-cn104.html |title=Census 2010 News | U.S. Census Bureau Delivers Massachusetts' 2010 Census Population Totals, Including First Look at Race and Hispanic Origin Data for Legislative Redistricting |publisher=2010.census.gov |date=2011-03-22 |accessdate=2012-04-28}} It is the fifth most populous city in the state, behind [[Boston]], [[Worcester, MA|Worcester]], [[Springfield, MA|Springfield]], and [[Lowell, Massachusetts|Lowell]]. Cambridge was one of the two [[county seat]]s of Middlesex County prior to the abolition of county government in 1997; [[Lowell, Massachusetts|Lowell]] was the other. ==History== {{See also|Timeline of Cambridge, Massachusetts history}} [[File:Formation of Massachusetts towns.svg|thumb|A map showing the original boundaries of Cambridge]] The site for what would become Cambridge was chosen in December 1630, because it was located safely upriver from Boston Harbor, which made it easily defensible from attacks by enemy ships. Also, the water from the local spring was so good that the local Native Americans believed it had medicinal properties.{{Citation needed|date=November 2009}} [[Thomas Dudley]], his daughter [[Anne Bradstreet]] and her husband Simon were among the first settlers of the town. The first houses were built in the spring of 1631. The settlement was initially referred to as "the newe towne".{{cite book|last=Drake|first=Samuel Adams|title=History of Middlesex County, Massachusetts|publisher=Estes and Lauriat|location=Boston|year=1880|volume=1|pages=305–16|url=http://books.google.com/books?id=QGolOAyd9RMC&pg=PA316&lpg=PA305&dq=newetowne&ct=result#PPA305,M1|accessdate=December 26, 2008}} Official Massachusetts records show the name capitalized as '''Newe Towne''' by 1632.{{cite book|title=Report on the Custody and Condition of the Public Records of Parishes|publisher=Massachusetts Secretary of the Commonwealth|url=http://books.google.com/books?id=IyYWAAAAYAAJ&pg=RA1-PA298&lpg=RA1-PA298&dq=%22Ordered+That+Newtowne+shall+henceforward+be+called%22|location=Boston|year=1889|page=298|accessdate=December 24, 2008}} Located at the first convenient [[Charles River]] crossing west of [[Boston]], Newe Towne was one of a number of towns (including Boston, [[Dorchester, Massachusetts|Dorchester]], [[Watertown, Massachusetts|Watertown]], and [[Weymouth, Massachusetts|Weymouth]]) founded by the 700 original [[Puritan]] colonists of the [[Massachusetts Bay Colony]] under governor [[John Winthrop]]. The original village site is in the heart of today's [[Harvard Square]]. The marketplace where farmers brought in crops from surrounding towns to sell survives today as the small park at the corner of John F. Kennedy (J.F.K.) and Winthrop Streets, then at the edge of a salt marsh, since filled. The town included a much larger area than the present city, with various outlying parts becoming independent towns over the years: [[Newton, Massachusetts|Newton (originally Cambridge Village, then Newtown)]] in 1688,{{cite book |last= Ritter |first= Priscilla R. |coauthors= Thelma Fleishman |title= Newton, Massachusetts 1679–1779: A Biographical Directory |year= 1982 |publisher= New England Historic Genealogical Society }} [[Lexington, Massachusetts|Lexington (Cambridge Farms)]] in 1712, and both [[Arlington, Massachusetts|West Cambridge (originally Menotomy)]] and [[Brighton, Massachusetts|Brighton (Little Cambridge)]] in 1807.{{cite web |url=http://www.brightonbot.com/history.php |title=A Short History of Allston-Brighton |first=Marchione |last=William P. |author= |authorlink= |coauthors= |date= |month= |year=2011 |work=Brighton-Allston Historical Society |publisher=Brighton Board of Trade |location= |page= |pages= |at= |language= |trans_title= |arxiv= |asin= |bibcode= |doi= |doibroken= |isbn= |issn= |jfm= |jstor= |lccn= |mr= |oclc= |ol= |osti= |pmc = |pmid= |rfc= |ssrn= |zbl= |id= |archiveurl= |archivedate= |deadurl= |accessdate=December 21, 2011 |quote= |ref= |separator= |postscript=}} Part of West Cambridge joined the new town of [[Belmont, Massachusetts|Belmont]] in 1859, and the rest of West Cambridge was renamed Arlington in 1867; Brighton was annexed by Boston in 1874. In the late 19th century, various schemes for annexing Cambridge itself to the City of Boston were pursued and rejected.{{cite news |title=ANNEXATION AND ITS FRUITS |author=Staff writer |first= |last= |authorlink= |url=http://query.nytimes.com/gst/abstract.html?res=9901E4DC173BEF34BC4D52DFB766838F669FDE |agency= |newspaper=[[The New York Times]] |publisher= |isbn= |issn= |pmid= |pmd= |bibcode= |doi= |date=January 15, 1874, Wednesday |page= 4 |pages= |accessdate=|archiveurl=http://query.nytimes.com/mem/archive-free/pdf?res=9901E4DC173BEF34BC4D52DFB766838F669FDE |archivedate=January 15, 1874 |ref= }}{{cite news |title=BOSTON'S ANNEXATION SCHEMES.; PROPOSAL TO ABSORB CAMBRIDGE AND OTHER NEAR-BY TOWNS |author=Staff writer |first= |last= |authorlink= |url=http://query.nytimes.com/gst/abstract.html?res=9C05E1DC1F39E233A25754C2A9659C94639ED7CF |agency= |newspaper=[[The New York Times]] |publisher= |isbn= |issn= |pmid= |pmd= |bibcode= |doi= |date=March 26, 1892, Wednesday |page= 11 |pages= |accessdate=August 21, 2010|archiveurl=http://query.nytimes.com/mem/archive-free/pdf?res=9C05E1DC1F39E233A25754C2A9659C94639ED7CF |archivedate=March 27, 1892 |ref= }} In 1636, [[Harvard College]] was founded by the colony to train [[minister (religion)|ministers]] and the new town was chosen for its site by [[Thomas Dudley]]. By 1638, the name "Newe Towne" had "compacted by usage into 'Newtowne'." In May 1638{{cite book|title=The Cambridge of Eighteen Hundred and Ninety-six|editor=Arthur Gilman, ed.|publisher=Committee on the Memorial Volume|location=Cambridge|year=1896|page=8}}{{cite web|author=Harvard News Office |url=http://news.harvard.edu/gazette/2002/05.02/02-history.html |title=''Harvard Gazette'' historical calendar giving May 12, 1638 as date of name change; certain other sources say May 2, 1638 or late 1637 |publisher=News.harvard.edu |date=2002-05-02 |accessdate=2012-04-28}} the name was changed to '''Cambridge''' in honor of the [[University of Cambridge|university]] in [[Cambridge, England]].{{cite book |last= Hannah Winthrop Chapter, D.A.R. |title= Historic Guide to Cambridge |edition= Second |year= 1907 |publisher= Hannah Winthrop Chapter, D.A.R. |location= Cambridge, Mass. |pages= 20–21 |quote= On October 15, 1637, the Great and General Court passed a vote that: "The college is ordered to bee at Newetowne." In this same year the name of Newetowne was changed to Cambridge, ("It is ordered that Newetowne shall henceforward be called Cambridge") in honor of the university in Cambridge, England, where many of the early settlers were educated. }} The first president ([[Henry Dunster]]), the first benefactor ([[John Harvard (clergyman)|John Harvard]]), and the first schoolmaster ([[Nathaniel Eaton]]) of Harvard were all Cambridge University alumni, as was the then ruling (and first) governor of the [[Massachusetts Bay Colony]], John Winthrop. In 1629, Winthrop had led the signing of the founding document of the city of Boston, which was known as the [[Cambridge Agreement]], after the university.{{cite web|url=http://www.winthropsociety.org/doc_cambr.php|publisher=The Winthrop Society|title=Descendants of the Great Migration|accessdate=September 8, 2008}} It was Governor Thomas Dudley who, in 1650, signed the charter creating the corporation which still governs Harvard College.{{cite web|url=http://hul.harvard.edu/huarc/charter.html |title=Harvard Charter of 1650, Harvard University Archives, Harvard University, harvard.edu |publisher=Hul.harvard.edu |date= |accessdate=2012-04-28}}{{cite book |last1= |first1= |authorlink1= |editor1-first= |editor1-last= |editor1-link= |others= |title=Constitution of the Commonwealth of Massachusetts|url=http://www.mass.gov/legis/const.htm |accessdate=December 13, 2009 |edition= |series= |volume= |date=September 1, 1779 |publisher=The General Court of Massachusetts |location= |isbn= |oclc= |doi= |page= |pages=|chapter=Chapter V: The University at Cambridge, and encouragement of literature, etc. |chapterurl= |ref= |bibcode= }} [[Image:Washington taking command of the American Army at Cambridge, 1775 - NARA - 532874.tif|thumb|right|George Washington in Cambridge, 1775]] Cambridge grew slowly as an agricultural village eight miles (13 km) by road from Boston, the capital of the colony. By the [[American Revolution]], most residents lived near the [[Cambridge Common|Common]] and Harvard College, with farms and estates comprising most of the town. Most of the inhabitants were descendants of the original Puritan colonists, but there was also a small elite of [[Anglicans|Anglican]] "worthies" who were not involved in village life, who made their livings from estates, investments, and trade, and lived in mansions along "the Road to Watertown" (today's [[Brattle Street (Cambridge, Massachusetts)|Brattle Street]], still known as [[Tory Row]]). In 1775, [[George Washington]] came up from [[Virginia]] to take command of fledgling volunteer American soldiers camped on the [[Cambridge Common]]—today called the birthplace of the [[U.S. Army]]. (The name of today's nearby Sheraton Commander Hotel refers to that event.) Most of the Tory estates were confiscated after the Revolution. On January 24, 1776, [[Henry Knox]] arrived with artillery captured from [[Fort Ticonderoga]], which enabled Washington to drive the British army out of Boston. [[File:Cambridge 1873 WardMap.jpg|thumb|300px|left|A map of Cambridge from 1873]] Between 1790 and 1840, Cambridge began to grow rapidly, with the construction of the [[West Boston Bridge]] in 1792, that connected Cambridge directly to Boston, making it no longer necessary to travel eight miles (13 km) through the [[Boston Neck]], [[Roxbury, Massachusetts|Roxbury]], and [[Brookline, Massachusetts|Brookline]] to cross the [[Charles River]]. A second bridge, the Canal Bridge, opened in 1809 alongside the new [[Middlesex Canal]]. The new bridges and roads made what were formerly estates and [[marsh]]land into prime industrial and residential districts. In the mid-19th century, Cambridge was the center of a literary revolution when it gave the country a new identity through poetry and literature. Cambridge was home to the famous Fireside Poets—so called because their poems would often be read aloud by families in front of their evening fires. In their day, the [[Fireside Poets]]—[[Henry Wadsworth Longfellow]], [[James Russell Lowell]], and [[Oliver Wendell Holmes, Sr.|Oliver Wendell Holmes]]—were as popular and influential as rock stars are today.{{Citation needed|date=November 2009}} Soon after, [[Toll road|turnpikes]] were built: the [[Cambridge and Concord Turnpike]] (today's Broadway and Concord Ave.), the [[Middlesex Turnpike (Massachusetts)|Middlesex Turnpike]] (Hampshire St. and [[Massachusetts Avenue (Boston)|Massachusetts Ave.]] northwest of [[Porter Square]]), and what are today's Cambridge, Main, and Harvard Streets were roads to connect various areas of Cambridge to the bridges. In addition, railroads crisscrossed the town during the same era, leading to the development of Porter Square as well as the creation of neighboring town [[Somerville, Massachusetts|Somerville]] from the formerly rural parts of [[Charlestown, Massachusetts|Charlestown]]. [[File:Middlesex Canal (Massachusetts) map, 1852.jpg|thumb|1852 Map of Boston area showing Cambridge and rail lines.]] Cambridge was incorporated as a city in 1846. This was despite noticeable tensions between East Cambridge, Cambridgeport, and Old Cambridge that stemmed from differences in in each area's culture, sources of income, and the national origins of the residents.Cambridge Considered: A Very Brief History of Cambridge, 1800-1900, Part I. http://cambridgeconsidered.blogspot.com/2011/01/very-brief-history-of-cambridge-1800.html The city's commercial center began to shift from Harvard Square to Central Square, which became the downtown of the city around this time. Between 1850 and 1900, Cambridge took on much of its present character—[[streetcar suburb]]an development along the turnpikes, with working-class and industrial neighborhoods focused on East Cambridge, comfortable middle-class housing being built on old estates in Cambridgeport and Mid-Cambridge, and upper-class enclaves near Harvard University and on the minor hills of the city. The coming of the railroad to North Cambridge and Northwest Cambridge then led to three major changes in the city: the development of massive brickyards and brickworks between Massachusetts Ave., Concord Ave. and [[Alewife Brook]]; the ice-cutting industry launched by [[Frederic Tudor]] on [[Fresh Pond, Cambridge, Massachusetts|Fresh Pond]]; and the carving up of the last estates into residential subdivisions to provide housing to the thousands of immigrants that arrived to work in the new industries. For many years, the city's largest employer was the [[New England Glass Company]], founded in 1818. By the middle of the 19th century it was the largest and most modern glassworks in the world. In 1888, all production was moved, by [[Edward Libbey|Edward Drummond Libbey]], to [[Toledo, Ohio]], where it continues today under the name Owens Illinois. Flint glassware with heavy lead content, produced by that company, is prized by antique glass collectors. There is none on public display in Cambridge, but there is a large collection in the [[Toledo Museum of Art]]. Among the largest businesses located in Cambridge was the firm of [[Carter's Ink Company]], whose neon sign long adorned the [[Charles River]] and which was for many years the largest manufacturer of ink in the world. By 1920, Cambridge was one of the main industrial cities of [[New England]], with nearly 120,000 residents. As industry in New England began to decline during the [[Great Depression]] and after World War II, Cambridge lost much of its industrial base. It also began the transition to being an intellectual, rather than an industrial, center. Harvard University had always been important in the city (both as a landowner and as an institution), but it began to play a more dominant role in the city's life and culture. Also, the move of the [[Massachusetts Institute of Technology]] from Boston in 1916 ensured Cambridge's status as an intellectual center of the United States. After the 1950s, the city's population began to decline slowly, as families tended to be replaced by single people and young couples. The 1980s brought a wave of high-technology startups, creating software such as [[Visicalc]] and [[Lotus 1-2-3]], and advanced computers, but many of these companies fell into decline with the fall of the minicomputer and [[DOS]]-based systems. However, the city continues to be home to many startups as well as a thriving biotech industry. By the end of the 20th century, Cambridge had one of the most expensive housing markets in the Northeastern United States. While maintaining much diversity in class, race, and age, it became harder and harder for those who grew up in the city to be able to afford to stay. The end of [[rent control]] in 1994 prompted many Cambridge renters to move to housing that was more affordable, in Somerville and other communities. In 2005, a reassessment of residential property values resulted in a disproportionate number of houses owned by non-affluent people jumping in value relative to other houses, with hundreds having their property tax increased by over 100%; this forced many homeowners in Cambridge to move elsewhere.Cambridge Chronicle, October 6, 13, 20, 27, 2005 As of 2012, Cambridge's mix of amenities and proximity to Boston has kept housing prices relatively stable. ==Geography== [[File:Charles River Cambridge USA.jpg|thumb|upright|A view from Boston of Harvard's [[Weld Boathouse]] and Cambridge in winter. The [[Charles River]] is in the foreground.]] According to the [[United States Census Bureau]], Cambridge has a total area of {{convert|7.1|sqmi|km2}}, of which {{convert|6.4|sqmi|km2}} of it is land and {{convert|0.7|sqmi|km2}} of it (9.82%) is water. ===Adjacent municipalities=== Cambridge is located in eastern Massachusetts, bordered by: *the city of [[Boston]] to the south (across the [[Charles River]]) and east *the city of [[Somerville, Massachusetts|Somerville]] to the north *the town of [[Arlington, Massachusetts|Arlington]] to the northwest *the town of [[Belmont, Massachusetts|Belmont]] and *the city of [[Watertown, Massachusetts|Watertown]] to the west The border between Cambridge and the neighboring city of [[Somerville, Massachusetts|Somerville]] passes through densely populated neighborhoods which are connected by the [[Red Line (MBTA)|MBTA Red Line]]. Some of the main squares, [[Inman Square|Inman]], [[Porter Square|Porter]], and to a lesser extent, [[Harvard Square|Harvard]], are very close to the city line, as are Somerville's [[Union Square (Somerville)|Union]] and [[Davis Square]]s. ===Neighborhoods=== ====Squares==== [[File:Centralsquarecambridgemass.jpg|thumb|[[Central Square (Cambridge)|Central Square]]]] [[File:Harvard square 2009j.JPG|thumb|[[Harvard Square]]]] [[File:Cambridge MA Inman Square.jpg|thumb|[[Inman Square]]]] Cambridge has been called the "City of Squares" by some,{{cite web|author=No Writer Attributed |url=http://www.thecrimson.com/article/1969/9/18/cambridge-a-city-of-squares-pcambridge/ |title="Cambridge: A City of Squares" Harvard Crimson, Sept. 18, 1969 |publisher=Thecrimson.com |date=1969-09-18 |accessdate=2012-04-28}}{{cite web|url=http://www.travelwritersmagazine.com/RonBernthal/Cambridge.html |title=Cambridge Journal: Massachusetts City No Longer in Boston's Shadow |publisher=Travelwritersmagazine.com |date= |accessdate=2012-04-28}} as most of its commercial districts are major street intersections known as [[Town square|squares]]. Each of the squares acts as a neighborhood center. These include: * [[Kendall Square]], formed by the junction of Broadway, Main Street, and Third Street, is also known as '''Technology Square''', a name shared with an office and laboratory building cluster in the neighborhood. Just over the [[Longfellow Bridge]] from Boston, at the eastern end of the [[Massachusetts Institute of Technology|MIT]] campus, it is served by the [[Kendall (MBTA station)|Kendall/MIT]] station on the [[Massachusetts Bay Transportation Authority|MBTA]] [[Red Line (MBTA)|Red Line]] subway. Most of Cambridge's large office towers are located here, giving the area somewhat of an office park feel. A flourishing [[biotech]] industry has grown up around this area. The "One Kendall Square" complex is nearby, but—confusingly—not actually in Kendall Square. Also, the "Cambridge Center" office complex is located here, and not at the actual center of Cambridge. * [[Central Square (Cambridge)|Central Square]], formed by the junction of Massachusetts Avenue, Prospect Street, and Western Avenue, is well known for its wide variety of ethnic restaurants. As recently as the late 1990s it was rather run-down; it underwent a controversial [[gentrification]] in recent years (in conjunction with the development of the nearby [[University Park at MIT]]), and continues to grow more expensive. It is served by the [[Central (MBTA station)|Central Station]] stop on the MBTA Red Line subway. '''Lafayette Square''', formed by the junction of Massachusetts Avenue, Columbia Street, Sidney Street, and Main Street, is considered part of the Central Square area. [[Cambridgeport]] is south of Central Square along Magazine Street and Brookline Street. * [[Harvard Square]], formed by the junction of Massachusetts Avenue, Brattle Street, and JFK Street. This is the primary site of [[Harvard University]], and is a major Cambridge shopping area. It is served by a [[Harvard (MBTA station)|Red Line station]]. Harvard Square was originally the northwestern terminus of the Red Line and a major transfer point to streetcars that also operated in a short [[Harvard Bus Tunnel|tunnel]]—which is still a major bus terminal, although the area under the Square was reconfigured dramatically in the 1980s when the Red Line was extended. The Harvard Square area includes '''Brattle Square''' and '''Eliot Square'''. A short distance away from the square lies the [[Cambridge Common]], while the neighborhood north of Harvard and east of Massachusetts Avenue is known as Agassiz in honor of the famed scientist [[Louis Agassiz]]. * [[Porter Square]], about a mile north on Massachusetts Avenue from Harvard Square, is formed by the junction of Massachusetts and Somerville Avenues, and includes part of the city of [[Somerville, Massachusetts|Somerville]]. It is served by the [[Porter (MBTA station)|Porter Square Station]], a complex housing a [[Red Line (MBTA)|Red Line]] stop and a [[Fitchburg Line]] [[MBTA commuter rail|commuter rail]] stop. [[Lesley University]]'s University Hall and Porter campus are located at Porter Square. * [[Inman Square]], at the junction of Cambridge and Hampshire streets in Mid-Cambridge. Inman Square is home to many diverse restaurants, bars, music venues and boutiques. The funky street scene still holds some urban flair, but was dressed up recently with Victorian streetlights, benches and bus stops. A new community park was installed and is a favorite place to enjoy some takeout food from the nearby restaurants and ice cream parlor. * [[Lechmere Square]], at the junction of Cambridge and First streets, adjacent to the CambridgeSide Galleria shopping mall. Perhaps best known as the northern terminus of the [[Massachusetts Bay Transportation Authority|MBTA]] [[Green Line (MBTA)|Green Line]] subway, at [[Lechmere (MBTA station)|Lechmere Station]]. ====Other neighborhoods==== The residential neighborhoods ([http://www.cambridgema.gov/CPD/publications/neighborhoods.cfm map]) in Cambridge border, but are not defined by the squares. These include: * [[East Cambridge, Massachusetts|East Cambridge]] (Area 1) is bordered on the north by the [[Somerville, Massachusetts|Somerville]] border, on the east by the Charles River, on the south by Broadway and Main Street, and on the west by the [[Grand Junction Railroad]] tracks. It includes the [[NorthPoint (Cambridge, Massachusetts)|NorthPoint]] development. * [[Massachusetts Institute of Technology|MIT]] Campus ([[MIT Campus (Area 2), Cambridge|Area 2]]) is bordered on the north by Broadway, on the south and east by the Charles River, and on the west by the Grand Junction Railroad tracks. * [[Wellington-Harrington]] (Area 3) is bordered on the north by the [[Somerville, Massachusetts|Somerville]] border, on the south and west by Hampshire Street, and on the east by the Grand Junction Railroad tracks. Referred to as "Mid-Block".{{clarify|What is? By whom? A full sentence would help.|date=September 2011}} * [[Area 4, Cambridge|Area 4]] is bordered on the north by Hampshire Street, on the south by Massachusetts Avenue, on the west by Prospect Street, and on the east by the Grand Junction Railroad tracks. Residents of Area 4 often refer to their neighborhood simply as "The Port", and refer to the area of Cambridgeport and Riverside as "The Coast". * [[Cambridgeport]] (Area 5) is bordered on the north by Massachusetts Avenue, on the south by the Charles River, on the west by River Street, and on the east by the Grand Junction Railroad tracks. * [[Mid-Cambridge]] (Area 6) is bordered on the north by Kirkland and Hampshire Streets and the [[Somerville, Massachusetts|Somerville]] border, on the south by Massachusetts Avenue, on the west by Peabody Street, and on the east by Prospect Street. * [[Riverside, Cambridge|Riverside]] (Area 7), an area sometimes referred to as "The Coast," is bordered on the north by Massachusetts Avenue, on the south by the Charles River, on the west by JFK Street, and on the east by River Street. * [[Agassiz, Cambridge, Massachusetts|Agassiz (Harvard North)]] (Area 8) is bordered on the north by the [[Somerville, Massachusetts|Somerville]] border, on the south and east by Kirkland Street, and on the west by Massachusetts Avenue. * [[Peabody, Cambridge, Massachusetts|Peabody]] (Area 9) is bordered on the north by railroad tracks, on the south by Concord Avenue, on the west by railroad tracks, and on the east by Massachusetts Avenue. The Avon Hill sub-neighborhood consists of the higher elevations bounded by Upland Road, Raymond Street, Linnaean Street and Massachusetts Avenue. * Brattle area/[[West Cambridge (neighborhood)|West Cambridge]] (Area 10) is bordered on the north by Concord Avenue and Garden Street, on the south by the Charles River and the [[Watertown, Massachusetts|Watertown]] border, on the west by Fresh Pond and the Collins Branch Library, and on the east by JFK Street. It includes the sub-neighborhoods of Brattle Street (formerly known as [[Tory Row]]) and Huron Village. * [[North Cambridge, Massachusetts|North Cambridge]] (Area 11) is bordered on the north by the [[Arlington, Massachusetts|Arlington]] and [[Somerville, Massachusetts|Somerville]] borders, on the south by railroad tracks, on the west by the [[Belmont, Massachusetts|Belmont]] border, and on the east by the [[Somerville, Massachusetts|Somerville]] border. * [[Cambridge Highlands]] (Area 12) is bordered on the north and east by railroad tracks, on the south by Fresh Pond, and on the west by the [[Belmont, Massachusetts|Belmont]] border. * [[Strawberry Hill, Cambridge|Strawberry Hill]] (Area 13) is bordered on the north by Fresh Pond, on the south by the [[Watertown, Massachusetts|Watertown]] border, on the west by the [[Belmont, Massachusetts|Belmont]] border, and on the east by railroad tracks. ===Parks and outdoors=== [[File:Alewife Brook Reservation.jpg|thumb|Alewife Brook Reservation]] Consisting largely of densely built residential space, Cambridge lacks significant tracts of public parkland. This is partly compensated for, however, by the presence of easily accessible open space on the university campuses, including [[Harvard Yard]] and MIT's Great Lawn, as well as the considerable open space of [[Mount Auburn Cemetery]]. At the western edge of Cambridge, the cemetery is well known as the first garden cemetery, for its distinguished inhabitants, for its superb landscaping (the oldest planned landscape in the country), and as a first-rate [[arboretum]]. Although known as a Cambridge landmark, much of the cemetery lies within the bounds of Watertown.http://www2.cambridgema.gov/CityOfCambridge_Content/documents/CambridgeStreetMap18x24_032007.pdf It is also a significant [[Important Bird Area]] (IBA) in the Greater Boston area. Public parkland includes the esplanade along the Charles River, which mirrors its [[Charles River Esplanade|Boston counterpart]], [[Cambridge Common]], a busy and historic public park immediately adjacent to the Harvard campus, and the [[Alewife Brook Reservation]] and [[Fresh Pond, Cambridge, Massachusetts|Fresh Pond]] in the western part of the city. ==Demographics== {{Historical populations | type=USA | align=right | 1790|2115 | 1800|2453 | 1810|2323 | 1820|3295 | 1830|6072 | 1840|8409 | 1850|15215 | 1860|26060 | 1870|39634 | 1880|52669 | 1890|70028 | 1900|91886 | 1910|104839 | 1920|109694 | 1930|113643 | 1940|110879 | 1950|120740 | 1960|107716 | 1970|100361 | 1980|95322 | 1990|95802 | 2000|101355 | 2010|105162 | footnote= {{Historical populations/Massachusetts municipalities references}}{{cite journal | title=1950 Census of Population | volume=1: Number of Inhabitants | at=Section 6, Pages 21-7 through 21-09, Massachusetts Table 4. Population of Urban Places of 10,000 or more from Earliest Census to 1920 | publisher=Bureau of the Census | accessdate=July 12, 2011 | year=1952 | url=http://www2.census.gov/prod2/decennial/documents/23761117v1ch06.pdf}} }} As of the census{{GR|2}} of 2010, there were 105,162 people, 44,032 households, and 17,420 families residing in the city. The population density was 16,422.08 people per square mile (6,341.98/km²), making Cambridge the fifth most densely populated city in the USCounty and City Data Book: 2000. Washington, DC: US Department of Commerce, Bureau of the Census. Table C-1. and the second most densely populated city in [[Massachusetts]] behind neighboring [[Somerville, Massachusetts|Somerville]].[http://www.boston.com/realestate/news/articles/2008/07/13/highest_population_density/ Highest Population Density, The Boston Globe] There were 47,291 housing units at an average density of 7,354.7 per square mile (2,840.3/km²). The racial makeup of the city was 66.60% [[White (U.S. Census)|White]], 11.70% [[Black (people)|Black]] or [[Race (United States Census)|African American]], 0.20% [[Native American (U.S. Census)|Native American]], 15.10% [[Asian (U.S. Census)|Asian]], 0.01% [[Pacific Islander (U.S. Census)|Pacific Islander]], 2.10% from [[Race (United States Census)|other races]], and 4.30% from two or more races. 7.60% of the population were [[Hispanics in the United States|Hispanic]] or [[Latino (U.S. Census)|Latino]] of any race. [[Non-Hispanic Whites]] were 62.1% of the population in 2010,{{cite web |url=http://quickfacts.census.gov/qfd/states/25/2511000.html |title=Cambridge (city), Massachusetts |work=State & County QuickFacts |publisher=U.S. Census Bureau}} down from 89.7% in 1970.{{cite web|title=Massachusetts - Race and Hispanic Origin for Selected Cities and Other Places: Earliest Census to 1990|publisher=U.S. Census Bureau|url=http://www.census.gov/population/www/documentation/twps0076/twps0076.html}} This rather closely parallels the average [[racial demographics of the United States]] as a whole, although Cambridge has significantly more Asians than the average, and fewer Hispanics and Caucasians. 11.0% were of [[irish people|Irish]], 7.2% English, 6.9% [[italians|Italian]], 5.5% [[West Indian]] and 5.3% [[germans|German]] ancestry according to [[Census 2000]]. 69.4% spoke English, 6.9% Spanish, 3.2% [[Standard Mandarin|Chinese]] or [[Standard Mandarin|Mandarin]], 3.0% [[portuguese language|Portuguese]], 2.9% [[French-based creole languages|French Creole]], 2.3% French, 1.5% [[korean language|Korean]], and 1.0% [[italian language|Italian]] as their first language. There were 44,032 households out of which 16.9% had children under the age of 18 living with them, 28.9% were married couples living together, 8.4% had a female householder with no husband present, and 60.4% were non-families. 40.7% of all households were made up of individuals and 9.6% had someone living alone who was 65 years of age or older. The average household size was 2.00 and the average family size was 2.76. In the city the population was spread out with 13.3% under the age of 18, 21.2% from 18 to 24, 38.6% from 25 to 44, 17.8% from 45 to 64, and 9.2% who were 65 years of age or older. The median age was 30.5 years. For every 100 females, there were 96.1 males. For every 100 females age 18 and over, there were 94.7 males. The median income for a household in the city was $47,979, and the median income for a family was $59,423 (these figures had risen to $58,457 and $79,533 respectively {{as of|2007|alt=as of a 2007 estimate}}{{cite web|url=http://factfinder.census.gov/servlet/ACSSAFFFacts?_event=Search&geo_id=16000US2418750&_geoContext=01000US%7C04000US24%7C16000US2418750&_street=&_county=cambridge&_cityTown=cambridge&_state=04000US25&_zip=&_lang=en&_sse=on&ActiveGeoDiv=geoSelect&_useEV=&pctxt=fph&pgsl=160&_submenuId=factsheet_1&ds_name=ACS_2007_3YR_SAFF&_ci_nbr=null&qr_name=null®=null%3Anull&_keyword=&_industry= |title=U.S. Census, 2000 |publisher=Factfinder.census.gov |date= |accessdate=2012-04-28}}). Males had a median income of $43,825 versus $38,489 for females. The per capita income for the city was $31,156. About 8.7% of families and 12.9% of the population were below the poverty line, including 15.1% of those under age 18 and 12.9% of those age 65 or over. Cambridge was ranked as one of the most liberal cities in America.{{cite web|author=Aug 16, 2005 12:00 AM |url=http://www.govpro.com/News/Article/31439/ |title=Study Ranks America’s Most Liberal and Conservative Cities |publisher=Govpro.com |date=2005-08-16 |accessdate=2012-04-28}} Locals living in and near the city jokingly refer to it as "The People's Republic of Cambridge."[http://www.universalhub.com/glossary/peoples_republic_the.html Wicked Good Guide to Boston English] Accessed February 2, 2009 For 2012, the residential property tax rate in Cambridge is $8.48 per $1,000.{{cite web|url=http://www.cambridgema.gov/finance/propertytaxinformation/fy12propertytaxinformation.aspx |title=FY12 Property Tax Information - City of Cambridge, Massachusetts |publisher=Cambridgema.gov |date= |accessdate=2012-04-28}} Cambridge enjoys the highest possible [[bond credit rating]], AAA, with all three Wall Street rating agencies.http://www.cambridgema.gov/CityOfCambridge_Content/documents/Understanding_Your_Taxes_2007.pdf Cambridge is noted for its diverse population, both racially and economically. Residents, known as ''Cantabrigians'', include affluent [[MIT]] and Harvard professors. The first legal applications in America for same-sex marriage licenses were issued at Cambridge's City Hall.{{cite web|url=http://www.boston.com/news/local/articles/2004/05/17/free_to_marry/ |title=Free to Marry |work=[[The Boston Globe]] |date=2004-05-17 |accessdate=2012-07-18}} Cambridge is also the birthplace of [[Thailand|Thai]] king [[Bhumibol Adulyadej|Bhumibol Adulyadej (Rama IX)]], who is the world's longest reigning monarch at age 82 (2010), as well as the longest reigning monarch in Thai history. He is also the first king of a foreign country to be born in the United States. ==Government== ===Federal and state representation=== {| class=wikitable ! colspan = 6 | Voter registration and party enrollment {{as of|lc=y|df=US|2008|10|15}}{{cite web|title = 2008 State Party Election Party Enrollment Statistics | publisher = Massachusetts Elections Division | format = PDF | accessdate = July 7, 2010 | url = http://www.sec.state.ma.us/ele/elepdf/st_county_town_enroll_breakdown_08.pdf}} |- ! colspan = 2 | Party ! Number of voters ! Percentage {{American politics/party colors/Democratic/row}} | [[Democratic Party (United States)|Democratic]] | style="text-align:center;"| 37,822 | style="text-align:center;"| 58.43% {{American politics/party colors/Republican/row}} | [[Republican Party (United States)|Republican]] | style="text-align:center;"| 3,280 | style="text-align:center;"| 5.07% {{American politics/party colors/Independent/row}} | Unaffiliated | style="text-align:center;"| 22,935 | style="text-align:center;"| 35.43% {{American politics/party colors/Libertarian/row}} | Minor Parties | style="text-align:center;"| 690 | style="text-align:center;"| 1.07% |- ! colspan = 2 | Total ! style="text-align:center;"| 64,727 ! style="text-align:center;"| 100% |} Cambridge is part of [[Massachusetts's 8th congressional district]], represented by Democrat [[Mike Capuano]], elected in 1998. The state's senior member of the [[United States Senate]] is Democrat [[John Kerry]], elected in 1984. The state's junior member is Republican [[Scott Brown]], [[United States Senate special election in Massachusetts, 2010|elected in 2010]] to fill the vacancy caused by the death of long-time Democratic Senator [[Ted Kennedy]]. The Governor of Massachusetts is Democrat [[Deval Patrick]], elected in 2006 and re-elected in 2010. On the state level, Cambridge is represented in six districts in the [[Massachusetts House of Representatives]]: the 24th Middlesex (which includes parts of Belmont and Arlington), the 25th and 26th Middlesex (the latter which includes a portion of Somerville), the 29th Middlesex (which includes a small part of Watertown), and the Eighth and Ninth Suffolk (both including parts of the City of Boston). The city is represented in the [[Massachusetts Senate]] as a part of the "First Suffolk and Middlesex" district (this contains parts of Boston, Revere and Winthrop each in Suffolk County); the "Middlesex, Suffolk and Essex" district, which includes Everett and Somerville, with Boston, Chelsea, and Revere of Suffolk, and Saugus in Essex; and the "Second Suffolk and Middlesex" district, containing parts of the City of Boston in Suffolk county, and Cambridge, Belmont and Watertown in Middlesex county.{{cite web|url=http://www.malegislature.gov/ |title=Index of Legislative Representation by City and Town, from |publisher=Mass.gov |date= |accessdate=2012-04-28}} In addition to the [[Cambridge Police Department (Massachusetts)|Cambridge Police Department]], the city is patrolled by the Fifth (Brighton) Barracks of Troop H of the [[Massachusetts State Police]].[http://www.mass.gov/?pageID=eopsterminal&L=5&L0=Home&L1=Law+Enforcement+%26+Criminal+Justice&L2=Law+Enforcement&L3=State+Police+Troops&L4=Troop+H&sid=Eeops&b=terminalcontent&f=msp_divisions_field_services_troops_troop_h_msp_field_troop_h_station_h5&csid=Eeops Station H-5, SP Brighton]{{dead link|date=April 2012}} Due, however, to close proximity, the city also practices functional cooperation with the Fourth (Boston) Barracks of Troop H, as well.[http://www.mass.gov/?pageID=eopsterminal&L=5&L0=Home&L1=Law+Enforcement+%26+Criminal+Justice&L2=Law+Enforcement&L3=State+Police+Troops&L4=Troop+H&sid=Eeops&b=terminalcontent&f=msp_divisions_field_services_troops_troop_h_msp_field_troop_h_station_h4&csid=Eeops Station H-4, SP Boston]{{dead link|date=April 2012}} ===City government=== [[File:CambridgeMACityHall1.jpg|thumb|right|[[Cambridge, Massachusetts City Hall|Cambridge City Hall]] in the 1980s]] Cambridge has a city government led by a [[List of mayors of Cambridge, Massachusetts|Mayor]] and nine-member City Council. There is also a six-member School Committee which functions alongside the Superintendent of public schools. The councilors and school committee members are elected every two years using the [[single transferable vote]] (STV) system.{{cite web|url=http://www.cambridgema.gov/election/Proportional_Representation.cfm |title=Proportional Representation Voting in Cambridge |publisher=Cambridgema.gov |date= |accessdate=2012-04-28}} Once a laborious process that took several days to complete by hand, ballot sorting and calculations to determine the outcome of elections are now quickly performed by computer, after the ballots have been [[Optical scan voting system|optically scanned]]. The mayor is elected by the city councilors from amongst themselves, and serves as the chair of City Council meetings. The mayor also sits on the School Committee. However, the Mayor is not the Chief Executive of the City. Rather, the City Manager, who is appointed by the City Council, serves in that capacity. Under the City's Plan E form of government the city council does not have the power to appoint or remove city officials who are under direction of the city manager. The city council and its individual members are also forbidden from giving orders to any subordinate of the city manager.http://www.cambridgema.gov/CityOfCambridge_Content/documents/planE.pdf [[Robert W. Healy]] is the City Manager; he has served in the position since 1981. In recent history, the media has highlighted the salary of the City Manager as being one of the highest in the State of Massachusetts.{{cite news |title=Cambridge city manager's salary almost as much as Obama's pay |url=http://www.wickedlocal.com/cambridge/features/x1837730973/Cambridge-city-managers-salary-almost-as-much-as-Obamas |agency= |newspaper=Wicked Local: Cambridge |publisher= |date=August 11, 2011 |accessdate=December 30, 2011 |quote= |archiveurl= |archivedate= |deadurl= |ref=}} The city council consists of:{{cite web|url=http://www.cambridgema.gov/ccouncil/citycouncilmembers.aspx |title=City of Cambridge – City Council Members |publisher=Cambridgema.gov |date= |accessdate=2012-04-28}}{{Refbegin|3}} *[[Leland Cheung]] (Jan. 2010–present) *Henrietta Davis (Jan. 1996–present)* *Marjorie C. Decker (Jan. 2000–present){{cite web |url= http://www.wickedlocal.com/cambridge/news/x738245499/Marjorie-Decker-announces-she-will-run-for-Alice-Wolfs-Cambridge-State-Representative-seat |title= Marjorie Decker announces she will run for Alice Wolf's Cambridge State Representative seat |date= 22 March 2012 |work= Wicked Local Cambridge |publisher= GateHouse Media, Inc. |accessdate= 4 April 2012 }} *Craig A. Kelley (Jan. 2006–present) *David Maher (Jan. 2000-Jan. 2006, Sept. 2007–present{{cite web|author=By ewelin, on September 5th, 2007 |url=http://www.cambridgehighlands.com/2007/09/david-p-maher-elected-to-fill-michael-sullivans-vacated-city-council-seat |title=David P. Maher Elected to fill Michael Sullivan’s Vacated City Council Seat • Cambridge Highlands Neighborhood Association |publisher=Cambridgehighlands.com |date=2007-09-05 |accessdate=2012-04-28}})** *[[Kenneth Reeves]] (Jan. 1990–present)** *[[E. Denise Simmons]] (Jan. 2002–present)** *[[Timothy J. Toomey, Jr.]] (Jan. 1990–present) *Minka vanBeuzekom (Jan. 2012–present){{Refend}} ''* = Current Mayor''
''** = former Mayor'' ===Fire Department=== The city of Cambridge is protected full-time by the 274 professional firefighters of the Cambridge Fire Department. The current Chief of Department is Gerald R. Reardon. The Cambridge Fire Department operates out of eight fire stations, located throughout the city, under the command of two divisions. The CFD also maintains and operates a front-line fire apparatus fleet of eight engines, four ladders, two Non-Transport Paramedic EMS units, a Haz-Mat unit, a Tactical Rescue unit, a Dive Rescue unit, two Marine units, and numerous special, support, and reserve units. John J. Gelinas, Chief of Operations, is in charge of day to day operation of the department.{{cite web|url=http://www2.cambridgema.gov/cfd/ |title=City of Cambridge Fire Department |publisher=.cambridgema.gov |date=2005-03-13 |accessdate=2012-06-26}} The CFD is rated as a Class 1 fire department by the [[Insurance Services Office]] (ISO), and is one of only 32 fire departments so rated, out of 37,000 departments in the United States. The other class 1 departments in New England are in [[Hartford, Connecticut]] and [[Milford, Connecticut]]. Class 1 signifies the highest level of fire protection according to various criteria.{{cite web|url=http://www2.cambridgema.gov/CFD/Class1FD.cfm |title=Class 1 Fire Department |publisher=.cambridgema.gov |date=1999-07-01 |accessdate=2012-06-26}} The CFD responds to approximately 15,000 emergency calls annually. {| class=wikitable |- valign=bottom ! Engine Company ! Ladder Company ! Special Unit ! Division ! Address ! Neighborhood |- | Engine 1 || Ladder 1 || || || 491 Broadway || Harvard Square |- | Engine 2 || Ladder 3 || Squad 2 || || 378 Massachusetts Ave. || Lafayette Square |- | Engine 3 || Ladder 2 || || || 175 Cambridge St. || East Cambridge |- | Engine 4 || || Squad 4 || || 2029 Massachusetts Ave. || Porter Square |- | Engine 5 || || || Division 1 || 1384 Cambridge St. || Inman Square |- | Engine 6 || || || || 176 River St. || Cambridgeport |- | Engine 8 || Ladder 4 || || Division 2 || 113 Garden St. || Taylor Square |- | Engine 9 || || || || 167 Lexington Ave || West Cambridge |- | Maintenance Facility || || || || 100 Smith Pl. || |} ===Water Department=== Cambridge is unusual among cities inside Route 128 in having a non-[[MWRA]] water supply. City water is obtained from [[Hobbs Brook]] (in [[Lincoln, Massachusetts|Lincoln]] and [[Waltham, Massachusetts|Waltham]]), [[Stony Brook (Boston)|Stony Brook]] (Waltham and [[Weston, Massachusetts|Weston]]), and [[Fresh Pond (Cambridge, Massachusetts)|Fresh Pond]] (Cambridge). The city owns over 1200 acres of land in other towns that includes these reservoirs and portions of their watershed.{{cite web|url=http://www2.cambridgema.gov/CWD/wat_lands.cfm |title=Cambridge Watershed Lands & Facilities |publisher=.cambridgema.gov |date= |accessdate=2012-04-28}} Water is treated at Fresh Pond, then pumped uphill to an elevation of {{convert|176|ft|m}} [[above sea level]] at the Payson Park Reservoir ([[Belmont, Massachusetts|Belmont]]); From there, the water is redistributed downhill via gravity to individual users in the city.{{cite web|url=http://www.cambridgema.gov/CityOfCambridge_Content/documents/CWD_March_2010.pdf |title=Water supply system |format=PDF |date= |accessdate=2012-04-28}}[http://www.cambridgema.gov/CWD/fpfaqs.cfm Is Fresh Pond really used for drinking water?], Cambridge Water Department ===County government=== Cambridge is a [[county seat]] of [[Middlesex County, Massachusetts]], along with [[Lowell, Massachusetts|Lowell]]. Though the county government was abolished in 1997, the county still exists as a geographical and political region. The employees of Middlesex County courts, jails, registries, and other county agencies now work directly for the state. At present, the county's registrars of [[Deed]]s and Probate remain in Cambridge; however, the Superior Court and District Attorney have had their base of operations transferred to [[Woburn, Massachusetts|Woburn]]. Third District court has shifted operations to [[Medford, Massachusetts|Medford]], and the Sheriff's office for the county is still awaiting a near-term relocation.{{cite news | url=http://www.boston.com/news/local/massachusetts/articles/2008/02/14/court_move_a_hassle_for_commuters/ |title=Court move a hassle for commuters |accessdate=July 25, 2009 |first=Eric |last=Moskowitz |authorlink= |coauthors= |date=February 14, 2008 |work=[[Boston Globe|The Boston Globe]] |pages= |archiveurl= |archivedate= |quote=In a little more than a month, Middlesex Superior Court will open in Woburn after nearly four decades at the Edward J. Sullivan Courthouse in Cambridge. With it, the court will bring the roughly 500 people who pass through its doors each day – the clerical staff, lawyers, judges, jurors, plaintiffs, defendants, and others who use or work in the system.}}{{cite news | url=http://www.wickedlocal.com/cambridge/homepage/x135741754/Cambridges-Middlesex-Jail-courts-may-be-shuttered-for-good |title=Cambridge's Middlesex Jail, courts may be shuttered for good |accessdate=July 25, 2009 |first=Charlie |last=Breitrose |authorlink= |coauthors= |date=July 7, 2009 |work=Wicked Local News: Cambridge |pages= |archiveurl= |archivedate= |quote=The courts moved out of the building to allow workers to remove asbestos. Superior Court moved to Woburn in March 2008, and in February, the Third District Court moved to Medford.}} ==Education== [[File:MIT Main Campus Aerial.jpg|thumb|Aerial view of part of [[MIT]]'s main campus]] [[File:Dunster House.jpg|thumb|[[Dunster House]], Harvard]] ===Higher education=== Cambridge is perhaps best known as an academic and intellectual center, owing to its colleges and universities, which include: *[[Cambridge College]] *[[Cambridge School of Culinary Arts]] *[[Episcopal Divinity School]] *[[Harvard University]] *[[Hult International Business School]] *[[Lesley University]] *[[Longy School of Music]] *[[Massachusetts Institute of Technology]] *[[Le Cordon Bleu College of Culinary Arts in Boston]] [[Nobel laureates by university affiliation|At least 129]] of the world's total 780 [[Nobel Prize]] winners have been, at some point in their careers, affiliated with universities in Cambridge. The [[American Academy of Arts and Sciences]] is also based in Cambridge. ===Primary and secondary public education=== The Cambridge Public School District encompasses 12 elementary schools that follow a variety of different educational systems and philosophies. All but one of the elementary schools extend up to the [[middle school]] grades as well. The 12 elementary schools are: *[[Amigos School]] *Baldwin School *Cambridgeport School *Fletcher-Maynard Academy *Graham and Parks Alternative School *Haggerty School *Kennedy-Longfellow School *King Open School *Martin Luther King, Jr. School *Morse School (a [[Core Knowledge Foundation|Core Knowledge]] school) *Peabody School *Tobin School (a [[Montessori school]]) There are three public high schools serving Cambridge students, including the [[Cambridge Rindge and Latin School]].{{cite web|url=http://www.cpsd.us/Web/PubInfo/SchoolsAtAGlance06-07.pdf|title=Cambridge Public Schools at a Glance|format=PDF}}{{dead link|date=June 2012}} and Community Charter School of Cambridge (www.ccscambridge.org) In 2003, the CRLS, also known as Rindge, came close to losing its educational accreditation when it was placed on probation by the [[New England Association of Schools and Colleges]].{{cite web|url=http://www.thecrimson.com/article.aspx?ref=512061|title=School Fights Achievement Gap|publisher=The Harvard Crimson|accessdate=May 14, 2009}} The school has improved under Principal Chris Saheed, graduation rates hover around 98%, and 70% of students gain college admission. Community Charter School of Cambridge serves 350 students, primarily from Boston and Cambridge, and is a tuition free public charter school with a college preparatory curriculum. All students from the class of 2009 and 2010 gained admission to college. Outside of the main public schools are public charter schools including: [[Benjamin Banneker Charter School]], which serves students in grades K-6,{{cite web|url=http://www.banneker.org/ |title=The Benjamin Banneker Charter Public School |publisher=Banneker.org |date=2012-03-01 |accessdate=2012-04-28}} [[Community Charter School of Cambridge]],{{cite web|url=http://www.ccscambridge.org/ |title=Community Charter School of Cambridge |publisher=Ccscambridge.org |date= |accessdate=2012-04-28}} which is located in Kendall Square and serves students in grades 7–12, and [[Prospect Hill Academy]], a [[charter school]] whose upper school is in [[Central Square (Cambridge)|Central Square]], though it is not a part of the Cambridge Public School District. ===Primary and secondary private education=== [[File:Cambridge Public Library, Cambridge, Massachusetts.JPG|thumb|right|[[Cambridge Public Library]] original building, part of an expanded facility]] There are also many private schools in the city including: *[[Boston Archdiocesan Choir School]] (BACS) *[[Buckingham Browne & Nichols]] (BB&N) *[[Cambridge montessori school|Cambridge Montessori School]] (CMS) *Cambridge [[Religious Society of Friends|Friends]] School. Thomas Waring served as founding headmaster of the school. *Fayerweather Street School (FSS)[http://www.fayerweather.org/ ] *[[International School of Boston]] (ISB, formerly École Bilingue) *[[Matignon High School]] *[[North Cambridge Catholic High School]] (re-branded as Cristo Rey Boston and relocated to Dorchester, MA in 2010) *[[Shady Hill School]] *St. Peter School ==Economy== [[File:Cambridge Skyline.jpg|thumb|Buildings of [[Kendall Square]], center of Cambridge's [[biotech]] economy, seen from the [[Charles River]]]] Manufacturing was an important part of the economy in the late 19th and early 20th century, but educational institutions are the city's biggest employers today. Harvard and [[Massachusetts Institute of Technology|MIT]] together employ about 20,000.[http://www2.cambridgema.gov/cdd/data/labor/top25/top25_2008.html Top 25 Cambridge Employers: 2008], City of Cambridge As a cradle of technological innovation, Cambridge was home to technology firms [[Analog Devices]], [[Akamai Technologies|Akamai]], [[BBN Technologies|Bolt, Beranek, and Newman (BBN Technologies)]] (now part of Raytheon), [[General Radio|General Radio (later GenRad)]], [[Lotus Development Corporation]] (now part of [[IBM]]), [[Polaroid Corporation|Polaroid]], [[Symbolics]], and [[Thinking Machines]]. In 1996, [[Polaroid Corporation|Polaroid]], [[Arthur D. Little]], and [[Lotus Development Corporation|Lotus]] were top employers with over 1,000 employees in Cambridge, but faded out a few years later. Health care and biotechnology firms such as [[Genzyme]], [[Biogen Idec]], [[Millennium Pharmaceuticals]], [[Sanofi]], [[Pfizer]] and [[Novartis]]{{cite news |title=Novartis doubles plan for Cambridge |author=Casey Ross and Robert Weisman |first= |last= |authorlink= |authorlink2= |url=http://articles.boston.com/2010-10-27/business/29323650_1_french-drug-maker-astrazeneca-plc-research-operations |agency= |newspaper=[[The Boston Globe]] |publisher= |isbn= |issn= |pmid= |pmd= |bibcode= |doi= |date=October 27, 2010 |page= |pages= |accessdate=April 12, 2011|quote=Already Cambridge’s largest corporate employer, the Swiss firm expects to hire an additional 200 to 300 employees over the next five years, bringing its total workforce in the city to around 2,300. Novartis’s global research operations are headquartered in Cambridge, across Massachusetts Avenue from the site of the new four-acre campus. |archiveurl= |archivedate= |ref=}} have significant presences in the city. Though headquartered in Switzerland, Novartis continues to expand its operations in Cambridge. Other major biotech and pharmaceutical firms expanding their presence in Cambridge include [[GlaxoSmithKline]], [[AstraZeneca]], [[Shire plc|Shire]], and [[Pfizer]].{{cite news|title=Novartis Doubles Plan for Cambridge|url=http://www.boston.com/business/healthcare/articles/2010/10/27/novartis_doubles_plan_for_cambridge/|accessdate=23 February 2012 | work=The Boston Globe|first1=Casey|last1=Ross|first2=Robert|last2=Weisman|date=October 27, 2010}} Most Biotech firms in Cambridge are located around [[Kendall Square]] and [[East Cambridge, Massachusetts|East Cambridge]], which decades ago were the city's center of manufacturing. A number of biotechnology companies are also located in [[University Park at MIT]], a new development in another former manufacturing area. None of the high technology firms that once dominated the economy was among the 25 largest employers in 2005, but by 2008 high tech companies [[Akamai Technologies|Akamai]] and [[ITA Software]] had grown to be among the largest 25 employers. [[Google]],{{cite web|url=http://www.google.com/corporate/address.html |title=Google Offices |publisher=Google.com |date= |accessdate=2012-07-18}} [[IBM Research]], and [[Microsoft Research]] maintain offices in Cambridge. In late January 2012—less than a year after acquiring [[Billerica, Massachusetts|Billerica]]-based analytic database management company, [[Vertica]]—[[Hewlett-Packard]] announced it would also be opening its first offices in Cambridge.{{cite web|last=Huang|first=Gregory|title=Hewlett-Packard Expands to Cambridge via Vertica’s "Big Data" Center|url=http://www.xconomy.com/boston/2012/01/23/hewlett-packard-expands-to-cambridge-via-verticas-big-data-center/?single_page=true}} Around this same time, e-commerce giants [[Staples Inc.|Staples]]{{cite web|title=Staples to bring e-commerce office to Cambridge's Kendall Square Read more: Staples to bring e-commerce office to Cambridge's Kendall Square - Cambridge, Massachusetts - Cambridge Chronicle http://www.wickedlocal.com/cambridge/news/x690035936/Staples-to-bring-E-commerce-office-to-Cambridges-Kendall-Square#ixzz1nDY39Who|url=http://www.wickedlocal.com/cambridge/news/x690035936/Staples-to-bring-E-commerce-office-to-Cambridges-Kendall-Square#axzz1kg3no7Zg}} and [[Amazon.com]]{{cite web|title=Amazon Seeks Brick-And-Mortar Presence In Boston Area|url=http://www.wbur.org/2011/12/22/amazon-boston}} said they would be opening research and innovation centers in Kendall Square. Video game developer [[Harmonix Music Systems]] is based in [[Central Square (Cambridge)|Central Square]]. The proximity of Cambridge's universities has also made the city a center for nonprofit groups and think tanks, including the [[National Bureau of Economic Research]], the [[Smithsonian Astrophysical Observatory]], the [[Lincoln Institute of Land Policy]], [[Cultural Survival]], and [[One Laptop per Child]]. In September 2011, an initiative by the City of Cambridge called the "[[Entrepreneur Walk of Fame]]" was launched. It seeks to highlight individuals who have made contributions to innovation in the global business community.{{cite news |title=Stars of invention |author= |first=Kathleen |last=Pierce |url=http://articles.boston.com/2011-09-16/business/30165912_1_gates-and-jobs-microsoft-granite-stars |agency= |newspaper=The Boston Globe|date=September 16, 2011 |page= |pages= |at= |accessdate=October 1, 2011}} ===Top employers=== The top ten employers in the city are:{{cite web|url=http://cambridgema.gov/citynewsandpublications/news/2012/01/fy11comprehensiveannualfinancialreportnowavailable.aspx |title=City of Cambridge, Massachusetts Comprehensive Annual Financial Report July 1, 2010—June 30, 2011 |publisher=Cambridgema.gov |date=2011-06-30 |accessdate=2012-04-28}} {| class="wikitable" |- ! # ! Employer ! # of employees |- | 1 |[[Harvard University]] |10,718 |- |2 |[[Massachusetts Institute of Technology]] |7,604 |- |3 |City of Cambridge |2,922 |- |4 |[[Novartis]] Institutes for BioMedical Research |2,095 |- |5 |[[Mount Auburn Hospital]] |1,665 |- |6 |[[Vertex Pharmaceuticals]] |1,600 |- |7 |[[Genzyme]] |1,504 |- |8 |[[Biogen Idec]] |1,350 |- |9 |[[Federal government of the United States|Federal Government]] |1,316 |- |10 |[[Pfizer]] |1,300 |} ==Transportation== {{See also|Boston transportation}} ===Road=== [[File:Harvard Square at Peabody Street and Mass Avenue.jpg|thumb|[[Massachusetts Avenue (Boston)|Massachusetts Avenue]] in [[Harvard Square]]]] Several major roads lead to Cambridge, including [[Massachusetts State Highway 2|Route 2]], [[Massachusetts State Highway 16|Route 16]] and the [[Massachusetts State Highway 28|McGrath Highway (Route 28)]]. The [[Massachusetts Turnpike]] does not pass through Cambridge, but provides access by an exit in nearby [[Allston, Massachusetts|Allston]]. Both [[U.S. Route 1]] and [[I-93 (MA)]] also provide additional access on the eastern end of Cambridge at Leverett Circle in [[Boston]]. [[Massachusetts State Highway 2A|Route 2A]] runs the length of the city, chiefly along Massachusetts Avenue. The Charles River forms the southern border of Cambridge and is crossed by 11 bridges connecting Cambridge to Boston, including the [[Longfellow Bridge]] and the [[Harvard Bridge]], eight of which are open to motorized road traffic. Cambridge has an irregular street network because many of the roads date from the colonial era. Contrary to popular belief, the road system did not evolve from longstanding cow-paths. Roads connected various village settlements with each other and nearby towns, and were shaped by geographic features, most notably streams, hills, and swampy areas. Today, the major "squares" are typically connected by long, mostly straight roads, such as Massachusetts Avenue between [[Harvard Square]] and [[Central Square (Cambridge)|Central Square]], or Hampshire Street between [[Kendall Square]] and [[Inman Square]]. ===Mass transit=== [[File:Central MBTA station.jpg|thumb|[[Central (MBTA)|Central station on the MBTA Red Line]]]] Cambridge is well served by the [[MBTA]], including the [[Porter (MBTA station)|Porter Square stop]] on the regional [[MBTA Commuter Rail|Commuter Rail]], the [[Lechmere (MBTA station)|Lechmere stop]] on the [[Green Line (MBTA)|Green Line]], and five stops on the [[Red Line (MBTA)|Red Line]] ([[Alewife Station (MBTA)|Alewife]], [[Porter (MBTA)|Porter Square]], [[Harvard (MBTA station)|Harvard Square]], [[Central (MBTA station)|Central Square]], and [[Kendall/MIT (MBTA station)|Kendall Square/MIT]]). Alewife Station, the current terminus of the Red Line, has a large multi-story parking garage (at a rate of $7 per day {{as of|lc=y|2009}}).{{cite web|url=http://www.mbta.com/schedules_and_maps/subway/lines/stations/?stopId=10029 |title=> Schedules & Maps > Subway > Alewife Station |publisher=MBTA |date= |accessdate=2012-04-28}} The [[Harvard Bus Tunnel]], under Harvard Square, reduces traffic congestion on the surface, and connects to the Red Line underground. This tunnel was originally opened for streetcars in 1912, and served trackless trolleys and buses as the routes were converted. The tunnel was partially reconfigured when the Red Line was extended to Alewife in the early 1980s. Outside of the state-owned transit agency, the city is also served by the Charles River Transportation Management Agency (CRTMA) shuttles which are supported by some of the largest companies operating in city, in addition to the municipal government itself.{{cite web |url=http://www.charlesrivertma.org/members.htm |title=Charles River TMA Members |author=Staff writer |date=(As of) January 1, 2013 |work=CRTMA |publisher= |language= |trans_title= |type= |archiveurl= |archivedate= |deadurl= |accessdate=January 1, 2013 |quote= |ref= |separator= |postscript=}} ===Cycling=== Cambridge has several [[bike path]]s, including one along the Charles River,{{cite web|url=http://www.mass.gov/dcr/parks/metroboston/maps/bikepaths_dudley.gif |title=Dr. Paul Dudley White Bikepath |date= |accessdate=2012-04-28}} and the [[Cambridge Linear Park|Linear Park]] connecting the [[Minuteman Bikeway]] at Alewife with the [[Somerville Community Path]]. Bike parking is common and there are bike lanes on many streets, although concerns have been expressed regarding the suitability of many of the lanes. On several central MIT streets, bike lanes transfer onto the sidewalk. Cambridge bans cycling on certain sections of sidewalk where pedestrian traffic is heavy.{{cite web|url=http://www.cambridgema.gov/cdd/et/bike/bike_ban.html |title=Sidewalk Bicycling Banned Areas – Cambridge Massachusetts |publisher=Cambridgema.gov |date= |accessdate=2012-04-28}}{{cite web|url=http://www.cambridgema.gov/cdd/et/bike/bike_reg.html |title=Traffic Regulations for Cyclists – Cambridge Massachusetts |publisher=Cambridgema.gov |date=1997-05-01 |accessdate=2012-04-28}} While ''[[Bicycling Magazine]]'' has rated Boston as one of the worst cities in the nation for bicycling (In their words, for "lousy roads, scarce and unconnected bike lanes and bike-friendly gestures from City Hall that go nowhere—such as hiring a bike coordinator in 2001, only to cut the position two years later"),[http://www.bicycling.com/article/1,6610,s1-2-16-14593-11,00.html Urban Treasures – bicycling.com]{{dead link|date=April 2012}} it has listed Cambridge as an honorable mention as one of the best[http://www.bicycling.com/article/1,6610,s1-2-16-14593-9,00.html Urban Treasures – bicycling.com]{{dead link|date=April 2012}} and was called by the magazine "Boston's Great Hope." Cambridge has an active, official bicycle committee. ===Walking=== [[File:Weeks Footbridge Cambridge, MA.jpg|thumb|The [[John W. Weeks Bridge|Weeks Bridge]] provides a pedestrian-only connection between Boston's Allston-Brighton neighborhood and Cambridge over the Charles River]] Walking is a popular activity in Cambridge. Per year 2000 data, of the communities in the U.S. with more than 100,000 residents, Cambridge has the highest percentage of commuters who walk to work.{{cite web|url=http://www.bikesatwork.com/carfree/census-lookup.php?state_select=ALL_STATES&lower_pop=100000&upper_pop=99999999&sort_num=2&show_rows=25&first_row=0 |title=The Carfree Census Database: Result of search for communities in any state with population over 100,000, sorted in descending order by % Pedestrian Commuters |publisher=Bikesatwork.com |date= |accessdate=2012-04-28}} Cambridge receives a "Walk Score" of 100 out of 100 possible points.[http://www.walkscore.com/get-score.php?street=cambridge%2C+ma&go=Go Walk Score site] Accessed July 28, 2009 Cambridge's major historic squares have been recently changed into a modern walking landscape, which has sparked a traffic calming program based on the needs of pedestrians rather than of motorists. ===Intercity=== The Boston intercity bus and train stations at [[South Station]], Boston, and [[Logan International Airport]] in [[East Boston]], are accessible by [[Red Line (MBTA)|subway]]. The [[Fitchburg Line]] rail service from [[Porter (MBTA station)|Porter Square]] connects to some western suburbs. Since October 2010, there has also been intercity bus service between [[Alewife (MBTA station)|Alewife Station]] (Cambridge) and [[New York City]].{{cite web|last=Thomas |first=Sarah |url=http://www.boston.com/yourtown/news/cambridge/2010/10/warren_mbta_welcome_world_wide.html |title=NYC-bound buses will roll from Newton, Cambridge |publisher=Boston.com |date=2010-10-19 |accessdate=2012-04-28}} ==Media== ===Newspapers=== Cambridge is served by several weekly newspapers. The most prominent is the ''[[Cambridge Chronicle]]'', which is also the oldest surviving weekly paper in the United States. ===Radio=== Cambridge is home to the following commercially licensed and student-run radio stations: {| class=wikitable |- ! [[Callsign]] !! Frequency !! City/town !! Licensee !! Format |- | [[WHRB]] || align=right | 95.3 FM || Cambridge (Harvard) || Harvard Radio Broadcasting Co., Inc. || [[Variety (US radio)|Musical variety]] |- | [[WJIB]] || align=right | 740 AM || Cambridge || Bob Bittner Broadcasting || [[Adult Standards]]/Pop |- | [[WMBR]] || align=right | 88.1 FM || Cambridge (MIT) || Technology Broadcasting Corporation || [[College radio]] |} ===Television=== Cambridge Community Television (CCTV) has served the Cambridge community since its inception in 1988. CCTV operates Cambridge's public access television facility and programs three television channels, 8, 9, and 96 on the Cambridge cable system (Comcast). ===Social media=== As of 2011, a growing number of social media efforts provide means for participatory engagement with the locality of Cambridge, such as Localocracy"Localocracy is an online town common where registered voters using real names can weigh in on local issues." [http://cambridge.localocracy.com/ Localocracy Cambridge, Massachusetts]. Accessed 2011-10-01 and [[foursquare (website)|Foursquare]]. ==Culture, art and architecture== [[File:Fogg.jpg|thumb|[[Fogg Museum]], Harvard]] ===Museums=== * [[Harvard Art Museum]], including the [[Busch-Reisinger Museum]], a collection of Germanic art the [[Fogg Art Museum]], a comprehensive collection of Western art, and the [[Arthur M. Sackler Museum]], a collection of Middle East and Asian art * [[Harvard Museum of Natural History]], including the [[Glass Flowers]] collection * [[Peabody Museum of Archaeology and Ethnology]], Harvard *[[Semitic Museum]], Harvard * [[MIT Museum]] * [[List Visual Arts Center]], MIT ===Public art=== Cambridge has a large and varied collection of permanent public art, both on city property (managed by the Cambridge Arts Council),{{cite web|url=http://www.cambridgema.gov/CAC/Public/overview.cfm |title=CAC Public Art Program |publisher=Cambridgema.gov |date=2007-03-13 |accessdate=2012-04-28}} and on the campuses of Harvard{{cite web|url=http://ofa.fas.harvard.edu/visualarts/pubart.php |title=Office for the Arts at Harvard: Public Art |publisher=Ofa.fas.harvard.edu |date= |accessdate=2012-04-28}} and MIT.{{cite web|url=http://listart.mit.edu/map |title=MIT Public Art Collection Map |publisher=Listart.mit.edu |date= |accessdate=2012-04-28}} Temporary public artworks are displayed as part of the annual Cambridge River Festival on the banks of the Charles River, during winter celebrations in Harvard and Central Squares, and at university campus sites. Experimental forms of public artistic and cultural expression include the Central Square World's Fair, the Somerville-based annual Honk! Festival,{{cite web|url=http://honkfest.org/ |title= Honk Fest}} and [[If This House Could Talk]],{{cite web|url=http://cambridgehistory.org/discover/ifthishousecouldtalk/index.html |title=The Cambridge Historical Society}} a neighborhood art and history event. {{or|date=April 2012}} {{Citation needed|date=April 2012}} An active tradition of street musicians and other performers in Harvard Square entertains an audience of tourists and local residents during the warmer months of the year. The performances are coordinated through a public process that has been developed collaboratively by the performers,{{cite web|url=http://www.buskersadvocates.org/ | title= Street Arts & Buskers Advocates}} city administrators, private organizations and business groups.{{cite web|url=http://harvardsquare.com/Home/Arts-and-Entertainment/Street-Arts-and-Buskers-Advocates.aspx |title=Street Arts and Buskers Advocates |publisher=Harvardsquare.com |date= |accessdate=2012-04-28}} [[File:Longfellow National Historic Site, Cambridge, Massachusetts.JPG|thumb|right|The [[Longfellow National Historic Site]]]] [[File:Wfm stata center.jpg|thumb|[[Stata Center]], MIT]] [[File:Simmons Hall, MIT, Cambridge, Massachusetts.JPG|thumb|[[List of MIT undergraduate dormitories|Simmons Hall]], MIT]] ===Architecture=== Despite intensive urbanization during the late 19th century and 20th century, Cambridge has preserved an unusual number of historic buildings, including some dating to the 17th century. The city also contains an abundance of innovative contemporary architecture, largely built by Harvard and MIT. ;Notable historic buildings in the city include: * The [[Asa Gray House]] (1810) * [[Austin Hall, Harvard University]] (1882–84) * [[Cambridge, Massachusetts City Hall|Cambridge City Hall]] (1888–89) * [[Cambridge Public Library]] (1888) * [[Christ Church, Cambridge]] (1761) * [[Cooper-Frost-Austin House]] (1689–1817) * [[Elmwood (Cambridge, Massachusetts)|Elmwood House]] (1767), residence of the [[President of Harvard University]] * [[First Church of Christ, Scientist (Cambridge, Massachusetts)|First Church of Christ, Scientist]] (1924–30) * [[The First Parish in Cambridge]] (1833) * [[Harvard-Epworth United Methodist Church]] (1891–93) * [[Harvard Lampoon Building]] (1909) * The [[Hooper-Lee-Nichols House]] (1685–1850) * [[Longfellow National Historic Site]] (1759), former home of poet [[Henry Wadsworth Longfellow]] * [[The Memorial Church of Harvard University]] (1932) * [[Memorial Hall, Harvard University]] (1870–77) * [[Middlesex County Courthouse (Massachusetts)|Middlesex County Courthouse]] (1814–48) * [[Urban Rowhouse (40-48 Pearl Street, Cambridge, Massachusetts)|Urban Rowhouse]] (1875) * [[spite house|O'Reilly Spite House]] (1908), built to spite a neighbor who would not sell his adjacent landBloom, Jonathan. (February 2, 2003) [[Boston Globe]] ''[http://nl.newsbank.com/nl-search/we/Archives?p_product=BG&p_theme=bg&p_action=search&p_maxdocs=200&p_topdoc=1&p_text_direct-0=0F907F2342522B5D&p_field_direct-0=document_id&p_perpage=10&p_sort=YMD_date:D Existing by the Thinnest of Margins. A Concord Avenue Landmark Gives New Meaning to Cozy.]'' Section: City Weekly; Page 11. Location: 260 Concord Ave, Cambridge, MA 02138. {{See also|List of Registered Historic Places in Cambridge, Massachusetts}} ;Contemporary architecture: * [[List of MIT undergraduate dormitories#Baker House|Baker House]] dormitory, MIT, by Finnish architect [[Alvar Aalto]], one of only two buildings by Aalto in the US * Harvard Graduate Center/Harkness Commons, by [[The Architects Collaborative]] (TAC, with [[Walter Gropius]]) * [[Carpenter Center for the Visual Arts]], Harvard, the only building in North America by [[Le Corbusier]] * [[Kresge Auditorium]], MIT, by [[Eero Saarinen]] * [[MIT Chapel]], by [[Eero Saarinen]] * [[Design Research Building]], by [[Benjamin Thompson and Associates]] * [[American Academy of Arts and Sciences]], by [[Kallmann McKinnell and Wood]], also architects of Boston City Hall * [[Arthur M. Sackler Museum]], Harvard, one of the few buildings in the U.S. by [[James Stirling (architect)|James Stirling]], winner of the [[Pritzker Prize]] * [[Stata Center]], MIT, by [[Frank Gehry]] * [[List of MIT undergraduate dormitories#Simmons Hall|Simmons Hall]], MIT, by [[Steven Holl]] ===Music=== The city has an active music scene from classical performances to the latest popular bands. ==Sister cities== Cambridge has 8 active, official [[Twin towns and sister cities|sister cities]], and an unofficial relationship with [[Cambridge]], England:"A message from the Peace Commission" [http://www.cambridgema.gov/peace/newsandpublications/news/detail.aspx?path=%2fsitecore%2fcontent%2fhome%2fpeace%2fnewsandpublications%2fnews%2f2008%2f02%2finformationoncambridgessistercities]. *{{Flagicon|PRT}} [[Coimbra]], [[Portugal]] *{{Flagicon|CUB}} [[Cienfuegos]], [[Cuba]] *{{Flagicon|ITA}} [[Gaeta]], [[Italy]] *{{Flagicon|IRL}} [[Galway]], [[Republic of Ireland|Ireland]] *{{Flagicon|ARM}} [[Yerevan]], [[Armenia]]{{cite web|url=http://www.cysca.org/ |title=Cambridge-Yerevan Sister City Association |publisher=Cysca.org |date= |accessdate=2012-04-28}} *{{Flagicon|SLV}} [[San José Las Flores, Chalatenango|San José Las Flores]], [[El Salvador]] *{{Flagicon|JPN}} [[Tsukuba, Ibaraki|Tsukuba Science City]], Japan *{{Flagicon|POL}} [[Kraków]], [[Poland]] *{{Flagicon|CHN}} [[Haidian District]], [[China]] Ten other official sister city relationships are inactive: [[Dublin]], Ireland; [[Ischia]], [[Catania]], and [[Florence]], Italy; [[Kraków]], Poland; [[Santo Domingo Oeste]], Dominican Republic; [[Southwark]], London, England; [[Yuseong]], Daejeon, Korea; and [[Haidian District|Haidian]], Beijing, China. There has also been an unofficial relationship with: *{{Flagicon|GBR}} [[Cambridge]], England, UK{{cite web|url=http://www.cambridgema.gov/peace/newsandpublications/news/detail.aspx?path=%2fsitecore%2fcontent%2fhome%2fpeace%2fnewsandpublications%2fnews%2f2008%2f02%2finformationoncambridgessistercities |title="Sister Cities", Cambridge Peace Commission |publisher=Cambridgema.gov |date=2008-02-15 |accessdate=2012-07-18}} ==Zip codes== *02138—Harvard Square/West Cambridge *02139—Central Square/Inman Square/MIT *02140—Porter Square/North Cambridge *02141—East Cambridge *02142—Kendall Square ==References== {{reflist|30em}} ==General references== * ''History of Middlesex County, Massachusetts'', [http://books.google.com/books?id=QGolOAyd9RMC&dq=intitle:History+intitle:of+intitle:Middlesex+intitle:County+intitle:Massachusetts&lr=&num=50&as_brr=0&source=gbs_other_versions_sidebar_s&cad=5 Volume 1 (A-H)], [http://books.google.com/books?id=hNaAnwRMedUC&pg=PA506&dq=intitle:History+intitle:of+intitle:Middlesex+intitle:County+intitle:Massachusetts&lr=&num=50&as_brr=0#PPA3,M1 Volume 2 (L-W)] factory by Samuel Adams Drake, published 1879–1880. ** [http://books.google.com/books?id=QGolOAyd9RMC&printsec=titlepage#PPA305,M1 Cambridge article] by Rev. Edward Abbott in volume 1, pages 305–358. *Eliot, Samuel Atkins. ''A History of Cambridge, Massachusetts: 1630–1913''. Cambridge: The Cambridge Tribune, 1913. *Hiestand, Emily. "Watershed: An Excursion in Four Parts" The Georgia Review Spring 1998 pages 7–28 *[[Lucius Robinson Paige|Paige, Lucius]]. ''History of Cambridge, Massachusetts: 1630–1877''. Cambridge: The Riverside Press, 1877. *Survey of Architectural History in Cambridge: Mid Cambridge, 1967, Cambridge Historical Commission, Cambridge, Mass.{{ISBN missing}} *Survey of Architectural History in Cambridge: Cambridgeport, 1971 ISBN 0-262-53013-9, Cambridge Historical Commission, Cambridge, Mass. *Survey of Architectural History in Cambridge: Old Cambridge, 1973 ISBN 0-262-53014-7, Cambridge Historical Commission, Cambridge, Mass. *Survey of Architectural History in Cambridge: Northwest Cambridge, 1977 ISBN 0-262-53032-5, Cambridge Historical Commission, Cambridge, Mass. *Survey of Architectural History in Cambridge: East Cambridge, 1988 (revised) ISBN 0-262-53078-3, Cambridge Historical Commission, Cambridge, Mass. *{{cite book|last=Sinclair|first=Jill|title=Fresh Pond: The History of a Cambridge Landscape|publisher=MIT Press|location=Cambridge, Mass.|date=April 2009|isbn=978-0-262-19591-1 }} *{{cite book|last=Seaburg|first=Alan|title=Cambridge on the Charles|url=http://books.google.com/books?id=c7_oCS782-8C|publisher=Anne Miniver Press|location=Billerica, Mass.|year=2001|author=Seaburg, A. and Dahill, T. and Rose, C.H.|isbn=978-0-9625794-9-3}} ==External links== {{Commons category}} {{Wikivoyage|Cambridge (Massachusetts)}} {{Portal|Boston}} {{Commons category|Cambridge, Massachusetts}} *{{Official website|http://www.cambridgema.gov/}} *[http://www.cambridge-usa.org/ Cambridge Office for Tourism] *[http://www.city-data.com/city/Cambridge-Massachusetts.html City-Data.com] *[http://www.epodunk.com/cgi-bin/genInfo.php?locIndex=2894 ePodunk: Profile for Cambridge, Massachusetts] *{{dmoz|Regional/North_America/United_States/Massachusetts/Localities/C/Cambridge}}
===Maps=== *[http://www.cambridgema.gov/GIS/FindMapAtlas.cfm Cambridge Maps] *[http://www.cambridgema.gov/GIS City of Cambridge Geographic Information System (GIS)] *[http://www.salemdeeds.com/atlases_results.asp?ImageType=index&atlastype=MassWorld&atlastown=&atlas=MASSACHUSETTS+1871&atlas_desc=MASSACHUSETTS+1871 ''1871 Atlas of Massachusetts''.] by Wall & Gray. [http://www.salemdeeds.com/atlases_pages.asp?ImageName=PAGE_0010_0011.jpg&atlastype=MassWorld&atlastown=&atlas=MASSACHUSETTS+1871&atlas_desc=MASSACHUSETTS+1871&pageprefix= Map of Massachusetts.] [http://www.salemdeeds.com/atlases_pages.asp?ImageName=PAGE_0044_0045.jpg&atlastype=MassWorld&atlastown=&atlas=MASSACHUSETTS+1871&atlas_desc=MASSACHUSETTS+1871&pageprefix= Map of Middlesex County.] *Dutton, E.P. [http://maps.bpl.org/details_10717/?srch_query=Dutton%2C+E.P.&srch_fields=all&srch_author=on&srch_style=exact&srch_fa=save&srch_ok=Go+Search Chart of Boston Harbor and Massachusetts Bay with Map of Adjacent Country.] Published 1867. A good map of roads and rail lines around Cambridge. *[http://www.citymap.com/cambridge/index.htm Cambridge Citymap – Community, Business, and Visitor Map.] *[http://docs.unh.edu/towns/CambridgeMassachusettsMapList.htm Old USGS maps of Cambridge area.] {{Greater Boston}} {{Middlesex County, Massachusetts}} {{Massachusetts}} {{New England}} {{Massachusetts cities and mayors of 100,000 population}} [[Category:Cambridge, Massachusetts| ]] [[Category:University towns in the United States]] [[Category:County seats in Massachusetts]] [[Category:Populated places established in 1630]] [[Category:Charles River]] [[Category:Place names of English origin in the United States]] [[af:Cambridge, Massachusetts]] [[ar:كامبريدج، ماساتشوستس]] [[zh-min-nan:Cambridge, Massachusetts]] [[be:Горад Кембрыдж, Масачусетс]] [[be-x-old:Кембрыдж (Масачусэтс)]] [[bg:Кеймбридж (Масачузетс)]] [[br:Cambridge (Massachusetts)]] [[ca:Cambridge (Massachusetts)]] [[cs:Cambridge (Massachusetts)]] [[cy:Cambridge, Massachusetts]] [[da:Cambridge (Massachusetts)]] [[de:Cambridge (Massachusetts)]] [[et:Cambridge (Massachusetts)]] [[es:Cambridge (Massachusetts)]] [[eo:Kembriĝo (Masaĉuseco)]] [[eu:Cambridge (Massachusetts)]] [[fa:کمبریج (ماساچوست)]] [[fr:Cambridge (Massachusetts)]] [[gd:Cambridge (MA)]] [[ko:케임브리지 (매사추세츠 주)]] [[hy:Քեմբրիջ (Մասաչուսեթս)]] [[id:Cambridge, Massachusetts]] [[it:Cambridge (Massachusetts)]] [[he:קיימברידג' (מסצ'וסטס)]] [[jv:Cambridge, Massachusetts]] [[kk:Кэмбридж (Массачусетс)]] [[kw:Cambridge, Massachusetts]] [[sw:Cambridge, Massachusetts]] [[ht:Cambridge, Massachusetts]] [[la:Cantabrigia (Massachusetta)]] [[lv:Keimbridža]] [[lb:Cambridge (Massachusetts)]] [[hu:Cambridge (Massachusetts)]] [[mr:केंब्रिज, मॅसेच्युसेट्स]] [[ms:Cambridge, Massachusetts]] [[nl:Cambridge (Massachusetts)]] [[ja:ケンブリッジ (マサチューセッツ州)]] [[no:Cambridge (Massachusetts)]] [[pl:Cambridge (Massachusetts)]] [[pt:Cambridge (Massachusetts)]] [[ro:Cambridge, Massachusetts]] [[ru:Кембридж (Массачусетс)]] [[scn:Cambridge (Massachusetts), USA]] [[simple:Cambridge, Massachusetts]] [[sk:Cambridge (Massachusetts)]] [[sl:Cambridge, Massachusetts]] [[sr:Кембриџ (Масачусетс)]] [[fi:Cambridge (Massachusetts)]] [[sv:Cambridge, Massachusetts]] [[tl:Cambridge, Massachusetts]] [[ta:கேம்பிரிஜ், மாசசூசெட்ஸ்]] [[th:เคมบริดจ์ (รัฐแมสซาชูเซตส์)]] [[tg:Кембриҷ (Массачусетс)]] [[tr:Cambridge, Massachusetts]] [[uk:Кембридж (Массачусетс)]] [[vi:Cambridge, Massachusetts]] [[vo:Cambridge (Massachusetts)]] [[war:Cambridge, Massachusetts]] [[yi:קעמברידזש, מאסאטשוסעטס]] [[zh:剑桥 (马萨诸塞州)]] \ No newline at end of file diff --git a/distribution/build.gradle b/distribution/build.gradle index f78ec0bce618d..253af219843c0 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -41,7 +41,7 @@ buildscript { } Collection distributions = project.subprojects.findAll { - it.path.contains(':tools') == false && it.name != 'bwc-zip' } + it.path.contains(':tools') == false && it.name != 'bwc' } /***************************************************************************** * Notice file * diff --git a/distribution/bwc-zip/build.gradle b/distribution/bwc/build.gradle similarity index 86% rename from distribution/bwc-zip/build.gradle rename to distribution/bwc/build.gradle index 22eb03b805b46..e31d0949bdf96 100644 --- a/distribution/bwc-zip/build.gradle +++ b/distribution/bwc/build.gradle @@ -17,7 +17,7 @@ * under the License. */ -import java.util.regex.Matcher + import org.elasticsearch.gradle.LoggedExec /** @@ -81,14 +81,19 @@ if (bwcVersion.endsWith('-SNAPSHOT')) { commandLine = ['git', 'checkout', "upstream/${bwcBranch}"] } + File bwcDeb = file("${checkoutDir}/distribution/zip/build/distributions/elasticsearch-${bwcVersion}.deb") + File bwcRpm = file("${checkoutDir}/distribution/zip/build/distributions/elasticsearch-${bwcVersion}.rpm") File bwcZip = file("${checkoutDir}/distribution/zip/build/distributions/elasticsearch-${bwcVersion}.zip") task buildBwcVersion(type: GradleBuild) { dependsOn checkoutBwcBranch dir = checkoutDir - tasks = [':distribution:zip:assemble'] + tasks = [':distribution:deb:assemble', ':distribution:rpm:assemble', ':distribution:zip:assemble'] } + artifacts { + 'default' file: bwcDeb, name: 'elasticsearch', type: 'deb', builtBy: buildBwcVersion + 'default' file: bwcRpm, name: 'elasticsearch', type: 'rpm', builtBy: buildBwcVersion 'default' file: bwcZip, name: 'elasticsearch', type: 'zip', builtBy: buildBwcVersion } } diff --git a/docs/build.gradle b/docs/build.gradle index 82d7b6a59a5b4..5f6e0d3ebb728 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -338,6 +338,58 @@ for (int i = 0; i < 50; i++) { } buildRestTests.setups['stackoverflow'] += """ """ +// Used by significant_text aggregation docs +buildRestTests.setups['news'] = ''' + - do: + indices.create: + index: news + body: + settings: + number_of_shards: 1 + number_of_replicas: 1 + mappings: + question: + properties: + source: + type: keyword + content: + type: text + - do: + bulk: + index: news + type: article + refresh: true + body: |''' + +// Make h5n1 strongly connected to bird flu + +for (int i = 0; i < 100; i++) { + buildRestTests.setups['news'] += """ + {"index":{}} + {"source": "very_relevant_$i", "content": "bird flu h5n1"}""" +} +for (int i = 0; i < 100; i++) { + buildRestTests.setups['news'] += """ + {"index":{}} + {"source": "filler_$i", "content": "bird dupFiller "}""" +} +for (int i = 0; i < 100; i++) { + buildRestTests.setups['news'] += """ + {"index":{}} + {"source": "filler_$i", "content": "flu dupFiller "}""" +} +for (int i = 0; i < 20; i++) { + buildRestTests.setups['news'] += """ + {"index":{}} + {"source": "partially_relevant_$i", "content": "elasticsearch dupFiller dupFiller dupFiller dupFiller pozmantier"}""" +} +for (int i = 0; i < 10; i++) { + buildRestTests.setups['news'] += """ + {"index":{}} + {"source": "partially_relevant_$i", "content": "elasticsearch logstash kibana"}""" +} +buildRestTests.setups['news'] += """ +""" // Used by some aggregations buildRestTests.setups['exams'] = ''' diff --git a/docs/plugins/plugin-script.asciidoc b/docs/plugins/plugin-script.asciidoc index 9eb6cae11692d..e89766f3c1fc3 100644 --- a/docs/plugins/plugin-script.asciidoc +++ b/docs/plugins/plugin-script.asciidoc @@ -165,12 +165,9 @@ can do this as follows: [source,sh] --------------------- -sudo bin/elasticsearch-plugin -Epath.conf=/path/to/custom/config/dir install +sudo CONF_DIR=/path/to/conf/dir bin/elasticsearch-plugin install --------------------- -You can also set the `CONF_DIR` environment variable to the custom config -directory path. - [float] === Proxy settings diff --git a/docs/reference/aggregations/bucket.asciidoc b/docs/reference/aggregations/bucket.asciidoc index f98fd52febaf3..7b9aafe4ea1b1 100644 --- a/docs/reference/aggregations/bucket.asciidoc +++ b/docs/reference/aggregations/bucket.asciidoc @@ -49,5 +49,7 @@ include::bucket/sampler-aggregation.asciidoc[] include::bucket/significantterms-aggregation.asciidoc[] +include::bucket/significanttext-aggregation.asciidoc[] + include::bucket/terms-aggregation.asciidoc[] diff --git a/docs/reference/aggregations/bucket/significanttext-aggregation.asciidoc b/docs/reference/aggregations/bucket/significanttext-aggregation.asciidoc new file mode 100644 index 0000000000000..cf282bb7fce86 --- /dev/null +++ b/docs/reference/aggregations/bucket/significanttext-aggregation.asciidoc @@ -0,0 +1,487 @@ +[[search-aggregations-bucket-significanttext-aggregation]] +=== Significant Text Aggregation + +experimental[] + +An aggregation that returns interesting or unusual occurrences of free-text terms in a set. +It is like the <> aggregation but differs in that: + +* It is specifically designed for use on type `text` fields +* It does not require field data or doc-values +* It re-analyzes text content on-the-fly meaning it can also filter duplicate sections of +noisy text that otherwise tend to skew statistics. + +WARNING: Re-analyzing _large_ result sets will require a lot of time and memory. It is recommended that the significant_text + aggregation is used as a child of either the <> or + <> aggregation to limit the analysis + to a _small_ selection of top-matching documents e.g. 200. This will typically improve speed, memory use and quality of + results. + +.Example use cases: +* Suggesting "H5N1" when users search for "bird flu" to help expand queries +* Suggesting keywords relating to stock symbol $ATI for use in an automated news classifier + +In these cases the words being selected are not simply the most popular terms in results. The most popular words tend to be +very boring (_and, of, the, we, I, they_ ...). +The significant words are the ones that have undergone a significant change in popularity measured between a _foreground_ and _background_ set. +If the term "H5N1" only exists in 5 documents in a 10 million document index and yet is found in 4 of the 100 documents that make up a user's search results +that is significant and probably very relevant to their search. 5/10,000,000 vs 4/100 is a big swing in frequency. + +experimental[The `significant_text` aggregation is new and may change in non-backwards compatible ways if we add further text-analysis features e.g. phrase detection] + +==== Basic use + +In the typical use case, the _foreground_ set of interest is a selection of the top-matching search results for a query +and the _background_set used for statistical comparisons is the index or indices from which the results were gathered. + +Example: + +[source,js] +-------------------------------------------------- +GET news/article/_search +{ + "query" : { + "match" : {"content" : "Bird flu"} + }, + "aggregations" : { + "my_sample" : { + "sampler" : { + "shard_size" : 100 + }, + "aggregations": { + "keywords" : { + "significant_text" : { "field" : "content" } + } + } + } + } +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:news] + + +Response: + +[source,js] +-------------------------------------------------- +{ + "took": 9, + "timed_out": false, + "_shards": ..., + "hits": ..., + "aggregations" : { + "my_sample": { + "doc_count": 100, + "keywords" : { + "doc_count": 100, + "buckets" : [ + { + "key": "h5n1", + "doc_count": 4, + "score": 4.71235374214817, + "bg_count": 5 + } + ... + ] + } + } + } +} +-------------------------------------------------- +// NOTCONSOLE + +The results show that "h5n1" is one of several terms strongly associated with bird flu. +It only occurs 5 times in our index as a whole (see the `bg_count`) and yet 4 of these +were lucky enough to appear in our 100 document sample of "bird flu" results. That suggests +a significant word and one which the user can potentially add to their search. + + +==== Dealing with noisy data using `filter_duplicate_text` +Free-text fields often contain a mix of original content and mechanical copies of text (cut-and-paste biographies, email reply chains, +retweets, boilerplate headers/footers, page navigation menus, sidebar news links, copyright notices, standard disclaimers, addresses). + +In real-world data these duplicate sections of text tend to feature heavily in `significant_text` results if they aren't filtered out. +Filtering near-duplicate text is a difficult task at index-time but we can cleanse the data on-the-fly at query time using the +`filter_duplicate_text` setting. + + +First let's look at an unfiltered real-world example using the http://research.signalmedia.co/newsir16/signal-dataset.html[Signal media dataset] of +a million news articles covering a wide variety of news. Here are the raw significant text results for a search for the articles +mentioning "elasticsearch": + + +[source,js] +-------------------------------------------------- +{ + ... + "aggregations": { + "sample": { + "doc_count": 35, + "keywords": { + "doc_count": 35, + "buckets": [ + { + "key": "elasticsearch", + "doc_count": 35, + "score": 28570.428571428572, + "bg_count": 35 + }, + ... + { + "key": "currensee", + "doc_count": 8, + "score": 6530.383673469388, + "bg_count": 8 + }, + ... + { + "key": "pozmantier", + "doc_count": 4, + "score": 3265.191836734694, + "bg_count": 4 + }, + ... + +} +-------------------------------------------------- +// NOTCONSOLE + +The uncleansed documents have thrown up some odd-looking terms that are, on the face of it, statistically +correlated with appearances of our search term "elasticsearch" e.g. "pozmantier". +We can drill down into examples of these documents to see why pozmantier is connected using this query: + +[source,js] +-------------------------------------------------- +GET news/article/_search +{ + "query": { + "simple_query_string": { + "query": "+elasticsearch +pozmantier" + } + }, + "_source": [ + "title", + "source" + ], + "highlight": { + "fields": { + "content": {} + } + } +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:news] +The results show a series of very similar news articles about a judging panel for a number of tech projects: + +[source,js] +-------------------------------------------------- +{ + ... + "hits": { + "hits": [ + { + ... + "_source": { + "source": "Presentation Master", + "title": "T.E.N. Announces Nominees for the 2015 ISE® North America Awards" + }, + "highlight": { + "content": [ + "City of San Diego Mike Pozmantier, Program Manager, Cyber Security Division, Department of", + " Janus, Janus ElasticSearch Security Visualization Engine " + ] + } + }, + { + ... + "_source": { + "source": "RCL Advisors", + "title": "T.E.N. Announces Nominees for the 2015 ISE(R) North America Awards" + }, + "highlight": { + "content": [ + "Mike Pozmantier, Program Manager, Cyber Security Division, Department of Homeland Security S&T", + "Janus, Janus ElasticSearch Security Visualization Engine" + ] + } + }, + ... +-------------------------------------------------- +// NOTCONSOLE +Mike Pozmantier was one of many judges on a panel and elasticsearch was used in one of many projects being judged. + +As is typical, this lengthy press release was cut-and-paste by a variety of news sites and consequently any rare names, numbers or +typos they contain become statistically correlated with our matching query. + +Fortunately similar documents tend to rank similarly so as part of examining the stream of top-matching documents the significant_text +aggregation can apply a filter to remove sequences of any 6 or more tokens that have already been seen. Let's try this same query now but +with the `filter_duplicate_text` setting turned on: + +[source,js] +-------------------------------------------------- +GET news/article/_search +{ + "query": { + "match": { + "content": "elasticsearch" + } + }, + "aggs": { + "sample": { + "sampler": { + "shard_size": 100 + }, + "aggs": { + "keywords": { + "significant_text": { + "field": "content", + "filter_duplicate_text": true + } + } + } + } + } +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:news] + +The results from analysing our deduplicated text are obviously of higher quality to anyone familiar with the elastic stack: + +[source,js] +-------------------------------------------------- +{ + ... + "aggregations": { + "sample": { + "doc_count": 35, + "keywords": { + "doc_count": 35, + "buckets": [ + { + "key": "elasticsearch", + "doc_count": 22, + "score": 11288.001166180758, + "bg_count": 35 + }, + { + "key": "logstash", + "doc_count": 3, + "score": 1836.648979591837, + "bg_count": 4 + }, + { + "key": "kibana", + "doc_count": 3, + "score": 1469.3020408163263, + "bg_count": 5 + } + ] + } + } + } +} +-------------------------------------------------- +// NOTCONSOLE + +Mr Pozmantier and other one-off associations with elasticsearch no longer appear in the aggregation +results as a consequence of copy-and-paste operations or other forms of mechanical repetition. + +If your duplicate or near-duplicate content is identifiable via a single-value indexed field (perhaps +a hash of the article's `title` text or an `original_press_release_url` field) then it would be more +efficient to use a parent <> aggregation +to eliminate these documents from the sample set based on that single key. The less duplicate content you can feed into +the significant_text aggregation up front the better in terms of performance. + + +.How are the significance scores calculated? +********************************** +The numbers returned for scores are primarily intended for ranking different suggestions sensibly rather than something easily +understood by end users. The scores are derived from the doc frequencies in _foreground_ and _background_ sets. In brief, a +term is considered significant if there is a noticeable difference in the frequency in which a term appears in the subset and +in the background. The way the terms are ranked can be configured, see "Parameters" section. + +********************************** + +.Use the _"like this but not this"_ pattern +********************************** +You can spot mis-categorized content by first searching a structured field e.g. `category:adultMovie` and use significant_text on the +text "movie_description" field. Take the suggested words (I'll leave them to your imagination) and then search for all movies NOT marked as category:adultMovie but containing these keywords. +You now have a ranked list of badly-categorized movies that you should reclassify or at least remove from the "familyFriendly" category. + +The significance score from each term can also provide a useful `boost` setting to sort matches. +Using the `minimum_should_match` setting of the `terms` query with the keywords will help control the balance of precision/recall in the result set i.e +a high setting would have a small number of relevant results packed full of keywords and a setting of "1" would produce a more exhaustive results set with all documents containing _any_ keyword. + +********************************** + + + +==== Limitations + + +===== No support for child aggregations +The significant_text aggregation intentionally does not support the addition of child aggregations because: + +* It would come with a high memory cost +* It isn't a generally useful feature and there is a workaround for those that need it + +The volume of candidate terms is generally very high and these are pruned heavily before the final +results are returned. Supporting child aggregations would generate additional churn and be inefficient. +Clients can always take the heavily-trimmed set of results from a `significant_text` request and +make a subsequent follow-up query using a `terms` aggregation with an `include` clause and child +aggregations to perform further analysis of selected keywords in a more efficient fashion. + + +===== Approximate counts +The counts of how many documents contain a term provided in results are based on summing the samples returned from each shard and +as such may be: + +* low if certain shards did not provide figures for a given term in their top sample +* high when considering the background frequency as it may count occurrences found in deleted documents + +Like most design decisions, this is the basis of a trade-off in which we have chosen to provide fast performance at the cost of some (typically small) inaccuracies. +However, the `size` and `shard size` settings covered in the next section provide tools to help control the accuracy levels. + +==== Parameters + +===== Significance heuristics + +This aggregation supports the same scoring heuristics (JLH, mutual_information, gnd, chi_square etc) as the <> aggregation + + +===== Size & Shard Size + +The `size` parameter can be set to define how many term buckets should be returned out of the overall terms list. By +default, the node coordinating the search process will request each shard to provide its own top term buckets +and once all shards respond, it will reduce the results to the final list that will then be returned to the client. +If the number of unique terms is greater than `size`, the returned list can be slightly off and not accurate +(it could be that the term counts are slightly off and it could even be that a term that should have been in the top +size buckets was not returned). + +To ensure better accuracy a multiple of the final `size` is used as the number of terms to request from each shard +using a heuristic based on the number of shards. To take manual control of this setting the `shard_size` parameter +can be used to control the volumes of candidate terms produced by each shard. + +Low-frequency terms can turn out to be the most interesting ones once all results are combined so the +significant_terms aggregation can produce higher-quality results when the `shard_size` parameter is set to +values significantly higher than the `size` setting. This ensures that a bigger volume of promising candidate terms are given +a consolidated review by the reducing node before the final selection. Obviously large candidate term lists +will cause extra network traffic and RAM usage so this is quality/cost trade off that needs to be balanced. If `shard_size` is set to -1 (the default) then `shard_size` will be automatically estimated based on the number of shards and the `size` parameter. + + +NOTE: `shard_size` cannot be smaller than `size` (as it doesn't make much sense). When it is, elasticsearch will + override it and reset it to be equal to `size`. + +===== Minimum document count + +It is possible to only return terms that match more than a configured number of hits using the `min_doc_count` option. +The Default value is 3. + +Terms that score highly will be collected on a shard level and merged with the terms collected from other shards in a second step. +However, the shard does not have the information about the global term frequencies available. The decision if a term is added to a +candidate list depends only on the score computed on the shard using local shard frequencies, not the global frequencies of the word. +The `min_doc_count` criterion is only applied after merging local terms statistics of all shards. In a way the decision to add the +term as a candidate is made without being very _certain_ about if the term will actually reach the required `min_doc_count`. +This might cause many (globally) high frequent terms to be missing in the final result if low frequent but high scoring terms populated +the candidate lists. To avoid this, the `shard_size` parameter can be increased to allow more candidate terms on the shards. +However, this increases memory consumption and network traffic. + +`shard_min_doc_count` parameter + +The parameter `shard_min_doc_count` regulates the _certainty_ a shard has if the term should actually be added to the candidate list or +not with respect to the `min_doc_count`. Terms will only be considered if their local shard frequency within the set is higher than the +`shard_min_doc_count`. If your dictionary contains many low frequent words and you are not interested in these (for example misspellings), +then you can set the `shard_min_doc_count` parameter to filter out candidate terms on a shard level that will with a reasonable certainty +not reach the required `min_doc_count` even after merging the local frequencies. `shard_min_doc_count` is set to `1` per default and has +no effect unless you explicitly set it. + + + + +WARNING: Setting `min_doc_count` to `1` is generally not advised as it tends to return terms that + are typos or other bizarre curiosities. Finding more than one instance of a term helps + reinforce that, while still rare, the term was not the result of a one-off accident. The + default value of 3 is used to provide a minimum weight-of-evidence. + Setting `shard_min_doc_count` too high will cause significant candidate terms to be filtered out on a shard level. + This value should be set much lower than `min_doc_count/#shards`. + + + +===== Custom background context + +The default source of statistical information for background term frequencies is the entire index and this +scope can be narrowed through the use of a `background_filter` to focus in on significant terms within a narrower +context: + +[source,js] +-------------------------------------------------- +GET news/article/_search +{ + "query" : { + "match" : { + "content" : "madrid" + } + }, + "aggs" : { + "tags" : { + "significant_text" : { + "field" : "content", + "background_filter": { + "term" : { "content" : "spain"} + } + } + } + } +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:news] + +The above filter would help focus in on terms that were peculiar to the city of Madrid rather than revealing +terms like "Spanish" that are unusual in the full index's worldwide context but commonplace in the subset of documents containing the +word "Spain". + +WARNING: Use of background filters will slow the query as each term's postings must be filtered to determine a frequency + + +===== Dealing with source and index mappings + +Ordinarily the indexed field name and the original JSON field being retrieved share the same name. +However with more complex field mappings using features like `copy_to` the source +JSON field(s) and the indexed field being aggregated can differ. +In these cases it is possible to list the JSON _source fields from which text +will be analyzed using the `source_fields` parameter: + +[source,js] +-------------------------------------------------- +GET news/article/_search +{ + "query" : { + "match" : { + "custom_all" : "elasticsearch" + } + }, + "aggs" : { + "tags" : { + "significant_text" : { + "field" : "custom_all", + "source_fields": ["content" , "title"] + } + } + } +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:news] + + +===== Filtering Values + +It is possible (although rarely required) to filter the values for which buckets will be created. This can be done using the `include` and +`exclude` parameters which are based on a regular expression string or arrays of exact terms. This functionality mirrors the features +described in the <> documentation. + + diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc index 0c2d5fd447312..0a0e9e4bfc30b 100644 --- a/docs/reference/docs/bulk.asciidoc +++ b/docs/reference/docs/bulk.asciidoc @@ -12,8 +12,8 @@ bulk requests and reindexing of documents from one index to another: Perl:: - See https://metacpan.org/pod/Search::Elasticsearch::Bulk[Search::Elasticsearch::Bulk] - and https://metacpan.org/pod/Search::Elasticsearch::Scroll[Search::Elasticsearch::Scroll] + See https://metacpan.org/pod/Search::Elasticsearch::Client::5_0::Bulk[Search::Elasticsearch::Client::5_0::Bulk] + and https://metacpan.org/pod/Search::Elasticsearch::Client::5_0::Scroll[Search::Elasticsearch::Client::5_0::Scroll] Python:: diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc index 025bad633064d..62a5e10ef0e53 100644 --- a/docs/reference/indices/aliases.asciidoc +++ b/docs/reference/indices/aliases.asciidoc @@ -382,7 +382,7 @@ found index aliases. Possible options: [horizontal] `index`:: - The index name to get aliases for. Partially names are + The index name to get aliases for. Partial names are supported via wildcards, also multiple index names can be specified separated with a comma. Also the alias name for an index can be used. diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc index e8150cd4dd90b..b42a8bea3f242 100644 --- a/docs/reference/ingest/ingest-node.asciidoc +++ b/docs/reference/ingest/ingest-node.asciidoc @@ -1738,6 +1738,63 @@ numeric fields `field_a` and `field_b` multiplied by the parameter param_c: -------------------------------------------------- // NOTCONSOLE +It is possible to use the Script Processor to manipulate document metadata like `_index` and `_type` during +ingestion. Here is an example of an Ingest Pipeline that renames the index and type to `my_index` no matter what +was provided in the original index request: + +[source,js] +-------------------------------------------------- +PUT _ingest/pipeline/my_index +{ + "description": "use index:my_index and type:my_type", + "processors": [ + { + "script": { + "inline": " ctx._index = 'my_index'; ctx._type = 'my_type' " + } + } + ] +} +-------------------------------------------------- +// CONSOLE + +Using the above pipeline, we can attempt to index a document into the `any_index` index. + +[source,js] +-------------------------------------------------- +PUT any_index/any_type/1?pipeline=my_index +{ + "message": "text" +} +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +The response from the above index request: + +[source,js] +-------------------------------------------------- +{ + "_index": "my_index", + "_type": "my_type", + "_id": "1", + "_version": 1, + "result": "created", + "_shards": { + "total": 2, + "successful": 1, + "failed": 0 + }, + "_seq_no": 0, + "_primary_term": 1, + "created": true +} +-------------------------------------------------- +// TESTRESPONSE + +In the above response, you can see that our document was actually indexed into `my_index` instead of +`any_index`. This type of manipulation is often convenient in pipelines that have various branches of transformation, +and depending on the progress made, indexed into different indices. [[set-processor]] === Set Processor diff --git a/docs/reference/mapping/types/text.asciidoc b/docs/reference/mapping/types/text.asciidoc index 357a41ae06ce0..0bb9b00a10293 100644 --- a/docs/reference/mapping/types/text.asciidoc +++ b/docs/reference/mapping/types/text.asciidoc @@ -7,7 +7,7 @@ description of a product. These fields are `analyzed`, that is they are passed t before being indexed. The analysis process allows Elasticsearch to search for individual words _within_ each full text field. Text fields are not used for sorting and seldom used for aggregations (although the -<> +<> is a notable exception). If you need to index structured content such as email addresses, hostnames, status diff --git a/docs/reference/search/request/collapse.asciidoc b/docs/reference/search/request/collapse.asciidoc index d91799946cf20..b4322e36f9343 100644 --- a/docs/reference/search/request/collapse.asciidoc +++ b/docs/reference/search/request/collapse.asciidoc @@ -70,8 +70,46 @@ GET /twitter/tweet/_search See <> for the complete list of supported options and the format of the response. +It is also possible to request multiple `inner_hits` for each collapsed hit. This can be useful when you want to get +multiple representations of the collapsed hits. + +[source,js] +-------------------------------------------------- +GET /twitter/tweet/_search +{ + "query": { + "match": { + "message": "elasticsearch" + } + }, + "collapse" : { + "field" : "user", <1> + "inner_hits": [ + { + "name": "most_liked", <2> + "size": 3, + "sort": ["likes"] + }, + { + "name": "most_recent", <3> + "size": 3, + "sort": [{ "date": "asc" }] + } + ] + }, + "sort": ["likes"] +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:twitter] +<1> collapse the result set using the "user" field +<2> return the three most liked tweets for the user +<3> return the three most recent tweets for the user + The expansion of the group is done by sending an additional query for each -collapsed hit returned in the response. +`inner_hit` request for each collapsed hit returned in the response. This can significantly slow things down +if you have too many groups and/or `inner_hit` requests. + The `max_concurrent_group_searches` request parameter can be used to control the maximum number of concurrent searches allowed in this phase. The default is based on the number of data nodes and the default search thread pool size. diff --git a/docs/reference/search/request/scroll.asciidoc b/docs/reference/search/request/scroll.asciidoc index 1b78c0bc9bacd..91f7b0ec85d6b 100644 --- a/docs/reference/search/request/scroll.asciidoc +++ b/docs/reference/search/request/scroll.asciidoc @@ -18,8 +18,8 @@ scrolled searches and reindexing of documents from one index to another: Perl:: - See https://metacpan.org/pod/Search::Elasticsearch::Bulk[Search::Elasticsearch::Bulk] - and https://metacpan.org/pod/Search::Elasticsearch::Scroll[Search::Elasticsearch::Scroll] + See https://metacpan.org/pod/Search::Elasticsearch::Client::5_0::Bulk[Search::Elasticsearch::Client::5_0::Bulk] + and https://metacpan.org/pod/Search::Elasticsearch::Client::5_0::Scroll[Search::Elasticsearch::Client::5_0::Scroll] Python:: diff --git a/docs/reference/search/suggesters/context-suggest.asciidoc b/docs/reference/search/suggesters/context-suggest.asciidoc index cc44f41771a54..064f919c25d36 100644 --- a/docs/reference/search/suggesters/context-suggest.asciidoc +++ b/docs/reference/search/suggesters/context-suggest.asciidoc @@ -28,8 +28,7 @@ PUT place "contexts": [ { <1> "name": "place_type", - "type": "category", - "path": "cat" + "type": "category" }, { <2> "name": "location", diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java index a23bdea7a1a2c..4200bbbebc169 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java @@ -23,10 +23,8 @@ import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptException; import org.elasticsearch.script.ScriptService; @@ -70,8 +68,8 @@ public final class ScriptProcessor extends AbstractProcessor { */ @Override public void execute(IngestDocument document) { - CompiledScript compiledScript = scriptService.compile(script, ScriptContext.INGEST); - ExecutableScript executableScript = scriptService.executable(compiledScript, script.getParams()); + ExecutableScript.Factory factory = scriptService.compile(script, ExecutableScript.INGEST_CONTEXT); + ExecutableScript executableScript = factory.newInstance(script.getParams()); executableScript.setNextVar("ctx", document.getSourceAndMetadata()); executableScript.run(); } @@ -134,7 +132,7 @@ public ScriptProcessor create(Map registry, String pr // verify script is able to be compiled before successfully creating processor. try { - scriptService.compile(script, ScriptContext.INGEST); + scriptService.compile(script, ExecutableScript.INGEST_CONTEXT); } catch (ScriptException e) { throw newConfigurationException(TYPE, processorTag, scriptPropertyUsed, e); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java index 5356d9c9e0b72..1004a41bcc592 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.RandomDocumentPicks; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; @@ -46,8 +45,10 @@ public void testScripting() throws Exception { ScriptService scriptService = mock(ScriptService.class); Script script = mockScript("_script"); + ExecutableScript.Factory factory = mock(ExecutableScript.Factory.class); ExecutableScript executableScript = mock(ExecutableScript.class); - when(scriptService.executable(any(CompiledScript.class), any())).thenReturn(executableScript); + when(scriptService.compile(script, ExecutableScript.INGEST_CONTEXT)).thenReturn(factory); + when(factory.newInstance(any())).thenReturn(executableScript); Map document = new HashMap<>(); document.put("bytes_in", randomInt()); diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java index f5d95f6c88a2b..f9cdae404572f 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java @@ -20,7 +20,6 @@ package org.elasticsearch.script.expression; import org.apache.lucene.expressions.Expression; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.GeneralScriptException; @@ -32,17 +31,16 @@ * of an {@link ExecutableScript}. */ public class ExpressionExecutableScript implements ExecutableScript { - public final CompiledScript compiledScript; + public final Expression expression; public final Map functionValuesMap; public final ReplaceableConstDoubleValues[] functionValuesArray; - public ExpressionExecutableScript(CompiledScript compiledScript, Map vars) { - this.compiledScript = compiledScript; - Expression expression = (Expression)this.compiledScript.compiled(); + public ExpressionExecutableScript(Expression expression, Map vars) { + this.expression = expression; int functionValuesLength = expression.variables.length; if (vars.size() != functionValuesLength) { - throw new GeneralScriptException("Error using " + compiledScript + ". " + + throw new GeneralScriptException("Error using " + expression + ". " + "The number of variables in an executable expression script [" + functionValuesLength + "] must match the number of variables in the variable map" + " [" + vars.size() + "]."); @@ -69,12 +67,12 @@ public void setNextVar(String name, Object value) { double doubleValue = ((Number)value).doubleValue(); functionValuesMap.get(name).setValue(doubleValue); } else { - throw new GeneralScriptException("Error using " + compiledScript + ". " + + throw new GeneralScriptException("Error using " + expression + ". " + "Executable expressions scripts can only process numbers." + " The variable [" + name + "] is not a number."); } } else { - throw new GeneralScriptException("Error using " + compiledScript + ". " + + throw new GeneralScriptException("Error using " + expression + ". " + "The variable [" + name + "] does not exist in the executable expressions script."); } } @@ -82,9 +80,9 @@ public void setNextVar(String name, Object value) { @Override public Object run() { try { - return ((Expression) compiledScript.compiled()).evaluate(functionValuesArray); + return expression.evaluate(functionValuesArray); } catch (Exception exception) { - throw new GeneralScriptException("Error evaluating " + compiledScript, exception); + throw new GeneralScriptException("Error evaluating " + expression, exception); } } } diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java index 071e5cf666d0c..fd80c56cdbe8a 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java @@ -19,15 +19,18 @@ package org.elasticsearch.script.expression; +import java.util.Collection; + import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; public class ExpressionPlugin extends Plugin implements ScriptPlugin { @Override - public ScriptEngine getScriptEngine(Settings settings) { + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { return new ExpressionScriptEngine(settings); } } diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java index 11a10247e0374..509e922a4cc6b 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java @@ -37,8 +37,8 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.script.ClassPermission; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.script.ScriptException; import org.elasticsearch.script.SearchScript; @@ -70,11 +70,11 @@ public String getType() { } @Override - public Object compile(String scriptName, String scriptSource, Map params) { + public T compile(String scriptName, String scriptSource, ScriptContext context, Map params) { // classloader created here final SecurityManager sm = System.getSecurityManager(); SpecialPermission.check(); - return AccessController.doPrivileged(new PrivilegedAction() { + Expression expr = AccessController.doPrivileged(new PrivilegedAction() { @Override public Expression run() { try { @@ -101,11 +101,17 @@ protected Class loadClass(String name, boolean resolve) throws ClassNotFoundE } } }); + if (context.instanceClazz.equals(SearchScript.class)) { + SearchScript.Factory factory = (p, lookup) -> newSearchScript(expr, lookup, p); + return context.factoryClazz.cast(factory); + } else if (context.instanceClazz.equals(ExecutableScript.class)) { + ExecutableScript.Factory factory = (p) -> new ExpressionExecutableScript(expr, p); + return context.factoryClazz.cast(factory); + } + throw new IllegalArgumentException("painless does not know how to handle context [" + context.name + "]"); } - @Override - public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map vars) { - Expression expr = (Expression)compiledScript.compiled(); + private SearchScript newSearchScript(Expression expr, SearchLookup lookup, @Nullable Map vars) { MapperService mapper = lookup.doc().mapperService(); // NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings, // instead of complicating SimpleBindings (which should stay simple) @@ -227,7 +233,7 @@ public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @ throw convertToScriptException("link error", expr.sourceText, variable, e); } } - return new ExpressionSearchScript(compiledScript, bindings, specialValue, needsScores); + return new ExpressionSearchScript(expr, bindings, specialValue, needsScores); } /** @@ -247,12 +253,4 @@ private ScriptException convertToScriptException(String message, String source, stack.add(pointer.toString()); throw new ScriptException(message, cause, stack, source, NAME); } - - @Override - public ExecutableScript executable(CompiledScript compiledScript, Map vars) { - return new ExpressionExecutableScript(compiledScript, vars); - } - - @Override - public void close() {} } diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java index 11a17d38c5993..2e6c010da6667 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java @@ -26,7 +26,6 @@ import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.Scorer; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.GeneralScriptException; import org.elasticsearch.script.LeafSearchScript; import org.elasticsearch.script.SearchScript; @@ -40,7 +39,7 @@ */ class ExpressionSearchScript implements SearchScript { - final CompiledScript compiledScript; + final Expression exprScript; final SimpleBindings bindings; final DoubleValuesSource source; final ReplaceableConstDoubleValueSource specialValue; // _value @@ -48,10 +47,10 @@ class ExpressionSearchScript implements SearchScript { Scorer scorer; int docid; - ExpressionSearchScript(CompiledScript c, SimpleBindings b, ReplaceableConstDoubleValueSource v, boolean needsScores) { - compiledScript = c; + ExpressionSearchScript(Expression e, SimpleBindings b, ReplaceableConstDoubleValueSource v, boolean needsScores) { + exprScript = e; bindings = b; - source = ((Expression)compiledScript.compiled()).getDoubleValuesSource(bindings); + source = exprScript.getDoubleValuesSource(bindings); specialValue = v; this.needsScores = needsScores; } @@ -81,7 +80,7 @@ public boolean advanceExact(int doc) throws IOException { try { return values.doubleValue(); } catch (Exception exception) { - throw new GeneralScriptException("Error evaluating " + compiledScript, exception); + throw new GeneralScriptException("Error evaluating " + exprScript, exception); } } @@ -100,7 +99,7 @@ public void setDocument(int d) { try { values.advanceExact(d); } catch (IOException e) { - throw new IllegalStateException("Can't advance to doc using " + compiledScript, e); + throw new IllegalStateException("Can't advance to doc using " + exprScript, e); } } @@ -121,7 +120,7 @@ public boolean advanceExact(int doc) throws IOException { } }); } catch (IOException e) { - throw new IllegalStateException("Can't get values using " + compiledScript, e); + throw new IllegalStateException("Can't get values using " + exprScript, e); } } @@ -137,7 +136,7 @@ public void setNextAggregationValue(Object value) { if (value instanceof Number) { specialValue.setValue(((Number)value).doubleValue()); } else { - throw new GeneralScriptException("Cannot use expression with text variable using " + compiledScript); + throw new GeneralScriptException("Cannot use expression with text variable using " + exprScript); } } } diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java index 49b5fa03957de..574879b1fd33e 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java @@ -21,9 +21,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexService; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ScriptException; -import org.elasticsearch.script.ScriptType; import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -44,8 +42,8 @@ public void setUp() throws Exception { } private SearchScript compile(String expression) { - Object compiled = service.compile(null, expression, Collections.emptyMap()); - return service.search(new CompiledScript(ScriptType.INLINE, "randomName", "expression", compiled), lookup, Collections.emptyMap()); + SearchScript.Factory factory = service.compile(null, expression, SearchScript.CONTEXT, Collections.emptyMap()); + return factory.newInstance(Collections.emptyMap(), lookup); } public void testNeedsScores() { diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java index 59f691e8894ca..d8d09ffba790a 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java @@ -33,7 +33,6 @@ import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.GeneralScriptException; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; @@ -514,9 +513,8 @@ public void testExecutableScripts() throws Exception { vars.put("xyz", -1); Expression expr = JavascriptCompiler.compile("a+b+xyz"); - CompiledScript compiledScript = new CompiledScript(ScriptType.INLINE, "", "expression", expr); - ExpressionExecutableScript ees = new ExpressionExecutableScript(compiledScript, vars); + ExpressionExecutableScript ees = new ExpressionExecutableScript(expr, vars); assertEquals((Double) ees.run(), 4.5, 0.001); ees.setNextVar("b", -2.5); @@ -532,7 +530,7 @@ public void testExecutableScripts() throws Exception { try { vars = new HashMap<>(); vars.put("a", 1); - ees = new ExpressionExecutableScript(compiledScript, vars); + ees = new ExpressionExecutableScript(expr, vars); ees.run(); fail("An incorrect number of variables were allowed to be used in an expression."); } catch (GeneralScriptException se) { @@ -545,7 +543,7 @@ public void testExecutableScripts() throws Exception { vars.put("a", 1); vars.put("b", 3); vars.put("c", -1); - ees = new ExpressionExecutableScript(compiledScript, vars); + ees = new ExpressionExecutableScript(expr, vars); ees.run(); fail("A variable was allowed to be set that does not exist in the expression."); } catch (GeneralScriptException se) { @@ -558,7 +556,7 @@ public void testExecutableScripts() throws Exception { vars.put("a", 1); vars.put("b", 3); vars.put("xyz", "hello"); - ees = new ExpressionExecutableScript(compiledScript, vars); + ees = new ExpressionExecutableScript(expr, vars); ees.run(); fail("A non-number was allowed to be use in the expression."); } catch (GeneralScriptException se) { diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java index fceb6fb98d9f5..bd294db6d2957 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java @@ -112,7 +112,7 @@ public MultiSearchTemplateRequest indicesOptions(IndicesOptions indicesOptions) @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - if (in.getVersion().onOrAfter(Version.V_5_5_0_UNRELEASED)) { + if (in.getVersion().onOrAfter(Version.V_5_5_0)) { maxConcurrentSearchRequests = in.readVInt(); } requests = in.readStreamableList(SearchTemplateRequest::new); @@ -121,7 +121,7 @@ public void readFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_5_5_0_UNRELEASED)) { + if (out.getVersion().onOrAfter(Version.V_5_5_0)) { out.writeVInt(maxConcurrentSearchRequests); } out.writeStreamableList(requests); diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java index 105fa3f6f803b..c05a88e9351db 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java @@ -33,16 +33,18 @@ import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.function.Supplier; public class MustachePlugin extends Plugin implements ScriptPlugin, ActionPlugin, SearchPlugin { @Override - public ScriptEngine getScriptEngine(Settings settings) { + public ScriptEngine getScriptEngine(Settings settings, Collection>contexts) { return new MustacheScriptEngine(); } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java index d5e35b6383f87..3363db47b4833 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java @@ -25,22 +25,16 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.logging.log4j.util.Supplier; import org.elasticsearch.SpecialPermission; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.FastStringReader; -import org.elasticsearch.common.io.UTF8StreamWriter; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.GeneralScriptException; import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; -import org.elasticsearch.script.SearchScript; -import org.elasticsearch.search.lookup.SearchLookup; import java.io.Reader; import java.io.StringWriter; -import java.lang.ref.SoftReference; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Collections; @@ -67,10 +61,15 @@ public final class MustacheScriptEngine implements ScriptEngine { * @return a compiled template object for later execution. * */ @Override - public Object compile(String templateName, String templateSource, Map params) { + public T compile(String templateName, String templateSource, ScriptContext context, Map params) { + if (context.instanceClazz.equals(ExecutableScript.class) == false) { + throw new IllegalArgumentException("mustache engine does not know how to handle context [" + context.name + "]"); + } final MustacheFactory factory = createMustacheFactory(params); Reader reader = new FastStringReader(templateSource); - return factory.compile(reader, "query-template"); + Mustache template = factory.compile(reader, "query-template"); + ExecutableScript.Factory compiled = p -> new MustacheExecutableScript(template, p); + return context.factoryClazz.cast(compiled); } private CustomMustacheFactory createMustacheFactory(Map params) { @@ -85,29 +84,12 @@ public String getType() { return NAME; } - @Override - public ExecutableScript executable(CompiledScript compiledScript, - @Nullable Map vars) { - return new MustacheExecutableScript(compiledScript, vars); - } - - @Override - public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, - @Nullable Map vars) { - throw new UnsupportedOperationException(); - } - - @Override - public void close() { - // Nothing to do here - } - /** * Used at query execution time by script service in order to execute a query template. * */ private class MustacheExecutableScript implements ExecutableScript { - /** Compiled template object wrapper. */ - private CompiledScript template; + /** Factory template. */ + private Mustache template; /** Parameters to fill above object with. */ private Map vars; @@ -115,7 +97,7 @@ private class MustacheExecutableScript implements ExecutableScript { * @param template the compiled template object wrapper * @param vars the parameters to fill above object with **/ - MustacheExecutableScript(CompiledScript template, Map vars) { + MustacheExecutableScript(Mustache template, Map vars) { this.template = template; this.vars = vars == null ? Collections.emptyMap() : vars; } @@ -132,7 +114,7 @@ public Object run() { // crazy reflection here SpecialPermission.check(); AccessController.doPrivileged((PrivilegedAction) () -> { - ((Mustache) template.compiled()).execute(writer, vars); + template.execute(writer, vars); return null; }); } catch (Exception e) { diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java index 90678aba85608..faf543e2a0e36 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java @@ -27,7 +27,6 @@ import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -35,18 +34,18 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.template.CompiledTemplate; -import org.elasticsearch.template.CompiledTemplate; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.io.IOException; import java.util.Collections; -import static org.elasticsearch.script.ScriptContext.SEARCH; +import static org.elasticsearch.script.ExecutableScript.CONTEXT; public class TransportSearchTemplateAction extends HandledTransportAction { @@ -102,7 +101,7 @@ static SearchRequest convert(SearchTemplateRequest searchTemplateRequest, Search NamedXContentRegistry xContentRegistry) throws IOException { Script script = new Script(searchTemplateRequest.getScriptType(), TEMPLATE_LANG, searchTemplateRequest.getScript(), searchTemplateRequest.getScriptParams() == null ? Collections.emptyMap() : searchTemplateRequest.getScriptParams()); - CompiledTemplate compiledScript = scriptService.compileTemplate(script, SEARCH); + CompiledTemplate compiledScript = scriptService.compileTemplate(script, ExecutableScript.CONTEXT); String source = compiledScript.run(script.getParams()); response.setSource(new BytesArray(source)); diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java index eab96a4e991b2..222334dbfda4f 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java @@ -19,9 +19,6 @@ package org.elasticsearch.script.mustache; -import com.github.mustachejava.Mustache; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptEngine; @@ -31,7 +28,6 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; -import static org.elasticsearch.script.ScriptType.INLINE; import static org.elasticsearch.script.mustache.CustomMustacheFactory.JSON_MIME_TYPE; import static org.elasticsearch.script.mustache.CustomMustacheFactory.PLAIN_TEXT_MIME_TYPE; import static org.elasticsearch.script.mustache.CustomMustacheFactory.X_WWW_FORM_URLENCODED_MIME_TYPE; @@ -64,10 +60,9 @@ public void testJsonEscapeEncoder() { final ScriptEngine engine = new MustacheScriptEngine(); final Map params = randomBoolean() ? singletonMap(Script.CONTENT_TYPE_OPTION, JSON_MIME_TYPE) : emptyMap(); - Mustache script = (Mustache) engine.compile(null, "{\"field\": \"{{value}}\"}", params); - CompiledScript compiled = new CompiledScript(INLINE, null, MustacheScriptEngine.NAME, script); + ExecutableScript.Factory factory = engine.compile(null, "{\"field\": \"{{value}}\"}", ExecutableScript.CONTEXT, params); - ExecutableScript executable = engine.executable(compiled, singletonMap("value", "a \"value\"")); + ExecutableScript executable = factory.newInstance(singletonMap("value", "a \"value\"")); assertThat(executable.run(), equalTo("{\"field\": \"a \\\"value\\\"\"}")); } @@ -75,10 +70,9 @@ public void testDefaultEncoder() { final ScriptEngine engine = new MustacheScriptEngine(); final Map params = singletonMap(Script.CONTENT_TYPE_OPTION, PLAIN_TEXT_MIME_TYPE); - Mustache script = (Mustache) engine.compile(null, "{\"field\": \"{{value}}\"}", params); - CompiledScript compiled = new CompiledScript(INLINE, null, MustacheScriptEngine.NAME, script); + ExecutableScript.Factory factory = engine.compile(null, "{\"field\": \"{{value}}\"}", ExecutableScript.CONTEXT, params); - ExecutableScript executable = engine.executable(compiled, singletonMap("value", "a \"value\"")); + ExecutableScript executable = factory.newInstance(singletonMap("value", "a \"value\"")); assertThat(executable.run(), equalTo("{\"field\": \"a \"value\"\"}")); } @@ -86,10 +80,9 @@ public void testUrlEncoder() { final ScriptEngine engine = new MustacheScriptEngine(); final Map params = singletonMap(Script.CONTENT_TYPE_OPTION, X_WWW_FORM_URLENCODED_MIME_TYPE); - Mustache script = (Mustache) engine.compile(null, "{\"field\": \"{{value}}\"}", params); - CompiledScript compiled = new CompiledScript(INLINE, null, MustacheScriptEngine.NAME, script); + ExecutableScript.Factory factory = engine.compile(null, "{\"field\": \"{{value}}\"}", ExecutableScript.CONTEXT, params); - ExecutableScript executable = engine.executable(compiled, singletonMap("value", "tilde~ AND date:[2016 FROM*]")); + ExecutableScript executable = factory.newInstance(singletonMap("value", "tilde~ AND date:[2016 FROM*]")); assertThat(executable.run(), equalTo("{\"field\": \"tilde%7E+AND+date%3A%5B2016+FROM*%5D\"}")); } } diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java index 30cf1fa04a424..3f623e1874d23 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java @@ -20,13 +20,10 @@ import com.github.mustachejava.MustacheFactory; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptType; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -58,8 +55,7 @@ public void testSimpleParameterReplace() { + "\"negative\": {\"term\": {\"body\": {\"value\": \"solr\"}" + "}}, \"negative_boost\": {{boost_val}} } }}"; Map vars = new HashMap<>(); vars.put("boost_val", "0.3"); - String o = (String) qe.executable(new CompiledScript(ScriptType.INLINE, "", "mustache", - qe.compile(null, template, compileParams)), vars).run(); + String o = (String) qe.compile(null, template, ExecutableScript.CONTEXT, compileParams).newInstance(vars).run(); assertEquals("GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}}," + "\"negative\": {\"term\": {\"body\": {\"value\": \"solr\"}}}, \"negative_boost\": 0.3 } }}", o); @@ -70,8 +66,7 @@ public void testSimpleParameterReplace() { Map vars = new HashMap<>(); vars.put("boost_val", "0.3"); vars.put("body_val", "\"quick brown\""); - String o = (String) qe.executable(new CompiledScript(ScriptType.INLINE, "", "mustache", - qe.compile(null, template, compileParams)), vars).run(); + String o = (String) qe.compile(null, template, ExecutableScript.CONTEXT, compileParams).newInstance(vars).run(); assertEquals("GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}}," + "\"negative\": {\"term\": {\"body\": {\"value\": \"\\\"quick brown\\\"\"}}}, \"negative_boost\": 0.3 } }}", o); @@ -86,9 +81,8 @@ public void testSimple() throws IOException { + "}"; XContentParser parser = createParser(JsonXContent.jsonXContent, templateString); Script script = Script.parse(parser); - CompiledScript compiledScript = new CompiledScript(ScriptType.INLINE, null, "mustache", - qe.compile(null, script.getIdOrCode(), Collections.emptyMap())); - ExecutableScript executableScript = qe.executable(compiledScript, script.getParams()); + ExecutableScript.Factory factory = qe.compile(null, script.getIdOrCode(), ExecutableScript.CONTEXT, Collections.emptyMap()); + ExecutableScript executableScript = factory.newInstance(script.getParams()); assertThat(executableScript.run(), equalTo("{\"match_all\":{}}")); } @@ -102,9 +96,8 @@ public void testParseTemplateAsSingleStringWithConditionalClause() throws IOExce + "}"; XContentParser parser = createParser(JsonXContent.jsonXContent, templateString); Script script = Script.parse(parser); - CompiledScript compiledScript = new CompiledScript(ScriptType.INLINE, null, "mustache", - qe.compile(null, script.getIdOrCode(), Collections.emptyMap())); - ExecutableScript executableScript = qe.executable(compiledScript, script.getParams()); + ExecutableScript.Factory factory = qe.compile(null, script.getIdOrCode(), ExecutableScript.CONTEXT, Collections.emptyMap()); + ExecutableScript executableScript = factory.newInstance(script.getParams()); assertThat(executableScript.run(), equalTo("{ \"match_all\":{} }")); } diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java index 316910a5cd531..235f97782a90d 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java @@ -18,13 +18,10 @@ */ package org.elasticsearch.script.mustache; -import com.github.mustachejava.Mustache; import com.github.mustachejava.MustacheException; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.test.ESTestCase; @@ -44,7 +41,6 @@ import static java.util.Collections.singleton; import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.script.ScriptType.INLINE; import static org.hamcrest.Matchers.both; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -64,9 +60,8 @@ public void testBasics() { + "}}, \"negative_boost\": {{boost_val}} } }}"; Map params = Collections.singletonMap("boost_val", "0.2"); - Mustache mustache = (Mustache) engine.compile(null, template, Collections.emptyMap()); - CompiledScript compiledScript = new CompiledScript(INLINE, "my-name", "mustache", mustache); - ExecutableScript result = engine.executable(compiledScript, params); + ExecutableScript.Factory factory = engine.compile(null, template, ExecutableScript.CONTEXT, Collections.emptyMap()); + ExecutableScript result = factory.newInstance(params); assertEquals( "Mustache templating broken", "GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}}," @@ -77,27 +72,27 @@ public void testBasics() { public void testArrayAccess() throws Exception { String template = "{{data.0}} {{data.1}}"; - CompiledScript mustache = new CompiledScript(INLINE, "inline", "mustache", engine.compile(null, template, Collections.emptyMap())); + ExecutableScript.Factory factory = engine.compile(null, template, ExecutableScript.CONTEXT, Collections.emptyMap()); Map vars = new HashMap<>(); Object data = randomFrom( new String[] { "foo", "bar" }, Arrays.asList("foo", "bar")); vars.put("data", data); - assertThat(engine.executable(mustache, vars).run(), equalTo("foo bar")); + assertThat(factory.newInstance(vars).run(), equalTo("foo bar")); // Sets can come out in any order Set setData = new HashSet<>(); setData.add("foo"); setData.add("bar"); vars.put("data", setData); - Object output = engine.executable(mustache, vars).run(); + Object output = factory.newInstance(vars).run(); assertThat(output, instanceOf(String.class)); assertThat((String)output, both(containsString("foo")).and(containsString("bar"))); } public void testArrayInArrayAccess() throws Exception { String template = "{{data.0.0}} {{data.0.1}}"; - CompiledScript mustache = new CompiledScript(INLINE, "inline", "mustache", engine.compile(null, template, Collections.emptyMap())); + ExecutableScript.Factory factory = engine.compile(null, template, ExecutableScript.CONTEXT, Collections.emptyMap()); Map vars = new HashMap<>(); Object data = randomFrom( new String[][] { new String[] { "foo", "bar" }}, @@ -105,25 +100,25 @@ public void testArrayInArrayAccess() throws Exception { singleton(new String[] { "foo", "bar" }) ); vars.put("data", data); - assertThat(engine.executable(mustache, vars).run(), equalTo("foo bar")); + assertThat(factory.newInstance(vars).run(), equalTo("foo bar")); } public void testMapInArrayAccess() throws Exception { String template = "{{data.0.key}} {{data.1.key}}"; - CompiledScript mustache = new CompiledScript(INLINE, "inline", "mustache", engine.compile(null, template, Collections.emptyMap())); + ExecutableScript.Factory factory = engine.compile(null, template, ExecutableScript.CONTEXT, Collections.emptyMap()); Map vars = new HashMap<>(); Object data = randomFrom( new Object[] { singletonMap("key", "foo"), singletonMap("key", "bar") }, Arrays.asList(singletonMap("key", "foo"), singletonMap("key", "bar"))); vars.put("data", data); - assertThat(engine.executable(mustache, vars).run(), equalTo("foo bar")); + assertThat(factory.newInstance(vars).run(), equalTo("foo bar")); // HashSet iteration order isn't fixed Set setData = new HashSet<>(); setData.add(singletonMap("key", "foo")); setData.add(singletonMap("key", "bar")); vars.put("data", setData); - Object output = engine.executable(mustache, vars).run(); + Object output = factory.newInstance(vars).run(); assertThat(output, instanceOf(String.class)); assertThat((String)output, both(containsString("foo")).and(containsString("bar"))); } @@ -134,14 +129,14 @@ public void testSizeAccessForCollectionsAndArrays() throws Exception { List randomList = Arrays.asList(generateRandomStringArray(10, 20, false)); String template = "{{data.array.size}} {{data.list.size}}"; - CompiledScript mustache = new CompiledScript(INLINE, "inline", "mustache", engine.compile(null, template, Collections.emptyMap())); + ExecutableScript.Factory factory = engine.compile(null, template, ExecutableScript.CONTEXT, Collections.emptyMap()); Map data = new HashMap<>(); data.put("array", randomArrayValues); data.put("list", randomList); Map vars = new HashMap<>(); vars.put("data", data); String expectedString = String.format(Locale.ROOT, "%s %s", randomArrayValues.length, randomList.size()); - assertThat(engine.executable(mustache, vars).run(), equalTo(expectedString)); + assertThat(factory.newInstance(vars).run(), equalTo(expectedString)); } public void testPrimitiveToJSON() throws Exception { @@ -376,12 +371,12 @@ public void testUrlEncoderWithJoin() { } private void assertScript(String script, Map vars, Matcher matcher) { - Object result = engine.executable(new CompiledScript(INLINE, "inline", "mustache", compile(script)), vars).run(); + Object result = compile(script).newInstance(vars).run(); assertThat(result, matcher); } - private Object compile(String script) { + private ExecutableScript.Factory compile(String script) { assertThat("cannot compile null or empty script", script, not(isEmptyOrNullString())); - return engine.compile(null, script, Collections.emptyMap()); + return engine.compile(null, script, ExecutableScript.CONTEXT, Collections.emptyMap()); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java index 62ea89acb06e2..0007b92314202 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java @@ -24,9 +24,11 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; import java.util.Arrays; +import java.util.Collection; import java.util.List; /** @@ -40,7 +42,7 @@ public final class PainlessPlugin extends Plugin implements ScriptPlugin { } @Override - public ScriptEngine getScriptEngine(Settings settings) { + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { return new PainlessScriptEngine(settings); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java index d362229c2180a..6bbda4ff0520c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java @@ -24,14 +24,12 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.painless.Compiler.Loader; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.LeafSearchScript; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.script.ScriptException; import org.elasticsearch.script.SearchScript; -import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.security.AccessControlContext; @@ -100,8 +98,25 @@ public String getType() { static final String INLINE_NAME = ""; @Override - public Object compile(String scriptName, final String scriptSource, final Map params) { - return compile(GenericElasticsearchScript.class, scriptName, scriptSource, params); + public T compile(String scriptName, String scriptSource, ScriptContext context, Map params) { + GenericElasticsearchScript painlessScript = compile(GenericElasticsearchScript.class, scriptName, scriptSource, params); + if (context.instanceClazz.equals(SearchScript.class)) { + SearchScript.Factory factory = (p, lookup) -> new SearchScript() { + @Override + public LeafSearchScript getLeafSearchScript(final LeafReaderContext context) throws IOException { + return new ScriptImpl(painlessScript, p, lookup.getLeafSearchLookup(context)); + } + @Override + public boolean needsScores() { + return painlessScript.uses$_score(); + } + }; + return context.factoryClazz.cast(factory); + } else if (context.instanceClazz.equals(ExecutableScript.class)) { + ExecutableScript.Factory factory = (p) -> new ScriptImpl(painlessScript, p, null); + return context.factoryClazz.cast(factory); + } + throw new IllegalArgumentException("painless does not know how to handle context [" + context.name + "]"); } T compile(Class iface, String scriptName, final String scriptSource, final Map params) { @@ -170,55 +185,6 @@ public T run() { } } - /** - * Retrieve an {@link ExecutableScript} for later use. - * @param compiledScript A previously compiled script. - * @param vars The variables to be used in the script. - * @return An {@link ExecutableScript} with the currently specified variables. - */ - @Override - public ExecutableScript executable(final CompiledScript compiledScript, final Map vars) { - return new ScriptImpl((GenericElasticsearchScript) compiledScript.compiled(), vars, null); - } - - /** - * Retrieve a {@link SearchScript} for later use. - * @param compiledScript A previously compiled script. - * @param lookup The object that ultimately allows access to search fields. - * @param vars The variables to be used in the script. - * @return An {@link SearchScript} with the currently specified variables. - */ - @Override - public SearchScript search(final CompiledScript compiledScript, final SearchLookup lookup, final Map vars) { - return new SearchScript() { - /** - * Get the search script that will have access to search field values. - * @param context The LeafReaderContext to be used. - * @return A script that will have the search fields from the current context available for use. - */ - @Override - public LeafSearchScript getLeafSearchScript(final LeafReaderContext context) throws IOException { - return new ScriptImpl((GenericElasticsearchScript) compiledScript.compiled(), vars, lookup.getLeafSearchLookup(context)); - } - - /** - * Whether or not the score is needed. - */ - @Override - public boolean needsScores() { - return ((GenericElasticsearchScript) compiledScript.compiled()).uses$_score(); - } - }; - } - - /** - * Action taken when the engine is closed. - */ - @Override - public void close() { - // Nothing to do. - } - private ScriptException convertToScriptException(String scriptName, String scriptSource, Throwable t) { // create a script stack: this is just the script portion List scriptStack = new ArrayList<>(); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java index 4611893b645d0..2549497c2f4b4 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java @@ -21,8 +21,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexService; -import org.elasticsearch.script.CompiledScript; -import org.elasticsearch.script.ScriptType; import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -41,26 +39,21 @@ public void testNeedsScores() { PainlessScriptEngine service = new PainlessScriptEngine(Settings.EMPTY); SearchLookup lookup = new SearchLookup(index.mapperService(), index.fieldData(), null); - Object compiled = service.compile(null, "1.2", Collections.emptyMap()); - SearchScript ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), - lookup, Collections.emptyMap()); + SearchScript.Factory factory = service.compile(null, "1.2", SearchScript.CONTEXT, Collections.emptyMap()); + SearchScript ss = factory.newInstance(Collections.emptyMap(), lookup); assertFalse(ss.needsScores()); - compiled = service.compile(null, "doc['d'].value", Collections.emptyMap()); - ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), - lookup, Collections.emptyMap()); + factory = service.compile(null, "doc['d'].value", SearchScript.CONTEXT, Collections.emptyMap()); + ss = factory.newInstance(Collections.emptyMap(), lookup); assertFalse(ss.needsScores()); - compiled = service.compile(null, "1/_score", Collections.emptyMap()); - ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), - lookup, Collections.emptyMap()); + factory = service.compile(null, "1/_score", SearchScript.CONTEXT, Collections.emptyMap()); + ss = factory.newInstance(Collections.emptyMap(), lookup); assertTrue(ss.needsScores()); - compiled = service.compile(null, "doc['d'].value * _score", Collections.emptyMap()); - ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), - lookup, Collections.emptyMap()); + factory = service.compile(null, "doc['d'].value * _score", SearchScript.CONTEXT, Collections.emptyMap()); + ss = factory.newInstance(Collections.emptyMap(), lookup); assertTrue(ss.needsScores()); - service.close(); } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java index 89385c8862cfc..37b1ff68ec542 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java @@ -19,9 +19,7 @@ package org.elasticsearch.painless; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.ScriptType; import java.util.Arrays; import java.util.Collections; @@ -81,10 +79,9 @@ public void testChangingVarsCrossExecution1() { Map ctx = new HashMap<>(); vars.put("ctx", ctx); - Object compiledScript = scriptEngine.compile(null, - "return ctx.value;", Collections.emptyMap()); - ExecutableScript script = scriptEngine.executable(new CompiledScript(ScriptType.INLINE, - "testChangingVarsCrossExecution1", "painless", compiledScript), vars); + ExecutableScript.Factory factory = + scriptEngine.compile(null, "return ctx.value;", ExecutableScript.CONTEXT, Collections.emptyMap()); + ExecutableScript script = factory.newInstance(vars); ctx.put("value", 1); Object o = script.run(); @@ -97,10 +94,9 @@ public void testChangingVarsCrossExecution1() { public void testChangingVarsCrossExecution2() { Map vars = new HashMap<>(); - Object compiledScript = scriptEngine.compile(null, "return params['value'];", Collections.emptyMap()); - - ExecutableScript script = scriptEngine.executable(new CompiledScript(ScriptType.INLINE, - "testChangingVarsCrossExecution2", "painless", compiledScript), vars); + ExecutableScript.Factory factory = + scriptEngine.compile(null, "return params['value'];", ExecutableScript.CONTEXT, Collections.emptyMap()); + ExecutableScript script = factory.newInstance(vars); script.setNextVar("value", 1); Object value = script.run(); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java index 27d7c5c94b71e..7e3eaf4c25413 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java @@ -25,10 +25,8 @@ import org.elasticsearch.common.lucene.ScorerAware; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.painless.antlr.Walker; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ScriptException; -import org.elasticsearch.script.ScriptType; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -87,9 +85,8 @@ public Object exec(String script, Map vars, Map c definition, null); } // test actual script execution - Object object = scriptEngine.compile(null, script, compileParams); - CompiledScript compiled = new CompiledScript(ScriptType.INLINE, getTestName(), "painless", object); - ExecutableScript executableScript = scriptEngine.executable(compiled, vars); + ExecutableScript.Factory factory = scriptEngine.compile(null, script, ExecutableScript.CONTEXT, compileParams); + ExecutableScript executableScript = factory.newInstance(vars); if (scorer != null) { ((ScorerAware)executableScript).setScorer(scorer); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java index 50fb925de68f3..f574236b351e1 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java @@ -126,7 +126,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeInt(maxChildren); out.writeVInt(scoreMode.ordinal()); out.writeNamedWriteable(query); - if (out.getVersion().before(Version.V_5_5_0_UNRELEASED)) { + if (out.getVersion().before(Version.V_5_5_0)) { final boolean hasInnerHit = innerHitBuilder != null; out.writeBoolean(hasInnerHit); if (hasInnerHit) { diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java index 559f5530c3e01..7bcbf7c611a1a 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java @@ -121,7 +121,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeString(type); out.writeBoolean(score); out.writeNamedWriteable(query); - if (out.getVersion().before(Version.V_5_5_0_UNRELEASED)) { + if (out.getVersion().before(Version.V_5_5_0)) { final boolean hasInnerHit = innerHitBuilder != null; out.writeBoolean(hasInnerHit); if (hasInnerHit) { diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java index 694c00e5c68e5..aef822ac00295 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java @@ -49,10 +49,8 @@ import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TypeFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.threadpool.ThreadPool; @@ -773,8 +771,8 @@ public RequestWrapper apply(RequestWrapper request, ScrollableHitSource.Hi return request; } if (executable == null) { - CompiledScript compiled = scriptService.compile(script, ScriptContext.UPDATE); - executable = scriptService.executable(compiled, params); + ExecutableScript.Factory factory = scriptService.compile(script, ExecutableScript.UPDATE_CONTEXT); + executable = factory.newInstance(params); } if (context == null) { context = new HashMap<>(); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java index 6ddf6daa8806c..955c99568c7d2 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java @@ -24,11 +24,9 @@ import org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.RequestWrapper; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ScriptService; import org.junit.Before; -import org.mockito.Matchers; import java.util.Map; import java.util.function.Consumer; @@ -36,6 +34,7 @@ import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -56,9 +55,9 @@ protected T applyScript(Consumer> IndexRequest index = new IndexRequest("index", "type", "1").source(singletonMap("foo", "bar")); ScrollableHitSource.Hit doc = new ScrollableHitSource.BasicHit("test", "type", "id", 0); ExecutableScript executableScript = new SimpleExecutableScript(scriptBody); - - when(scriptService.executable(any(CompiledScript.class), Matchers.>any())) - .thenReturn(executableScript); + ExecutableScript.Factory factory = params -> executableScript; + when(scriptService.compile(any(), eq(ExecutableScript.CONTEXT))).thenReturn(factory); + when(scriptService.compile(any(), eq(ExecutableScript.UPDATE_CONTEXT))).thenReturn(factory); AbstractAsyncBulkByScrollAction action = action(scriptService, request().setScript(mockScript(""))); RequestWrapper result = action.buildScriptApplier().apply(AbstractAsyncBulkByScrollAction.wrap(index), doc); return (result != null) ? (T) result.self() : null; diff --git a/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java b/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java index b06743b95e740..9166c0678fbe4 100644 --- a/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java +++ b/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java @@ -21,24 +21,19 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.util.Collection; import java.util.Map; -import java.util.Objects; -import java.util.function.Function; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; -import org.apache.lucene.search.Scorer; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; -import org.elasticsearch.script.CompiledScript; -import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.LeafSearchScript; +import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.script.SearchScript; -import org.elasticsearch.search.lookup.SearchLookup; /** * An example script plugin that adds a {@link ScriptEngine} implementing expert scoring. @@ -46,7 +41,7 @@ public class ExpertScriptPlugin extends Plugin implements ScriptPlugin { @Override - public ScriptEngine getScriptEngine(Settings settings) { + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { return new MyExpertScriptEngine(); } @@ -59,10 +54,13 @@ public String getType() { } @Override - public Function,SearchScript> compile(String scriptName, String scriptSource, Map params) { + public T compile(String scriptName, String scriptSource, ScriptContext context, Map params) { + if (context.equals(SearchScript.CONTEXT) == false) { + throw new IllegalArgumentException(getType() + " scripts cannot be used for context [" + context.name + "]"); + } // we use the script "source" as the script identifier if ("pure_df".equals(scriptSource)) { - return p -> new SearchScript() { + SearchScript.Factory factory = (p, lookup) -> new SearchScript() { final String field; final String term; { @@ -117,24 +115,15 @@ public boolean needsScores() { return false; } }; + return context.factoryClazz.cast(factory); } throw new IllegalArgumentException("Unknown script name " + scriptSource); } @Override - @SuppressWarnings("unchecked") - public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map params) { - Function,SearchScript> scriptFactory = (Function,SearchScript>) compiledScript.compiled(); - return scriptFactory.apply(params); - } - - @Override - public ExecutableScript executable(CompiledScript compiledScript, @Nullable Map params) { - throw new UnsupportedOperationException(); + public void close() { + // optionally close resources } - - @Override - public void close() {} } // end::expert_engine } diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 2848f1b2479ac..cf4a400a7591a 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -137,7 +137,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { Files.exists(path.resolve("bin").resolve("hdfs.dll"))) { fixtureSupported = true } else { - throw new IllegalStateException("HADOOP_HOME: " + path.toString() + " is invalid, does not contain hadoop native libraries in $HADOOP_HOME/bin"); + throw new IllegalStateException("HADOOP_HOME: ${path} is invalid, does not contain hadoop native libraries in \$HADOOP_HOME/bin"); } } } else { diff --git a/qa/full-cluster-restart/build.gradle b/qa/full-cluster-restart/build.gradle new file mode 100644 index 0000000000000..bbae27eaa36f3 --- /dev/null +++ b/qa/full-cluster-restart/build.gradle @@ -0,0 +1,98 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.elasticsearch.gradle.test.RestIntegTestTask +import org.elasticsearch.gradle.Version + +apply plugin: 'elasticsearch.standalone-test' + +// This is a top level task which we will add dependencies to below. +// It is a single task that can be used to backcompat tests against all versions. +task bwcTest { + description = 'Runs backwards compatibility tests.' + group = 'verification' +} + +for (Version version : indexCompatVersions) { + String baseName = "v${version}" + + Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) { + mustRunAfter(precommit) + } + tasks.getByName("${baseName}#oldClusterTestRunner").configure { + systemProperty 'tests.is_old_cluster', 'true' + systemProperty 'tests.old_cluster_version', version.toString().minus("-SNAPSHOT") + systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo") + } + + Object extension = extensions.findByName("${baseName}#oldClusterTestCluster") + configure(extensions.findByName("${baseName}#oldClusterTestCluster")) { + distribution = 'zip' + bwcVersion = version + numBwcNodes = 2 + numNodes = 2 + clusterName = 'full-cluster-restart' + if (version.onOrAfter('5.3.0')) { + setting 'http.content_type.required', 'true' + } + } + + + Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask) { + dependsOn(oldClusterTest, "${baseName}#oldClusterTestCluster#node0.stop") + } + + configure(extensions.findByName("${baseName}#upgradedClusterTestCluster")) { + dependsOn oldClusterTest, + "${baseName}#oldClusterTestCluster#node0.stop", + "${baseName}#oldClusterTestCluster#node1.stop" + distribution = 'zip' + clusterName = 'full-cluster-restart' + numNodes = 2 + dataDir = { nodeNum -> oldClusterTest.nodes[nodeNum].dataDir } + } + + tasks.getByName("${baseName}#upgradedClusterTestRunner").configure { + systemProperty 'tests.is_old_cluster', 'false' + systemProperty 'tests.old_cluster_version', version.toString().minus("-SNAPSHOT") + systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo") + } + + Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") { + dependsOn = [upgradedClusterTest] + } + + /* Delay this change because the task we need to modify isn't created until + * after projects are evaluated. */ + gradle.projectsEvaluated { + // Disable cleaning the repository so we can test loading a snapshot + tasks.getByName("${baseName}#upgradedClusterTestCluster#prepareCluster.cleanShared").enabled = false + } + + bwcTest.dependsOn(versionBwcTest) +} + +test.enabled = false // no unit tests for rolling upgrades, only the rest integration test + +// basic integ tests includes testing bwc against the most recent version +task integTest { + dependsOn = ["v${indexCompatVersions[-1]}#bwcTest"] +} + +check.dependsOn(integTest) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java new file mode 100644 index 0000000000000..ae901c308d506 --- /dev/null +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -0,0 +1,265 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.upgrades; + +import org.apache.http.ParseException; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.Version; +import org.elasticsearch.common.Booleans; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; +import static org.hamcrest.Matchers.containsString; + +/** + * Tests to run before and after a full cluster restart. This is run twice, + * one with {@code tests.is_old_cluster} set to {@code true} against a cluster + * of an older version. The cluster is shutdown and a cluster of the new + * version is started with the same data directories and then this is rerun + * with {@code tests.is_old_cluster} set to {@code false}. + */ +public class FullClusterRestartIT extends ESRestTestCase { + private static final String REPO = "/_snapshot/repo"; + + private final boolean runningAgainstOldCluster = Booleans.parseBoolean(System.getProperty("tests.is_old_cluster")); + private final Version oldClusterVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); + private final boolean supportsLenientBooleans = oldClusterVersion.onOrAfter(Version.V_6_0_0_alpha1); + + @Override + protected boolean preserveIndicesUponCompletion() { + return true; + } + + @Override + protected boolean preserveReposUponCompletion() { + return true; + } + + /** + * Tests that a single document survives. Super basic smoke test. + */ + public void testSingleDoc() throws IOException { + String docLocation = "/" + getTestName().toLowerCase(Locale.ROOT) + "/doc/1"; + String doc = "{\"test\": \"test\"}"; + + if (runningAgainstOldCluster) { + client().performRequest("PUT", docLocation, singletonMap("refresh", "true"), + new StringEntity(doc, ContentType.APPLICATION_JSON)); + } + + assertThat(EntityUtils.toString(client().performRequest("GET", docLocation).getEntity()), containsString(doc)); + } + + public void testRandomDocumentsAndSnapshot() throws IOException { + String testName = getTestName().toLowerCase(Locale.ROOT); + String index = testName + "_data"; + String infoDocument = "/" + testName + "_info/doc/info"; + + int count; + boolean shouldHaveTranslog; + if (runningAgainstOldCluster) { + count = between(200, 300); + /* We've had bugs in the past where we couldn't restore + * an index without a translog so we randomize whether + * or not we have one. */ + shouldHaveTranslog = randomBoolean(); + logger.info("Creating {} documents", count); + indexRandomDocuments(index, count, true); + createSnapshot(); + // Explicitly flush so we're sure to have a bunch of documents in the Lucene index + client().performRequest("POST", "/_flush"); + if (shouldHaveTranslog) { + // Update a few documents so we are sure to have a translog + indexRandomDocuments(index, count / 10, false /* Flushing here would invalidate the whole thing....*/); + } + + // Record how many documents we built so we can compare later + XContentBuilder infoDoc = JsonXContent.contentBuilder().startObject(); + infoDoc.field("count", count); + infoDoc.field("should_have_translog", shouldHaveTranslog); + infoDoc.endObject(); + client().performRequest("PUT", infoDocument, singletonMap("refresh", "true"), + new StringEntity(infoDoc.string(), ContentType.APPLICATION_JSON)); + } else { + // Load the number of documents that were written to the old cluster + String doc = EntityUtils.toString( + client().performRequest("GET", infoDocument, singletonMap("filter_path", "_source")).getEntity()); + Matcher m = Pattern.compile("\"count\":(\\d+)").matcher(doc); + assertTrue(doc, m.find()); + count = Integer.parseInt(m.group(1)); + m = Pattern.compile("\"should_have_translog\":(true|false)").matcher(doc); + assertTrue(doc, m.find()); + shouldHaveTranslog = Booleans.parseBoolean(m.group(1)); + } + + // Count the documents in the index to make sure we have as many as we put there + String countResponse = EntityUtils.toString( + client().performRequest("GET", "/" + index + "/_search", singletonMap("size", "0")).getEntity()); + assertThat(countResponse, containsString("\"total\":" + count)); + + if (false == runningAgainstOldCluster) { + assertTranslogRecoveryStatistics(index, shouldHaveTranslog); + } + + restoreSnapshot(index, count); + + // TODO finish adding tests for the things in OldIndexBackwardsCompatibilityIT + } + + // TODO tests for upgrades after shrink. We've had trouble with shrink in the past. + + private void indexRandomDocuments(String index, int count, boolean flushAllowed) throws IOException { + for (int i = 0; i < count; i++) { + XContentBuilder doc = JsonXContent.contentBuilder().startObject(); { + doc.field("string", randomAlphaOfLength(10)); + doc.field("int", randomInt(100)); + doc.field("float", randomFloat()); + // be sure to create a "proper" boolean (True, False) for the first document so that automapping is correct + doc.field("bool", i > 0 && supportsLenientBooleans ? randomLenientBoolean() : randomBoolean()); + doc.field("field.with.dots", randomAlphaOfLength(10)); + // TODO a binary field + } + doc.endObject(); + client().performRequest("POST", "/" + index + "/doc/" + i, emptyMap(), + new StringEntity(doc.string(), ContentType.APPLICATION_JSON)); + if (rarely()) { + client().performRequest("POST", "/_refresh"); + } + if (flushAllowed && rarely()) { + client().performRequest("POST", "/_flush"); + } + } + } + + private void createSnapshot() throws IOException { + XContentBuilder repoConfig = JsonXContent.contentBuilder().startObject(); { + repoConfig.field("type", "fs"); + repoConfig.startObject("settings"); { + repoConfig.field("compress", randomBoolean()); + repoConfig.field("location", System.getProperty("tests.path.repo")); + } + repoConfig.endObject(); + } + repoConfig.endObject(); + client().performRequest("PUT", REPO, emptyMap(), new StringEntity(repoConfig.string(), ContentType.APPLICATION_JSON)); + + client().performRequest("PUT", REPO + "/snap", singletonMap("wait_for_completion", "true")); + } + + private void assertTranslogRecoveryStatistics(String index, boolean shouldHaveTranslog) throws ParseException, IOException { + boolean restoredFromTranslog = false; + boolean foundPrimary = false; + Map params = new HashMap<>(); + params.put("h", "index,shard,type,stage,translog_ops_recovered"); + params.put("s", "index,shard,type"); + String recoveryResponse = EntityUtils.toString(client().performRequest("GET", "/_cat/recovery/" + index, params).getEntity()); + for (String line : recoveryResponse.split("\n")) { + // Find the primaries + foundPrimary = true; + if (false == line.contains("done") && line.contains("existing_store")) { + continue; + } + /* Mark if we see a primary that looked like it restored from the translog. + * Not all primaries will look like this all the time because we modify + * random documents when we want there to be a translog and they might + * not be spread around all the shards. */ + Matcher m = Pattern.compile("(\\d+)$").matcher(line); + assertTrue(line, m.find()); + int translogOps = Integer.parseInt(m.group(1)); + if (translogOps > 0) { + restoredFromTranslog = true; + } + } + assertTrue("expected to find a primary but didn't\n" + recoveryResponse, foundPrimary); + assertEquals("mismatch while checking for translog recovery\n" + recoveryResponse, shouldHaveTranslog, restoredFromTranslog); + + String currentLuceneVersion = Version.CURRENT.luceneVersion.toString(); + String bwcLuceneVersion = oldClusterVersion.luceneVersion.toString(); + if (shouldHaveTranslog && false == currentLuceneVersion.equals(bwcLuceneVersion)) { + int numCurrentVersion = 0; + int numBwcVersion = 0; + params.clear(); + params.put("h", "prirep,shard,index,version"); + params.put("s", "prirep,shard,index"); + String segmentsResponse = EntityUtils.toString( + client().performRequest("GET", "/_cat/segments/" + index, params).getEntity()); + for (String line : segmentsResponse.split("\n")) { + if (false == line.startsWith("p")) { + continue; + } + Matcher m = Pattern.compile("(\\d+\\.\\d+\\.\\d+)$").matcher(line); + assertTrue(line, m.find()); + String version = m.group(1); + if (currentLuceneVersion.equals(version)) { + numCurrentVersion++; + } else if (bwcLuceneVersion.equals(version)) { + numBwcVersion++; + } else { + fail("expected version to be one of [" + currentLuceneVersion + "," + bwcLuceneVersion + "] but was " + line); + } + } + assertNotEquals("expected at least 1 current segment after translog recovery", 0, numCurrentVersion); + assertNotEquals("expected at least 1 old segment", 0, numBwcVersion); + } + } + + private void restoreSnapshot(String index, int count) throws ParseException, IOException { + if (false == runningAgainstOldCluster) { + /* Remove any "restored" indices from the old cluster run of this test. + * We intentionally don't remove them while running this against the + * old cluster so we can test starting the node with a restored index + * in the cluster. */ + client().performRequest("DELETE", "/restored_*"); + } + + if (runningAgainstOldCluster) { + // TODO restoring the snapshot seems to fail! This seems like a bug. + XContentBuilder restoreCommand = JsonXContent.contentBuilder().startObject(); + restoreCommand.field("include_global_state", false); + restoreCommand.field("indices", index); + restoreCommand.field("rename_pattern", index); + restoreCommand.field("rename_replacement", "restored_" + index); + restoreCommand.endObject(); + client().performRequest("POST", REPO + "/snap/_restore", singletonMap("wait_for_completion", "true"), + new StringEntity(restoreCommand.string(), ContentType.APPLICATION_JSON)); + + String countResponse = EntityUtils.toString( + client().performRequest("GET", "/restored_" + index + "/_search", singletonMap("size", "0")).getEntity()); + assertThat(countResponse, containsString("\"total\":" + count)); + } + + } + + private Object randomLenientBoolean() { + return randomFrom(new Object[] {"off", "no", "0", 0, "false", false, "on", "yes", "1", 1, "true", true}); + } +} diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java index 0954682094cb3..1216b25178eba 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java @@ -207,7 +207,7 @@ public void testSeqNoCheckpoints() throws Exception { .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2) .put("index.routing.allocation.include._name", bwcNames); - final boolean checkGlobalCheckpoints = nodes.getMaster().getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED); + final boolean checkGlobalCheckpoints = nodes.getMaster().getVersion().onOrAfter(Version.V_6_0_0_alpha1); logger.info("master version is [{}], global checkpoints will be [{}]", nodes.getMaster().getVersion(), checkGlobalCheckpoints ? "checked" : "not be checked"); final String index = "test"; @@ -287,7 +287,7 @@ private void assertSeqNoOnShards(String index, Nodes nodes, boolean checkGlobalC final long expectedGlobalCkp; final long expectMaxSeqNo; logger.info("primary resolved to node {}", primaryShard.getNode()); - if (primaryShard.getNode().getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (primaryShard.getNode().getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { expectMaxSeqNo = numDocs - 1; expectedGlobalCkp = numDocs - 1; } else { @@ -295,7 +295,7 @@ private void assertSeqNoOnShards(String index, Nodes nodes, boolean checkGlobalC expectMaxSeqNo = SequenceNumbersService.NO_OPS_PERFORMED; } for (Shard shard : shards) { - if (shard.getNode().getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (shard.getNode().getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { final SeqNoStats seqNoStats = shard.getSeqNoStats(); logger.info("stats for {}, primary [{}]: [{}]", shard.getNode(), shard.isPrimary(), seqNoStats); assertThat("max_seq no on " + shard.getNode() + " is wrong", seqNoStats.getMaxSeqNo(), equalTo(expectMaxSeqNo)); @@ -324,7 +324,7 @@ private List buildShards(String index, Nodes nodes, RestClient client) th final Boolean primary = ObjectPath.evaluate(shard, "routing.primary"); final Node node = nodes.getSafe(nodeId); final SeqNoStats seqNoStats; - if (node.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { + if (node.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { Integer maxSeqNo = ObjectPath.evaluate(shard, "seq_no.max_seq_no"); Integer localCheckpoint = ObjectPath.evaluate(shard, "seq_no.local_checkpoint"); Integer globalCheckpoint = ObjectPath.evaluate(shard, "seq_no.global_checkpoint"); diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 74e830e61ba28..03cbf24bdcf26 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -61,7 +61,9 @@ for (Version version : wireCompatVersions) { distribution = 'zip' clusterName = 'rolling-upgrade' unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() } - dataDir = "${-> oldClusterTest.nodes[1].dataDir}" + /* Override the data directory so the new node always gets the node we + * just stopped's data directory. */ + dataDir = { nodeNumber -> oldClusterTest.nodes[1].dataDir } setting 'repositories.url.allowed_urls', 'http://snapshot.test*' } @@ -79,7 +81,9 @@ for (Version version : wireCompatVersions) { distribution = 'zip' clusterName = 'rolling-upgrade' unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() } - dataDir = "${-> oldClusterTest.nodes[0].dataDir}" + /* Override the data directory so the new node always gets the node we + * just stopped's data directory. */ + dataDir = { nodeNumber -> oldClusterTest.nodes[0].dataDir} setting 'repositories.url.allowed_urls', 'http://snapshot.test*' } diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java index 18f04195898c6..c009c46c57e7a 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java +++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.mustache.MustacheScriptEngine; import org.elasticsearch.test.ESTestCase; @@ -38,7 +39,7 @@ public abstract class AbstractScriptTestCase extends ESTestCase { public void init() throws Exception { MustacheScriptEngine engine = new MustacheScriptEngine(); Map engines = Collections.singletonMap(engine.getType(), engine); - ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptContext.BUILTINS); + ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); templateService = new InternalTemplateService(scriptService); } diff --git a/qa/verify-version-constants/build.gradle b/qa/verify-version-constants/build.gradle new file mode 100644 index 0000000000000..d3b0f7f99cf7c --- /dev/null +++ b/qa/verify-version-constants/build.gradle @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.test.RestIntegTestTask + +apply plugin: 'elasticsearch.standalone-test' + +// This is a top level task which we will add dependencies to below. +// It is a single task that can be used to backcompat tests against all versions. +task bwcTest { + description = 'Runs backwards compatibility tests.' + group = 'verification' +} + +for (Version version : indexCompatVersions) { + String baseName = "v${version}" + Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) { + mustRunAfter(precommit) + } + configure(extensions.findByName("${baseName}#oldClusterTestCluster")) { + distribution = 'zip' + bwcVersion = version + numBwcNodes = 1 + numNodes = 1 + clusterName = 'verify-version-constants' + if (version.onOrAfter('5.3.0')) { + setting 'http.content_type.required', 'true' + } + } + + Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") { + dependsOn = [oldClusterTest] + } + + bwcTest.dependsOn(versionBwcTest) +} + +test.enabled = false + +task integTest { + dependsOn = ["v${indexCompatVersions[-1]}#bwcTest"] +} + +check.dependsOn(integTest) diff --git a/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java b/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java new file mode 100644 index 0000000000000..a26237962bf4b --- /dev/null +++ b/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.qa.verify_version_constants; + +import org.elasticsearch.Version; +import org.elasticsearch.client.Response; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.yaml.ObjectPath; + +import java.io.IOException; +import java.text.ParseException; + +import static org.hamcrest.CoreMatchers.equalTo; + +public class VerifyVersionConstantsIT extends ESRestTestCase { + + public void testLuceneVersionConstant() throws IOException, ParseException { + final Response response = client().performRequest("GET", "/"); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + final ObjectPath objectPath = ObjectPath.createFromResponse(response); + final String elasticsearchVersionString = objectPath.evaluate("version.number").toString(); + final Version elasticsearchVersion = Version.fromString(elasticsearchVersionString); + final String luceneVersionString = objectPath.evaluate("version.lucene_version").toString(); + final org.apache.lucene.util.Version luceneVersion = org.apache.lucene.util.Version.parse(luceneVersionString); + assertThat(elasticsearchVersion.luceneVersion, equalTo(luceneVersion)); + } + +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/90_sig_text.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/90_sig_text.yml new file mode 100644 index 0000000000000..bfbf171e8cc34 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/90_sig_text.yml @@ -0,0 +1,176 @@ +--- +"Default index": + + - skip: + version: " - 5.99.99" + reason: this uses a new feature that has been added in 6.0.0 + + - do: + indices.create: + index: goodbad + body: + settings: + number_of_shards: "1" + mappings: + doc: + properties: + text: + type: text + fielddata: false + class: + type: keyword + + - do: + index: + index: goodbad + type: doc + id: 1 + body: { text: "good", class: "good" } + - do: + index: + index: goodbad + type: doc + id: 2 + body: { text: "good", class: "good" } + - do: + index: + index: goodbad + type: doc + id: 3 + body: { text: "bad", class: "bad" } + - do: + index: + index: goodbad + type: doc + id: 4 + body: { text: "bad", class: "bad" } + - do: + index: + index: goodbad + type: doc + id: 5 + body: { text: "good bad", class: "good" } + - do: + index: + index: goodbad + type: doc + id: 6 + body: { text: "good bad", class: "bad" } + - do: + index: + index: goodbad + type: doc + id: 7 + body: { text: "bad", class: "bad" } + + + + - do: + indices.refresh: + index: [goodbad] + + - do: + search: + index: goodbad + type: doc + + - match: {hits.total: 7} + + - do: + search: + index: goodbad + type: doc + body: {"aggs": {"class": {"terms": {"field": "class"},"aggs": {"sig_text": {"significant_text": {"field": "text"}}}}}} + + - match: {aggregations.class.buckets.0.sig_text.buckets.0.key: "bad"} + - match: {aggregations.class.buckets.1.sig_text.buckets.0.key: "good"} + +--- +"Dedup noise": + + - skip: + version: " - 5.99.99" + reason: this uses a new feature that has been added in 6.0.0 + + - do: + indices.create: + index: goodbad + body: + settings: + number_of_shards: "1" + mappings: + doc: + properties: + text: + type: text + fielddata: false + class: + type: keyword + + - do: + index: + index: goodbad + type: doc + id: 1 + body: { text: "good noisewords1 g1 g2 g3 g4 g5 g6", class: "good" } + - do: + index: + index: goodbad + type: doc + id: 2 + body: { text: "good noisewords2 g1 g2 g3 g4 g5 g6", class: "good" } + - do: + index: + index: goodbad + type: doc + id: 3 + body: { text: "bad noisewords3 b1 b2 b3 b4 b5 b6", class: "bad" } + - do: + index: + index: goodbad + type: doc + id: 4 + body: { text: "bad noisewords4 b1 b2 b3 b4 b5 b6", class: "bad" } + - do: + index: + index: goodbad + type: doc + id: 5 + body: { text: "good bad noisewords5 gb1 gb2 gb3 gb4 gb5 gb6", class: "good" } + - do: + index: + index: goodbad + type: doc + id: 6 + body: { text: "good bad noisewords6 gb1 gb2 gb3 gb4 gb5 gb6", class: "bad" } + - do: + index: + index: goodbad + type: doc + id: 7 + body: { text: "bad noisewords7 b1 b2 b3 b4 b5 b6", class: "bad" } + + + + - do: + indices.refresh: + index: [goodbad] + + - do: + search: + index: goodbad + type: doc + + - match: {hits.total: 7} + + - do: + search: + index: goodbad + type: doc + body: {"aggs": {"class": {"terms": {"field": "class"},"aggs": {"sig_text": {"significant_text": {"field": "text", "filter_duplicate_text": true}}}}}} + + - match: {aggregations.class.buckets.0.sig_text.buckets.0.key: "bad"} + - length: { aggregations.class.buckets.0.sig_text.buckets: 1 } + - match: {aggregations.class.buckets.1.sig_text.buckets.0.key: "good"} + - length: { aggregations.class.buckets.1.sig_text.buckets: 1 } + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml index 9d3fc349a23ba..77ba43198bf5e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml @@ -107,8 +107,8 @@ setup: "field collapsing and inner_hits": - skip: - version: " - 5.2.99" - reason: this uses a new API that has been added in 5.3 + version: " - 5.99.99" + reason: disable this test temporary due to a pending backport (#24517) - do: search: @@ -265,3 +265,62 @@ setup: - match: { hits.total: 6 } - length: { hits.hits: 0 } + +--- +"field collapsing and multiple inner_hits": + + - skip: + version: " - 5.4.99" + reason: Multiple inner_hits is a new feature added in 5.5 + + - do: + search: + index: test + type: test + body: + collapse: { + field: numeric_group, + inner_hits: [ + { name: sub_hits_asc, size: 2, sort: [{ sort: asc }] }, + { name: sub_hits_desc, size: 1, sort: [{ sort: desc }] } + ] + } + sort: [{ sort: desc }] + + - match: { hits.total: 6 } + - length: { hits.hits: 3 } + - match: { hits.hits.0._index: test } + - match: { hits.hits.0._type: test } + - match: { hits.hits.0.fields.numeric_group: [3] } + - match: { hits.hits.0.sort: [36] } + - match: { hits.hits.0._id: "6" } + - match: { hits.hits.0.inner_hits.sub_hits_asc.hits.total: 1 } + - length: { hits.hits.0.inner_hits.sub_hits_asc.hits.hits: 1 } + - match: { hits.hits.0.inner_hits.sub_hits_asc.hits.hits.0._id: "6" } + - match: { hits.hits.0.inner_hits.sub_hits_desc.hits.total: 1 } + - length: { hits.hits.0.inner_hits.sub_hits_desc.hits.hits: 1 } + - match: { hits.hits.0.inner_hits.sub_hits_desc.hits.hits.0._id: "6" } + - match: { hits.hits.1._index: test } + - match: { hits.hits.1._type: test } + - match: { hits.hits.1.fields.numeric_group: [1] } + - match: { hits.hits.1.sort: [24] } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.1.inner_hits.sub_hits_asc.hits.total: 3 } + - length: { hits.hits.1.inner_hits.sub_hits_asc.hits.hits: 2 } + - match: { hits.hits.1.inner_hits.sub_hits_asc.hits.hits.0._id: "2" } + - match: { hits.hits.1.inner_hits.sub_hits_asc.hits.hits.1._id: "1" } + - match: { hits.hits.1.inner_hits.sub_hits_desc.hits.total: 3 } + - length: { hits.hits.1.inner_hits.sub_hits_desc.hits.hits: 1 } + - match: { hits.hits.1.inner_hits.sub_hits_desc.hits.hits.0._id: "3" } + - match: { hits.hits.2._index: test } + - match: { hits.hits.2._type: test } + - match: { hits.hits.2.fields.numeric_group: [25] } + - match: { hits.hits.2.sort: [10] } + - match: { hits.hits.2._id: "4" } + - match: { hits.hits.2.inner_hits.sub_hits_asc.hits.total: 2 } + - length: { hits.hits.2.inner_hits.sub_hits_asc.hits.hits: 2 } + - match: { hits.hits.2.inner_hits.sub_hits_asc.hits.hits.0._id: "5" } + - match: { hits.hits.2.inner_hits.sub_hits_asc.hits.hits.1._id: "4" } + - match: { hits.hits.2.inner_hits.sub_hits_desc.hits.total: 2 } + - length: { hits.hits.2.inner_hits.sub_hits_desc.hits.hits: 1 } + - match: { hits.hits.2.inner_hits.sub_hits_desc.hits.hits.0._id: "4" } diff --git a/settings.gradle b/settings.gradle index 7977686a24755..9fc33b59f326f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -15,7 +15,7 @@ List projects = [ 'client:benchmark', 'benchmarks', 'distribution:integ-test-zip', - 'distribution:bwc-zip', + 'distribution:bwc', 'distribution:zip', 'distribution:tar', 'distribution:deb', @@ -62,6 +62,7 @@ List projects = [ 'plugins:store-smb', 'qa:auto-create-index', 'qa:evil-tests', + 'qa:full-cluster-restart', 'qa:mixed-cluster', 'qa:multi-cluster-search', 'qa:no-bootstrap-tests', @@ -76,6 +77,7 @@ List projects = [ 'qa:smoke-test-reindex-with-all-modules', 'qa:smoke-test-tribe-node', 'qa:vagrant', + 'qa:verify-version-constants', 'qa:wildfly' ] diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index dbfa77635ab33..515e01c0409e3 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -489,7 +489,8 @@ protected Engine.Index indexDoc(IndexShard shard, String type, String id, String index = shard.prepareIndexOnReplica( SourceToParse.source(shard.shardId().getIndexName(), type, id, new BytesArray(source), xContentType), - randomInt(1 << 10), 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false); + shard.seqNoStats().getMaxSeqNo() + 1, shard.getPrimaryTerm(), 0, + VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false); } shard.index(index); return index; diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java index c34a858791bad..97d8ef8122b74 100644 --- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java +++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java @@ -21,7 +21,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Scorer; -import org.elasticsearch.common.Nullable; import org.elasticsearch.search.lookup.LeafSearchLookup; import org.elasticsearch.search.lookup.SearchLookup; @@ -67,7 +66,7 @@ public String getType() { } @Override - public Object compile(String name, String source, Map params) { + public T compile(String name, String source, ScriptContext context, Map params) { // Scripts are always resolved using the script's source. For inline scripts, it's easy because they don't have names and the // source is always provided. For stored and file scripts, the source of the script must match the key of a predefined script. Function, Object> script = scripts.get(source); @@ -75,23 +74,15 @@ public Object compile(String name, String source, Map params) { throw new IllegalArgumentException("No pre defined script matching [" + source + "] for script with name [" + name + "], " + "did you declare the mocked script?"); } - return new MockCompiledScript(name, params, source, script); - } - - @Override - public ExecutableScript executable(CompiledScript compiledScript, @Nullable Map vars) { - MockCompiledScript compiled = (MockCompiledScript) compiledScript.compiled(); - return compiled.createExecutableScript(vars); - } - - @Override - public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map vars) { - MockCompiledScript compiled = (MockCompiledScript) compiledScript.compiled(); - return compiled.createSearchScript(vars, lookup); - } - - @Override - public void close() throws IOException { + MockCompiledScript mockCompiled = new MockCompiledScript(name, params, source, script); + if (context.instanceClazz.equals(SearchScript.class)) { + SearchScript.Factory factory = mockCompiled::createSearchScript; + return context.factoryClazz.cast(factory); + } else if (context.instanceClazz.equals(ExecutableScript.class)) { + ExecutableScript.Factory factory = mockCompiled::createExecutableScript; + return context.factoryClazz.cast(factory); + } + throw new IllegalArgumentException("mock script engine does not know how to handle context [" + context.name + "]"); } public class MockCompiledScript { diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptPlugin.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptPlugin.java index ff022e1d73173..cd951a3b53f5c 100644 --- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptPlugin.java +++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptPlugin.java @@ -23,6 +23,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; +import java.util.Collection; import java.util.Map; import java.util.function.Function; @@ -34,7 +35,7 @@ public abstract class MockScriptPlugin extends Plugin implements ScriptPlugin { public static final String NAME = MockScriptEngine.NAME; @Override - public ScriptEngine getScriptEngine(Settings settings) { + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { return new MockScriptEngine(pluginScriptLang(), pluginScripts()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java index c38efca52037a..354501fff4a17 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.test; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; @@ -93,7 +94,11 @@ public void testSerialization() throws IOException { * Serialize the given instance and asserts that both are equal */ protected T assertSerialization(T testInstance) throws IOException { - T deserializedInstance = copyInstance(testInstance); + return assertSerialization(testInstance, Version.CURRENT); + } + + protected T assertSerialization(T testInstance, Version version) throws IOException { + T deserializedInstance = copyInstance(testInstance, version); assertEquals(testInstance, deserializedInstance); assertEquals(testInstance.hashCode(), deserializedInstance.hashCode()); assertNotSame(testInstance, deserializedInstance); @@ -101,10 +106,16 @@ protected T assertSerialization(T testInstance) throws IOException { } protected T copyInstance(T instance) throws IOException { + return copyInstance(instance, Version.CURRENT); + } + + protected T copyInstance(T instance, Version version) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { + output.setVersion(version); instance.writeTo(output); try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), getNamedWriteableRegistry())) { + in.setVersion(version); return instanceReader().read(in); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index f7f641c6a292c..256227779aa0e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -94,6 +94,7 @@ import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; @@ -1195,7 +1196,7 @@ public static TestAnalysis createTestAnalysis(IndexSettings indexSettings, Setti public static ScriptModule newTestScriptModule() { return new ScriptModule(Settings.EMPTY, singletonList(new ScriptPlugin() { @Override - public ScriptEngine getScriptEngine(Settings settings) { + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { return new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap("1", script -> "1")); } })); diff --git a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java index 5d8af02a06621..9f93a63919daa 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java @@ -20,60 +20,101 @@ package org.elasticsearch.test; import org.elasticsearch.Version; +import org.elasticsearch.common.collect.Tuple; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Random; -import java.util.Set; -import java.util.stream.Collectors; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static java.util.Collections.unmodifiableList; /** Utilities for selecting versions in tests */ public class VersionUtils { - - private static final List RELEASED_VERSIONS; - private static final List UNRELEASED_VERSIONS; - - static { - final Field[] declaredFields = Version.class.getFields(); - final Set releasedIdsSet = new HashSet<>(); - final Set unreleasedIdsSet = new HashSet<>(); - for (final Field field : declaredFields) { + /** + * Sort versions that have backwards compatibility guarantees from + * those that don't. Doesn't actually check whether or not the versions + * are released, instead it relies on gradle to have already checked + * this which it does in {@code :core:verifyVersions}. So long as the + * rules here match up with the rules in gradle then this should + * produce sensible results. + * @return a tuple containing versions with backwards compatibility + * guarantees in v1 and versions without the guranteees in v2 + */ + static Tuple, List> resolveReleasedVersions(Version current, Class versionClass) { + Field[] fields = versionClass.getFields(); + List versions = new ArrayList<>(fields.length); + for (final Field field : fields) { final int mod = field.getModifiers(); - if (Modifier.isStatic(mod) && Modifier.isFinal(mod) && Modifier.isPublic(mod)) { - if (field.getType() == Version.class) { - final int id; - try { - id = ((Version) field.get(null)).id; - } catch (final IllegalAccessException e) { - throw new RuntimeException(e); - } - assert field.getName().matches("(V(_\\d+)+(_(alpha|beta|rc)\\d+)?(_UNRELEASED)?|CURRENT)") : field.getName(); - // note that below we remove CURRENT and add it to released; we do it this way because there are two constants that - // correspond to CURRENT, CURRENT itself and the actual version that CURRENT points to - if (field.getName().equals("CURRENT") || field.getName().endsWith("UNRELEASED")) { - unreleasedIdsSet.add(id); - } else { - releasedIdsSet.add(id); - } - } + if (false == Modifier.isStatic(mod) && Modifier.isFinal(mod) && Modifier.isPublic(mod)) { + continue; + } + if (field.getType() != Version.class) { + continue; + } + assert field.getName().matches("(V(_\\d+)+(_(alpha|beta|rc)\\d+)?|CURRENT)") : field.getName(); + if ("CURRENT".equals(field.getName())) { + continue; + } + try { + versions.add(((Version) field.get(null))); + } catch (final IllegalAccessException e) { + throw new RuntimeException(e); } } + Collections.sort(versions); + assert versions.get(versions.size() - 1).equals(current) : "The highest version must be the current one " + + "but was [" + versions.get(versions.size() - 1) + "] and current was [" + current + "]"; + + if (current.revision != 0) { + /* If we are in a stable branch there should be no unreleased version constants + * because we don't expect to release any new versions in older branches. If there + * are extra constants then gradle will yell about it. */ + return new Tuple<>(unmodifiableList(versions), emptyList()); + } + + /* If we are on a patch release then we know that at least the version before the + * current one is unreleased. If it is released then gradle would be complaining. */ + int unreleasedIndex = versions.size() - 2; + while (true) { + if (unreleasedIndex < 0) { + throw new IllegalArgumentException("Couldn't find first non-alpha release"); + } + /* Technically we don't support backwards compatiblity for alphas, betas, + * and rcs. But the testing infrastructure requires that we act as though we + * do. This is a difference between the gradle and Java logic but should be + * fairly safe as it is errs on us being more compatible rather than less.... + * Anyway, the upshot is that we never declare alphas as unreleased, no + * matter where they are in the list. */ + if (versions.get(unreleasedIndex).isRelease()) { + break; + } + unreleasedIndex--; + } - // treat CURRENT as released for BWC testing - unreleasedIdsSet.remove(Version.CURRENT.id); - releasedIdsSet.add(Version.CURRENT.id); + Version unreleased = versions.remove(unreleasedIndex); + if (unreleased.revision == 0) { + /* If the last unreleased version is itself a patch release then gradle enforces + * that there is yet another unreleased version before that. */ + unreleasedIndex--; + Version earlierUnreleased = versions.remove(unreleasedIndex); + return new Tuple<>(unmodifiableList(versions), unmodifiableList(Arrays.asList(earlierUnreleased, unreleased))); + } + return new Tuple<>(unmodifiableList(versions), singletonList(unreleased)); + } - // unreleasedIdsSet and releasedIdsSet should be disjoint - assert unreleasedIdsSet.stream().filter(releasedIdsSet::contains).collect(Collectors.toSet()).isEmpty(); + private static final List RELEASED_VERSIONS; + private static final List UNRELEASED_VERSIONS; - RELEASED_VERSIONS = - Collections.unmodifiableList(releasedIdsSet.stream().sorted().map(Version::fromId).collect(Collectors.toList())); - UNRELEASED_VERSIONS = - Collections.unmodifiableList(unreleasedIdsSet.stream().sorted().map(Version::fromId).collect(Collectors.toList())); + static { + Tuple, List> versions = resolveReleasedVersions(Version.CURRENT, Version.class); + RELEASED_VERSIONS = versions.v1(); + UNRELEASED_VERSIONS = versions.v2(); } /** diff --git a/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java b/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java new file mode 100644 index 0000000000000..4b59abc0a31c4 --- /dev/null +++ b/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java @@ -0,0 +1,162 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.test; + +import org.elasticsearch.Version; +import org.elasticsearch.common.collect.Tuple; + +import java.util.Arrays; +import java.util.List; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; + +public class VersionUtilsTests extends ESTestCase { + + public void testAllVersionsSorted() { + List allVersions = VersionUtils.allReleasedVersions(); + for (int i = 0, j = 1; j < allVersions.size(); ++i, ++j) { + assertTrue(allVersions.get(i).before(allVersions.get(j))); + } + } + + public void testRandomVersionBetween() { + // full range + Version got = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), Version.CURRENT); + assertTrue(got.onOrAfter(VersionUtils.getFirstVersion())); + assertTrue(got.onOrBefore(Version.CURRENT)); + got = VersionUtils.randomVersionBetween(random(), null, Version.CURRENT); + assertTrue(got.onOrAfter(VersionUtils.getFirstVersion())); + assertTrue(got.onOrBefore(Version.CURRENT)); + got = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), null); + assertTrue(got.onOrAfter(VersionUtils.getFirstVersion())); + assertTrue(got.onOrBefore(Version.CURRENT)); + + // sub range + got = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, + Version.V_6_0_0_alpha2); + assertTrue(got.onOrAfter(Version.V_5_0_0)); + assertTrue(got.onOrBefore(Version.V_6_0_0_alpha2)); + + // unbounded lower + got = VersionUtils.randomVersionBetween(random(), null, Version.V_6_0_0_alpha2); + assertTrue(got.onOrAfter(VersionUtils.getFirstVersion())); + assertTrue(got.onOrBefore(Version.V_6_0_0_alpha2)); + got = VersionUtils.randomVersionBetween(random(), null, VersionUtils.allReleasedVersions().get(0)); + assertTrue(got.onOrAfter(VersionUtils.getFirstVersion())); + assertTrue(got.onOrBefore(VersionUtils.allReleasedVersions().get(0))); + + // unbounded upper + got = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, null); + assertTrue(got.onOrAfter(Version.V_5_0_0)); + assertTrue(got.onOrBefore(Version.CURRENT)); + got = VersionUtils.randomVersionBetween(random(), VersionUtils.getPreviousVersion(), null); + assertTrue(got.onOrAfter(VersionUtils.getPreviousVersion())); + assertTrue(got.onOrBefore(Version.CURRENT)); + + // range of one + got = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getFirstVersion()); + assertEquals(got, VersionUtils.getFirstVersion()); + got = VersionUtils.randomVersionBetween(random(), Version.CURRENT, Version.CURRENT); + assertEquals(got, Version.CURRENT); + got = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0_alpha2, + Version.V_6_0_0_alpha2); + assertEquals(got, Version.V_6_0_0_alpha2); + + // implicit range of one + got = VersionUtils.randomVersionBetween(random(), null, VersionUtils.getFirstVersion()); + assertEquals(got, VersionUtils.getFirstVersion()); + got = VersionUtils.randomVersionBetween(random(), Version.CURRENT, null); + assertEquals(got, Version.CURRENT); + } + + static class TestReleaseBranch { + public static final Version V_5_3_0 = Version.fromString("5.3.0"); + public static final Version V_5_3_1 = Version.fromString("5.3.1"); + public static final Version V_5_3_2 = Version.fromString("5.3.2"); + public static final Version V_5_4_0 = Version.fromString("5.4.0"); + public static final Version V_5_4_1 = Version.fromString("5.4.1"); + public static final Version CURRENT = V_5_4_1; + } + public void testResolveReleasedVersionsForReleaseBranch() { + Tuple, List> t = VersionUtils.resolveReleasedVersions(TestReleaseBranch.CURRENT, TestReleaseBranch.class); + List released = t.v1(); + List unreleased = t.v2(); + assertEquals(Arrays.asList(TestReleaseBranch.V_5_3_0, TestReleaseBranch.V_5_3_1, TestReleaseBranch.V_5_3_2, + TestReleaseBranch.V_5_4_0, TestReleaseBranch.V_5_4_1), released); + assertEquals(emptyList(), unreleased); + } + + static class TestStableBranch { + public static final Version V_5_3_0 = Version.fromString("5.3.0"); + public static final Version V_5_3_1 = Version.fromString("5.3.1"); + public static final Version V_5_3_2 = Version.fromString("5.3.2"); + public static final Version V_5_4_0 = Version.fromString("5.4.0"); + public static final Version CURRENT = V_5_4_0; + } + public void testResolveReleasedVersionsForUnreleasedStableBranch() { + Tuple, List> t = VersionUtils.resolveReleasedVersions(TestStableBranch.CURRENT, + TestStableBranch.class); + List released = t.v1(); + List unreleased = t.v2(); + assertEquals( + Arrays.asList(TestStableBranch.V_5_3_0, TestStableBranch.V_5_3_1, TestStableBranch.V_5_4_0), + released); + assertEquals(singletonList(TestStableBranch.V_5_3_2), unreleased); + } + + static class TestStableBranchBehindStableBranch { + public static final Version V_5_3_0 = Version.fromString("5.3.0"); + public static final Version V_5_3_1 = Version.fromString("5.3.1"); + public static final Version V_5_3_2 = Version.fromString("5.3.2"); + public static final Version V_5_4_0 = Version.fromString("5.4.0"); + public static final Version V_5_5_0 = Version.fromString("5.5.0"); + public static final Version CURRENT = V_5_5_0; + } + public void testResolveReleasedVersionsForStableBtranchBehindStableBranch() { + Tuple, List> t = VersionUtils.resolveReleasedVersions(TestStableBranchBehindStableBranch.CURRENT, + TestStableBranchBehindStableBranch.class); + List released = t.v1(); + List unreleased = t.v2(); + assertEquals(Arrays.asList(TestStableBranchBehindStableBranch.V_5_3_0, TestStableBranchBehindStableBranch.V_5_3_1, + TestStableBranchBehindStableBranch.V_5_5_0), released); + assertEquals(Arrays.asList(TestStableBranchBehindStableBranch.V_5_3_2, Version.V_5_4_0), unreleased); + } + + static class TestUnstableBranch { + public static final Version V_5_3_0 = Version.fromString("5.3.0"); + public static final Version V_5_3_1 = Version.fromString("5.3.1"); + public static final Version V_5_3_2 = Version.fromString("5.3.2"); + public static final Version V_5_4_0 = Version.fromString("5.4.0"); + public static final Version V_6_0_0_alpha1 = Version.fromString("6.0.0-alpha1"); + public static final Version V_6_0_0_alpha2 = Version.fromString("6.0.0-alpha2"); + public static final Version CURRENT = V_6_0_0_alpha2; + } + public void testResolveReleasedVersionsForUnstableBranch() { + Tuple, List> t = VersionUtils.resolveReleasedVersions(TestUnstableBranch.CURRENT, + TestUnstableBranch.class); + List released = t.v1(); + List unreleased = t.v2(); + assertEquals(Arrays.asList(TestUnstableBranch.V_5_3_0, TestUnstableBranch.V_5_3_1, + TestUnstableBranch.V_6_0_0_alpha1, TestUnstableBranch.V_6_0_0_alpha2), released); + assertEquals(Arrays.asList(TestUnstableBranch.V_5_3_2, TestUnstableBranch.V_5_4_0), unreleased); + } + + // TODO add a test that compares gradle and VersionUtils.java in a followup +} diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java deleted file mode 100644 index 5641917bc45ad..0000000000000 --- a/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.test; - -import org.elasticsearch.Version; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.VersionUtils; - -import java.util.List; - -public class VersionUtilsTests extends ESTestCase { - - public void testAllVersionsSorted() { - List allVersions = VersionUtils.allReleasedVersions(); - for (int i = 0, j = 1; j < allVersions.size(); ++i, ++j) { - assertTrue(allVersions.get(i).before(allVersions.get(j))); - } - } - - public void testRandomVersionBetween() { - // full range - Version got = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), Version.CURRENT); - assertTrue(got.onOrAfter(VersionUtils.getFirstVersion())); - assertTrue(got.onOrBefore(Version.CURRENT)); - got = VersionUtils.randomVersionBetween(random(), null, Version.CURRENT); - assertTrue(got.onOrAfter(VersionUtils.getFirstVersion())); - assertTrue(got.onOrBefore(Version.CURRENT)); - got = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), null); - assertTrue(got.onOrAfter(VersionUtils.getFirstVersion())); - assertTrue(got.onOrBefore(Version.CURRENT)); - - // sub range - got = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, - Version.V_6_0_0_alpha2_UNRELEASED); - assertTrue(got.onOrAfter(Version.V_5_0_0)); - assertTrue(got.onOrBefore(Version.V_6_0_0_alpha2_UNRELEASED)); - - // unbounded lower - got = VersionUtils.randomVersionBetween(random(), null, Version.V_6_0_0_alpha2_UNRELEASED); - assertTrue(got.onOrAfter(VersionUtils.getFirstVersion())); - assertTrue(got.onOrBefore(Version.V_6_0_0_alpha2_UNRELEASED)); - got = VersionUtils.randomVersionBetween(random(), null, VersionUtils.allReleasedVersions().get(0)); - assertTrue(got.onOrAfter(VersionUtils.getFirstVersion())); - assertTrue(got.onOrBefore(VersionUtils.allReleasedVersions().get(0))); - - // unbounded upper - got = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, null); - assertTrue(got.onOrAfter(Version.V_5_0_0)); - assertTrue(got.onOrBefore(Version.CURRENT)); - got = VersionUtils.randomVersionBetween(random(), VersionUtils.getPreviousVersion(), null); - assertTrue(got.onOrAfter(VersionUtils.getPreviousVersion())); - assertTrue(got.onOrBefore(Version.CURRENT)); - - // range of one - got = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getFirstVersion()); - assertEquals(got, VersionUtils.getFirstVersion()); - got = VersionUtils.randomVersionBetween(random(), Version.CURRENT, Version.CURRENT); - assertEquals(got, Version.CURRENT); - got = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0_alpha2_UNRELEASED, - Version.V_6_0_0_alpha2_UNRELEASED); - assertEquals(got, Version.V_6_0_0_alpha2_UNRELEASED); - - // implicit range of one - got = VersionUtils.randomVersionBetween(random(), null, VersionUtils.getFirstVersion()); - assertEquals(got, VersionUtils.getFirstVersion()); - got = VersionUtils.randomVersionBetween(random(), Version.CURRENT, null); - assertEquals(got, Version.CURRENT); - } -}