Tests: improve back compat tests by adding delete-by-query in the tra…

…nsaction log on upgrade

Closes #10266

dev-tools/create-bwc-index.py

@@ -69,6 +69,34 @@ def index_documents(es, index_name, type, num_docs):
   logging.info('Flushing index')
   es.indices.flush(index=index_name)
 
+def delete_by_query(es, version, index_name, doc_type):
+
+  logging.info('Deleting long_sort:[10..20] docs')
+
+  query = {'query':
+           {'range':
+            {'long_sort':
+             {'gte': 10,
+              'lte': 20}}}}
+
+  if version.startswith('0.90.') or version in ('1.0.0.Beta1', '1.0.0.Beta2'):
+    # TODO #10262: we can't write DBQ into the translog for these old versions until we fix this back-compat bug:
+
+    # #4074: these versions don't expect to see the top-level 'query' to count/delete_by_query:
+    query = query['query']
+    return
+
+  deleted_count = es.count(index=index_name, doc_type=doc_type, body=query)['count']
+
+  result = es.delete_by_query(index=index_name,
+                              doc_type=doc_type,
+                              body=query)
+
+  # make sure no shards failed:
+  assert result['_indices'][index_name]['_shards']['failed'] == 0, 'delete by query failed: %s' % result
+
+  logging.info('Deleted %d docs' % deleted_count)
+
 def run_basic_asserts(es, index_name, type, num_docs):
   count = es.count(index=index_name)['count']
   assert count == num_docs, 'Expected %r but got %r documents' % (num_docs, count)
@@ -150,7 +178,7 @@ def generate_index(client, version):
       }
     }
     # completion type was added in 0.90.3
-    if not version in ['0.90.0.Beta1', '0.90.0.RC1', '0.90.0.RC2', '0.90.0', '0.90.1', '0.90.2']:
+    if version not in ['0.90.0.Beta1', '0.90.0.RC1', '0.90.0.RC2', '0.90.0', '0.90.1', '0.90.2']:
       mappings['analyzer_type1']['properties']['completion_with_index_analyzer'] = {
         'type': 'completion',
         'index_analyzer': 'standard'
@@ -312,6 +340,12 @@ def main():
     generate_index(client, cfg.version)
     if cfg.snapshot_supported:
       snapshot_index(client, cfg)
+
+    # 10067: get a delete-by-query into the translog on upgrade.  We must do
+    # this after the snapshot, because it calls flush.  Otherwise the index
+    # will already have the deletions applied on upgrade.
+    delete_by_query(client, cfg.version, 'test', 'doc')
+
   finally:
     if 'node' in vars():
       logging.info('Shutting down node with pid %d', node.pid)

dev-tools/get-bwc-version.py

@@ -61,7 +61,11 @@ def main():
   else:
     filename = '%s.tar.gz' % version_dir
 
-  url = 'https://download.elasticsearch.org/elasticsearch/elasticsearch/%s' % filename
+  if c.version == '1.2.0':
+    # 1.2.0 was pulled from download.elasticsearch.org because of routing bug:
+    url = 'http://central.maven.org/maven2/org/elasticsearch/elasticsearch/1.2.0/%s' % filename
+  else:
+    url = 'https://download.elasticsearch.org/elasticsearch/elasticsearch/%s' % filename
   print('Downloading %s' % url)
   urllib.request.urlretrieve(url, filename)
 

src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityTests.java

@@ -116,16 +116,17 @@ void assertOldIndexWorks(String index) throws Exception {
         assertBasicSearchWorks();
         assertRealtimeGetWorks();
         assertNewReplicasWork();
-        assertUpgradeWorks(isLatestLuceneVersion(index));
+        Version version = extractVersion(index);
+        assertUpgradeWorks(isLatestLuceneVersion(version));
+        assertDeleteByQueryWorked(version);
         unloadIndex();
     }
 
     Version extractVersion(String index) {
         return Version.fromString(index.substring(index.indexOf('-') + 1, index.lastIndexOf('.')));
     }
 
-    boolean isLatestLuceneVersion(String index) {
-        Version version = extractVersion(index);
+    boolean isLatestLuceneVersion(Version version) {
         return version.luceneVersion.major == Version.CURRENT.luceneVersion.major &&
                version.luceneVersion.minor == Version.CURRENT.luceneVersion.minor;
     }
@@ -186,6 +187,16 @@ void assertNewReplicasWork() throws Exception {
             .execute().actionGet());
         waitNoPendingTasksOnAll(); // make sure the replicas are removed before going on
     }
+
+    // #10067: create-bwc-index.py deleted any doc with long_sort:[10-20]
+    void assertDeleteByQueryWorked(Version version) throws Exception {
+        if (version.onOrBefore(Version.V_1_0_0_Beta2)) {
+            // TODO: remove this once #10262 is fixed
+            return;
+        }
+        SearchRequestBuilder searchReq = client().prepareSearch("test").setQuery(QueryBuilders.queryStringQuery("long_sort:[10 TO 20]"));
+        assertEquals(0, searchReq.get().getHits().getTotalHits());
+    }
 
     void assertUpgradeWorks(boolean alreadyLatest) throws Exception {
         HttpRequestBuilder httpClient = httpClient();