Merge pull request #437 from PanDAWMS/es-7.10.2

ES v7.10.2

Utils/API/server/lib/dkb/api/storages/es/common.py

@@ -35,7 +35,7 @@
                          'query')
 
 # Default datetime format for ES
-ES_DATE_FORMAT = '%d-%m-%y %H:%M:%S'
+ES_DATE_FORMAT = '%d-%m-%Y %H:%M:%S'
 
 # ES configuration
 CONFIG = None

Utils/API/server/lib/dkb/api/storages/es_nested/common.py

@@ -35,7 +35,7 @@
                          'query')
 
 # Default datetime format for ES
-ES_DATE_FORMAT = '%d-%m-%y %H:%M:%S'
+ES_DATE_FORMAT = '%d-%m-%Y %H:%M:%S'
 
 # ES configuration
 CONFIG = None

Utils/Dataflow/data4es/016_task2es/README

@@ -6,7 +6,6 @@
 --------------
 Add ES-specific service fields to task metadata:
 * _id = taskid
-* _type = 'task'
 
 2. Samples
 ----------

Utils/Dataflow/data4es/016_task2es/task2es.py

@@ -31,7 +31,6 @@ def add_es_index_info(data):
 
     Add fields:
       _id => taskid
-      _type => 'task'
 
     Return value:
       False -- update failed, skip the record
@@ -42,7 +41,6 @@ def add_es_index_info(data):
     if not data.get('taskid'):
         return False
     data['_id'] = data['taskid']
-    data['_type'] = 'task'
     return True
 
 

Utils/Dataflow/data4es/019_esFormat/esFormat.php

@@ -63,7 +63,7 @@ function check_input($row) {
   :return: TRUE if input is correct, FALSE if it is not
   :rtype: bool
   */
-  $required_fields = array('_id', '_type');
+  $required_fields = array('_id');
 
   if (!is_array($row)) {
     fwrite(STDERR, "(WARN) Failed to decode message.\n");
@@ -181,7 +181,6 @@ function constructActionJson($row) {
   $action = Array(
     $act => Array(
       '_index' => $index,
-      '_type'  => $row['_type'],
       '_id'    => $row['_id'],
     )
   );

Utils/Dataflow/data4es/040_progress/stage.py

@@ -96,7 +96,6 @@ def progress_data(data, granularity=86400):
     rounded_ts_sec = int((rounded_ts - datetime(1970, 1, 1)).total_seconds())
     rounded_ts_ms = rounded_ts_sec * 10**3
     result['_id'] = '%s_%s' % (rounded_ts_ms, result['taskid'])
-    result['_type'] = 'task_progress'
     result['_index'] = 'progress'
 
     return result

Utils/Elasticsearch/mapping/progress.mapping

@@ -1,10 +1,10 @@
 {
-  "template": "*_progress",
-  "settings": {
-    "number_of_shards": 4
-  },
-  "mappings": {
-    "task_progress": {
+  "index_patterns": "progress_*",
+  "template": {
+    "settings": {
+      "number_of_shards": 4
+    },
+    "mappings": {
       "properties": {
         "taskid": {
           "type": "integer",
@@ -16,7 +16,7 @@
         },
         "date": {
           "type": "date",
-          "format": "dd-MM-yy HH:mm:ss||epoch_millis"
+          "format": "dd-MM-yyyy HH:mm:ss||epoch_millis"
         },
         "processed_events": {
           "type": "long"

Utils/Elasticsearch/mapping/tasks.mapping

@@ -1,51 +1,51 @@
 {
-  "template": "tasks_*",
-  "settings": {
-    "number_of_shards": 4,
-    "analysis": {
-      "analyzer": {
-        "dsname": {
-          "char_filter": ["dsname_wildcarded", "remove_wildcard"],
-          "tokenizer": "dsname_tokenizer",
-          "filter": ["lowercase"]
-        },
-        "dsname_fields": {
-          "char_filter": ["dsname_wildcarded", "remove_wildcard"],
-          "tokenizer": "dsname_field_tokenizer",
-          "filter": "lowercase"
-        },
-        "dsname_fields_wildcarded": {
-          "char_filter": ["dsname_wildcarded"],
-          "tokenizer": "dsname_field_tokenizer",
-          "filter": "lowercase"
-        }
-      },
-      "tokenizer": {
-        "dsname_tokenizer": {
-          "type": "pattern",
-          "pattern": "\\.+|_+"
-        },
-        "dsname_field_tokenizer": {
-          "type": "pattern",
-          "pattern": "\\.+"
-        }
-      },
-      "char_filter": {
-        "dsname_wildcarded": {
-          "type": "pattern_replace",
-          "pattern": "[^a-zA-Z0-9_/.*?-]",
-          "replacement": ""
-        },
-        "remove_wildcard": {
-          "type": "pattern_replace",
-          "pattern": "[?*]",
-          "replacement": ""
+  "index_patterns": ["tasks_*"],
+  "template": {
+    "settings": {
+      "number_of_shards": 4,
+      "analysis": {
+        "analyzer": {
+          "dsname": {
+            "char_filter": ["dsname_wildcarded", "remove_wildcard"],
+            "tokenizer": "dsname_tokenizer",
+            "filter": ["lowercase"]
+          },
+          "dsname_fields": {
+            "char_filter": ["dsname_wildcarded", "remove_wildcard"],
+            "tokenizer": "dsname_field_tokenizer",
+            "filter": "lowercase"
+          },
+          "dsname_fields_wildcarded": {
+            "char_filter": ["dsname_wildcarded"],
+            "tokenizer": "dsname_field_tokenizer",
+            "filter": "lowercase"
+          }
+        },
+        "tokenizer": {
+          "dsname_tokenizer": {
+            "type": "pattern",
+            "pattern": "\\.+|_+"
+          },
+          "dsname_field_tokenizer": {
+            "type": "pattern",
+            "pattern": "\\.+"
+          }
+        },
+        "char_filter": {
+          "dsname_wildcarded": {
+            "type": "pattern_replace",
+            "pattern": "[^a-zA-Z0-9_/.*?-]",
+            "replacement": ""
+          },
+          "remove_wildcard": {
+            "type": "pattern_replace",
+            "pattern": "[?*]",
+            "replacement": ""
+          }
         }
       }
-    }
-  },
-  "mappings": {
-    "task": {
+    },
+    "mappings": {
       "properties": {
         "site": {
           "type": "text",
@@ -192,15 +192,15 @@
         },
         "task_timestamp": {
           "type": "date",
-          "format": "dd-MM-yy HH:mm:ss"
+          "format": "dd-MM-yyyy HH:mm:ss"
         },
         "start_time": {
           "type": "date",
-          "format": "dd-MM-yy HH:mm:ss"
+          "format": "dd-MM-yyyy HH:mm:ss"
         },
         "end_time": {
           "type": "date",
-          "format": "dd-MM-yy HH:mm:ss"
+          "format": "dd-MM-yyyy HH:mm:ss"
         },
         "hashtag_list": {
           "type": "keyword",

Utils/Elasticsearch/tools/load_mapping.sh

@@ -29,4 +29,4 @@ shift $((OPTIND-1))
   && echo "You must provide path to mapping file!" \
   || curl $ES_AUTH -H "Content-Type: application/x-ndjson" \
           -XPUT --data-binary "@${1}" \
-          "${ES_PROTO}://${ES_HOST}:${ES_PORT}${ES_PATH}/_template/`basename ${1%.*}`?pretty"
+          "${ES_PROTO}://${ES_HOST}:${ES_PORT}${ES_PATH}/_index_template/`basename ${1%.*}`?pretty"