refactored dashboard

ohsu-comp-bio · Feb 4, 2019 · 43d2c39 · 43d2c39
1 parent 7108de6
commit 43d2c39
Show file tree

Hide file tree

Showing 3 changed files with 439 additions and 308 deletions.
diff --git a/kibana/everything.json b/kibana/everything.json
diff --git a/util/elastic/index-setup.sh b/util/elastic/index-setup.sh
@@ -6,46 +6,11 @@ if [ -z "$ES" ]
     ES=http://localhost:9200
 fi
 echo setting up $ES
-curl  -X DELETE $ES"/associations-new"
+curl  -X DELETE $ES"/associations"
 echo deleted
 
-# shards disk.indices disk.used disk.avail disk.total disk.percent host      ip        node
-#     21        9.7gb    20.4gb     42.2gb     62.7gb           32 127.0.0.1 127.0.0.1 -BRXEWH
-#     21                                                                               UNASSIGNED
 
-# curl -XPUT $ES"/associations-new" -H 'Content-Type: application/json' -d'
-# {
-#   "settings" : {
-#       "index" : {
-#         "mapping.total_fields.limit": 30000
-#       }
-#   },
-#
-#   "mappings": {
-#     "association": {
-#       "dynamic_templates": [
-#         {
-#           "strings": {
-#             "match_mapping_type": "string",
-#             "mapping": {
-#               "type": "text",
-#               "fields": {
-#                 "keyword": {
-#                   "type":  "keyword",
-#                   "ignore_above": 1024,
-#                   "store": true
-#                 }
-#               }
-#             }
-#           }
-#         }
-#       ]
-#     }
-#   }
-# }'
-echo created
-
-curl -XPUT $ES"/associations-new" -H 'Content-Type: application/json' -d'
+curl -XPUT $ES"/associations" -H 'Content-Type: application/json' -d'
 {
   "settings":{
     "index":{
@@ -94,53 +59,60 @@ curl -XPUT $ES"/associations-new" -H 'Content-Type: application/json' -d'
         "molecularmatch_trials":{
           "type":"keyword",
           "store":true, "index":false, "ignore_above":0
+        },
+        "evidence": {
+          "properties": {
+            "evidenceType": {
+              "properties": {
+                "id": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+        },
+        "features": {
+          "properties": {
+            "protein_domains": {
+              "properties": {
+                "name": {
+                  "type":"keyword"
+                }
+              }
+            }
+          }
+        },
+        "association": {
+          "properties": {
+            "description": {
+              "type": "text"
+            }
+          }
         }
       },
-      "dynamic_templates":[
+      "dynamic_templates": [
         {
-          "strings":{
-            "match_mapping_type":"string",
-            "mapping":{
-              "type":"text",
-              "fields":{
-                "keyword":{
-                  "type":"keyword",
-                  "ignore_above":1024,
-                  "store":true
-                }
-              }
+          "strings_as_keywords": {
+            "match_mapping_type": "string",
+            "mapping": {
+              "type": "keyword"
             }
           }
         }
       ]
+
     }
   }
 }
 '
 echo created
 
 
-# curl  -X PUT $ES"/associations-new/_settings" -d'
+
+# curl -XPOST  $ES"/_aliases" -H 'Content-Type: application/json' -d'
 # {
-#   "index.mapping.total_fields.limit": 30000
+#     "actions" : [
+#         { "add" : { "index" : "associations-new", "alias" : "associations" } }
+#     ]
 # }'
-# echo settings
-#
-# curl  -X PUT $ES"/associations-new" -d'
-# {
-#     "settings" : {
-#         "index" : {
-#           "mapping.total_fields.limit": 30000
-#         }
-#     }
-# }
-# '
-# echo created
-
-curl -XPOST  $ES"/_aliases" -H 'Content-Type: application/json' -d'
-{
-    "actions" : [
-        { "add" : { "index" : "associations-new", "alias" : "associations" } }
-    ]
-}'
-echo aliased
+# echo aliased
diff --git a/util/elastic/put_index.py b/util/elastic/put_index.py
@@ -21,6 +21,9 @@ def _stdin_actions(args):
         count = count + 1
         if count > args.skip:
             hit = json.loads(line)
+            if hit['source'] == 'litvar':
+                continue
+
             # hit = _fix_features(hit)
             # hit = _fix_genes(hit)
             # hit = _stringify(hit)
@@ -96,7 +99,8 @@ def _from_stdin(args):
     if not args.dry_run:
         print bulk(client,
                    (d for d in _stdin_actions(args)),
-                   request_timeout=120
+                   request_timeout=120,
+                   max_chunk_bytes=10485760
                    )
     else:
         for d in _stdin_actions(args):