Remove index management (over manual management), update index docume…

…ntation for ES7.

* Add term vectors to allow large content fields.
* Add MinHash/fingerprint to content.
* Default query fields, prevents “Field expansion matches too many fields”.

config/indexes.go

@@ -1,7 +1,6 @@
 package config
 
 import (
-    "encoding/json"
     "github.com/ipfs-search/ipfs-search/index/elasticsearch"
 )
 
@@ -10,36 +9,15 @@ type Indexes map[string]*elasticsearch.Config
 
 // IndexesDefaults returns the default indexes.
 func IndexesDefaults() Indexes {
-    var indexSettings, fileMapping, dirMapping, invalidMapping map[string]interface{}
-
-    if err := json.Unmarshal([]byte(indexSettingsJSON), &indexSettings); err != nil {
-        panic(err)
-    }
-    if err := json.Unmarshal([]byte(fileMappingJSON), &fileMapping); err != nil {
-        panic(err)
-    }
-    if err := json.Unmarshal([]byte(dirMappingJSON), &dirMapping); err != nil {
-        panic(err)
-    }
-    if err := json.Unmarshal([]byte(invalidMappingJSON), &invalidMapping); err != nil {
-        panic(err)
-    }
-
     return Indexes{
         "files": &elasticsearch.Config{
-            Name:     "ipfs_files_v0",
-            Settings: indexSettings,
-            Mapping:  fileMapping,
+            Name: "ipfs_files",
         },
         "directories": &elasticsearch.Config{
-            Name:     "ipfs_directories_v0",
-            Settings: indexSettings,
-            Mapping:  dirMapping,
+            Name: "ipfs_directories",
         },
         "invalids": &elasticsearch.Config{
-            Name:     "ipfs_invalids_v0",
-            Settings: indexSettings,
-            Mapping:  invalidMapping,
+            Name: "ipfs_invalids",
         },
     }
 }

crawler/factory/factory.go

@@ -50,10 +50,6 @@ func New(ctx context.Context, config *Config, errc chan<- error) (*Factory, erro
 	log.Printf("Connected to ElasticSearch.")
 
 	indexes := elasticsearch.NewMulti(es, config.Indexes["files"], config.Indexes["directories"], config.Indexes["invalids"])
-	if err := index.EnsureExistsAndUpdatedMulti(ctx, indexes...); err != nil {
-		return nil, err
-	}
-	log.Printf("Index configuration up to date.")
 
 	return &Factory{
 		crawlerConfig:  config.CrawlerConfig,

docs/indices/directories.json

@@ -0,0 +1,60 @@
+{
+    "settings": {
+        "index": {
+            "refresh_interval": "15m",
+            "number_of_shards": "20"
+        }
+    },
+    "mappings": {
+        "dynamic": "strict",
+        "properties": {
+            "first-seen": {
+                "type": "date",
+                "format": "date_time_no_millis"
+            },
+            "last-seen": {
+                "type": "date",
+                "format": "date_time_no_millis"
+            },
+            "links": {
+                "dynamic": true,
+                "properties": {
+                    "Hash": {
+                        "type": "keyword",
+                        "index": true
+                    },
+                    "Name": {
+                        "type": "text"
+                    },
+                    "Size": {
+                        "type": "long",
+                        "ignore_malformed": true
+                    },
+                    "Type": {
+                        "type": "keyword"
+                    }
+                }
+            },
+            "size": {
+                "type": "long",
+                "ignore_malformed": true
+            },
+            "references": {
+                "properties": {
+                    "name": {
+                        "type": "text",
+                        "index": true
+                    },
+                    "hash": {
+                        "type": "keyword",
+                        "index": true
+                    },
+                    "parent_hash": {
+                        "type": "keyword",
+                        "index": true
+                    }
+                }
+            }
+        }
+    }
+}