Manage Fleet system indices within Elasticsearch (#71413)

This commit moves the management of seven system indices that are used by Fleet to an Elasticsearch plugin. The mappings were taken largely untouched from the Fleet server project, with the exception of adding a _meta field with a version key to enable the system indices infrastructure to manage these indices. Co-authored-by: Aleksandr Maus <aleksandr.maus@elastic.co> Backport of #70689
elastic · Apr 7, 2021 · 5c0a773 · 5c0a773
1 parent 955bd8d
commit 5c0a773
Show file tree

Hide file tree

Showing 14 changed files with 828 additions and 11 deletions.
diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java
@@ -279,6 +279,9 @@ private Version readMappingVersion(SystemIndexDescriptor descriptor, MappingMeta
         } catch (ElasticsearchParseException e) {
             logger.error(new ParameterizedMessage("Cannot parse the mapping for index [{}]", indexName), e);
             throw new ElasticsearchException("Cannot parse the mapping for index [{}]", e, indexName);
+        } catch (IllegalArgumentException e) {
+            logger.error(new ParameterizedMessage("Cannot parse the mapping for index [{}]", indexName), e);
+            throw e;
         }
     }
 

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
@@ -95,6 +95,7 @@ public static Map<String, String> filterSecurityHeaders(Map<String, String> head
     public static final String STACK_ORIGIN = "stack";
     public static final String SEARCHABLE_SNAPSHOTS_ORIGIN = "searchable_snapshots";
     public static final String LOGSTASH_MANAGEMENT_ORIGIN = "logstash_management";
+    public static final String FLEET_ORIGIN = "fleet";
 
     private ClientHelper() {}
 

diff --git a/x-pack/plugin/core/src/main/resources/fleet-actions.json b/x-pack/plugin/core/src/main/resources/fleet-actions.json
@@ -0,0 +1,40 @@
+{
+  "settings": {
+    "auto_expand_replicas": "0-1"
+  },
+  "mappings": {
+    "_doc" : {
+      "dynamic": false,
+      "_meta": {
+        "version": "${fleet.version}"
+      },
+      "properties": {
+        "action_id": {
+          "type": "keyword"
+        },
+        "agents": {
+          "type": "keyword"
+        },
+        "data": {
+          "enabled": false,
+          "type": "object"
+        },
+        "expiration": {
+          "type": "date"
+        },
+        "input_type": {
+          "type": "keyword"
+        },
+        "@timestamp": {
+          "type": "date"
+        },
+        "type": {
+          "type": "keyword"
+        },
+        "user_id" : {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}
diff --git a/x-pack/plugin/core/src/main/resources/fleet-agents.json b/x-pack/plugin/core/src/main/resources/fleet-agents.json
@@ -0,0 +1,231 @@
+{
+  "settings": {
+    "auto_expand_replicas": "0-1"
+  },
+  "mappings": {
+    "_doc" : {
+      "dynamic": false,
+      "_meta": {
+        "version": "${fleet.version}"
+      },
+      "properties": {
+        "access_api_key_id": {
+          "type": "keyword"
+        },
+        "action_seq_no": {
+          "type": "integer",
+          "index": false
+        },
+        "active": {
+          "type": "boolean"
+        },
+        "agent": {
+          "properties": {
+            "id": {
+              "type": "keyword"
+            },
+            "version": {
+              "type": "keyword"
+            }
+          }
+        },
+        "default_api_key": {
+          "type": "keyword"
+        },
+        "default_api_key_id": {
+          "type": "keyword"
+        },
+        "enrolled_at": {
+          "type": "date"
+        },
+        "last_checkin": {
+          "type": "date"
+        },
+        "last_checkin_status": {
+          "type": "keyword"
+        },
+        "last_updated": {
+          "type": "date"
+        },
+        "local_metadata": {
+          "properties": {
+            "elastic": {
+              "properties": {
+                "agent": {
+                  "properties": {
+                    "build": {
+                      "properties": {
+                        "original": {
+                          "type": "text",
+                          "fields": {
+                            "keyword": {
+                              "type": "keyword",
+                              "ignore_above": 256
+                            }
+                          }
+                        }
+                      }
+                    },
+                    "id": {
+                      "type": "keyword"
+                    },
+                    "log_level": {
+                      "type": "keyword"
+                    },
+                    "snapshot": {
+                      "type": "boolean"
+                    },
+                    "upgradeable": {
+                      "type": "boolean"
+                    },
+                    "version": {
+                      "type": "text",
+                      "fields": {
+                        "keyword": {
+                          "type": "keyword",
+                          "ignore_above": 16
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            },
+            "host": {
+              "properties": {
+                "architecture": {
+                  "type": "keyword"
+                },
+                "hostname": {
+                  "type": "text",
+                  "fields": {
+                    "keyword": {
+                      "type": "keyword",
+                      "ignore_above": 256
+                    }
+                  }
+                },
+                "id": {
+                  "type": "keyword"
+                },
+                "ip": {
+                  "type": "text",
+                  "fields": {
+                    "keyword": {
+                      "type": "keyword",
+                      "ignore_above": 64
+                    }
+                  }
+                },
+                "mac": {
+                  "type": "text",
+                  "fields": {
+                    "keyword": {
+                      "type": "keyword",
+                      "ignore_above": 17
+                    }
+                  }
+                },
+                "name": {
+                  "type": "text",
+                  "fields": {
+                    "keyword": {
+                      "type": "keyword",
+                      "ignore_above": 256
+                    }
+                  }
+                }
+              }
+            },
+            "os": {
+              "properties": {
+                "family": {
+                  "type": "keyword"
+                },
+                "full": {
+                  "type": "text",
+                  "fields": {
+                    "keyword": {
+                      "type": "keyword",
+                      "ignore_above": 128
+                    }
+                  }
+                },
+                "kernel": {
+                  "type": "text",
+                  "fields": {
+                    "keyword": {
+                      "type": "keyword",
+                      "ignore_above": 128
+                    }
+                  }
+                },
+                "name": {
+                  "type": "text",
+                  "fields": {
+                    "keyword": {
+                      "type": "keyword",
+                      "ignore_above": 256
+                    }
+                  }
+                },
+                "platform": {
+                  "type": "keyword"
+                },
+                "version": {
+                  "type": "text",
+                  "fields": {
+                    "keyword": {
+                      "type": "keyword",
+                      "ignore_above": 32
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "packages": {
+          "type": "keyword"
+        },
+        "policy_coordinator_idx": {
+          "type": "integer"
+        },
+        "policy_id": {
+          "type": "keyword"
+        },
+        "policy_output_permissions_hash": {
+          "type": "keyword"
+        },
+        "policy_revision_idx": {
+          "type": "integer"
+        },
+        "shared_id": {
+          "type": "keyword"
+        },
+        "type": {
+          "type": "keyword"
+        },
+        "unenrolled_at": {
+          "type": "date"
+        },
+        "unenrollment_started_at": {
+          "type": "date"
+        },
+        "updated_at": {
+          "type": "date"
+        },
+        "upgrade_started_at": {
+          "type": "date"
+        },
+        "upgraded_at": {
+          "type": "date"
+        },
+        "user_provided_metadata": {
+          "type": "object",
+          "enabled": false
+        }
+      }
+    }
+  }
+}
diff --git a/x-pack/plugin/core/src/main/resources/fleet-artifacts.json b/x-pack/plugin/core/src/main/resources/fleet-artifacts.json
@@ -0,0 +1,55 @@
+{
+  "settings": {
+    "auto_expand_replicas": "0-1"
+  },
+  "mappings": {
+    "_doc" : {
+      "dynamic": false,
+      "_meta": {
+        "version": "${fleet.version}"
+      },
+      "properties": {
+        "identifier": {
+          "type": "keyword"
+        },
+        "compression_algorithm": {
+          "type": "keyword",
+          "index": false
+        },
+        "encryption_algorithm": {
+          "type": "keyword",
+          "index": false
+        },
+        "encoded_sha256": {
+          "type": "keyword"
+        },
+        "encoded_size": {
+          "type": "long",
+          "index": false
+        },
+        "decoded_sha256": {
+          "type": "keyword"
+        },
+        "decoded_size": {
+          "type": "long",
+          "index": false
+        },
+        "created": {
+          "type": "date"
+        },
+        "package_name": {
+          "type": "keyword"
+        },
+        "type": {
+          "type": "keyword"
+        },
+        "relative_url": {
+          "type": "keyword"
+        },
+        "body": {
+          "type": "binary"
+        }
+      }
+    }
+  }
+}
diff --git a/x-pack/plugin/core/src/main/resources/fleet-enrollment-api-keys.json b/x-pack/plugin/core/src/main/resources/fleet-enrollment-api-keys.json
@@ -0,0 +1,39 @@
+{
+  "settings": {
+    "auto_expand_replicas": "0-1"
+  },
+  "mappings": {
+    "_doc" : {
+      "dynamic": false,
+      "_meta": {
+        "version": "${fleet.version}"
+      },
+      "properties": {
+        "active": {
+          "type": "boolean"
+        },
+        "api_key": {
+          "type": "keyword"
+        },
+        "api_key_id": {
+          "type": "keyword"
+        },
+        "created_at": {
+          "type": "date"
+        },
+        "expire_at": {
+          "type": "date"
+        },
+        "name": {
+          "type": "keyword"
+        },
+        "policy_id": {
+          "type": "keyword"
+        },
+        "updated_at": {
+          "type": "date"
+        }
+      }
+    }
+  }
+}