Remove duplicated fields from the fields.yml

elastic · Nov 10, 2016 · 3cec8e3 · 3cec8e3
1 parent 4878158
commit 3cec8e3
Show file tree

Hide file tree

Showing 9 changed files with 63 additions and 180 deletions.
diff --git a/libbeat/scripts/generate_index_pattern.py b/libbeat/scripts/generate_index_pattern.py
@@ -13,6 +13,8 @@
 import os
 import errno
 
+unique_fields = []
+
 def fields_to_json(section, path, output):
 
     for field in section["fields"]:
@@ -29,6 +31,13 @@ def fields_to_json(section, path, output):
 
 def field_to_json(desc, path, output):
 
+    global unique_fields
+
+    if path in unique_fields:
+        print "ERROR: Field", path, "is duplicated. Please delete it and try again. Fields already are", unique_fields
+    else:
+        unique_fields.append(path)
+
     field = {
         "name": path,
         "count": 0,

diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc
@@ -822,9 +822,9 @@ type: keyword
 Representing the node ip.
 
 [float]
-=== cassandra.response.event.host
+=== cassandra.response.event.port
 
-type: keyword
+type: long
 
 Representing the node port.
 
@@ -1156,6 +1156,14 @@ example: udp
 The transport protocol used for the transaction. If not specified, then tcp is assumed.
 
 
+[float]
+=== type
+
+required: True
+
+The type of the transaction (for example, HTTP, MySQL, Redis, or RUM) or "flow" in case of flows.
+
+
 [float]
 === port
 
@@ -1289,6 +1297,14 @@ example: amazon.co.uk.
 
 The effective top-level domain (eTLD) plus one more label. For example, the eTLD+1 for "foo.bar.golang.org." is "golang.org.". The data for determining the eTLD comes from an embedded copy of the data from http://publicsuffix.org.
 
+[float]
+=== dns.answers
+
+type: dict
+
+An array containing a dictionary about each answer section returned by the server.
+
+
 [float]
 === dns.answers_count
 
@@ -1369,28 +1385,6 @@ example: IN
 
 The class of DNS data contained in this resource record.
 
-[float]
-=== dns.answers
-
-type: dict
-
-An array containing a dictionary about each answer section returned by the server.
-
-
-[float]
-=== dns.answers.ttl
-
-type: long
-
-The time interval in seconds that this resource record may be cached before it should be discarded. Zero values mean that the data should not be cached.
-
-
-[float]
-=== dns.answers.data
-
-The data describing the resource. The meaning of this data depends on the type and class of the resource record.
-
-
 [float]
 === dns.additionals
 
@@ -1477,20 +1471,6 @@ These fields contain data about the flow itself.
 
 
 
-[float]
-=== @timestamp
-
-type: date
-
-example: 2015-01-24 14:06:05.071000
-
-format: YYYY-MM-DDTHH:MM:SS.milliZ
-
-required: True
-
-The timestamp of the event, as measured by the Beat. The precision is in milliseconds. The timezone is UTC.
-
-
 [float]
 === start_time
 
@@ -1519,14 +1499,6 @@ required: True
 The time, the most recent processed packet for the flow has been seen.
 
 
-[float]
-=== type
-
-required: True
-
-Indicates the event to be a flow event. This field is always set to "flow".
-
-
 [float]
 === final
 
@@ -1769,12 +1741,6 @@ Total number of bytes
 ICMP id used in ICMP based flow.
 
 
-[float]
-=== transport
-
-The transport protocol used by the flow. If known, one of "udp" or "tcp".
-
-
 [float]
 === connection_id
 
@@ -2644,28 +2610,6 @@ These fields contain data about the transaction itself.
 
 
 
-[float]
-=== @timestamp
-
-type: date
-
-example: 2015-01-24 14:06:05.071000
-
-format: YYYY-MM-DDTHH:MM:SS.milliZ
-
-required: True
-
-The timestamp of the event, as measured either by the Beat or by a common collector point. The precision is in milliseconds. The timezone is UTC.
-
-
-[float]
-=== type
-
-required: True
-
-The type of the transaction (for example, HTTP, MySQL, Redis, or RUM).
-
-
 [float]
 === direction
 

diff --git a/packetbeat/etc/fields.yml b/packetbeat/etc/fields.yml
@@ -73,6 +73,11 @@
         tcp is assumed.
       example: udp
 
+    - name: type
+      description: >
+        The type of the transaction (for example, HTTP, MySQL, Redis, or RUM) or "flow" in case of flows.
+      required: true
+
     - name: port
       description: >
         The layer 4 port of the process that served the transaction.
@@ -96,15 +101,6 @@
   description: >
     These fields contain data about the flow itself.
   fields:
-    - name: "@timestamp"
-      type: date
-      required: true
-      format: YYYY-MM-DDTHH:MM:SS.milliZ
-      example: 2015-01-24T14:06:05.071Z
-      description: >
-        The timestamp of the event, as measured by the Beat. The precision is in
-        milliseconds. The timezone is UTC.
-
     - name: "start_time"
       type: date
       required: true
@@ -121,11 +117,6 @@
       description: >
         The time, the most recent processed packet for the flow has been seen.
 
-    - name: type
-      description: >
-        Indicates the event to be a flow event. This field is always set to "flow".
-      required: true
-
     - name: final
       description: >
         Indicates if event is last event in flow. If final is false, the event
@@ -301,10 +292,6 @@
       description: >
         ICMP id used in ICMP based flow.
 
-    - name: transport
-      description: >
-        The transport protocol used by the flow. If known, one of "udp" or "tcp".
-
     - name: connection_id
       description: >
         optional TCP connection id
@@ -314,20 +301,6 @@
   description: >
     These fields contain data about the transaction itself.
   fields:
-    - name: "@timestamp"
-      type: date
-      required: true
-      format: YYYY-MM-DDTHH:MM:SS.milliZ
-      example: 2015-01-24T14:06:05.071Z
-      description: >
-        The timestamp of the event, as measured either by the Beat or
-        by a common collector point. The precision is in milliseconds.
-        The timezone is UTC.
-
-    - name: type
-      description: >
-        The type of the transaction (for example, HTTP, MySQL, Redis, or RUM).
-      required: true
 
     - name: direction
       required: true
@@ -885,8 +858,8 @@
                 - name: host
                   type: keyword
                   description: Representing the node ip.
-                - name: host
-                  type: keyword
+                - name: port
+                  type: long
                   description: Representing the node port.
                 - name: schema_change
                   type: group
@@ -1072,11 +1045,18 @@
             data from http://publicsuffix.org.
           example: amazon.co.uk.
 
+        - name: answers
+          type: dict
+          description: >
+            An array containing a dictionary about each answer section returned by
+            the server.
+
         - name: answers_count
           type: long
           description: >
             The number of resource records contained in the `dns.answers` field.
 
+
         - name: answers.name
           description: The domain name to which this resource record pertains.
           example: example.com.
@@ -1126,24 +1106,6 @@
           description: The class of DNS data contained in this resource record.
           example: IN
 
-        - name: answers
-          type: dict
-          description: >
-            An array containing a dictionary about each answer section returned by
-            the server.
-
-        - name: answers.ttl
-          description: >
-            The time interval in seconds that this resource record may be cached
-            before it should be discarded. Zero values mean that the data should
-            not be cached.
-          type: long
-
-        - name: answers.data
-          description: >
-            The data describing the resource. The meaning of this data depends
-            on the type and class of the resource record.
-
         - name: additionals
           type: dict
           description: >

diff --git a/packetbeat/etc/fields_base.yml b/packetbeat/etc/fields_base.yml
@@ -73,6 +73,11 @@
         tcp is assumed.
       example: udp
 
+    - name: type
+      description: >
+        The type of the transaction (for example, HTTP, MySQL, Redis, or RUM) or "flow" in case of flows.
+      required: true
+
     - name: port
       description: >
         The layer 4 port of the process that served the transaction.
@@ -96,15 +101,6 @@
   description: >
     These fields contain data about the flow itself.
   fields:
-    - name: "@timestamp"
-      type: date
-      required: true
-      format: YYYY-MM-DDTHH:MM:SS.milliZ
-      example: 2015-01-24T14:06:05.071Z
-      description: >
-        The timestamp of the event, as measured by the Beat. The precision is in
-        milliseconds. The timezone is UTC.
-
     - name: "start_time"
       type: date
       required: true
@@ -121,11 +117,6 @@
       description: >
         The time, the most recent processed packet for the flow has been seen.
 
-    - name: type
-      description: >
-        Indicates the event to be a flow event. This field is always set to "flow".
-      required: true
-
     - name: final
       description: >
         Indicates if event is last event in flow. If final is false, the event
@@ -301,10 +292,6 @@
       description: >
         ICMP id used in ICMP based flow.
 
-    - name: transport
-      description: >
-        The transport protocol used by the flow. If known, one of "udp" or "tcp".
-
     - name: connection_id
       description: >
         optional TCP connection id
@@ -314,20 +301,6 @@
   description: >
     These fields contain data about the transaction itself.
   fields:
-    - name: "@timestamp"
-      type: date
-      required: true
-      format: YYYY-MM-DDTHH:MM:SS.milliZ
-      example: 2015-01-24T14:06:05.071Z
-      description: >
-        The timestamp of the event, as measured either by the Beat or
-        by a common collector point. The precision is in milliseconds.
-        The timezone is UTC.
-
-    - name: type
-      description: >
-        The type of the transaction (for example, HTTP, MySQL, Redis, or RUM).
-      required: true
 
     - name: direction
       required: true

diff --git a/packetbeat/etc/kibana/index-pattern/packetbeat.json b/packetbeat/etc/kibana/index-pattern/packetbeat.json
diff --git a/packetbeat/packetbeat.template-es2x.json b/packetbeat/packetbeat.template-es2x.json
@@ -334,6 +334,9 @@
                       "index": "not_analyzed",
                       "type": "string"
                     },
+                    "port": {
+                      "type": "long"
+                    },
                     "schema_change": {
                       "properties": {
                         "args": {

diff --git a/packetbeat/packetbeat.template.json b/packetbeat/packetbeat.template.json
@@ -294,6 +294,9 @@
                       "ignore_above": 1024,
                       "type": "keyword"
                     },
+                    "port": {
+                      "type": "long"
+                    },
                     "schema_change": {
                       "properties": {
                         "args": {