[ti_misp] Incorrect misp.event.publish_date

[andrewkroh]

The misp.event.publish_date field is being interpreted incorrectly by Elasticsearch. A raw value of 1685972314 is represented as "1970-01-20T12:19:32.314Z". It should be 2023-06-06T14:17:34Z.

"fields": {
    "misp.event.publish_timestamp": [
      "1970-01-20T12:19:32.314Z"
    ]
}

Event

{
  "@timestamp": "2023-06-05T13:38:34.000Z",
  "agent": {
    "ephemeral_id": "99308eca-376f-44b3-84b5-1e62d57c867c",
    "id": "3b4885c5-66eb-4b06-a771-04c7f3b9ed82",
    "name": "docker-fleet-agent",
    "type": "filebeat",
    "version": "8.7.1"
  },
  "data_stream": {
    "dataset": "ti_misp.threat",
    "namespace": "default",
    "type": "logs"
  },
  "ecs": {
    "version": "8.7.0"
  },
  "elastic_agent": {
    "id": "3b4885c5-66eb-4b06-a771-04c7f3b9ed82",
    "snapshot": false,
    "version": "8.7.1"
  },
  "event": {
    "agent_id_status": "verified",
    "category": "threat",
    "created": "2023-06-05T13:41:12.845Z",
    "dataset": "ti_misp.threat",
    "ingested": "2023-06-05T13:41:13Z",
    "kind": "enrichment",
    "original": "{\"Event\":{\"Attribute\":{\"Galaxy\":[],\"ShadowAttribute\":[],\"category\":\"Payload delivery\",\"comment\":\"Phishing email\",\"deleted\":false,\"disable_correlation\":false,\"distribution\":\"5\",\"event_id\":\"122\",\"first_seen\":null,\"id\":\"26879\",\"last_seen\":null,\"object_id\":\"0\",\"object_relation\":null,\"sharing_group_id\":\"0\",\"timestamp\":\"1464003945\",\"to_ids\":true,\"type\":\"sha256\",\"uuid\":\"5742ed69-d374-40b6-8f10-48ff950d210f\",\"value\":\"4d5e0eddcd014c63123f6a46af7e53b5ac25a7ff7de86f56277fe39bff32c7b5\"},\"Galaxy\":[],\"Object\":{\"Attribute\":{\"Galaxy\":[],\"ShadowAttribute\":[],\"category\":\"Other\",\"comment\":\"Group 2: 6/1/2012 – 7/10/2012 - BS2005\",\"deleted\":false,\"disable_correlation\":false,\"distribution\":\"5\",\"event_id\":\"122\",\"first_seen\":null,\"id\":\"26917\",\"last_seen\":null,\"object_id\":\"10\",\"object_relation\":\"last-submission\",\"sharing_group_id\":\"0\",\"timestamp\":\"1515750510\",\"to_ids\":false,\"type\":\"datetime\",\"uuid\":\"5a58846e-4330-4878-b3c4-4aa502de0b81\",\"value\":\"2016-11-27T14:06:22.000000+0000\"},\"ObjectReference\":[],\"comment\":\"\",\"deleted\":false,\"description\":\"VirusTotal report\",\"distribution\":\"0\",\"event_id\":\"122\",\"first_seen\":null,\"id\":\"10\",\"last_seen\":null,\"meta-category\":\"misc\",\"name\":\"virustotal-report\",\"sharing_group_id\":\"0\",\"template_uuid\":\"d7dd0154-e04f-4c34-a2fb-79f3a3a52aa4\",\"template_version\":\"1\",\"timestamp\":\"1515750510\",\"uuid\":\"b7768dd0-8628-45a0-a1c0-30ff2c345300\"},\"Org\":{\"id\":\"1\",\"local\":true,\"name\":\"ORGNAME\",\"uuid\":\"647de191-50fc-493a-9f52-009fac110002\"},\"Orgc\":{\"id\":\"3\",\"local\":false,\"name\":\"CIRCL\",\"uuid\":\"55f6ea5e-2c60-40e5-964f-47a8950d210f\"},\"RelatedEvent\":[{\"Event\":{\"Org\":{\"id\":\"1\",\"name\":\"ORGNAME\",\"uuid\":\"647de191-50fc-493a-9f52-009fac110002\"},\"Orgc\":{\"id\":\"3\",\"name\":\"CIRCL\",\"uuid\":\"55f6ea5e-2c60-40e5-964f-47a8950d210f\"},\"analysis\":\"2\",\"date\":\"2016-05-25\",\"distribution\":\"3\",\"id\":\"129\",\"info\":\"OSINT - CVE-2015-2545: overview of current threats\",\"org_id\":\"1\",\"orgc_id\":\"3\",\"published\":true,\"threat_level_id\":\"2\",\"timestamp\":\"1469608718\",\"uuid\":\"57460863-76dc-4272-8116-4ea302de0b81\"}},{\"Event\":{\"Org\":{\"id\":\"1\",\"name\":\"ORGNAME\",\"uuid\":\"647de191-50fc-493a-9f52-009fac110002\"},\"Orgc\":{\"id\":\"3\",\"name\":\"CIRCL\",\"uuid\":\"55f6ea5e-2c60-40e5-964f-47a8950d210f\"},\"analysis\":\"2\",\"date\":\"2016-05-09\",\"distribution\":\"3\",\"id\":\"113\",\"info\":\"OSINT - Exploring CVE-2015-2545 and its users\",\"org_id\":\"1\",\"orgc_id\":\"3\",\"published\":true,\"threat_level_id\":\"3\",\"timestamp\":\"1463502585\",\"uuid\":\"5730965a-fa18-43d4-8692-4296950d210f\"}},{\"Event\":{\"Org\":{\"id\":\"1\",\"name\":\"ORGNAME\",\"uuid\":\"647de191-50fc-493a-9f52-009fac110002\"},\"Orgc\":{\"id\":\"3\",\"name\":\"CIRCL\",\"uuid\":\"55f6ea5e-2c60-40e5-964f-47a8950d210f\"},\"analysis\":\"2\",\"date\":\"2016-04-28\",\"distribution\":\"3\",\"id\":\"106\",\"info\":\"OSINT - PLATINUM Targeted attacks in South and Southeast Asia\",\"org_id\":\"1\",\"orgc_id\":\"3\",\"published\":true,\"threat_level_id\":\"3\",\"timestamp\":\"1464773185\",\"uuid\":\"57221ede-4084-4c2b-9463-4e1e950d210f\"}},{\"Event\":{\"Org\":{\"id\":\"1\",\"name\":\"ORGNAME\",\"uuid\":\"647de191-50fc-493a-9f52-009fac110002\"},\"Orgc\":{\"id\":\"3\",\"name\":\"CIRCL\",\"uuid\":\"55f6ea5e-2c60-40e5-964f-47a8950d210f\"},\"analysis\":\"2\",\"date\":\"2016-04-22\",\"distribution\":\"3\",\"id\":\"92\",\"info\":\"OSINT - New Poison Ivy RAT Variant Targets Hong Kong Pro-Democracy Activists\",\"org_id\":\"1\",\"orgc_id\":\"3\",\"published\":true,\"threat_level_id\":\"2\",\"timestamp\":\"1461356749\",\"uuid\":\"571a87f2-13e0-4396-83e5-4780950d210f\"}}],\"ShadowAttribute\":[],\"Tag\":[{\"colour\":\"#ffffff\",\"exportable\":true,\"hide_tag\":false,\"id\":\"4\",\"local\":0,\"name\":\"tlp:white\",\"numerical_value\":null,\"user_id\":\"0\"},{\"colour\":\"#004646\",\"exportable\":true,\"hide_tag\":false,\"id\":\"3\",\"local\":0,\"name\":\"type:OSINT\",\"numerical_value\":null,\"user_id\":\"0\"}],\"analysis\":\"2\",\"attribute_count\":\"110\",\"date\":\"2016-05-23\",\"disable_correlation\":false,\"distribution\":\"3\",\"extends_uuid\":\"\",\"id\":\"122\",\"info\":\"OSINT - Operation Ke3chang Resurfaces With New TidePool Malware\",\"locked\":false,\"org_id\":\"1\",\"orgc_id\":\"3\",\"proposal_email_lock\":false,\"publish_timestamp\":\"1685972314\",\"published\":true,\"sharing_group_id\":\"0\",\"threat_level_id\":\"2\",\"timestamp\":\"1685972314\",\"uuid\":\"5742ea44-5ff4-4634-99c9-4b32950d210f\"}}",
    "type": "indicator"
  },
  "input": {
    "type": "httpjson"
  },
  "misp": {
    "attribute": {
      "category": "Other",
      "comment": "Group 2: 6/1/2012 – 7/10/2012 - BS2005",
      "deleted": false,
      "disable_correlation": false,
      "distribution": 5,
      "event_id": "122",
      "id": "26917",
      "object_id": "10",
      "object_relation": "last-submission",
      "sharing_group_id": "0",
      "timestamp": "2018-01-12T09:48:30.000Z",
      "to_ids": false,
      "type": "datetime",
      "uuid": "5a58846e-4330-4878-b3c4-4aa502de0b81",
      "value": "2016-11-27T14:06:22.000000+0000"
    },
    "context": {
      "attribute": {
        "category": "Payload delivery",
        "comment": "Phishing email",
        "deleted": false,
        "disable_correlation": false,
        "distribution": 5,
        "event_id": "122",
        "id": "26879",
        "object_id": "0",
        "sharing_group_id": "0",
        "timestamp": "2016-05-23T11:45:45.000Z",
        "to_ids": true,
        "type": "sha256",
        "uuid": "5742ed69-d374-40b6-8f10-48ff950d210f",
        "value": "4d5e0eddcd014c63123f6a46af7e53b5ac25a7ff7de86f56277fe39bff32c7b5"
      }
    },
    "event": {
      "attribute_count": 110,
      "date": "2016-05-23",
      "disable_correlation": false,
      "distribution": 3,
      "extends_uuid": "",
      "id": "122",
      "info": "OSINT - Operation Ke3chang Resurfaces With New TidePool Malware",
      "locked": false,
      "org_id": "1",
      "orgc_id": "3",
      "proposal_email_lock": false,
      "publish_timestamp": "1685972314",
      "published": true,
      "sharing_group_id": "0",
      "threat_level_id": 2,
      "uuid": "5742ea44-5ff4-4634-99c9-4b32950d210f"
    },
    "object": {
      "comment": "",
      "deleted": false,
      "description": "VirusTotal report",
      "distribution": 0,
      "event_id": "122",
      "id": "10",
      "meta_category": "misc",
      "name": "virustotal-report",
      "sharing_group_id": "0",
      "template_uuid": "d7dd0154-e04f-4c34-a2fb-79f3a3a52aa4",
      "template_version": "1",
      "timestamp": "2018-01-12T09:48:30.000Z",
      "uuid": "b7768dd0-8628-45a0-a1c0-30ff2c345300"
    },
    "orgc": {
      "id": "3",
      "local": false,
      "name": "CIRCL",
      "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f"
    }
  },
  "tags": [
    "preserve_original_event",
    "forwarded",
    "test1",
    "tlp:white",
    "type:OSINT"
  ],
  "threat": {
    "feed": {
      "name": "MISP"
    },
    "indicator": {
      "marking": {
        "tlp": [
          "WHITE"
        ]
      },
      "provider": "misp",
      "scanner_stats": 2
    }
  }
}

[elasticmachine]

Pinging @elastic/security-external-integrations (Team:Security-External Integrations)

[chrisberkhout]

The issue here seems to be seconds versus milliseconds since the epoch. The value was being left as an integer timestamp and stored in a date field, which Kibana then presents as a millisecond timestamp. See the PR for details of the change.

For reference, the interpretation proposed in the issue description seems slightly off, perhaps due to a cut and paste error:

     raw: 1685972314
 current: 1970-01-20T12:19:32.314Z (UNIX timestamp in milliseconds)
proposed: 2023-06-06T14:17:34.000Z
 adopted: 2023-06-05T13:38:34.000Z (UNIX timestamp in seconds, 1 day 39 minutes earlier than proposed)