From 8854c2576dc17da9da54e54b116779bd2131617c Mon Sep 17 00:00:00 2001 From: Samuel Hassine Date: Fri, 5 Jul 2019 02:23:08 +0200 Subject: [PATCH] #116 No more install in Docker entrypoint, #114 Correct volumes config, #109 Use workers to import, #113 Fix, #85 More Observables types --- README.md | 2 +- opencti-docker/config/.config | 0 opencti-docker/docker-compose.yml | 35 ++- opencti-docker/integration/Dockerfile | 10 +- opencti-docker/integration/entrypoint.sh | 11 +- opencti-docker/volumes/esdata/.esdata | 0 opencti-docker/volumes/grakndata/.grakndata | 0 opencti-docker/worker/Dockerfile | 3 + opencti-docker/worker/entrypoint.sh | 12 +- opencti-documentation/website/core/Footer.js | 6 +- opencti-front/package.json | 26 +- .../components/connector/ConnectorsStats.js | 219 ++++++++++++++++ .../components/report/ReportAddObservable.js | 20 +- .../StixObservablesRightBar.js | 4 +- opencti-front/src/utils/Localization.js | 18 ++ opencti-front/src/utils/String.js | 3 + opencti-front/yarn.lock | 144 ++++++----- opencti-graphql/config/default.json | 2 + opencti-graphql/config/schema/opencti.graphql | 16 ++ .../1562235292773-update_stix_observables.js | 35 +++ opencti-graphql/package.json | 11 +- opencti-graphql/src/database/rabbitmq.js | 19 ++ opencti-graphql/src/domain/connector.js | 44 ++-- opencti-graphql/src/domain/stixEntity.js | 4 +- opencti-graphql/src/domain/stixObservable.js | 2 +- opencti-graphql/src/opencti.gql | 17 ++ opencti-graphql/src/resolvers/connector.js | 5 +- opencti-graphql/yarn.lock | 237 ++++++++++++++++-- opencti-integration/config.yml.docker.sample | 6 + opencti-integration/config.yml.sample | 6 + opencti-integration/connectors | 2 +- opencti-integration/connectors_scheduler.py | 65 +++-- opencti-integration/requirements.txt | 1 + opencti-worker/config.yml.docker.sample | 13 +- opencti-worker/config.yml.sample | 3 - opencti-worker/grakn_dumper.py | 2 +- 36 files changed, 807 insertions(+), 196 deletions(-) delete mode 100644 opencti-docker/config/.config delete mode 100644 opencti-docker/volumes/esdata/.esdata delete mode 100644 opencti-docker/volumes/grakndata/.grakndata create mode 100644 opencti-front/src/private/components/connector/ConnectorsStats.js create mode 100644 opencti-graphql/migrations/1562235292773-update_stix_observables.js diff --git a/README.md b/README.md index 0514b0c3826f..7a70ab82f439 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ - +

diff --git a/opencti-docker/config/.config b/opencti-docker/config/.config deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/opencti-docker/docker-compose.yml b/opencti-docker/docker-compose.yml index acb3cf03b2dd..c38f22da89cd 100644 --- a/opencti-docker/docker-compose.yml +++ b/opencti-docker/docker-compose.yml @@ -5,25 +5,18 @@ services: ports: - 48555:48555 volumes: - - ./volumes/grakndata:/grakn-core-all-linux/server/db + - grakndata:/grakn-core-all-linux/server/db restart: always redis: image: redis restart: always elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:6.7.1 - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 - hard: 65536 volumes: - - ./volumes/esdata:/usr/share/elasticsearch/data + - esdata:/usr/share/elasticsearch/data restart: always rabbitmq: - image: rabbitmq + image: rabbitmq:management restart: always worker: image: opencti/worker @@ -31,8 +24,11 @@ services: - OPENCTI_HOSTNAME=opencti - OPENCTI_PORT=8080 - RABBITMQ_HOSTNAME=rabbitmq + - RABBITMQ_PORT=5672 + - RABBITMQ_USERNAME=guest + - RABBITMQ_PASSWORD=guest volumes: - - "./config:/opt/opencti/shared_config:rw" + - config:/opt/opencti/shared_config:rw depends_on: - opencti restart: always @@ -41,8 +37,12 @@ services: environment: - OPENCTI_HOSTNAME=opencti - OPENCTI_PORT=8080 + - RABBITMQ_HOSTNAME=rabbitmq + - RABBITMQ_PORT=5672 + - RABBITMQ_USERNAME=guest + - RABBITMQ_PASSWORD=guest volumes: - - "./config:/opt/opencti/shared_config:rw" + - config:/opt/opencti/shared_config:rw depends_on: - opencti restart: always @@ -64,13 +64,22 @@ services: - ELASTICSEARCH__PORT=9200 - RABBITMQ__HOSTNAME=rabbitmq - RABBITMQ__PORT=5672 + - RABBITMQ__PORT_MANAGEMENT=15672 + - RABBITMQ__MANAGEMENT_SSL=false + - RABBITMQ__USERNAME=guest + - RABBITMQ__PASSWORD=guest ports: - "8080:8080" volumes: - - "./config:/opt/opencti/shared_config:rw" + - config:/opt/opencti/shared_config:rw depends_on: - grakn - redis - elasticsearch - rabbitmq restart: always + +volumes: + grakndata: + esdata: + config: diff --git a/opencti-docker/integration/Dockerfile b/opencti-docker/integration/Dockerfile index 16a0ffc7e50e..28270b11ca32 100644 --- a/opencti-docker/integration/Dockerfile +++ b/opencti-docker/integration/Dockerfile @@ -8,7 +8,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get -y upgrade && apt-g RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip curl rsyslog wget netcat locales # Download archive -RUN wget -O /opt/opencti.tar.gz https://releases.opencti.io/opencti-20190702.tar.gz +RUN wget -O /opt/opencti.tar.gz https://releases.opencti.io/opencti-20190704.tar.gz RUN cd /opt && tar xvfz opencti.tar.gz # Set the locale @@ -20,7 +20,13 @@ ENV LC_ALL en_US.UTF-8 # Prepare configuration RUN cd /opt/opencti/integration && rm -f config.yml && ln -s /opt/opencti/shared_config/config_integration.yml config.yml +# Install Python modules +RUN cd /opt/opencti/integration && pip3 install -r requirements.txt + +# Install Python modules of each connectors +RUN cd /opt/opencti/integration && find connectors -name requirements.txt -exec pip3 install -r {} \; + # Expose and entrypoint COPY entrypoint.sh / RUN chmod +x /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] +ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/opencti-docker/integration/entrypoint.sh b/opencti-docker/integration/entrypoint.sh index f595fa8be9e8..c7504d9c0a77 100755 --- a/opencti-docker/integration/entrypoint.sh +++ b/opencti-docker/integration/entrypoint.sh @@ -3,13 +3,6 @@ # Start log /etc/init.d/rsyslog start -# Install Python modules -cd /opt/opencti/integration -pip3 install -r requirements.txt - -# Install Python modules of each connectors -find connectors -name requirements.txt -exec pip3 install -r {} \; - # Check configuration while [ ! -f /opt/opencti/shared_config/token ] do @@ -28,6 +21,10 @@ cp config.yml.docker.sample config.yml.sample sed -i -e "s/REPLACE_API_KEY/$(cat /opt/opencti/shared_config/token)/g" config.yml.sample sed -i -e "s/OPENCTI_HOSTNAME/${OPENCTI_HOSTNAME}/g" config.yml.sample sed -i -e "s/OPENCTI_PORT/${OPENCTI_PORT}/g" config.yml.sample +sed -i -e "s/RABBITMQ_HOSTNAME/${RABBITMQ_HOSTNAME}/g" config.yml.sample +sed -i -e "s/RABBITMQ_PORT/${RABBITMQ_PORT}/g" config.yml.sample +sed -i -e "s/RABBITMQ_USERNAME/${RABBITMQ_USERNAME}/g" config.yml.sample +sed -i -e "s/RABBITMQ_PASSWORD/${RABBITMQ_PASSWORD}/g" config.yml.sample cp config.yml.sample /opt/opencti/shared_config/config_integration.yml # Start diff --git a/opencti-docker/volumes/esdata/.esdata b/opencti-docker/volumes/esdata/.esdata deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/opencti-docker/volumes/grakndata/.grakndata b/opencti-docker/volumes/grakndata/.grakndata deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/opencti-docker/worker/Dockerfile b/opencti-docker/worker/Dockerfile index 92b3bdba2258..e0b2fb0c25ad 100644 --- a/opencti-docker/worker/Dockerfile +++ b/opencti-docker/worker/Dockerfile @@ -20,6 +20,9 @@ ENV LC_ALL en_US.UTF-8 # Prepare configuration RUN cd /opt/opencti/worker && rm -f config.yml && ln -s /opt/opencti/shared_config/config_worker.yml config.yml +# Install Python modules +RUN cd /opt/opencti/worker && pip3 install -r requirements.txt + # Expose and entrypoint COPY entrypoint.sh / RUN chmod +x /entrypoint.sh diff --git a/opencti-docker/worker/entrypoint.sh b/opencti-docker/worker/entrypoint.sh index ccb6a20d3f30..3638a2938fe8 100755 --- a/opencti-docker/worker/entrypoint.sh +++ b/opencti-docker/worker/entrypoint.sh @@ -3,10 +3,6 @@ # Start log /etc/init.d/rsyslog start -# Install Python modules -cd /opt/opencti/worker -pip3 install -r requirements.txt - # Check configuration while [ ! -f /opt/opencti/shared_config/token ] do @@ -26,9 +22,17 @@ sed -i -e "s/REPLACE_API_KEY/$(cat /opt/opencti/shared_config/token)/g" config.y sed -i -e "s/OPENCTI_HOSTNAME/${OPENCTI_HOSTNAME}/g" config.yml.sample sed -i -e "s/OPENCTI_PORT/${OPENCTI_PORT}/g" config.yml.sample sed -i -e "s/RABBITMQ_HOSTNAME/${RABBITMQ_HOSTNAME}/g" config.yml.sample +sed -i -e "s/RABBITMQ_PORT/${RABBITMQ_PORT}/g" config.yml.sample +sed -i -e "s/RABBITMQ_USERNAME/${RABBITMQ_USERNAME}/g" config.yml.sample +sed -i -e "s/RABBITMQ_PASSWORD/${RABBITMQ_PASSWORD}/g" config.yml.sample cp config.yml.sample /opt/opencti/shared_config/config_worker.yml # Start cd /opt/opencti/worker python3 worker_import.py & +python3 worker_import.py & +python3 worker_import.py & +python3 worker_import.py & +python3 worker_import.py & +python3 worker_export.py & python3 worker_export.py \ No newline at end of file diff --git a/opencti-documentation/website/core/Footer.js b/opencti-documentation/website/core/Footer.js index d87742ec5fb7..3ce069ab2388 100644 --- a/opencti-documentation/website/core/Footer.js +++ b/opencti-documentation/website/core/Footer.js @@ -37,13 +37,13 @@ class Footer extends React.Component {
Docs
- + Install using Docker - + Usage introduction - + Development environment
diff --git a/opencti-front/package.json b/opencti-front/package.json index dc48ef585e01..2b798a8464e3 100644 --- a/opencti-front/package.json +++ b/opencti-front/package.json @@ -4,10 +4,10 @@ "author": "Luatix", "license": "AGPL-3.0-or-later", "dependencies": { - "@date-io/moment": "^1.3.6", - "@material-ui/core": "^4.1.1", - "@material-ui/icons": "^4.2.0", - "@material-ui/lab": "^4.0.0-alpha.16", + "@date-io/moment": "^1.3.8", + "@material-ui/core": "^4.1.3", + "@material-ui/icons": "^4.2.1", + "@material-ui/lab": "^4.0.0-alpha.18", "@material-ui/pickers": "^3.1.2", "apollo-link": "^1.2.12", "apollo-link-ws": "^1.0.18", @@ -16,14 +16,14 @@ "countdown": "^2.6.0", "dagre": "^0.8.4", "formik": "^1.5.7", - "formik-material-ui": "^0.0.18", - "graphql": "^14.3.1", + "formik-material-ui": "^0.0.19", + "graphql": "^14.4.1", "http-proxy-middleware": "^0.19.1", "js-cookie": "^2.2.0", "js-file-download": "^0.4.7", "jsonwebtoken": "^8.5.1", "lodash": "^4.17.11", - "mdi-material-ui": "^6.0.0", + "mdi-material-ui": "^6.1.0", "moment": "^2.24.0", "moment-timezone": "^0.5.25", "object-hash": "^1.3.1", @@ -35,18 +35,18 @@ "react-dom": "^16.8.6", "react-grid-layout": "^0.16.6", "react-intl": "^2.9.0", - "react-markdown": "^4.0.8", + "react-markdown": "^4.1.0", "react-particles-js": "^2.6.0", - "react-relay": "^4.0.0", - "react-relay-network-modern": "^3.0.3", - "react-router-dom": "^5.0.0", + "react-relay": "^5.0.0", + "react-relay-network-modern": "^4.0.2", + "react-router-dom": "^5.0.1", "react-scripts": "^3.0.1", "react-select": "^3.0.4", "react-virtualized": "^9.21.1", "react-window-dimensions": "^1.0.2", "recharts": "^1.6.2", - "relay-compiler": "^4.0.0", - "relay-runtime": "^4.0.0", + "relay-compiler": "^5.0.0", + "relay-runtime": "^5.0.0", "rxjs": "^6.5.2", "storm-react-diagrams": "^5.2.1", "subscriptions-transport-ws": "^0.9.16", diff --git a/opencti-front/src/private/components/connector/ConnectorsStats.js b/opencti-front/src/private/components/connector/ConnectorsStats.js new file mode 100644 index 000000000000..d6bbc20b934f --- /dev/null +++ b/opencti-front/src/private/components/connector/ConnectorsStats.js @@ -0,0 +1,219 @@ +import React, { Component } from 'react'; +import * as PropTypes from 'prop-types'; +import { + compose, map, assoc, mapObjIndexed, values, +} from 'ramda'; +import { Formik, Field, Form } from 'formik'; +import graphql from 'babel-plugin-relay/macro'; +import { withStyles } from '@material-ui/core/styles/index'; +import Paper from '@material-ui/core/Paper'; +import Button from '@material-ui/core/Button'; +import MenuItem from '@material-ui/core/MenuItem'; +import Typography from '@material-ui/core/Typography'; +import { + commitMutation, + MESSAGING$, + QueryRenderer, +} from '../../relay/environment'; + +const styles = () => ({ + container: { + margin: 0, + }, + panel: { + width: '50%', + margin: '0 auto', + marginBottom: 30, + padding: '20px 20px 20px 20px', + textAlign: 'left', + borderRadius: 6, + }, +}); + +const connectorsStatsQuery = graphql` + query ConnectorsStatsQuery { + connectorsStats { + name + messages + messages_ready + messages_unacknowledged + consumers + message_stats { + deliver_details { + rate + } + } + } + } +`; + +class ConnectorsStats extends Component { + constructor(props) { + super(props); + this.state = { refetch: false }; + } + + render() { + const { classes, t } = this.props; + return ( +
+ { + if (props) { + const { connectorsStats } = props; + return connectorsStats.map((connector) => { + const connectorConfigTemplate = JSON.parse( + Buffer.from(connector.config_template, 'base64').toString( + 'ascii', + ), + ); + let config = map( + x => x.default, + connectorConfigTemplate.fields, + ); + if (connector.config) { + config = JSON.parse( + Buffer.from(connector.config, 'base64').toString('ascii'), + ); + } + const fields = values( + mapObjIndexed( + (value, key, obj) => assoc('name', key, value), + connectorConfigTemplate.fields, + ), + ); + return ( + + ( +
+ + {connectorConfigTemplate.name} + + +
+ {fields.map((field) => { + if (field.type === 'select') { + return ( + + {field.options.map(option => ( + + {option.label} + + ))} + + ); + } + if (field.type === 'switch') { + return ( + + ); + } + return ( + + ); + })} +
+ +
+ + )} + /> + + ); + }); + } + return
 
; + }} + /> +
+ ); + } +} + +ConnectorsStats.propTypes = { + classes: PropTypes.object, + t: PropTypes.func, +}; + +export default compose( + inject18n, + withStyles(styles), +)(ConnectorsStats); diff --git a/opencti-front/src/private/components/report/ReportAddObservable.js b/opencti-front/src/private/components/report/ReportAddObservable.js index 25b7a762d5c3..abc071a2cee2 100644 --- a/opencti-front/src/private/components/report/ReportAddObservable.js +++ b/opencti-front/src/private/components/report/ReportAddObservable.js @@ -546,12 +546,26 @@ class ReportAddObservable extends Component { containerstyle={{ width: '100%' }} > {t('Domain')} + {t('Email address')} + {t('Email subject')} + {t('File name')} + {t('File path')} + {t('File MD5 hash')} + {t('File SHA1 hash')} + {t('File SHA256 hash')} {t('IPv4 address')} {t('IPv6 address')} - {t('URL')} - {t('Email address')} {t('Mutex')} - {t('File hash')} + {t('PDB Path')} + {t('Registry key')} + {t('Registry key value')} + {t('Mutex')} + {t('URL')} + {t('Windows Service Name')} + {t('Windows Service Display Name')} + {t('Windows Scheduled Task')} + {t('X509 Certificate Issuer')} + {t('X509 Certificate Serial number')} - + - + { return str; } const trimmedStr = str.substr(0, limit); + if (!trimmedStr.includes(' ')) { + return `${trimmedStr}...`; + } return `${trimmedStr.substr( 0, Math.min(trimmedStr.length, trimmedStr.lastIndexOf(' ')), diff --git a/opencti-front/yarn.lock b/opencti-front/yarn.lock index e92a49d62215..ceb6c4ea74a2 100644 --- a/opencti-front/yarn.lock +++ b/opencti-front/yarn.lock @@ -956,10 +956,17 @@ resolved "https://registry.yarnpkg.com/@csstools/normalize.css/-/normalize.css-9.0.1.tgz#c27b391d8457d1e893f1eddeaf5e5412d12ffbb5" integrity sha512-6It2EVfGskxZCQhuykrfnALg7oVeiI6KclWSmGDqB0AiInVrTGB9Jp9i4/Ad21u9Jde/voVQz6eFX/eSg/UsPA== -"@date-io/moment@^1.3.6": - version "1.3.6" - resolved "https://registry.yarnpkg.com/@date-io/moment/-/moment-1.3.6.tgz#ee6d911c400e66c0bfe6cd782be36834f529de03" - integrity sha512-aaWDAPaiwmya/W8ERd19BohO8mgNwPP/l/nTwcG7V+WZKQjDotOfcIHJaDSLKbDsA5WDP6kxDrbXE15Jztvu/g== +"@date-io/core@^1.3.8": + version "1.3.8" + resolved "https://registry.yarnpkg.com/@date-io/core/-/core-1.3.8.tgz#a20da7125f1444631dd96a6e8baa8982665aaa95" + integrity sha512-XlvpzhipF/badZEJPAfgXRX9bFG2UpFl1T6O0hn1RzWJf3yexf9/M2wcEm6jh7RVyrlSayttkXtGPqVPdoJQ7Q== + +"@date-io/moment@^1.3.8": + version "1.3.8" + resolved "https://registry.yarnpkg.com/@date-io/moment/-/moment-1.3.8.tgz#e0d4599891782ac811be763d514408949b3e09b0" + integrity sha512-1caePZfVFeL8bJLQVZVBqYQfgF+4SVJZBTDC1o7bMTtId7/yfIFm3ySxHJrfpJa4ACdF667Ld4wXEfvZz98tuA== + dependencies: + "@date-io/core" "^1.3.8" "@emotion/cache@^10.0.9": version "10.0.9" @@ -1234,15 +1241,15 @@ "@types/istanbul-reports" "^1.1.1" "@types/yargs" "^12.0.9" -"@material-ui/core@^4.1.1": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@material-ui/core/-/core-4.1.1.tgz#11e703f3d44d84750c9b829f13bb08ad8949e12a" - integrity sha512-dZVuVFqKnG3uf+s32U5wMTAXYBGBM6e2LF4fz4ud9woaYcthRiEFJTg2idt0j1jBMg99gqLuznR5+A9TCQbgxQ== +"@material-ui/core@^4.1.3": + version "4.1.3" + resolved "https://registry.yarnpkg.com/@material-ui/core/-/core-4.1.3.tgz#0f8ffae88ac668399fbfc66efa1ed37c4ce29a7d" + integrity sha512-cSrI3yZ2L1XVi5IyO0/dPGUq1FaDBKTL9ygDOlvojD6bGuD/Qu17dCFAD8/mHV4CPNQMJYUKLYyOmqtDYdEB9g== dependencies: "@babel/runtime" "^7.2.0" - "@material-ui/styles" "^4.1.1" - "@material-ui/system" "^4.2.0" - "@material-ui/types" "^4.1.0" + "@material-ui/styles" "^4.1.2" + "@material-ui/system" "^4.3.0" + "@material-ui/types" "^4.1.1" "@material-ui/utils" "^4.1.0" "@types/react-transition-group" "^2.0.16" clsx "^1.0.2" @@ -1258,23 +1265,24 @@ react-transition-group "^4.0.0" warning "^4.0.1" -"@material-ui/icons@^4.2.0": - version "4.2.0" - resolved "https://registry.yarnpkg.com/@material-ui/icons/-/icons-4.2.0.tgz#cd0b6521288c5e6c0b084d6defec37a4491e0c51" - integrity sha512-v+rz61KzH+qR8x17BrfOF73f75x+wUNiBhv9tsKnEed+ElROMK2dqfMAlsdgEP+wgGl4VOcxzUQqWHcaApZ+CA== +"@material-ui/icons@^4.2.1": + version "4.2.1" + resolved "https://registry.yarnpkg.com/@material-ui/icons/-/icons-4.2.1.tgz#fe2f1c4f60c24256d244a69d86d0c00e8ed4037e" + integrity sha512-FvSD5lUBJ66frI4l4AYAPy2CH14Zs2Dgm0o3oOMr33BdQtOAjCgbdOcvPBeaD1w6OQl31uNW3CKOE8xfPNxvUQ== dependencies: "@babel/runtime" "^7.2.0" -"@material-ui/lab@^4.0.0-alpha.16": - version "4.0.0-alpha.16" - resolved "https://registry.yarnpkg.com/@material-ui/lab/-/lab-4.0.0-alpha.16.tgz#0d5f4e7e7eda057b732d26fe603b38cce6adf3b2" - integrity sha512-NKaIMQd9bUI+Iny6AduYA0KBkVq5XAVWspNtem4qmPTr4aFbh90IZ1a8twN77hHHJm35MRBy4Sq+Y7I07JGw0g== +"@material-ui/lab@^4.0.0-alpha.18": + version "4.0.0-alpha.18" + resolved "https://registry.yarnpkg.com/@material-ui/lab/-/lab-4.0.0-alpha.18.tgz#e1b8f51d177190f8431b1d07da9f78f560727ac8" + integrity sha512-YdsKIyTTZPQ5w4oWflTYBFbrdzN6nx6qqzgIw10Gkjsk5MZronOyxMx+HRxPtgmtsgXudlA7v4JqptSWl5Pxug== dependencies: "@babel/runtime" "^7.2.0" "@material-ui/utils" "^4.1.0" clsx "^1.0.2" keycode "^2.1.9" prop-types "^15.7.2" + warning "^4.0.1" "@material-ui/pickers@^3.1.2": version "3.1.2" @@ -1287,14 +1295,14 @@ rifm "^0.7.0" tslib "^1.9.3" -"@material-ui/styles@^4.1.1": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@material-ui/styles/-/styles-4.1.1.tgz#16e9d4770bd18b85c25fd3cb925043af5fa37f36" - integrity sha512-BmtfLRY0CqAkYPdcFmNcD1/zyU6ATRx9vaBxJ31//YVxfRsyPOuQXW6fvAoDvQt/hbZpTR4E0K/+4D3wHHTdHQ== +"@material-ui/styles@^4.1.2": + version "4.1.2" + resolved "https://registry.yarnpkg.com/@material-ui/styles/-/styles-4.1.2.tgz#f22cb0d8f60a5e4a4b92f1ffb4b56f825ff89034" + integrity sha512-IRwhGI3OzxMKIDXnYl/vi9FD/i5ZktVP2m/5PIf/HVdvhqUZGIzqR2OB/9f3W1hc+kKKExdOPlpZpVihIsJWkA== dependencies: "@babel/runtime" "^7.2.0" "@emotion/hash" "^0.7.1" - "@material-ui/types" "^4.1.0" + "@material-ui/types" "^4.1.1" "@material-ui/utils" "^4.1.0" clsx "^1.0.2" csstype "^2.5.2" @@ -1311,20 +1319,22 @@ prop-types "^15.7.2" warning "^4.0.1" -"@material-ui/system@^4.2.0": - version "4.2.0" - resolved "https://registry.yarnpkg.com/@material-ui/system/-/system-4.2.0.tgz#1c0759f1168bf871a7970239dc781b7bf9dfd010" - integrity sha512-t51525FWVDjca/3UPwN99vqyvbfGNtBVesGYH2UpxVgKOdiP1ZINeHhBrZ8h4uOu5ZwgO4aceuk1TuM9uMttYw== +"@material-ui/system@^4.3.0": + version "4.3.0" + resolved "https://registry.yarnpkg.com/@material-ui/system/-/system-4.3.0.tgz#6812049bf9257f8936c5de8f18b7142a67048247" + integrity sha512-VQh3mWZSmzm1JR7Ci35AHKwOhhxHHMrBWCdP4mh7UAwSdjWBE6s2Y9Y0iJiqMoEsHP64vU3W1JpsW2AgkUeHsQ== dependencies: "@babel/runtime" "^7.2.0" deepmerge "^3.0.0" prop-types "^15.7.2" warning "^4.0.1" -"@material-ui/types@^4.1.0": - version "4.1.0" - resolved "https://registry.yarnpkg.com/@material-ui/types/-/types-4.1.0.tgz#58e60d86f5a554e60c06a4a91f2030ac2c56a5fd" - integrity sha512-F4z7GOAeEucPjrrhJ2PHBhMZjhggE6Jjnzmap5W2PdZ3TSWNlqucB+oOzT6EzWRkHDDhVmANMU8QMfT/kcZtOg== +"@material-ui/types@^4.1.1": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@material-ui/types/-/types-4.1.1.tgz#b65e002d926089970a3271213a3ad7a21b17f02b" + integrity sha512-AN+GZNXytX9yxGi0JOfxHrRTbhFybjUJ05rnsBVjcB+16e466Z0Xe5IxawuOayVZgTBNDxmPKo5j4V6OnMtaSQ== + dependencies: + "@types/react" "*" "@material-ui/utils@^4.1.0": version "4.1.0" @@ -5858,10 +5868,10 @@ format@^0.2.2: resolved "https://registry.yarnpkg.com/format/-/format-0.2.2.tgz#d6170107e9efdc4ed30c9dc39016df942b5cb58b" integrity sha1-1hcBB+nv3E7TDJ3DkBbflCtctYs= -formik-material-ui@^0.0.18: - version "0.0.18" - resolved "https://registry.yarnpkg.com/formik-material-ui/-/formik-material-ui-0.0.18.tgz#40c4b09881af9c065690a9d1f9bb6efacf8aabd7" - integrity sha512-NQwcWZ5spUYiCc7hRP0b6WzQfIvbidIw1dBS4TcOvLqiwkVtamVPOZWO9mkLOQboAyCn/OKR5CVcdtFaslQmcg== +formik-material-ui@^0.0.19: + version "0.0.19" + resolved "https://registry.yarnpkg.com/formik-material-ui/-/formik-material-ui-0.0.19.tgz#f02b568507031881f897ca14ebbfcbeb4f043c17" + integrity sha512-1EgNO5jjMyEijEjzX187hk73odxOJ2Rlm6TBCmEg3hTEazrKO8Isfdx79ItyStMIvxdDI41p4UBgAOhyCSOsYg== formik@^1.5.7: version "1.5.7" @@ -6193,10 +6203,10 @@ graphlib@^2.1.7: dependencies: lodash "^4.17.5" -graphql@^14.3.1: - version "14.3.1" - resolved "https://registry.yarnpkg.com/graphql/-/graphql-14.3.1.tgz#b3aa50e61a841ada3c1f9ccda101c483f8e8c807" - integrity sha512-FZm7kAa3FqKdXy8YSSpAoTtyDFMIYSpCDOr+3EqlI1bxmtHu+Vv/I2vrSeT1sBOEnEniX3uo4wFhFdS/8XN6gA== +graphql@^14.4.1: + version "14.4.1" + resolved "https://registry.yarnpkg.com/graphql/-/graphql-14.4.1.tgz#7a7818d3f63f66b9528ba5416b6c88460db62280" + integrity sha512-g4HUH26CohlMjaHneXMAtvG3QtO6peJIUTFxrPW4g5LNnXkUuFoBI6Bk1c14Q5kW8+FyjM/tTbePTgpiVB/2hQ== dependencies: iterall "^1.2.2" @@ -8459,10 +8469,10 @@ mdast-add-list-metadata@1.0.1: dependencies: unist-util-visit-parents "1.1.2" -mdi-material-ui@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/mdi-material-ui/-/mdi-material-ui-6.0.0.tgz#76ff2ebbd2ce9012f700dc09a1a2b4b1ef116bd6" - integrity sha512-AVSWSH09aj6w6zf1zYtnLwyok//GTrAnZsFf5Yk4VuBFhzxKChhIyZ+KEwA2JDAipyztAHfgYgZE388egVKVnw== +mdi-material-ui@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/mdi-material-ui/-/mdi-material-ui-6.1.0.tgz#e380726e5a7d50341dfc5d38683800e6df6c0a49" + integrity sha512-sRNB8p+71OlU1HhH7Ao0DT0IPXwVa0YaHBbVT4/DC+Nc9ea91qd1ukx5Vuq6X+0wy+ylmBqbFw8nmlBBb9k6Lg== mdn-data@~1.1.0: version "1.1.4" @@ -10877,10 +10887,10 @@ react-lifecycles-compat@^3.0.2, react-lifecycles-compat@^3.0.4: resolved "https://registry.yarnpkg.com/react-lifecycles-compat/-/react-lifecycles-compat-3.0.4.tgz#4f1a273afdfc8f3488a8c516bfda78f872352362" integrity sha512-fBASbA6LnOU9dOU2eW7aQ8xmYBSXUIWr+UmF9b1efZBazGNO+rcXT/icdKnYm2pTwcRylVUYwW7H1PHfLekVzA== -react-markdown@^4.0.8: - version "4.0.8" - resolved "https://registry.yarnpkg.com/react-markdown/-/react-markdown-4.0.8.tgz#e3621b5becaac82a651008d7bc8390d3e4e438c0" - integrity sha512-Z6oa648rufvzyO0KwYJ/9p9AsdYGIluqK6OlpJ35ouJ8HPF0Ko1WDNdyymjDSHxNrkb7HDyEcIDJCQs8NlET5A== +react-markdown@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/react-markdown/-/react-markdown-4.1.0.tgz#7fdf840ecbabc803f28156f7411c726b58f25a73" + integrity sha512-EOHsEAN+aoP8UVz7vTHx6Z63GJfhrO9KItKlfsiBtVVS9tmSWtUaBTw73+2SObrWiOiE2Cs9qUBL7ORsvVhOrA== dependencies: html-to-react "^1.3.4" mdast-add-list-metadata "1.0.1" @@ -10917,20 +10927,20 @@ react-popper@^1.3.3: typed-styles "^0.0.7" warning "^4.0.2" -react-relay-network-modern@^3.0.3: - version "3.0.4" - resolved "https://registry.yarnpkg.com/react-relay-network-modern/-/react-relay-network-modern-3.0.4.tgz#caf61ca7398a8109f3d2d7377954ecaba66c8df8" - integrity sha512-d1ttgGGRrjvntUeStKOGO4tsJy2P7hQ+T98DzdTa8QFEx8//0+/IPc1TUtFk0GVTUQJXG66cQw7EknwDRmRLfg== +react-relay-network-modern@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/react-relay-network-modern/-/react-relay-network-modern-4.0.2.tgz#f87df3c9b4c03c5139e8374110e216ce5e837bf5" + integrity sha512-eTeMEk+4t4z32dHsf5nEi/7Dp74ygDptwAS4bSOgishq1qOAjs/BngppO9ehBbI4Y6YV6wjulSAGamde4XngEQ== -react-relay@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/react-relay/-/react-relay-4.0.0.tgz#000f63ca3ae179b4415ee11765d04bc2fe4278da" - integrity sha512-Z7O+zM5oUmUWqOiPv7Z3L8a9cNCXare8wkUfDgtTBl048wXXtkAWTHTP0QZdViRSLmNHmEwHo/y0lBpZ7iy6SQ== +react-relay@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/react-relay/-/react-relay-5.0.0.tgz#66af68e8e5fad05879a3f21f895a0296ef2741a8" + integrity sha512-gpUvedaCaPVPT0nMrTbev2TzrU0atgq2j/zAnGHiR9WgqRXwtHsK6FWFN65HRbopO2DzuJx9VZ2I3VO6uL5EMA== dependencies: "@babel/runtime" "^7.0.0" fbjs "^1.0.0" nullthrows "^1.1.0" - relay-runtime "4.0.0" + relay-runtime "5.0.0" react-resizable@1.x: version "1.8.0" @@ -10961,7 +10971,7 @@ react-resize-detector@^4.0.5: raf-schd "^4.0.0" resize-observer-polyfill "^1.5.1" -react-router-dom@^5.0.0: +react-router-dom@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-5.0.1.tgz#ee66f4a5d18b6089c361958e443489d6bab714be" integrity sha512-zaVHSy7NN0G91/Bz9GD4owex5+eop+KvgbxXsP/O+iW1/Ln+BrJ8QiIR5a6xNPtrdTvLkxqlDClx13QO1uB8CA== @@ -11465,10 +11475,10 @@ relateurl@0.2.x: resolved "https://registry.yarnpkg.com/relateurl/-/relateurl-0.2.7.tgz#54dbf377e51440aca90a4cd274600d3ff2d888a9" integrity sha1-VNvzd+UUQKypCkzSdGANP/LYiKk= -relay-compiler@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/relay-compiler/-/relay-compiler-4.0.0.tgz#d97dd2819d3b92669599c9cfec8049c9b487be67" - integrity sha512-IljjuroVEBWfNgZHb9Qm2YeAH6GRagM64eRYIdyVN4kitY1u1Hitle4000y3A1XRwjgrEuD4CuryFK0ktGaszA== +relay-compiler@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/relay-compiler/-/relay-compiler-5.0.0.tgz#ca2514bda20ff829550ac87f126d07a1517bf6de" + integrity sha512-q8gKlPRTJe/TwtIokbdXehy1SxDFIyLBZdsfg60J4OcqyviIx++Vhc+h4lXzBY8LsBVaJjTuolftYcXJLhlE6g== dependencies: "@babel/core" "^7.0.0" "@babel/generator" "^7.0.0" @@ -11484,7 +11494,7 @@ relay-compiler@^4.0.0: fbjs "^1.0.0" immutable "~3.7.6" nullthrows "^1.1.0" - relay-runtime "4.0.0" + relay-runtime "5.0.0" signedsource "^1.0.0" yargs "^9.0.0" @@ -11498,10 +11508,10 @@ relay-devtools@^1.4.0: update-notifier "^2.2.0" ws "^3.1.0" -relay-runtime@4.0.0, relay-runtime@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/relay-runtime/-/relay-runtime-4.0.0.tgz#5cf50f52e3ea56ec81f0b5fed05b03e99a9e3906" - integrity sha512-Fd5nAMNfySINAYPf1zgtcYbCk33pO5ANiYfaMhKYzukfc2GCmn6RP7NEpG69GxVezu/E1aOxo72t+Y1NspDV8A== +relay-runtime@5.0.0, relay-runtime@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/relay-runtime/-/relay-runtime-5.0.0.tgz#7c688ee621d6106a2cd9f3a3706eb6d717c7f660" + integrity sha512-lrC2CwfpWWHBAN608eENAt5Bc5zqXXE2O9HSo8tc6Gy5TxfK+fU+x9jdwXQ2mXxVPgANYtYeKzU5UTfcX0aDEw== dependencies: "@babel/runtime" "^7.0.0" fbjs "^1.0.0" diff --git a/opencti-graphql/config/default.json b/opencti-graphql/config/default.json index 74b929f09b91..c53f6d34a1bb 100644 --- a/opencti-graphql/config/default.json +++ b/opencti-graphql/config/default.json @@ -24,6 +24,8 @@ "rabbitmq": { "hostname": "localhost", "port": 5672, + "port_management": 15672, + "management_ssl": false, "username": "guest", "password": "guest" }, diff --git a/opencti-graphql/config/schema/opencti.graphql b/opencti-graphql/config/schema/opencti.graphql index 65b9722fdcb7..6b20a3273106 100644 --- a/opencti-graphql/config/schema/opencti.graphql +++ b/opencti-graphql/config/schema/opencti.graphql @@ -85,6 +85,21 @@ type EditUserContext { } ### Stats +type DeliverDetails { + rate: Int +} +type MessagesStats { + deliver_details: DeliverDetails +} +type QueueStats { + name: String + messages: Int + messages_ready: Int + messages_unacknowledged: Int + consumers: Int, + message_stats: MessagesStats +} + enum StatsOperation { count sum @@ -2057,6 +2072,7 @@ type Query { settings: Settings # Connectors connectors: [Connector] @auth(for: ROLE_ADMIN) + connectorsStats: [QueueStats] @auth(for: ROLE_ADMIN) # StixDomainEntities stixDomainEntity(id: String): StixDomainEntity @auth stixDomainEntities( diff --git a/opencti-graphql/migrations/1562235292773-update_stix_observables.js b/opencti-graphql/migrations/1562235292773-update_stix_observables.js new file mode 100644 index 000000000000..ece5f7d2d872 --- /dev/null +++ b/opencti-graphql/migrations/1562235292773-update_stix_observables.js @@ -0,0 +1,35 @@ +import { queryMultiple, updateAttribute } from '../src/database/grakn'; +import { index } from '../src/database/elasticSearch'; +import { logger } from '../src/config/conf'; + +module.exports.up = async next => { + const resultPromise = Promise.all( + ['observable'].map(async entityType => { + const query = `match $x isa entity; $x has stix_id $sid; $sid contains "${entityType}"; get $x;`; + const entities = await queryMultiple(query, ['x']); + logger.info('Entities loaded'); + const updatePromise = Promise.all( + entities.map(entity => { + return updateAttribute(entity.x.id, { + key: 'stix_id', + value: [entity.x.stix_id.replace(entityType, 'indicator')] + }).then(stixDomainEntity => { + index('stix-observables', 'stix_observable', stixDomainEntity); + }); + }) + ); + await Promise.resolve(updatePromise).then(() => { + logger.info('Entities updated'); + return Promise.resolve(true); + }); + }) + ); + await Promise.resolve(resultPromise).then(() => { + logger.info('Migration complete'); + }); + next(); +}; + +module.exports.down = async next => { + next(); +}; diff --git a/opencti-graphql/package.json b/opencti-graphql/package.json index 5c9c46f8fc0b..b0df200a23cb 100644 --- a/opencti-graphql/package.json +++ b/opencti-graphql/package.json @@ -32,8 +32,9 @@ "@godaddy/terminus": "^4.1.2", "amqplib": "^0.5.3", "apollo-errors": "^1.9.0", - "apollo-server-express": "^2.5.0", + "apollo-server-express": "^2.7.0-alpha.3", "await": "^0.2.6", + "axios": "^0.19.0", "bcryptjs": "^2.4.3", "body-parser": "^1.19.0", "bytes": "^3.1.0", @@ -43,19 +44,19 @@ "cors": "^2.8.5", "crypto": "^1.0.1", "dotenv": "^8.0.0", - "elasticsearch": "^16.0.0", + "elasticsearch": "^16.1.1", "express": "^4.17.1", "grakn-client": "^1.5.3", - "graphql": "^14.3.1", + "graphql": "^14.4.1", "graphql-constraint-directive": "^1.4.0", "graphql-import": "^0.7.1", "graphql-iso-date": "^3.6.1", "graphql-redis-subscriptions": "^2.1.0", "graphql-relay": "^0.6.0", "graphql-subscriptions": "^1.1.0", - "graphql-tools": "^4.0.4", + "graphql-tools": "^4.0.5", "helmet": "^3.18.0", - "ioredis": "^4.9.5", + "ioredis": "^4.11.1", "iterall": "^1.2.2", "jsonwebtoken": "^8.5.1", "merge-graphql-schemas": "^1.5.8", diff --git a/opencti-graphql/src/database/rabbitmq.js b/opencti-graphql/src/database/rabbitmq.js index 2231aa5c794f..921501eaa5c2 100644 --- a/opencti-graphql/src/database/rabbitmq.js +++ b/opencti-graphql/src/database/rabbitmq.js @@ -1,4 +1,5 @@ import amqp from 'amqplib'; +import axios from 'axios'; import conf, { logger } from '../config/conf'; export const send = (exchangeName, key, message) => { @@ -26,3 +27,21 @@ export const send = (exchangeName, key, message) => { }); } }; + +export const statsQueues = () => { + const baseURL = `http${ + conf.get('rabbitmq:management_ssl') === true ? 's' : '' + }://${conf.get('rabbitmq:hostname')}:${conf.get('rabbitmq:port_management')}`; + return axios + .get('/api/queues', { + baseURL, + withCredentials: true, + auth: { + username: conf.get('rabbitmq:username'), + password: conf.get('rabbitmq:password') + } + }) + .then(response => { + return response.data; + }); +}; diff --git a/opencti-graphql/src/domain/connector.js b/opencti-graphql/src/domain/connector.js index aa154a0cc35c..288e056a8c2e 100644 --- a/opencti-graphql/src/domain/connector.js +++ b/opencti-graphql/src/domain/connector.js @@ -1,13 +1,15 @@ import { readdirSync, readFileSync, lstatSync } from 'fs'; import { join, basename } from 'path'; import uuid from 'uuid/v4'; -import { isNil, filter } from 'ramda'; +import { isNil, filter, includes } from 'ramda'; import { commitWriteTx, queryOne, takeWriteTx, updateAttribute } from '../database/grakn'; +import { statsQueues } from '../database/rabbitmq'; + import conf, { logger } from '../config/conf'; export const getConnectors = async () => { @@ -18,24 +20,23 @@ export const getConnectors = async () => { .map(name => join(source, name)) .filter(isDirectory) .map(connector => basename(connector)); - return filter(n => n !== '.github', getDirectories(path)).map( - async connector => { - const connectorConfigTemplate = readFileSync( - `${path}/${connector}/config.json` - ); - const connectorObject = await queryOne( - `match $x isa Connector; $x has connector_identifier "${connector}"; get $x;`, - ['x'] - ); - return { - identifier: connector, - config_template: Buffer.from(connectorConfigTemplate).toString( - 'base64' - ), - config: connectorObject ? connectorObject.x.connector_config : null - }; - } - ); + return filter( + n => !includes(n, ['.github', 'test']), + getDirectories(path) + ).map(async connector => { + const connectorConfigTemplate = readFileSync( + `${path}/${connector}/config.json` + ); + const connectorObject = await queryOne( + `match $x isa Connector; $x has connector_identifier "${connector}"; get $x;`, + ['x'] + ); + return { + identifier: connector, + config_template: Buffer.from(connectorConfigTemplate).toString('base64'), + config: connectorObject ? connectorObject.x.connector_config : null + }; + }); }; export const updateConfig = async (identifier, config) => { @@ -58,3 +59,8 @@ export const updateConfig = async (identifier, config) => { await updateAttribute(id, { key: 'connector_config', value: [config] }); return Promise.resolve(true); }; + +export const getStatistics = async () => { + const importStatistics = await statsQueues(); + return importStatistics; +}; diff --git a/opencti-graphql/src/domain/stixEntity.js b/opencti-graphql/src/domain/stixEntity.js index 47c632fd07f8..4b9e6d92fcee 100644 --- a/opencti-graphql/src/domain/stixEntity.js +++ b/opencti-graphql/src/domain/stixEntity.js @@ -7,8 +7,8 @@ export const importData = async (type, file) => { JSON.stringify({ type, file_name: file.name, - content: file.base64, + content: file.base64 }) ); return true; -}; \ No newline at end of file +}; diff --git a/opencti-graphql/src/domain/stixObservable.js b/opencti-graphql/src/domain/stixObservable.js index d6d6b7ba4831..510bece7b5fa 100644 --- a/opencti-graphql/src/domain/stixObservable.js +++ b/opencti-graphql/src/domain/stixObservable.js @@ -156,7 +156,7 @@ export const addStixObservable = async (user, stixObservable) => { has stix_id "${ stixObservable.stix_id ? escapeString(stixObservable.stix_id) - : `observable--${uuid()}` + : `indicator--${uuid()}` }", has entity_type "${escapeString(stixObservable.type.toLowerCase())}", has name "", diff --git a/opencti-graphql/src/opencti.gql b/opencti-graphql/src/opencti.gql index aabd9e4837e3..8dcb8823b50e 100644 --- a/opencti-graphql/src/opencti.gql +++ b/opencti-graphql/src/opencti.gql @@ -765,10 +765,27 @@ define Mutex sub Stix-Observable; File sub Stix-Observable; + File-Name sub File; + File-Path sub File; File-MD5 sub File; File-SHA1 sub File; File-SHA256 sub File; + Registry-Key sub Stix-Observable; + Registry-Key-Value sub Registry-Key; + + PDB-Path sub Stix-Observable; + + Windows-Service sub Stix-Observable; + Windows-Service-Name sub Windows-Service; + Windows-Service-Display-Name sub Windows-Service; + + Windows-Scheduled-Task sub Stix-Observable; + + X509-Certificate sub Stix-Observable; + X509-Certificate-Issuer sub X509-Certificate; + X509-Certificate-Serial-Number sub X509-Certificate; + Stix-Observable-Data sub entity, abstract, has internal_id, diff --git a/opencti-graphql/src/resolvers/connector.js b/opencti-graphql/src/resolvers/connector.js index 0c57447348fd..16e78224f273 100644 --- a/opencti-graphql/src/resolvers/connector.js +++ b/opencti-graphql/src/resolvers/connector.js @@ -1,8 +1,9 @@ -import { getConnectors, updateConfig } from '../domain/connector'; +import { getConnectors, updateConfig, getStatistics } from '../domain/connector'; const connectorResolvers = { Query: { - connectors: () => getConnectors() + connectors: () => getConnectors(), + connectorsStats: () => getStatistics() }, Mutation: { connectorConfig: (_, { identifier, config }) => diff --git a/opencti-graphql/yarn.lock b/opencti-graphql/yarn.lock index d7ca7e4dae4e..2590f6dd312c 100644 --- a/opencti-graphql/yarn.lock +++ b/opencti-graphql/yarn.lock @@ -2,13 +2,18 @@ # yarn lockfile v1 -"@apollographql/apollo-tools@^0.3.6-alpha.1": +"@apollographql/apollo-tools@^0.3.6", "@apollographql/apollo-tools@^0.3.6-alpha.1": version "0.3.7" resolved "https://registry.yarnpkg.com/@apollographql/apollo-tools/-/apollo-tools-0.3.7.tgz#3bc9c35b9fff65febd4ddc0c1fc04677693a3d40" integrity sha512-+ertvzAwzkYmuUtT8zH3Zi6jPdyxZwOgnYaZHY7iLnMVJDhQKWlkyjLMF8wyzlPiEdDImVUMm5lOIBZo7LkGlg== dependencies: apollo-env "0.5.1" +"@apollographql/graphql-playground-html@1.6.20": + version "1.6.20" + resolved "https://registry.yarnpkg.com/@apollographql/graphql-playground-html/-/graphql-playground-html-1.6.20.tgz#bf9f2acdf319c0959fad8ec1239741dd2ead4e8d" + integrity sha512-3LWZa80HcP70Pl+H4KhLDJ7S0px+9/c8GTXdl6SpunRecUaB27g/oOQnAjNHLHdbWuGE0uyqcuGiTfbKB3ilaQ== + "@apollographql/graphql-playground-html@^1.6.6": version "1.6.18" resolved "https://registry.yarnpkg.com/@apollographql/graphql-playground-html/-/graphql-playground-html-1.6.18.tgz#c0763b4b5a64d4a5316ae6adc2a3a61af58df402" @@ -986,6 +991,11 @@ dependencies: "@types/express" "*" +"@types/debug@0.0.31": + version "0.0.31" + resolved "https://registry.yarnpkg.com/@types/debug/-/debug-0.0.31.tgz#bac8d8aab6a823e91deb7f79083b2a35fa638f33" + integrity sha512-LS1MCPaQKqspg7FvexuhmDbWUhE2yIJ+4AgVIyObfc06/UKZ8REgxGNjZc82wPLWmbeOm7S+gSsLgo75TanG4A== + "@types/events@*": version "3.0.0" resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7" @@ -999,7 +1009,7 @@ "@types/node" "*" "@types/range-parser" "*" -"@types/express@*", "@types/express@4.16.1": +"@types/express@*": version "4.16.1" resolved "https://registry.yarnpkg.com/@types/express/-/express-4.16.1.tgz#d756bd1a85c34d87eaf44c888bad27ba8a4b7cf0" integrity sha512-V0clmJow23WeyblmACoxbHBu2JKlE5TiIme6Lem14FnPW9gsttyHtk6wq7njcdIWH1njAaFgR8gW09lgY98gQg== @@ -1008,6 +1018,15 @@ "@types/express-serve-static-core" "*" "@types/serve-static" "*" +"@types/express@4.17.0": + version "4.17.0" + resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.0.tgz#49eaedb209582a86f12ed9b725160f12d04ef287" + integrity sha512-CjaMu57cjgjuZbh9DpkloeGxV45CnMGlVd+XpG7Gm9QgVrd7KFq+X4HY0vM+2v0bczS48Wg7bvnMY5TN+Xmcfw== + dependencies: + "@types/body-parser" "*" + "@types/express-serve-static-core" "*" + "@types/serve-static" "*" + "@types/glob@^7.1.1": version "7.1.1" resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575" @@ -1216,6 +1235,14 @@ apollo-cache-control@0.6.0: apollo-server-env "2.3.0" graphql-extensions "0.6.0" +apollo-cache-control@0.7.5-alpha.2: + version "0.7.5-alpha.2" + resolved "https://registry.yarnpkg.com/apollo-cache-control/-/apollo-cache-control-0.7.5-alpha.2.tgz#1ea6107f11dad01e9f54bac5527590f4c40017e2" + integrity sha512-DZkQ8m/W49FeVRTql1I1jGt6TI+tBSfXwAf0Rs1aTZ1xYHeYHp+5ZRFfCnK1rXejwPcL1+tHKU4XQkpu4SqLMg== + dependencies: + apollo-server-env "2.4.0" + graphql-extensions "0.8.0-alpha.2" + apollo-datasource@0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/apollo-datasource/-/apollo-datasource-0.4.0.tgz#f042641fd2593fa5f4f002fc30d1fb1a20284df8" @@ -1224,6 +1251,14 @@ apollo-datasource@0.4.0: apollo-server-caching "0.4.0" apollo-server-env "2.3.0" +apollo-datasource@0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/apollo-datasource/-/apollo-datasource-0.5.0.tgz#7a8c97e23da7b9c15cb65103d63178ab19eca5e9" + integrity sha512-SVXxJyKlWguuDjxkY/WGlC/ykdsTmPxSF0z8FenagcQ91aPURXzXP1ZDz5PbamY+0iiCRubazkxtTQw4GWTFPg== + dependencies: + apollo-server-caching "0.4.0" + apollo-server-env "2.4.0" + apollo-engine-reporting-protobuf@0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/apollo-engine-reporting-protobuf/-/apollo-engine-reporting-protobuf-0.3.0.tgz#2c764c054ff9968387cf16115546e0d5b04ee9f1" @@ -1231,6 +1266,13 @@ apollo-engine-reporting-protobuf@0.3.0: dependencies: protobufjs "^6.8.6" +apollo-engine-reporting-protobuf@0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/apollo-engine-reporting-protobuf/-/apollo-engine-reporting-protobuf-0.3.1.tgz#a581257fa8e3bb115ce38bf1b22e052d1475ad69" + integrity sha512-Ui3nPG6BSZF8BEqxFs6EkX6mj2OnFLMejxEHSOdM82bakyeouCGd7J0fiy8AD6liJoIyc4X7XfH4ZGGMvMh11A== + dependencies: + protobufjs "^6.8.6" + apollo-engine-reporting@1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/apollo-engine-reporting/-/apollo-engine-reporting-1.1.0.tgz#10def3d3bf3f11ddb24765c19d9c81e30cb9d55c" @@ -1243,6 +1285,18 @@ apollo-engine-reporting@1.1.0: async-retry "^1.2.1" graphql-extensions "0.6.0" +apollo-engine-reporting@1.4.0-alpha.3: + version "1.4.0-alpha.3" + resolved "https://registry.yarnpkg.com/apollo-engine-reporting/-/apollo-engine-reporting-1.4.0-alpha.3.tgz#d2c9abf7da84ae49581ba70528e92ba211ceef74" + integrity sha512-3yFzsegRw7JabXu9mRLrpnxV99QAaoqkMaMwteTJEumYzNCKl+RQR7WQSnQBYH0UIBEX/UuqDOnDtWRXmRi9ZA== + dependencies: + apollo-engine-reporting-protobuf "0.3.1" + apollo-graphql "^0.3.3" + apollo-server-core "2.7.0-alpha.3" + apollo-server-env "2.4.0" + async-retry "^1.2.1" + graphql-extensions "0.8.0-alpha.3" + apollo-env@0.4.1-register.1: version "0.4.1-register.1" resolved "https://registry.yarnpkg.com/apollo-env/-/apollo-env-0.4.1-register.1.tgz#e8c94e21a5b3f9c45088dec47862dfe2026111c2" @@ -1277,6 +1331,14 @@ apollo-graphql@^0.2.1-alpha.1: apollo-env "0.4.1-register.1" lodash.sortby "^4.7.0" +apollo-graphql@^0.3.3: + version "0.3.3" + resolved "https://registry.yarnpkg.com/apollo-graphql/-/apollo-graphql-0.3.3.tgz#ce1df194f6e547ad3ce1e35b42f9c211766e1658" + integrity sha512-t3CO/xIDVsCG2qOvx2MEbuu4b/6LzQjcBBwiVnxclmmFyAxYCIe7rpPlnLHSq7HyOMlCWDMozjoeWfdqYSaLqQ== + dependencies: + apollo-env "0.5.1" + lodash.sortby "^4.7.0" + apollo-link@^1.2.2, apollo-link@^1.2.3: version "1.2.11" resolved "https://registry.yarnpkg.com/apollo-link/-/apollo-link-1.2.11.tgz#493293b747ad3237114ccd22e9f559e5e24a194d" @@ -1320,6 +1382,32 @@ apollo-server-core@2.5.0: subscriptions-transport-ws "^0.9.11" ws "^6.0.0" +apollo-server-core@2.7.0-alpha.3: + version "2.7.0-alpha.3" + resolved "https://registry.yarnpkg.com/apollo-server-core/-/apollo-server-core-2.7.0-alpha.3.tgz#d0619d39d01dd62708108e04675dc7aed47bda4b" + integrity sha512-N9j2yLNSIaP2oKcwocZ9056eFvYoKGDYFflS0HYGS8qu8q/JcaEbLHbGwHWaQgdYZn71W/1Fj32BUwSDiJyyPg== + dependencies: + "@apollographql/apollo-tools" "^0.3.6" + "@apollographql/graphql-playground-html" "1.6.20" + "@types/ws" "^6.0.0" + apollo-cache-control "0.7.5-alpha.2" + apollo-datasource "0.5.0" + apollo-engine-reporting "1.4.0-alpha.3" + apollo-server-caching "0.4.0" + apollo-server-env "2.4.0" + apollo-server-errors "2.3.0" + apollo-server-plugin-base "0.6.0-alpha.3" + apollo-tracing "0.7.4-alpha.2" + fast-json-stable-stringify "^2.0.0" + graphql-extensions "0.8.0-alpha.3" + graphql-subscriptions "^1.0.0" + graphql-tag "^2.9.2" + graphql-tools "^4.0.0" + graphql-upload "^8.0.2" + sha.js "^2.4.11" + subscriptions-transport-ws "^0.9.11" + ws "^6.0.0" + apollo-server-env@2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/apollo-server-env/-/apollo-server-env-2.3.0.tgz#f0bf4484a6cc331a8c13763ded56e91beb16ba17" @@ -1328,27 +1416,37 @@ apollo-server-env@2.3.0: node-fetch "^2.1.2" util.promisify "^1.0.0" +apollo-server-env@2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/apollo-server-env/-/apollo-server-env-2.4.0.tgz#6611556c6b627a1636eed31317d4f7ea30705872" + integrity sha512-7ispR68lv92viFeu5zsRUVGP+oxsVI3WeeBNniM22Cx619maBUwcYTIC3+Y3LpXILhLZCzA1FASZwusgSlyN9w== + dependencies: + node-fetch "^2.1.2" + util.promisify "^1.0.0" + apollo-server-errors@2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/apollo-server-errors/-/apollo-server-errors-2.3.0.tgz#700622b66a16dffcad3b017e4796749814edc061" integrity sha512-rUvzwMo2ZQgzzPh2kcJyfbRSfVKRMhfIlhY7BzUfM4x6ZT0aijlgsf714Ll3Mbf5Fxii32kD0A/DmKsTecpccw== -apollo-server-express@^2.5.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/apollo-server-express/-/apollo-server-express-2.5.0.tgz#ff6cbd3fcb8933f6316c5a5edd4db12d9a56fa65" - integrity sha512-2gd3VWIqji2jyDYMTTqKzVU4/znjEjugtLUmPgVl5SoBvJSMTsO7VgJv+roBubZGDK8jXXUEXr2a33RtIeHe4g== +apollo-server-express@^2.7.0-alpha.3: + version "2.7.0-alpha.3" + resolved "https://registry.yarnpkg.com/apollo-server-express/-/apollo-server-express-2.7.0-alpha.3.tgz#4de51baddf904602fddc85239f0a9a74759ec990" + integrity sha512-aPuNGYBpqDkJfHYC+Mxpg3gYf1wr4UCyOV1kPzGISdgTsVfJfVh8VbpqAlPfbjFzc230ETFdZ3maO1zesRMOig== dependencies: - "@apollographql/graphql-playground-html" "^1.6.6" + "@apollographql/graphql-playground-html" "1.6.20" "@types/accepts" "^1.3.5" "@types/body-parser" "1.17.0" "@types/cors" "^2.8.4" - "@types/express" "4.16.1" + "@types/express" "4.17.0" accepts "^1.3.5" - apollo-server-core "2.5.0" + apollo-server-core "2.7.0-alpha.3" body-parser "^1.18.3" + compose-middleware "^5.0.1" cors "^2.8.4" graphql-subscriptions "^1.0.0" graphql-tools "^4.0.0" + parseurl "^1.3.2" type-is "^1.6.16" apollo-server-plugin-base@0.4.0: @@ -1356,6 +1454,11 @@ apollo-server-plugin-base@0.4.0: resolved "https://registry.yarnpkg.com/apollo-server-plugin-base/-/apollo-server-plugin-base-0.4.0.tgz#38a3c37767043873dd1b07143d4e70eecbb09562" integrity sha512-iD7ARNtwnvHGd1EMPK0CuodM8d8hgDvFwTfIDzJY04QIQ6/KrBFaWhnCXJsy+HMb47GovwBbq67IK6eb2WJgBg== +apollo-server-plugin-base@0.6.0-alpha.3: + version "0.6.0-alpha.3" + resolved "https://registry.yarnpkg.com/apollo-server-plugin-base/-/apollo-server-plugin-base-0.6.0-alpha.3.tgz#39ccfb13129a8639cd4d93ce5e831222912c706f" + integrity sha512-t+tbXTOvhSfzSzrbwYAT0DInTg+nnALB9LoMdF9mgXZnXz4/X8P4MKvCUaUL8t0CksTySCHsRJk+dK0X0Nhwjw== + apollo-server-testing@^2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/apollo-server-testing/-/apollo-server-testing-2.5.0.tgz#6c4c386ddbcc5e555a02afc2c625955150827969" @@ -1371,6 +1474,14 @@ apollo-tracing@0.6.0: apollo-server-env "2.3.0" graphql-extensions "0.6.0" +apollo-tracing@0.7.4-alpha.2: + version "0.7.4-alpha.2" + resolved "https://registry.yarnpkg.com/apollo-tracing/-/apollo-tracing-0.7.4-alpha.2.tgz#472a75d54b8f973879db319600761c979ec3565d" + integrity sha512-9mreZuowCNZFxkGz0S2tteTyvbAD+B8nzNo8P+NRmgX0mEKhz8+be3ud6wCpsG8L2A46fpSej5LRaJySjZBEqg== + dependencies: + apollo-server-env "2.4.0" + graphql-extensions "0.8.0-alpha.2" + apollo-utilities@^1.0.1, apollo-utilities@^1.2.1: version "1.3.0" resolved "https://registry.yarnpkg.com/apollo-utilities/-/apollo-utilities-1.3.0.tgz#9803724c07ac94ca11dc26397edb58735d2b0211" @@ -1425,6 +1536,11 @@ array-flatten@1.1.1: resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2" integrity sha1-ml9pkFGx5wczKPKgCJaLZOopVdI= +array-flatten@^2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-2.1.2.tgz#24ef80a28c1a893617e2149b0c6d0d788293b099" + integrity sha512-hNfzcOV8W4NdualtqBFPyVO+54DSJuZGY9qT4pRroB6S9e3iiido2ISIC5h9R2sPJ8H3FHCIiEnsv1lPXO3KtQ== + array-includes@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.0.3.tgz#184b48f62d92d7452bb31b323165c7f8bd02266d" @@ -1542,6 +1658,14 @@ aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f" integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ== +axios@^0.19.0: + version "0.19.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.19.0.tgz#8e09bff3d9122e133f7b8101c8fbdd00ed3d2ab8" + integrity sha512-1uvKqKQta3KBxIz14F2v06AEHZ/dIoeKfbTRkK1E5oqjDnuEerLmYTgJB5AiQZHJcljpg1TuRzdjDR06qNk0DQ== + dependencies: + follow-redirects "1.5.10" + is-buffer "^2.0.2" + babel-jest@^24.8.0: version "24.8.0" resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-24.8.0.tgz#5c15ff2b28e20b0f45df43fe6b7f2aae93dba589" @@ -2002,6 +2126,15 @@ component-emitter@^1.2.1: resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.3.0.tgz#16e4070fba8ae29b679f2215853ee181ab2eabc0" integrity sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg== +compose-middleware@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/compose-middleware/-/compose-middleware-5.0.1.tgz#4c0adb751afdde45d637a7a0b361095e510fafff" + integrity sha512-Rcv19QgPOtYHu8wDJsu4ehSfkqSXjQLwKRXhIy9TFiIijSZz330ORyLCeirb4sPuBBbDNC5lUvQLuM72vWjKSQ== + dependencies: + "@types/debug" "0.0.31" + array-flatten "^2.1.2" + debug "^4.1.0" + compressible@~2.0.16: version "2.0.17" resolved "https://registry.yarnpkg.com/compressible/-/compressible-2.0.17.tgz#6e8c108a16ad58384a977f3a482ca20bff2f38c1" @@ -2197,6 +2330,13 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9, debug@~2.6. dependencies: ms "2.0.0" +debug@=3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" + integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g== + dependencies: + ms "2.0.0" + debug@^3.1.0, debug@^3.2.6: version "3.2.6" resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" @@ -2400,10 +2540,10 @@ ee-first@1.1.1: resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= -elasticsearch@^16.0.0: - version "16.0.0" - resolved "https://registry.yarnpkg.com/elasticsearch/-/elasticsearch-16.0.0.tgz#39cf4d45bd806b443c0970379fce7cb216c233e4" - integrity sha512-R2pHVWdJs9L2+ZghwAHjAEyQq4B0WVkWVPxlUtLMTyeBXlh4Y0Z+3VnW2+zaud4PRFBiAaTyKzfxD7TgMqpMJA== +elasticsearch@^16.1.1: + version "16.1.1" + resolved "https://registry.yarnpkg.com/elasticsearch/-/elasticsearch-16.1.1.tgz#1eb657e4ffe6fa2a4b3107b6f24a4118fc3e8393" + integrity sha512-OF2fIjcTPfq/4Tj6k4/SZr2IIlfWlBBQoy/em225mfevYFW1abN3nyXKWldXGV+eWI6LWNqB8lb3hAP4d6Rh/Q== dependencies: agentkeepalive "^3.4.1" chalk "^1.0.0" @@ -2968,6 +3108,13 @@ flatted@^2.0.0: resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.0.tgz#55122b6536ea496b4b44893ee2608141d10d9916" integrity sha512-R+H8IZclI8AAkSBRQJLVOsxwAoHd6WC40b4QTNWIjzAa6BXOBfQcM587MXDTVPeYaopFNWHUFLx7eNmHDSxMWg== +follow-redirects@1.5.10: + version "1.5.10" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.10.tgz#7b7a9f9aea2fdff36786a94ff643ed07f4ff5e2a" + integrity sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ== + dependencies: + debug "=3.1.0" + for-in@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" @@ -3186,6 +3333,20 @@ graphql-extensions@0.6.0: dependencies: "@apollographql/apollo-tools" "^0.3.6-alpha.1" +graphql-extensions@0.8.0-alpha.2: + version "0.8.0-alpha.2" + resolved "https://registry.yarnpkg.com/graphql-extensions/-/graphql-extensions-0.8.0-alpha.2.tgz#36c1c336acd2d353e1f0c2fdf9fe333b9d3aed6d" + integrity sha512-723VFlfcmgs5ThbpDXF5sDqr/ME7y9rZhjf3adJrEs/g7yqm84GNcO65KNrEkgqb8HsrwePWGcdhKYEGmC1vqw== + dependencies: + "@apollographql/apollo-tools" "^0.3.6" + +graphql-extensions@0.8.0-alpha.3: + version "0.8.0-alpha.3" + resolved "https://registry.yarnpkg.com/graphql-extensions/-/graphql-extensions-0.8.0-alpha.3.tgz#f429e8e6a122d949c8ce1f6b00aa4ad2d14fc22c" + integrity sha512-ZAze2LgSQ0zLoH8bjymuB59JPZPlpSrSnddft/yuzNZCEptbalQ1HpQjdbapDf7QcE9T9VyNdfoZMG+45lhSVA== + dependencies: + "@apollographql/apollo-tools" "^0.3.6" + graphql-import@^0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/graphql-import/-/graphql-import-0.7.1.tgz#4add8d91a5f752d764b0a4a7a461fcd93136f223" @@ -3238,7 +3399,7 @@ graphql-tools@^3.0.1: iterall "^1.1.3" uuid "^3.1.0" -graphql-tools@^4.0.0, graphql-tools@^4.0.4: +graphql-tools@^4.0.0: version "4.0.4" resolved "https://registry.yarnpkg.com/graphql-tools/-/graphql-tools-4.0.4.tgz#ca08a63454221fdde825fe45fbd315eb2a6d566b" integrity sha512-chF12etTIGVVGy3fCTJ1ivJX2KB7OSG4c6UOJQuqOHCmBQwTyNgCDuejZKvpYxNZiEx7bwIjrodDgDe9RIkjlw== @@ -3249,6 +3410,17 @@ graphql-tools@^4.0.0, graphql-tools@^4.0.4: iterall "^1.1.3" uuid "^3.1.0" +graphql-tools@^4.0.5: + version "4.0.5" + resolved "https://registry.yarnpkg.com/graphql-tools/-/graphql-tools-4.0.5.tgz#d2b41ee0a330bfef833e5cdae7e1f0b0d86b1754" + integrity sha512-kQCh3IZsMqquDx7zfIGWBau42xe46gmqabwYkpPlCLIjcEY1XK+auP7iGRD9/205BPyoQdY8hT96MPpgERdC9Q== + dependencies: + apollo-link "^1.2.3" + apollo-utilities "^1.0.1" + deprecated-decorator "^0.1.6" + iterall "^1.1.3" + uuid "^3.1.0" + graphql-upload@^8.0.2: version "8.0.6" resolved "https://registry.yarnpkg.com/graphql-upload/-/graphql-upload-8.0.6.tgz#90fb6246962d953b64d9ddabd6472d8e8b116ee0" @@ -3259,10 +3431,10 @@ graphql-upload@^8.0.2: http-errors "^1.7.2" object-path "^0.11.4" -graphql@^14.3.1: - version "14.3.1" - resolved "https://registry.yarnpkg.com/graphql/-/graphql-14.3.1.tgz#b3aa50e61a841ada3c1f9ccda101c483f8e8c807" - integrity sha512-FZm7kAa3FqKdXy8YSSpAoTtyDFMIYSpCDOr+3EqlI1bxmtHu+Vv/I2vrSeT1sBOEnEniX3uo4wFhFdS/8XN6gA== +graphql@^14.4.1: + version "14.4.1" + resolved "https://registry.yarnpkg.com/graphql/-/graphql-14.4.1.tgz#7a7818d3f63f66b9528ba5416b6c88460db62280" + integrity sha512-g4HUH26CohlMjaHneXMAtvG3QtO6peJIUTFxrPW4g5LNnXkUuFoBI6Bk1c14Q5kW8+FyjM/tTbePTgpiVB/2hQ== dependencies: iterall "^1.2.2" @@ -3577,7 +3749,22 @@ invert-kv@^2.0.0: resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02" integrity sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA== -ioredis@^4.6.3, ioredis@^4.9.5: +ioredis@^4.11.1: + version "4.11.1" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.11.1.tgz#cf14cf923b10a772d7e1eafed89f707e05760f48" + integrity sha512-Qnp7ecb3WeaL7ojeSlb0UBRXmsRMMFcjM+PaAcap8FLLf1NznRD6x96/PS2DEqoRfdM9WVffAjIIYuUp+q3zEw== + dependencies: + cluster-key-slot "^1.0.6" + debug "^4.1.1" + denque "^1.1.0" + lodash.defaults "^4.2.0" + lodash.flatten "^4.4.0" + redis-commands "1.5.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.0.1" + +ioredis@^4.6.3: version "4.9.5" resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.9.5.tgz#0bbba0a9faae93485d3231e1b819d2d4e23271d9" integrity sha512-L9MVfvX4F3LScTMEgriCGixzqinJsYy7Mt0NPX8RyuOTmx5JW0744pM4Ze2KVQcP3J0zvKYZ1LywAB6KIq7PYg== @@ -3633,6 +3820,11 @@ is-buffer@^1.1.5: resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== +is-buffer@^2.0.2: + version "2.0.3" + resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-2.0.3.tgz#4ecf3fcf749cbd1e472689e109ac66261a25e725" + integrity sha512-U15Q7MXTuZlrbymiz95PJpZxu8IlipAp4dtS3wOdgPXx3mqBnslrWU14kxfHB+Py/+2PVKSr37dMAgM2A4uArw== + is-callable@^1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.1.4.tgz#1e1adf219e1eeb684d691f9d6a05ff0d30a24d75" @@ -5289,7 +5481,7 @@ parse5@4.0.0: resolved "https://registry.yarnpkg.com/parse5/-/parse5-4.0.0.tgz#6d78656e3da8d78b4ec0b906f7c08ef1dfe3f608" integrity sha512-VrZ7eOd3T1Fk4XWNXMgiGBK/z0MG48BWG2uQNU4I72fkQuKUTZpl+u9k+CxEG0twMVzSmXEEz12z5Fnw1jIQFA== -parseurl@~1.3.3: +parseurl@^1.3.2, parseurl@~1.3.3: version "1.3.3" resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== @@ -5768,6 +5960,11 @@ redis-commands@1.4.0: resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.4.0.tgz#52f9cf99153efcce56a8f86af986bd04e988602f" integrity sha512-cu8EF+MtkwI4DLIT0x9P8qNTLFhQD4jLfxLR0cCNkeGzs87FN6879JOJwNQR/1zD7aSYNbU0hgsV9zGY71Itvw== +redis-commands@1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.5.0.tgz#80d2e20698fe688f227127ff9e5164a7dd17e785" + integrity sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg== + redis-errors@^1.0.0, redis-errors@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad" diff --git a/opencti-integration/config.yml.docker.sample b/opencti-integration/config.yml.docker.sample index a340635556ce..c917b17d0ce7 100644 --- a/opencti-integration/config.yml.docker.sample +++ b/opencti-integration/config.yml.docker.sample @@ -2,3 +2,9 @@ opencti: api_url: 'http://OPENCTI_HOSTNAME:OPENCTI_PORT' api_key: 'REPLACE_API_KEY' verbose: true + +rabbitmq: + hostname: 'RABBITMQ_HOSTNAME' + port: RABBITMQ_PORT + username: 'RABBITMQ_USERNAME' + password: 'RABBITMQ_PASSWORD' diff --git a/opencti-integration/config.yml.sample b/opencti-integration/config.yml.sample index ca66f834f2dd..e01ab32ee9d9 100644 --- a/opencti-integration/config.yml.sample +++ b/opencti-integration/config.yml.sample @@ -2,3 +2,9 @@ opencti: api_url: 'http://REPLACE_API_URL' api_key: 'REPLACE_API_KEY' verbose: true + +rabbitmq: + hostname: 'localhost' + port: 5672 + username: 'guest' + password: 'guest' diff --git a/opencti-integration/connectors b/opencti-integration/connectors index 7ed4851e31a5..caf99f2666d8 160000 --- a/opencti-integration/connectors +++ b/opencti-integration/connectors @@ -1 +1 @@ -Subproject commit 7ed4851e31a5e7df79724af585c1bd96aa7cb762 +Subproject commit caf99f2666d87926d3413d6c4863110a9e25a159 diff --git a/opencti-integration/connectors_scheduler.py b/opencti-integration/connectors_scheduler.py index a958284eb59c..d6a1d03e4712 100644 --- a/opencti-integration/connectors_scheduler.py +++ b/opencti-integration/connectors_scheduler.py @@ -6,6 +6,7 @@ import base64 import importlib import time +import pika import schedule from logger import Logger from pycti import OpenCTI @@ -31,29 +32,54 @@ def __init__(self, verbose=True): self.config['opencti']['verbose'] ) + def send_stix2_bundle(self, bundle): + print(bundle) + self.logger.log('Sending a message to the import workers') + # Prepare + message = { + 'type': 'import.stix2.bundle', + 'content': base64.b64encode(bundle.encode('utf-8')).decode('utf-8') + } + + # Initialize the RabbitMQ connection + credentials = pika.PlainCredentials(self.config['rabbitmq']['username'], self.config['rabbitmq']['password']) + connection = pika.BlockingConnection(pika.ConnectionParameters( + host=self.config['rabbitmq']['hostname'], + port=self.config['rabbitmq']['port'], + virtual_host='/', + credentials=credentials + )) + channel = connection.channel() + channel.exchange_declare(exchange='opencti', exchange_type='topic', durable=True) + channel.basic_publish('opencti', 'import.stix2.bundle', json.dumps(message)) + connection.close() + def init_connectors(self): self.logger.log('Configuring connectors') connectors = self.opencti.get_connectors() for connector in connectors: - if connector['config'] is not None: - connector_config = json.loads(base64.b64decode(connector['config'])) - config = self.config - config[connector['identifier']] = connector_config - - if connector['identifier'] not in self.connectors: - connector_module = importlib.import_module('connectors.' + connector['identifier'] + '.' + connector['identifier']) - connector_class = getattr(connector_module, connector['identifier'].capitalize()) - self.connectors[connector['identifier']] = {"config": config, "instance": connector_class(config)} - self.logger.log('Connector ' + connector['identifier'] + ' initialized') - else: - self.connectors[connector['identifier']]['instance'].set_config(config) - self.connectors[connector['identifier']]['config'] = config - self.logger.log('Connector ' + connector['identifier'] + ' configured') - - if 'triggered' in connector_config and connector_config['triggered'] is True: - connector_config['triggered'] = False - self.opencti.update_connector_config(connector['identifier'], connector_config) - self.run_connector(connector['identifier']) + try: + if connector['config'] is not None: + connector_config = json.loads(base64.b64decode(connector['config'])) + config = self.config + config[connector['identifier']] = connector_config + + if connector['identifier'] not in self.connectors: + connector_module = importlib.import_module('connectors.' + connector['identifier'] + '.' + connector['identifier']) + connector_class = getattr(connector_module, connector['identifier'].capitalize()) + self.connectors[connector['identifier']] = {"config": config, "instance": connector_class(config, self)} + self.logger.log('Connector ' + connector['identifier'] + ' initialized') + else: + self.connectors[connector['identifier']]['instance'].set_config(config) + self.connectors[connector['identifier']]['config'] = config + self.logger.log('Connector ' + connector['identifier'] + ' configured') + + if 'triggered' in connector_config and connector_config['triggered'] is True: + connector_config['triggered'] = False + self.opencti.update_connector_config(connector['identifier'], connector_config) + self.run_connector(connector['identifier']) + except Exception as e: + self.logger.log('Unable to initialize ' + connector['identifier'] + ': {' + str(e) + '}') def run_connector(self, identifier): try: @@ -76,6 +102,7 @@ def run_connectors(self): schedule.every().wednesday.at("04:30").do(self.run_connector, identifier=identifier) elif connector_config['cron'] == 'monthly': schedule.every(30).day.at("04:30").do(self.run_connector, identifier=identifier) + self.run_connector('misp') while True: schedule.run_pending() time.sleep(1) diff --git a/opencti-integration/requirements.txt b/opencti-integration/requirements.txt index 6568e0525dbe..9145595c9dd0 100644 --- a/opencti-integration/requirements.txt +++ b/opencti-integration/requirements.txt @@ -3,3 +3,4 @@ pycti schedule python-dateutil PyYAML +pika \ No newline at end of file diff --git a/opencti-worker/config.yml.docker.sample b/opencti-worker/config.yml.docker.sample index 670897b80ea2..118e6aaca684 100644 --- a/opencti-worker/config.yml.docker.sample +++ b/opencti-worker/config.yml.docker.sample @@ -1,6 +1,3 @@ -dumper: - file: './dump.gql' - opencti: api_url: 'http://OPENCTI_HOSTNAME:OPENCTI_PORT' api_key: 'REPLACE_API_KEY' @@ -8,14 +5,14 @@ opencti: grakn: hostname: 'GRAKN_HOSTNAME' - port: 48555 + port: GRAKN_PORT elasticsearch: hostname: 'ELASTICSEARCH_HOSTNAME' - port: 9200 + port: ELASTICSEARCH_PORT rabbitmq: hostname: 'RABBITMQ_HOSTNAME' - port: 5672 - username: 'guest' - password: 'guest' + port: RABBITMQ_PORT + username: 'RABBITMQ_USERNAME' + password: 'RABBITMQ_PASSWORD' diff --git a/opencti-worker/config.yml.sample b/opencti-worker/config.yml.sample index ff36e6704b13..e7411c2b13c6 100644 --- a/opencti-worker/config.yml.sample +++ b/opencti-worker/config.yml.sample @@ -1,6 +1,3 @@ -dumper: - file: './dump.gql' - opencti: api_url: 'http://localhost:8080' api_key: 'REPLACE_API_KEY' diff --git a/opencti-worker/grakn_dumper.py b/opencti-worker/grakn_dumper.py index e72a0f2c3b42..89ce98fd9f4e 100644 --- a/opencti-worker/grakn_dumper.py +++ b/opencti-worker/grakn_dumper.py @@ -18,7 +18,7 @@ def __init__(self): self.session = self.grakn.session(keyspace='grakn') # Open the dump file - self.dump_file = open(self.config['dumper']['file'], 'w') + self.dump_file = open('./dump.gql', 'w') # Entities self.entities = {}