From 4d9d99c0f839754cac44be18afb8abfe8a25c9d9 Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Wed, 19 Jul 2017 13:06:15 -0400 Subject: [PATCH 1/3] METRON-1053 Relocate Metron Docker --- README.md | 4 ++ .../metron-docker}/.gitignore | 0 .../metron-docker}/README.md | 13 +++--- .../metron-docker}/compose/docker-compose.yml | 7 +-- .../compose/elasticsearch/Dockerfile | 0 .../elasticsearch/docker-entrypoint.sh | 0 .../metron-docker}/compose/hadoop/Dockerfile | 5 ++- .../compose/hadoop/conf/core-site.xml | 0 .../compose/hadoop/conf/hdfs-site.xml | 0 .../compose/hadoop/docker-entrypoint.sh | 0 .../metron-docker}/compose/hbase/Dockerfile | 8 ++-- .../compose/hbase/bin/init-commands.txt | 0 .../compose/hbase/bin/init-hbase.sh | 0 .../metron-docker}/compose/hbase/bin/start.sh | 0 .../hbase/conf/enrichment-extractor.json | 0 .../compose/hbase/conf/hbase-site.docker.xml | 0 .../hbase/conf/threatintel-extractor.json | 0 .../compose/hbase/data/enrichments.csv | 0 .../compose/hbase/data/threatintel.csv | 0 .../metron-docker}/compose/kafkazk/Dockerfile | 19 ++++---- .../compose/kafkazk/bin/create-topic.sh | 0 .../compose/kafkazk/bin/init-kafka.sh | 0 .../compose/kafkazk/bin/init-zk.sh | 0 .../compose/kafkazk/bin/output-data.sh | 0 .../compose/kafkazk/bin/produce-data.sh | 0 .../compose/kafkazk/bin/run-consumer.sh | 0 .../compose/kafkazk/bin/start.sh | 0 .../compose/kafkazk/conf/global.json | 0 .../compose/kafkazk/data/BroExampleOutput.txt | 0 .../kafkazk/data/SquidExampleOutput.txt | 0 .../metron-docker}/compose/kibana/Dockerfile | 0 .../compose/kibana/conf/kibana-index.json | 0 .../compose/kibana/images/metron.svg | 0 .../metron-docker}/compose/storm/Dockerfile | 0 .../storm/bin/start_docker_parser_topology.sh | 0 .../metron-docker}/conf/.env | 0 .../metron-docker}/install-metron.sh | 0 .../metron-docker}/pom.xml | 0 .../scripts/create-docker-machine.sh | 0 .../metron-docker}/scripts/wait-for-it.sh | 0 metron-contrib/pom.xml | 44 +++++++++++++++++++ pom.xml | 2 +- 42 files changed, 77 insertions(+), 25 deletions(-) rename {metron-docker => metron-contrib/metron-docker}/.gitignore (100%) rename {metron-docker => metron-contrib/metron-docker}/README.md (90%) rename {metron-docker => metron-contrib/metron-docker}/compose/docker-compose.yml (90%) rename {metron-docker => metron-contrib/metron-docker}/compose/elasticsearch/Dockerfile (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/elasticsearch/docker-entrypoint.sh (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/hadoop/Dockerfile (91%) rename {metron-docker => metron-contrib/metron-docker}/compose/hadoop/conf/core-site.xml (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/hadoop/conf/hdfs-site.xml (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/hadoop/docker-entrypoint.sh (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/hbase/Dockerfile (92%) rename {metron-docker => metron-contrib/metron-docker}/compose/hbase/bin/init-commands.txt (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/hbase/bin/init-hbase.sh (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/hbase/bin/start.sh (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/hbase/conf/enrichment-extractor.json (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/hbase/conf/hbase-site.docker.xml (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/hbase/conf/threatintel-extractor.json (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/hbase/data/enrichments.csv (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/hbase/data/threatintel.csv (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/kafkazk/Dockerfile (67%) rename {metron-docker => metron-contrib/metron-docker}/compose/kafkazk/bin/create-topic.sh (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/kafkazk/bin/init-kafka.sh (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/kafkazk/bin/init-zk.sh (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/kafkazk/bin/output-data.sh (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/kafkazk/bin/produce-data.sh (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/kafkazk/bin/run-consumer.sh (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/kafkazk/bin/start.sh (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/kafkazk/conf/global.json (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/kafkazk/data/BroExampleOutput.txt (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/kafkazk/data/SquidExampleOutput.txt (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/kibana/Dockerfile (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/kibana/conf/kibana-index.json (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/kibana/images/metron.svg (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/storm/Dockerfile (100%) rename {metron-docker => metron-contrib/metron-docker}/compose/storm/bin/start_docker_parser_topology.sh (100%) rename {metron-docker => metron-contrib/metron-docker}/conf/.env (100%) rename {metron-docker => metron-contrib/metron-docker}/install-metron.sh (100%) rename {metron-docker => metron-contrib/metron-docker}/pom.xml (100%) rename {metron-docker => metron-contrib/metron-docker}/scripts/create-docker-machine.sh (100%) rename {metron-docker => metron-contrib/metron-docker}/scripts/wait-for-it.sh (100%) create mode 100644 metron-contrib/pom.xml diff --git a/README.md b/README.md index dcea8fabee..8599f0f3b8 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,10 @@ Metron's code: Option 3 is more likely to have the latest code. +# Getting Started + +To start exploring the capabilities of Apache Metron [follow these instructions to launch Metron in a single-node VM](metron-deployment/vagrant/full-dev-platform) on your own hardware. + # Building Metron Build the full project and run tests: diff --git a/metron-docker/.gitignore b/metron-contrib/metron-docker/.gitignore similarity index 100% rename from metron-docker/.gitignore rename to metron-contrib/metron-docker/.gitignore diff --git a/metron-docker/README.md b/metron-contrib/metron-docker/README.md similarity index 90% rename from metron-docker/README.md rename to metron-contrib/metron-docker/README.md index 32892f8f42..54cb63b38f 100644 --- a/metron-docker/README.md +++ b/metron-contrib/metron-docker/README.md @@ -1,6 +1,8 @@ # Metron Docker -Metron Docker is a [Docker Compose](https://docs.docker.com/compose/overview/) application that is intended for development and integration testing of Metron. Use this instead of Vagrant when: +Metron Docker is a [Docker Compose](https://docs.docker.com/compose/overview/) application that is intended only for development and integration testing of Metron. These images can quickly spin-up the underlying components on which Apache Metron runs. + +None of the core Metron components are setup or launched automatically with these Docker images. You will need to manually setup and start the Metron components that you require. You should not expect to see telemetry being parsed, enriched, or indexed. If you are looking to try-out, experiment or demo Metron capabilities on a single node, then the [Vagrant-driven VM](../../metron-deployment/vagrant/full-dev-platform) is what you need. Use this instead of Vagrant when: - You want an environment that can be built and spun up quickly - You need to frequently rebuild and restart services @@ -10,7 +12,7 @@ Metron Docker includes these images that have been customized for Metron: - Kafka (with Zookeeper) - HBase - - Storm (with all topologies deployed) + - Storm - Elasticsearch - Kibana - HDFS @@ -32,8 +34,9 @@ $ mvn clean install -DskipTests You are welcome to use an existing Docker host but we prefer one with more resources. You can create one of those with this script: ``` -$ export METRON_DOCKER_HOME=$METRON_HOME/metron-docker -$ cd $METRON_DOCKER_HOME && ./scripts/create-docker-machine.sh +$ export METRON_DOCKER_HOME=$METRON_HOME/metron-contrib/metron-docker +$ cd $METRON_DOCKER_HOME +$ ./scripts/create-docker-machine.sh ``` This will create a host called "metron-machine". Anytime you want to run Docker commands against this host, make sure you run this first to set the Docker environment variables: @@ -201,7 +204,7 @@ $ docker-compose exec kafkazk ./bin/produce-data.sh /data/BroExampleOutput.txt b Open a separate console session and verify the sensor is running by consuming a message from Kafka: ``` -$ export METRON_DOCKER_HOME=$METRON_HOME/metron-docker +$ export METRON_DOCKER_HOME=$METRON_HOME/metron-contrib/metron-docker $ cd $METRON_DOCKER_HOME/compose $ docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic bro ``` diff --git a/metron-docker/compose/docker-compose.yml b/metron-contrib/metron-docker/compose/docker-compose.yml similarity index 90% rename from metron-docker/compose/docker-compose.yml rename to metron-contrib/metron-docker/compose/docker-compose.yml index efc4227a04..5831b44b03 100644 --- a/metron-docker/compose/docker-compose.yml +++ b/metron-contrib/metron-docker/compose/docker-compose.yml @@ -34,7 +34,7 @@ services: ports: - "16010:16010" volumes: - - "/opt/hbase-1.1.6/conf" + - "/opt/hbase/conf" depends_on: - kafkazk hadoop: @@ -43,7 +43,7 @@ services: ports: - "50070:50070" volumes: - - "/opt/hadoop-2.7.3/etc/hadoop" + - "/opt/hadoop/etc/hadoop" command: tail -f /dev/null storm: build: @@ -56,7 +56,8 @@ services: - "8081:8081" environment: ZOOKEEPER_ADDR: kafkazk - CONFIG_TOPOLOGY_CLASSPATH: "/opt/hbase-1.1.6/conf:/opt/hadoop-2.7.3/etc/hadoop" + CONFIG_TOPOLOGY_CLASSPATH: "/opt/hbase/conf:/opt/hadoop/etc/hadoop" + CONFIG_TOPOLOGY_CLASSPATH: "/opt/hbase/conf:/opt/hadoop/etc/hadoop" volumes_from: - hbase - hadoop diff --git a/metron-docker/compose/elasticsearch/Dockerfile b/metron-contrib/metron-docker/compose/elasticsearch/Dockerfile similarity index 100% rename from metron-docker/compose/elasticsearch/Dockerfile rename to metron-contrib/metron-docker/compose/elasticsearch/Dockerfile diff --git a/metron-docker/compose/elasticsearch/docker-entrypoint.sh b/metron-contrib/metron-docker/compose/elasticsearch/docker-entrypoint.sh similarity index 100% rename from metron-docker/compose/elasticsearch/docker-entrypoint.sh rename to metron-contrib/metron-docker/compose/elasticsearch/docker-entrypoint.sh diff --git a/metron-docker/compose/hadoop/Dockerfile b/metron-contrib/metron-docker/compose/hadoop/Dockerfile similarity index 91% rename from metron-docker/compose/hadoop/Dockerfile rename to metron-contrib/metron-docker/compose/hadoop/Dockerfile index caddbd9258..9a0a3ce656 100644 --- a/metron-docker/compose/hadoop/Dockerfile +++ b/metron-contrib/metron-docker/compose/hadoop/Dockerfile @@ -16,11 +16,12 @@ # FROM openjdk:7 -ENV HADOOP_PREFIX=/opt/hadoop-2.7.3 +ENV HADOOP_PREFIX=/opt/hadoop ENV HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop ENV HADOOP_IDENT_STRING=root -RUN curl -sL http://archive.apache.org/dist/hadoop/core/hadoop-2.7.3/hadoop-2.7.3.tar.gz | tar -xzC /opt +RUN curl -sL http://archive.apache.org/dist/hadoop/core/hadoop-2.7.3/hadoop-2.7.3.tar.gz | tar -xzC /tmp +RUN mv /tmp/hadoop-2.7.3 /opt/hadoop COPY ./conf/* $HADOOP_CONF_DIR/ RUN mkdir -p /hadoop/dfs/name \ diff --git a/metron-docker/compose/hadoop/conf/core-site.xml b/metron-contrib/metron-docker/compose/hadoop/conf/core-site.xml similarity index 100% rename from metron-docker/compose/hadoop/conf/core-site.xml rename to metron-contrib/metron-docker/compose/hadoop/conf/core-site.xml diff --git a/metron-docker/compose/hadoop/conf/hdfs-site.xml b/metron-contrib/metron-docker/compose/hadoop/conf/hdfs-site.xml similarity index 100% rename from metron-docker/compose/hadoop/conf/hdfs-site.xml rename to metron-contrib/metron-docker/compose/hadoop/conf/hdfs-site.xml diff --git a/metron-docker/compose/hadoop/docker-entrypoint.sh b/metron-contrib/metron-docker/compose/hadoop/docker-entrypoint.sh similarity index 100% rename from metron-docker/compose/hadoop/docker-entrypoint.sh rename to metron-contrib/metron-docker/compose/hadoop/docker-entrypoint.sh diff --git a/metron-docker/compose/hbase/Dockerfile b/metron-contrib/metron-docker/compose/hbase/Dockerfile similarity index 92% rename from metron-docker/compose/hbase/Dockerfile rename to metron-contrib/metron-docker/compose/hbase/Dockerfile index dd857904f9..7e1d4e5191 100644 --- a/metron-docker/compose/hbase/Dockerfile +++ b/metron-contrib/metron-docker/compose/hbase/Dockerfile @@ -20,7 +20,7 @@ ARG METRON_VERSION ENV METRON_VERSION $METRON_VERSION ENV JAVA_HOME /usr -ENV HBASE_HOME /opt/hbase-1.1.6 +ENV HBASE_HOME /opt/hbase ENV HBASE_MANAGES_ZK false ENV METRON_HOME /usr/metron/$METRON_VERSION/ @@ -28,8 +28,8 @@ ADD ./data /data ADD ./data-management /data-management RUN mkdir -p $METRON_HOME RUN tar -xzf /data-management/metron-data-management-$METRON_VERSION-archive.tar.gz -C /usr/metron/$METRON_VERSION/ -ADD http://archive.apache.org/dist/hbase/1.1.6/hbase-1.1.6-bin.tar.gz /opt/hbase-1.1.6-bin.tar.gz -RUN tar -xzf /opt/hbase-1.1.6-bin.tar.gz -C /opt +ADD http://archive.apache.org/dist/hbase/1.1.6/hbase-1.1.6-bin.tar.gz /tmp +RUN mv /tmp/hbase-1.1.6 /opt/hbase RUN yum install -y java-1.8.0-openjdk lsof ADD ./conf/enrichment-extractor.json /conf/enrichment-extractor.json ADD ./conf/threatintel-extractor.json /conf/threatintel-extractor.json @@ -39,5 +39,5 @@ RUN chmod 755 $HBASE_HOME/bin/wait-for-it.sh EXPOSE 8080 8085 9090 9095 16000 16010 16201 16301 -WORKDIR /opt/hbase-1.1.6 +WORKDIR /opt/hbase CMD ./bin/start.sh diff --git a/metron-docker/compose/hbase/bin/init-commands.txt b/metron-contrib/metron-docker/compose/hbase/bin/init-commands.txt similarity index 100% rename from metron-docker/compose/hbase/bin/init-commands.txt rename to metron-contrib/metron-docker/compose/hbase/bin/init-commands.txt diff --git a/metron-docker/compose/hbase/bin/init-hbase.sh b/metron-contrib/metron-docker/compose/hbase/bin/init-hbase.sh similarity index 100% rename from metron-docker/compose/hbase/bin/init-hbase.sh rename to metron-contrib/metron-docker/compose/hbase/bin/init-hbase.sh diff --git a/metron-docker/compose/hbase/bin/start.sh b/metron-contrib/metron-docker/compose/hbase/bin/start.sh similarity index 100% rename from metron-docker/compose/hbase/bin/start.sh rename to metron-contrib/metron-docker/compose/hbase/bin/start.sh diff --git a/metron-docker/compose/hbase/conf/enrichment-extractor.json b/metron-contrib/metron-docker/compose/hbase/conf/enrichment-extractor.json similarity index 100% rename from metron-docker/compose/hbase/conf/enrichment-extractor.json rename to metron-contrib/metron-docker/compose/hbase/conf/enrichment-extractor.json diff --git a/metron-docker/compose/hbase/conf/hbase-site.docker.xml b/metron-contrib/metron-docker/compose/hbase/conf/hbase-site.docker.xml similarity index 100% rename from metron-docker/compose/hbase/conf/hbase-site.docker.xml rename to metron-contrib/metron-docker/compose/hbase/conf/hbase-site.docker.xml diff --git a/metron-docker/compose/hbase/conf/threatintel-extractor.json b/metron-contrib/metron-docker/compose/hbase/conf/threatintel-extractor.json similarity index 100% rename from metron-docker/compose/hbase/conf/threatintel-extractor.json rename to metron-contrib/metron-docker/compose/hbase/conf/threatintel-extractor.json diff --git a/metron-docker/compose/hbase/data/enrichments.csv b/metron-contrib/metron-docker/compose/hbase/data/enrichments.csv similarity index 100% rename from metron-docker/compose/hbase/data/enrichments.csv rename to metron-contrib/metron-docker/compose/hbase/data/enrichments.csv diff --git a/metron-docker/compose/hbase/data/threatintel.csv b/metron-contrib/metron-docker/compose/hbase/data/threatintel.csv similarity index 100% rename from metron-docker/compose/hbase/data/threatintel.csv rename to metron-contrib/metron-docker/compose/hbase/data/threatintel.csv diff --git a/metron-docker/compose/kafkazk/Dockerfile b/metron-contrib/metron-docker/compose/kafkazk/Dockerfile similarity index 67% rename from metron-docker/compose/kafkazk/Dockerfile rename to metron-contrib/metron-docker/compose/kafkazk/Dockerfile index 77b6b7ee88..ebade482b3 100644 --- a/metron-docker/compose/kafkazk/Dockerfile +++ b/metron-contrib/metron-docker/compose/kafkazk/Dockerfile @@ -22,18 +22,17 @@ ARG METRON_VERSION ENV METRON_VERSION $METRON_VERSION ENV METRON_HOME /usr/metron/$METRON_VERSION/ -ENV ZK_CLIENT_JARS /opt/kafka_2.11-0.10.0.0/libs +ENV ZK_CLIENT_JARS /opt/kafka/libs -ADD https://archive.apache.org/dist/kafka/0.10.0.0/kafka_2.11-0.10.0.0.tgz /opt/kafka_2.11-0.10.0.0.tgz -RUN tar -xzf /opt/kafka_2.11-0.10.0.0.tgz -C /opt -RUN echo -n 'advertised.listeners=PLAINTEXT://' >> /opt/kafka_2.11-0.10.0.0/config/server.properties -RUN echo $DOCKER_HOST | sed "s/^$/"$BROKER_IP_ADDR":/g" | sed "s/tcp:\\/\\///g" | sed "s/:.*/:9092/g" >> /opt/kafka_2.11-0.10.0.0/config/server.properties -RUN echo 'delete.topic.enable=true' >> /opt/kafka_2.11-0.10.0.0/config/server.properties +RUN curl -sL https://archive.apache.org/dist/kafka/0.10.0.0/kafka_2.11-0.10.0.0.tgz | tar -xzC /tmp +RUN mv /tmp/kafka_2.11-0.10.0.0 /opt/kafka +RUN echo -n 'advertised.listeners=PLAINTEXT://' >> /opt/kafka/config/server.properties +RUN echo $DOCKER_HOST | sed "s/^$/"$BROKER_IP_ADDR":/g" | sed "s/tcp:\\/\\///g" | sed "s/:.*/:9092/g" >> /opt/kafka/config/server.properties +RUN echo 'delete.topic.enable=true' >> /opt/kafka/config/server.properties RUN yum install -y java-1.8.0-openjdk lsof - RUN mkdir -p $METRON_HOME -ADD ./bin /opt/kafka_2.11-0.10.0.0/bin -RUN chmod 755 /opt/kafka_2.11-0.10.0.0/bin/wait-for-it.sh +ADD ./bin /opt/kafka/bin +RUN chmod 755 /opt/kafka/bin/wait-for-it.sh COPY ./data /data/ COPY ./packages/* /packages/ RUN find /packages -type f -name '*.tar.gz' -exec tar -xzf {} -C /usr/metron/$METRON_VERSION/ \; @@ -41,5 +40,5 @@ ADD ./conf /$METRON_HOME/config/zookeeper EXPOSE 2181 9092 -WORKDIR /opt/kafka_2.11-0.10.0.0 +WORKDIR /opt/kafka CMD ./bin/start.sh diff --git a/metron-docker/compose/kafkazk/bin/create-topic.sh b/metron-contrib/metron-docker/compose/kafkazk/bin/create-topic.sh similarity index 100% rename from metron-docker/compose/kafkazk/bin/create-topic.sh rename to metron-contrib/metron-docker/compose/kafkazk/bin/create-topic.sh diff --git a/metron-docker/compose/kafkazk/bin/init-kafka.sh b/metron-contrib/metron-docker/compose/kafkazk/bin/init-kafka.sh similarity index 100% rename from metron-docker/compose/kafkazk/bin/init-kafka.sh rename to metron-contrib/metron-docker/compose/kafkazk/bin/init-kafka.sh diff --git a/metron-docker/compose/kafkazk/bin/init-zk.sh b/metron-contrib/metron-docker/compose/kafkazk/bin/init-zk.sh similarity index 100% rename from metron-docker/compose/kafkazk/bin/init-zk.sh rename to metron-contrib/metron-docker/compose/kafkazk/bin/init-zk.sh diff --git a/metron-docker/compose/kafkazk/bin/output-data.sh b/metron-contrib/metron-docker/compose/kafkazk/bin/output-data.sh similarity index 100% rename from metron-docker/compose/kafkazk/bin/output-data.sh rename to metron-contrib/metron-docker/compose/kafkazk/bin/output-data.sh diff --git a/metron-docker/compose/kafkazk/bin/produce-data.sh b/metron-contrib/metron-docker/compose/kafkazk/bin/produce-data.sh similarity index 100% rename from metron-docker/compose/kafkazk/bin/produce-data.sh rename to metron-contrib/metron-docker/compose/kafkazk/bin/produce-data.sh diff --git a/metron-docker/compose/kafkazk/bin/run-consumer.sh b/metron-contrib/metron-docker/compose/kafkazk/bin/run-consumer.sh similarity index 100% rename from metron-docker/compose/kafkazk/bin/run-consumer.sh rename to metron-contrib/metron-docker/compose/kafkazk/bin/run-consumer.sh diff --git a/metron-docker/compose/kafkazk/bin/start.sh b/metron-contrib/metron-docker/compose/kafkazk/bin/start.sh similarity index 100% rename from metron-docker/compose/kafkazk/bin/start.sh rename to metron-contrib/metron-docker/compose/kafkazk/bin/start.sh diff --git a/metron-docker/compose/kafkazk/conf/global.json b/metron-contrib/metron-docker/compose/kafkazk/conf/global.json similarity index 100% rename from metron-docker/compose/kafkazk/conf/global.json rename to metron-contrib/metron-docker/compose/kafkazk/conf/global.json diff --git a/metron-docker/compose/kafkazk/data/BroExampleOutput.txt b/metron-contrib/metron-docker/compose/kafkazk/data/BroExampleOutput.txt similarity index 100% rename from metron-docker/compose/kafkazk/data/BroExampleOutput.txt rename to metron-contrib/metron-docker/compose/kafkazk/data/BroExampleOutput.txt diff --git a/metron-docker/compose/kafkazk/data/SquidExampleOutput.txt b/metron-contrib/metron-docker/compose/kafkazk/data/SquidExampleOutput.txt similarity index 100% rename from metron-docker/compose/kafkazk/data/SquidExampleOutput.txt rename to metron-contrib/metron-docker/compose/kafkazk/data/SquidExampleOutput.txt diff --git a/metron-docker/compose/kibana/Dockerfile b/metron-contrib/metron-docker/compose/kibana/Dockerfile similarity index 100% rename from metron-docker/compose/kibana/Dockerfile rename to metron-contrib/metron-docker/compose/kibana/Dockerfile diff --git a/metron-docker/compose/kibana/conf/kibana-index.json b/metron-contrib/metron-docker/compose/kibana/conf/kibana-index.json similarity index 100% rename from metron-docker/compose/kibana/conf/kibana-index.json rename to metron-contrib/metron-docker/compose/kibana/conf/kibana-index.json diff --git a/metron-docker/compose/kibana/images/metron.svg b/metron-contrib/metron-docker/compose/kibana/images/metron.svg similarity index 100% rename from metron-docker/compose/kibana/images/metron.svg rename to metron-contrib/metron-docker/compose/kibana/images/metron.svg diff --git a/metron-docker/compose/storm/Dockerfile b/metron-contrib/metron-docker/compose/storm/Dockerfile similarity index 100% rename from metron-docker/compose/storm/Dockerfile rename to metron-contrib/metron-docker/compose/storm/Dockerfile diff --git a/metron-docker/compose/storm/bin/start_docker_parser_topology.sh b/metron-contrib/metron-docker/compose/storm/bin/start_docker_parser_topology.sh similarity index 100% rename from metron-docker/compose/storm/bin/start_docker_parser_topology.sh rename to metron-contrib/metron-docker/compose/storm/bin/start_docker_parser_topology.sh diff --git a/metron-docker/conf/.env b/metron-contrib/metron-docker/conf/.env similarity index 100% rename from metron-docker/conf/.env rename to metron-contrib/metron-docker/conf/.env diff --git a/metron-docker/install-metron.sh b/metron-contrib/metron-docker/install-metron.sh similarity index 100% rename from metron-docker/install-metron.sh rename to metron-contrib/metron-docker/install-metron.sh diff --git a/metron-docker/pom.xml b/metron-contrib/metron-docker/pom.xml similarity index 100% rename from metron-docker/pom.xml rename to metron-contrib/metron-docker/pom.xml diff --git a/metron-docker/scripts/create-docker-machine.sh b/metron-contrib/metron-docker/scripts/create-docker-machine.sh similarity index 100% rename from metron-docker/scripts/create-docker-machine.sh rename to metron-contrib/metron-docker/scripts/create-docker-machine.sh diff --git a/metron-docker/scripts/wait-for-it.sh b/metron-contrib/metron-docker/scripts/wait-for-it.sh similarity index 100% rename from metron-docker/scripts/wait-for-it.sh rename to metron-contrib/metron-docker/scripts/wait-for-it.sh diff --git a/metron-contrib/pom.xml b/metron-contrib/pom.xml new file mode 100644 index 0000000000..324c7932bd --- /dev/null +++ b/metron-contrib/pom.xml @@ -0,0 +1,44 @@ + + + + + 4.0.0 + metron-contrib + pom + metron-contrib + + org.apache.metron + Metron + 0.4.1 + + Contributed Tooling for Metron + https://metron.apache.org/ + + scm:git:https://git-wip-us.apache.org/repos/asf/metron.git + scm:git:https://git-wip-us.apache.org/repos/asf/metron.git + HEAD + https://git-wip-us.apache.org/repos/asf/metron + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + metron-docker + + diff --git a/pom.xml b/pom.xml index 5079655f8d..50d6cef8c5 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ metron-analytics metron-platform metron-deployment - metron-docker + metron-contrib metron-interface site-book metron-stellar From 1fd060ed0ae5c63fa7838874d62e43af51667ac5 Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Wed, 19 Jul 2017 13:34:37 -0400 Subject: [PATCH 2/3] Fixed the pom --- metron-contrib/metron-docker/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metron-contrib/metron-docker/pom.xml b/metron-contrib/metron-docker/pom.xml index ed2969812a..8245e64983 100644 --- a/metron-contrib/metron-docker/pom.xml +++ b/metron-contrib/metron-docker/pom.xml @@ -20,7 +20,7 @@ metron-docker org.apache.metron - Metron + metron-contrib 0.4.1 Metron Docker @@ -309,4 +309,4 @@ - \ No newline at end of file + From aa613bac98bdfeaaecf1105ffd1fefb0cf055568 Mon Sep 17 00:00:00 2001 From: kylerichardson Date: Tue, 25 Jul 2017 09:29:04 -0400 Subject: [PATCH 3/3] Corrected relative paths for pom copy resources --- .../metron-docker/compose/hbase/Dockerfile | 2 +- metron-contrib/metron-docker/pom.xml | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/metron-contrib/metron-docker/compose/hbase/Dockerfile b/metron-contrib/metron-docker/compose/hbase/Dockerfile index 7e1d4e5191..bab2e673c3 100644 --- a/metron-contrib/metron-docker/compose/hbase/Dockerfile +++ b/metron-contrib/metron-docker/compose/hbase/Dockerfile @@ -28,7 +28,7 @@ ADD ./data /data ADD ./data-management /data-management RUN mkdir -p $METRON_HOME RUN tar -xzf /data-management/metron-data-management-$METRON_VERSION-archive.tar.gz -C /usr/metron/$METRON_VERSION/ -ADD http://archive.apache.org/dist/hbase/1.1.6/hbase-1.1.6-bin.tar.gz /tmp +RUN curl -sL http://archive.apache.org/dist/hbase/1.1.6/hbase-1.1.6-bin.tar.gz | tar -xzC /tmp RUN mv /tmp/hbase-1.1.6 /opt/hbase RUN yum install -y java-1.8.0-openjdk lsof ADD ./conf/enrichment-extractor.json /conf/enrichment-extractor.json diff --git a/metron-contrib/metron-docker/pom.xml b/metron-contrib/metron-docker/pom.xml index 8245e64983..650c41757f 100644 --- a/metron-contrib/metron-docker/pom.xml +++ b/metron-contrib/metron-docker/pom.xml @@ -45,7 +45,7 @@ ${project.basedir}/compose/kafkazk/packages - ../metron-platform/metron-common/target/ + ${project.parent.parent.basedir}/metron-platform/metron-common/target/ *.tar.gz @@ -63,7 +63,7 @@ ${project.basedir}/compose/kafkazk/packages - ../metron-platform/metron-parsers/target/ + ${project.parent.parent.basedir}/metron-platform/metron-parsers/target/ *.tar.gz @@ -81,7 +81,7 @@ ${project.basedir}/compose/kafkazk/packages - ../metron-platform/metron-enrichment/target/ + ${project.parent.parent.basedir}/metron-platform/metron-enrichment/target/ *.tar.gz @@ -99,7 +99,7 @@ ${project.basedir}/compose/kafkazk/packages - ../metron-platform/metron-indexing/target/ + ${project.parent.parent.basedir}/metron-platform/metron-indexing/target/ *.tar.gz @@ -117,7 +117,7 @@ ${project.basedir}/compose/elasticsearch/es_templates - ../metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/ + ${project.parent.parent.basedir}/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/ *.template @@ -135,7 +135,7 @@ ${project.basedir}/compose/hbase/data-management - ../metron-platform/metron-data-management/target/ + ${project.parent.parent.basedir}/metron-platform/metron-data-management/target/ *.tar.gz @@ -153,7 +153,7 @@ ${project.basedir}/compose/storm/parser - ${project.basedir}/../metron-platform/metron-parsers/target/ + ${project.parent.parent.basedir}/metron-platform/metron-parsers/target/ *.tar.gz @@ -171,7 +171,7 @@ ${project.basedir}/compose/storm/enrichment - ${project.basedir}/../metron-platform/metron-enrichment/target/ + ${project.parent.parent.basedir}/metron-platform/metron-enrichment/target/ *.tar.gz @@ -189,7 +189,7 @@ ${project.basedir}/compose/storm/indexing - ${project.basedir}/../metron-platform/metron-indexing/target/ + ${project.parent.parent.basedir}/metron-platform/metron-indexing/target/ *.tar.gz @@ -207,7 +207,7 @@ ${project.basedir}/compose/storm/elasticsearch - ../metron-platform/metron-elasticsearch/target/ + ${project.parent.parent.basedir}/metron-platform/metron-elasticsearch/target/ *.tar.gz