ISO and Docker image improvements (#35)

* Build DEB package for python3.7 rather than just make altinstall * Updated build ISO process to create a package for python3.7 rather than just make altinstall * working on making malcolm iso buster-based * bump version to 1.3.1 * genericize gitignore * enable net.ipv4.ip_forward for docker * issue #34: don't HUP netsniff-ng based on PCAP files that predate the netsniff-ng process being examined * remove development package required for pip before finishing docker build
cisagov · Jul 11, 2019 · 1af097b · 1af097b
1 parent 0a144b4
commit 1af097b
Show file tree

Hide file tree

Showing 22 changed files with 144 additions and 147 deletions.
diff --git a/.gitignore b/.gitignore
@@ -12,7 +12,6 @@
 /nginx/htpasswd
 
 # development
-docker-compose-seth*
 .vagrant
 malcolm_*images.tar.gz
 *.iso

diff --git a/Dockerfiles/moloch.Dockerfile b/Dockerfiles/moloch.Dockerfile
@@ -199,6 +199,7 @@ RUN sed -i "s/stretch main/stretch main contrib non-free/" /etc/apt/sources.list
     ln -sf $MOLOCHDIR/bin/npm /usr/local/bin/npm && \
       ln -sf $MOLOCHDIR/bin/node /usr/local/bin/node && \
       ln -sf $MOLOCHDIR/bin/npx /usr/local/bin/npx && \
+    apt-get -q -y --purge remove gcc gcc-6 cpp cpp-6 libssl1.0-dev && \
     apt-get -q -y autoremove && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

diff --git a/README.md b/README.md
@@ -108,16 +108,16 @@ You can then observe that the images have been retrieved by running `docker imag
 ```
 $ docker images
 REPOSITORY                                          TAG                 IMAGE ID            CREATED             SIZE
-malcolmnetsec/nginx-proxy                           1.3.0               xxxxxxxxxxxx        16 hours ago        53MB
-malcolmnetsec/file-upload                           1.3.0               xxxxxxxxxxxx        16 hours ago        214MB
-malcolmnetsec/pcap-capture                          1.3.0               xxxxxxxxxxxx        17 hours ago        111MB
-malcolmnetsec/file-monitor                          1.3.0               xxxxxxxxxxxx        17 hours ago        353MB
-malcolmnetsec/moloch                                1.3.0               xxxxxxxxxxxx        17 hours ago        1.04GB
-malcolmnetsec/filebeat-oss                          1.3.0               xxxxxxxxxxxx        17 hours ago        454MB
-malcolmnetsec/curator                               1.3.0               xxxxxxxxxxxx        17 hours ago        303MB
-malcolmnetsec/logstash-oss                          1.3.0               xxxxxxxxxxxx        17 hours ago        1.14GB
-malcolmnetsec/elastalert                            1.3.0               xxxxxxxxxxxx        17 hours ago        268MB
-malcolmnetsec/kibana-oss                            1.3.0               xxxxxxxxxxxx        17 hours ago        850MB
+malcolmnetsec/nginx-proxy                           1.3.1               xxxxxxxxxxxx        16 hours ago        53MB
+malcolmnetsec/file-upload                           1.3.1               xxxxxxxxxxxx        16 hours ago        214MB
+malcolmnetsec/pcap-capture                          1.3.1               xxxxxxxxxxxx        17 hours ago        111MB
+malcolmnetsec/file-monitor                          1.3.1               xxxxxxxxxxxx        17 hours ago        353MB
+malcolmnetsec/moloch                                1.3.1               xxxxxxxxxxxx        17 hours ago        1.04GB
+malcolmnetsec/filebeat-oss                          1.3.1               xxxxxxxxxxxx        17 hours ago        454MB
+malcolmnetsec/curator                               1.3.1               xxxxxxxxxxxx        17 hours ago        303MB
+malcolmnetsec/logstash-oss                          1.3.1               xxxxxxxxxxxx        17 hours ago        1.14GB
+malcolmnetsec/elastalert                            1.3.1               xxxxxxxxxxxx        17 hours ago        268MB
+malcolmnetsec/kibana-oss                            1.3.1               xxxxxxxxxxxx        17 hours ago        850MB
 docker.elastic.co/elasticsearch/elasticsearch-oss   6.8.1               xxxxxxxxxxxx        3 weeks ago         765MB
 ```
 
@@ -1231,16 +1231,16 @@ Pulling nginx-proxy   ... done
 
 user@host:~/Malcolm$ docker images
 REPOSITORY                                          TAG                 IMAGE ID            CREATED             SIZE
-malcolmnetsec/nginx-proxy                           1.3.0               xxxxxxxxxxxx        16 hours ago        53MB
-malcolmnetsec/file-upload                           1.3.0               xxxxxxxxxxxx        16 hours ago        214MB
-malcolmnetsec/pcap-capture                          1.3.0               xxxxxxxxxxxx        17 hours ago        111MB
-malcolmnetsec/file-monitor                          1.3.0               xxxxxxxxxxxx        17 hours ago        353MB
-malcolmnetsec/curator                               1.3.0               xxxxxxxxxxxx        17 hours ago        303MB
-malcolmnetsec/moloch                                1.3.0               xxxxxxxxxxxx        17 hours ago        1.04GB
-malcolmnetsec/filebeat-oss                          1.3.0               xxxxxxxxxxxx        17 hours ago        454MB
-malcolmnetsec/logstash-oss                          1.3.0               xxxxxxxxxxxx        17 hours ago        1.14GB
-malcolmnetsec/elastalert                            1.3.0               xxxxxxxxxxxx        17 hours ago        268MB
-malcolmnetsec/kibana-oss                            1.3.0               xxxxxxxxxxxx        17 hours ago        850MB
+malcolmnetsec/nginx-proxy                           1.3.1               xxxxxxxxxxxx        16 hours ago        53MB
+malcolmnetsec/file-upload                           1.3.1               xxxxxxxxxxxx        16 hours ago        214MB
+malcolmnetsec/pcap-capture                          1.3.1               xxxxxxxxxxxx        17 hours ago        111MB
+malcolmnetsec/file-monitor                          1.3.1               xxxxxxxxxxxx        17 hours ago        353MB
+malcolmnetsec/curator                               1.3.1               xxxxxxxxxxxx        17 hours ago        303MB
+malcolmnetsec/moloch                                1.3.1               xxxxxxxxxxxx        17 hours ago        1.04GB
+malcolmnetsec/filebeat-oss                          1.3.1               xxxxxxxxxxxx        17 hours ago        454MB
+malcolmnetsec/logstash-oss                          1.3.1               xxxxxxxxxxxx        17 hours ago        1.14GB
+malcolmnetsec/elastalert                            1.3.1               xxxxxxxxxxxx        17 hours ago        268MB
+malcolmnetsec/kibana-oss                            1.3.1               xxxxxxxxxxxx        17 hours ago        850MB
 docker.elastic.co/elasticsearch/elasticsearch-oss   6.8.1               xxxxxxxxxxxx        3 weeks ago         765MB
 ```
 

diff --git a/docker-compose-standalone-zeek-live.yml b/docker-compose-standalone-zeek-live.yml
@@ -104,7 +104,7 @@ services:
       - ./elasticsearch:/usr/share/elasticsearch/data:delegated
       - ./elasticsearch-backup:/opt/elasticsearch/backup:delegated
   kibana:
-    image: malcolmnetsec/kibana-oss:1.3.0
+    image: malcolmnetsec/kibana-oss:1.3.1
     hostname: kibana
     environment:
       << : *kibana-variables
@@ -126,7 +126,7 @@ services:
         retries: 3
         start_period: 200s
   elastalert:
-    image: malcolmnetsec/elastalert:1.3.0
+    image: malcolmnetsec/elastalert:1.3.1
     hostname: elastalert
     environment:
       ELASTICSEARCH_URL : 'http://elasticsearch:9200'
@@ -149,7 +149,7 @@ services:
       - ./elastalert/config/config.json:/opt/elastalert-server/config/config.json
       - ./elastalert/rules/:/opt/elastalert/rules/
   curator:
-    image: malcolmnetsec/curator:1.3.0
+    image: malcolmnetsec/curator:1.3.1
     hostname: curator
     environment:
       << : *curator-variables
@@ -158,7 +158,7 @@ services:
     depends_on:
       - elasticsearch
   logstash:
-    image: malcolmnetsec/logstash-oss:1.3.0
+    image: malcolmnetsec/logstash-oss:1.3.1
     hostname: logstash
     environment:
       << : *logstash-variables
@@ -185,7 +185,7 @@ services:
       - ./cidr-map.txt:/usr/share/logstash/config/cidr-map.txt:ro
       - ./host-map.txt:/usr/share/logstash/config/host-map.txt:ro
   filebeat:
-    image: malcolmnetsec/filebeat-oss:1.3.0
+    image: malcolmnetsec/filebeat-oss:1.3.1
     hostname: filebeat
     environment:
       << : *common-upload-variables
@@ -209,7 +209,7 @@ services:
       - ./filebeat/certs/client.crt:/certs/client.crt:ro
       - ./filebeat/certs/client.key:/certs/client.key:ro
   moloch:
-    image: malcolmnetsec/moloch:1.3.0
+    image: malcolmnetsec/moloch:1.3.1
     hostname: moloch
     env_file:
       - ./auth.env
@@ -241,7 +241,7 @@ services:
       - ./moloch-logs:/data/moloch/logs
       - ./moloch-raw:/data/moloch/raw
   file-monitor:
-    image: malcolmnetsec/file-monitor:1.3.0
+    image: malcolmnetsec/file-monitor:1.3.1
     hostname: filemon
     environment:
       << : *zeek-file-extraction-variables
@@ -251,7 +251,7 @@ services:
       - ./zeek-logs/extract_files:/data/zeek/extract_files
       - ./zeek-logs/current:/data/zeek/logs
   pcap-capture:
-    image: malcolmnetsec/pcap-capture:1.3.0
+    image: malcolmnetsec/pcap-capture:1.3.1
     network_mode: host
     ulimits:
       memlock:
@@ -267,7 +267,7 @@ services:
     volumes:
       - ./pcap/upload:/pcap
   upload:
-    image: malcolmnetsec/file-upload:1.3.0
+    image: malcolmnetsec/file-upload:1.3.1
     hostname: upload
     env_file:
       - ./auth.env
@@ -283,7 +283,7 @@ services:
     volumes:
       - ./pcap/upload:/var/www/upload/server/php/chroot/files
   nginx-proxy:
-    image: malcolmnetsec/nginx-proxy:1.3.0
+    image: malcolmnetsec/nginx-proxy:1.3.1
     hostname: nginx-proxy
     depends_on:
       - moloch

diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml
@@ -104,7 +104,7 @@ services:
       - ./elasticsearch:/usr/share/elasticsearch/data:delegated
       - ./elasticsearch-backup:/opt/elasticsearch/backup:delegated
   kibana:
-    image: malcolmnetsec/kibana-oss:1.3.0
+    image: malcolmnetsec/kibana-oss:1.3.1
     hostname: kibana
     environment:
       << : *kibana-variables
@@ -126,7 +126,7 @@ services:
         retries: 3
         start_period: 200s
   elastalert:
-    image: malcolmnetsec/elastalert:1.3.0
+    image: malcolmnetsec/elastalert:1.3.1
     hostname: elastalert
     environment:
       ELASTICSEARCH_URL : 'http://elasticsearch:9200'
@@ -149,7 +149,7 @@ services:
       - ./elastalert/config/config.json:/opt/elastalert-server/config/config.json
       - ./elastalert/rules/:/opt/elastalert/rules/
   curator:
-    image: malcolmnetsec/curator:1.3.0
+    image: malcolmnetsec/curator:1.3.1
     hostname: curator
     environment:
       << : *curator-variables
@@ -158,7 +158,7 @@ services:
     depends_on:
       - elasticsearch
   logstash:
-    image: malcolmnetsec/logstash-oss:1.3.0
+    image: malcolmnetsec/logstash-oss:1.3.1
     hostname: logstash
     environment:
       << : *logstash-variables
@@ -185,7 +185,7 @@ services:
       - ./cidr-map.txt:/usr/share/logstash/config/cidr-map.txt:ro
       - ./host-map.txt:/usr/share/logstash/config/host-map.txt:ro
   filebeat:
-    image: malcolmnetsec/filebeat-oss:1.3.0
+    image: malcolmnetsec/filebeat-oss:1.3.1
     hostname: filebeat
     environment:
       << : *common-upload-variables
@@ -209,7 +209,7 @@ services:
       - ./filebeat/certs/client.crt:/certs/client.crt:ro
       - ./filebeat/certs/client.key:/certs/client.key:ro
   moloch:
-    image: malcolmnetsec/moloch:1.3.0
+    image: malcolmnetsec/moloch:1.3.1
     hostname: moloch
     env_file:
       - ./auth.env
@@ -241,7 +241,7 @@ services:
       - ./moloch-logs:/data/moloch/logs
       - ./moloch-raw:/data/moloch/raw
   file-monitor:
-    image: malcolmnetsec/file-monitor:1.3.0
+    image: malcolmnetsec/file-monitor:1.3.1
     hostname: filemon
     environment:
       << : *zeek-file-extraction-variables
@@ -251,7 +251,7 @@ services:
       - ./zeek-logs/extract_files:/data/zeek/extract_files
       - ./zeek-logs/current:/data/zeek/logs
   pcap-capture:
-    image: malcolmnetsec/pcap-capture:1.3.0
+    image: malcolmnetsec/pcap-capture:1.3.1
     network_mode: host
     ulimits:
       memlock:
@@ -267,7 +267,7 @@ services:
     volumes:
       - ./pcap/upload:/pcap
   upload:
-    image: malcolmnetsec/file-upload:1.3.0
+    image: malcolmnetsec/file-upload:1.3.1
     hostname: upload
     env_file:
       - ./auth.env
@@ -283,7 +283,7 @@ services:
     volumes:
       - ./pcap/upload:/var/www/upload/server/php/chroot/files
   nginx-proxy:
-    image: malcolmnetsec/nginx-proxy:1.3.0
+    image: malcolmnetsec/nginx-proxy:1.3.1
     hostname: nginx-proxy
     depends_on:
       - moloch

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -107,7 +107,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfiles/kibana.Dockerfile
-    image: malcolmnetsec/kibana-oss:1.3.0
+    image: malcolmnetsec/kibana-oss:1.3.1
     hostname: kibana
     environment:
       << : *kibana-variables
@@ -132,7 +132,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfiles/elastalert.Dockerfile
-    image: malcolmnetsec/elastalert:1.3.0
+    image: malcolmnetsec/elastalert:1.3.1
     hostname: elastalert
     environment:
       ELASTICSEARCH_URL : 'http://elasticsearch:9200'
@@ -158,7 +158,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfiles/curator.Dockerfile
-    image: malcolmnetsec/curator:1.3.0
+    image: malcolmnetsec/curator:1.3.1
     hostname: curator
     environment:
       << : *curator-variables
@@ -172,7 +172,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfiles/logstash.Dockerfile
-    image: malcolmnetsec/logstash-oss:1.3.0
+    image: malcolmnetsec/logstash-oss:1.3.1
     hostname: logstash
     environment:
       << : *logstash-variables
@@ -209,7 +209,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfiles/filebeat.Dockerfile
-    image: malcolmnetsec/filebeat-oss:1.3.0
+    image: malcolmnetsec/filebeat-oss:1.3.1
     hostname: filebeat
     environment:
       << : *common-upload-variables
@@ -237,7 +237,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfiles/moloch.Dockerfile
-    image: malcolmnetsec/moloch:1.3.0
+    image: malcolmnetsec/moloch:1.3.1
     hostname: moloch
     env_file:
       - ./auth.env
@@ -275,7 +275,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfiles/file-monitor.Dockerfile
-    image: malcolmnetsec/file-monitor:1.3.0
+    image: malcolmnetsec/file-monitor:1.3.1
     hostname: filemon
     environment:
       << : *zeek-file-extraction-variables
@@ -288,7 +288,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfiles/pcap-capture.Dockerfile
-    image: malcolmnetsec/pcap-capture:1.3.0
+    image: malcolmnetsec/pcap-capture:1.3.1
     network_mode: host
     ulimits:
       memlock:
@@ -307,7 +307,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfiles/file-upload.Dockerfile
-    image: malcolmnetsec/file-upload:1.3.0
+    image: malcolmnetsec/file-upload:1.3.1
     hostname: upload
     env_file:
       - ./auth.env
@@ -326,7 +326,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfiles/nginx.Dockerfile
-    image: malcolmnetsec/nginx-proxy:1.3.0
+    image: malcolmnetsec/nginx-proxy:1.3.1
     hostname: nginx-proxy
     depends_on:
       - moloch

diff --git a/iso-build/build.sh b/iso-build/build.sh
@@ -2,7 +2,7 @@
 
 IMAGE_NAME=malcolm
 IMAGE_VERSION=1.0.0
-IMAGE_DISTRIBUTION=stretch
+IMAGE_DISTRIBUTION=buster
 
 BUILD_ERROR_CODE=1
 
@@ -73,7 +73,7 @@ if [ -d "$WORKDIR" ]; then
   # make sure we install the newer kernel, firmwares, and kernel headers
   echo "linux-image-$(uname -r)" > ./config/package-lists/kernel.list.chroot
   echo "linux-headers-$(uname -r)" >> ./config/package-lists/kernel.list.chroot
-  echo "linux-compiler-gcc-6-x86=$(dpkg -s linux-compiler-gcc-6-x86 | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot
+  echo "linux-compiler-gcc-8-x86=$(dpkg -s linux-compiler-gcc-8-x86 | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot
   echo "linux-kbuild-4.19=$(dpkg -s linux-kbuild-4.19 | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot
   echo "firmware-linux=$(dpkg -s firmware-linux | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot
   echo "firmware-linux-nonfree=$(dpkg -s firmware-linux-nonfree | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot
@@ -152,7 +152,7 @@ if [ -d "$WORKDIR" ]; then
     --binary-images iso-hybrid \
     --bootloaders "syslinux,grub-efi" \
     --chroot-filesystem squashfs \
-    --backports true \
+    --backports false \
     --security true \
     --updates true \
     --source false \

diff --git a/iso-build/config/archives/docker.list.binary b/iso-build/config/archives/docker.list.binary
@@ -1,2 +1,2 @@
-deb [arch=amd64] https://download.docker.com/linux/debian stretch stable
+deb [arch=amd64] https://download.docker.com/linux/debian buster stable
 
diff --git a/iso-build/config/archives/docker.list.chroot b/iso-build/config/archives/docker.list.chroot
@@ -1,2 +1,2 @@
-deb [arch=amd64] https://download.docker.com/linux/debian stretch stable
+deb [arch=amd64] https://download.docker.com/linux/debian buster stable
 
diff --git a/iso-build/config/hooks/normal/0168-python-install.hook.chroot b/iso-build/config/hooks/normal/0168-python-install.hook.chroot