Merge pull request #60 from automationator/master

Minor updates
IntegralDefense · May 15, 2019 · 94a42ef · 94a42ef
2 parents 72b7710 + 7408c7a
commit 94a42ef
Show file tree

Hide file tree

Showing 17 changed files with 130 additions and 17 deletions.
diff --git a/bin/import-crits-campaigns-PROD.sh b/bin/import-crits-campaigns-PROD.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Error if the campaigns.json path does not exist
+if [[ ! -f "$1" ]]
+then
+    echo "Could not find CRITS campaigns JSON: $1"
+    exit 1
+fi
+
+# Create the import directory if it does not exist
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+import_dir="$DIR/../services/web/import"
+if [[ ! -d "$import_dir" ]]
+then
+    mkdir "$import_dir"
+fi
+
+# Copy the campaigns.json file into the import directory
+cp "$1" "$import_dir/campaigns.json"
+
+docker-compose -f docker-compose-PROD.yml build
+docker-compose -f docker-compose-PROD.yml run web-prod pypy3 manage.py import-crits-campaigns
+
+# Delete the campaigns.json file from the container.
+rm "$import_dir/campaigns.json"
diff --git a/bin/import-crits-indicators-PROD.sh b/bin/import-crits-indicators-PROD.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Error if the indicators.json path does not exist
+if [[ ! -f "$1" ]]
+then
+    echo "Could not find CRITS indicators JSON: $1"
+    exit 1
+fi
+
+# Create the import directory if it does not exist
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+import_dir="$DIR/../services/web/import"
+if [[ ! -d "$import_dir" ]]
+then
+    mkdir "$import_dir"
+fi
+
+# Copy the indicators.json file into the import directory
+cp "$1" "$import_dir/indicators.json"
+
+docker-compose -f docker-compose-PROD.yml build
+docker-compose -f docker-compose-PROD.yml run web-prod pypy3 manage.py import-crits-indicators
+
+# Delete the indicators.json file from the container.
+rm "$import_dir/indicators.json"
diff --git a/docker-compose-DEV.yml b/docker-compose-DEV.yml
@@ -5,6 +5,9 @@ services:
         build:
             context: ./services/web
             dockerfile: Dockerfile-DEV
+            args:
+                - http_proxy
+                - https_proxy
         networks:
             - dev
         restart: on-failure
@@ -21,6 +24,9 @@ services:
         build:
             context: ./services/db
             dockerfile: Dockerfile-DEV
+            args:
+                - http_proxy
+                - https_proxy
         networks:
             - dev
         restart: on-failure
@@ -36,6 +42,9 @@ services:
         build:
             context: ./services/nginx
             dockerfile: Dockerfile-DEV
+            args:
+                - http_proxy
+                - https_proxy
         networks:
             - dev
         restart: on-failure

diff --git a/docker-compose-PROD.yml b/docker-compose-PROD.yml
@@ -5,6 +5,9 @@ services:
         build:
             context: ./services/web
             dockerfile: Dockerfile-PROD
+            args:
+                - http_proxy
+                - https_proxy
         networks:
             - prod
         restart: on-failure
@@ -19,6 +22,9 @@ services:
         build:
             context: ./services/db
             dockerfile: Dockerfile-PROD
+            args:
+                - http_proxy
+                - https_proxy
         networks:
             - prod
         restart: on-failure
@@ -28,12 +34,15 @@ services:
             - mysql-prod:/var/lib/mysql
         env_file:
             - ./services/db/docker-PROD.env
-        command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --explicit_defaults_for_timestamp
+        command: --explicit_defaults_for_timestamp
 
     nginx-prod:
         build:
             context: ./services/nginx
             dockerfile: Dockerfile-PROD
+            args:
+                - http_proxy
+                - https_proxy
         networks:
             - prod
         restart: on-failure

diff --git a/docker-compose-TEST.yml b/docker-compose-TEST.yml
@@ -5,6 +5,9 @@ services:
         build:
             context: ./services/web
             dockerfile: Dockerfile-TEST
+            args:
+                - http_proxy
+                - https_proxy
         networks:
             - test
         restart: on-failure
@@ -19,19 +22,25 @@ services:
         build:
             context: ./services/db
             dockerfile: Dockerfile-TEST
+            args:
+                - http_proxy
+                - https_proxy
         networks:
             - test
         restart: on-failure
         volumes:
             - mysql-test:/var/lib/mysql
         env_file:
             - ./services/db/docker-TEST.env
-        command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --explicit_defaults_for_timestamp
+        command: --explicit_defaults_for_timestamp
 
     nginx-test:
         build:
             context: ./services/nginx
             dockerfile: Dockerfile-TEST
+            args:
+                - http_proxy
+                - https_proxy
         networks:
             - test
         restart: on-failure

diff --git a/services/db/Dockerfile-DEV b/services/db/Dockerfile-DEV
@@ -1,5 +1,7 @@
-#FROM mysql:5.7
 FROM mysql:8.0.15
 
+ARG http_proxy
+ARG https_proxy
+
 ADD create.sql /docker-entrypoint-initdb.d
 ADD conf.d/ /etc/mysql/conf.d
diff --git a/services/db/Dockerfile-PROD b/services/db/Dockerfile-PROD
@@ -1,5 +1,7 @@
-#FROM mysql:5.7
-FROM mysql:8.0
+FROM mysql:8.0.15
+
+ARG http_proxy
+ARG https_proxy
 
 ADD create.sql /docker-entrypoint-initdb.d
 ADD conf.d/ /etc/mysql/conf.d
diff --git a/services/db/Dockerfile-TEST b/services/db/Dockerfile-TEST
@@ -1,5 +1,7 @@
-#FROM mysql:5.7
-FROM mysql:8.0
+FROM mysql:8.0.15
+
+ARG http_proxy
+ARG https_proxy
 
 ADD create.sql /docker-entrypoint-initdb.d
 ADD conf.d/ /etc/mysql/conf.d
diff --git a/services/nginx/Dockerfile-DEV b/services/nginx/Dockerfile-DEV
@@ -1,5 +1,8 @@
 FROM nginx:1.15.6-alpine
 
+ARG http_proxy
+ARG https_proxy
+
 RUN rm /etc/nginx/conf.d/default.conf
 COPY ./server-DEV.conf /etc/nginx/conf.d
 ADD certs/ /etc/nginx/certs
diff --git a/services/nginx/Dockerfile-PROD b/services/nginx/Dockerfile-PROD
@@ -1,5 +1,8 @@
 FROM nginx:1.15.6-alpine
 
+ARG http_proxy
+ARG https_proxy
+
 RUN rm /etc/nginx/conf.d/default.conf
 COPY ./server-PROD.conf /etc/nginx/conf.d
 ADD certs/ /etc/nginx/certs
diff --git a/services/nginx/Dockerfile-TEST b/services/nginx/Dockerfile-TEST
@@ -1,5 +1,8 @@
 FROM nginx:1.15.6-alpine
 
+ARG http_proxy
+ARG https_proxy
+
 RUN rm /etc/nginx/conf.d/default.conf
 COPY ./server-TEST.conf /etc/nginx/conf.d
 ADD certs/ /etc/nginx/certs
diff --git a/services/web/Dockerfile-DEV b/services/web/Dockerfile-DEV
@@ -1,5 +1,8 @@
 FROM pypy:3
 
+ARG http_proxy
+ARG https_proxy
+
 RUN apt-get update && apt-get install -y mysql-client
 
 WORKDIR /usr/src/app

diff --git a/services/web/Dockerfile-PROD b/services/web/Dockerfile-PROD
@@ -1,5 +1,8 @@
 FROM pypy:3
 
+ARG http_proxy
+ARG https_proxy
+
 RUN apt-get update && apt-get install -y mysql-client
 
 WORKDIR /usr/src/app

diff --git a/services/web/Dockerfile-TEST b/services/web/Dockerfile-TEST
@@ -1,5 +1,8 @@
 FROM pypy:3
 
+ARG http_proxy
+ARG https_proxy
+
 RUN apt-get update && apt-get install -y mysql-client
 
 WORKDIR /usr/src/app

diff --git a/services/web/project/api/routes/indicator.py b/services/web/project/api/routes/indicator.py
@@ -485,6 +485,7 @@ def read_indicators():
     :query confidence: Confidence value
     :query created_after: Parsable date or datetime in GMT. Ex: YYYY-MM-DD or YYYY-MM-DD HH:MM:SS
     :query created_before: Parsable date or datetime in GMT. Ex: YYYY-MM-DD or YYYY-MM-DD HH:MM:SS
+    :query exact_value: Exact indicator value to find. Does not use a wildcard search.
     :query impact: Impact value
     :query modified_after: Parsable date or datetime in GMT. Ex: YYYY-MM-DD or YYYY-MM-DD HH:MM:SS
     :query modified_before: Parsable date or datetime in GMT. Ex: YYYY-MM-DD or YYYY-MM-DD HH:MM:SS
@@ -528,6 +529,10 @@ def read_indicators():
             created_before = datetime.date.min
         filters.add(Indicator.created_time < created_before)
 
+    # Exact value filter
+    if 'exact_value' in request.args:
+        filters.add(Indicator.value == request.args.get('exact_value'))
+
     # Impact filter
     if 'impact' in request.args:
         filters.add(Indicator.impact.has(IndicatorImpact.value == request.args.get('impact')))

diff --git a/services/web/project/tests/api/test_indicator.py b/services/web/project/tests/api/test_indicator.py
@@ -1020,11 +1020,23 @@ def test_read_with_filters(client):
     assert response['items'][0]['user'] == 'analyst'
 
     # Filter by value
-    request = client.get('/api/indicators?value=1.1')
+    request = client.get('/api/indicators?value=abcd')
+    response = json.loads(request.data.decode())
+    assert request.status_code == 200
+    assert len(response['items']) == 2
+
+    # Filter by exact value (success)
+    request = client.get('/api/indicators?exact_value=abcd@abcd.com')
     response = json.loads(request.data.decode())
     assert request.status_code == 200
     assert len(response['items']) == 1
-    assert response['items'][0]['value'] == '1.1.1.1'
+    assert response['items'][0]['value'] == 'abcd@abcd.com'
+
+    # Filter by exact value (failure)
+    request = client.get('/api/indicators?exact_value=abcd')
+    response = json.loads(request.data.decode())
+    assert request.status_code == 200
+    assert len(response['items']) == 0
 
     # Filter by intel reference
     request = client.get('/api/indicators?reference={}'.format(urllib.parse.quote('https://your.wiki/display/events/20190501+somebadsite.local+-+Bad+Guy')))

diff --git a/setup.py b/setup.py
@@ -66,14 +66,12 @@
 
 FLASK_SECRET_KEY = os.urandom(48).hex()
 FLASK_SECURITY_PASSWORD_SALT = os.urandom(48).hex()
-JWT_SECRET_KEY = os.urandom(48).hex()
 WEB_DOCKER_ENV = """FLASK_APP=project
 FLASK_ENV={environment}
 APP_SETTINGS={config}
 DATABASE_URL=mysql+pymysql://{user}:{password}@db:3306/SIP?charset=utf8mb4
 SECRET_KEY={secret_key}
 SECURITY_PASSWORD_SALT={salt}
-JWT_SECRET_KEY={jwt_key}
 """
 
 
@@ -208,8 +206,7 @@
                                        user=MYSQL_USER,
                                        password=MYSQL_PASS,
                                        secret_key=FLASK_SECRET_KEY,
-                                       salt=FLASK_SECURITY_PASSWORD_SALT,
-                                       jwt_key=JWT_SECRET_KEY)
+                                       salt=FLASK_SECURITY_PASSWORD_SALT)
         with open(WEB_DOCKER_ENV_DEV, 'w') as f:
             f.write(output)
 
@@ -220,8 +217,7 @@
                                        user=MYSQL_USER,
                                        password=MYSQL_PASS,
                                        secret_key=FLASK_SECRET_KEY,
-                                       salt=FLASK_SECURITY_PASSWORD_SALT,
-                                       jwt_key=JWT_SECRET_KEY)
+                                       salt=FLASK_SECURITY_PASSWORD_SALT)
         with open(WEB_DOCKER_ENV_TEST, 'w') as f:
             f.write(output)
 
@@ -232,8 +228,7 @@
                                        user=MYSQL_USER,
                                        password=MYSQL_PASS,
                                        secret_key=FLASK_SECRET_KEY,
-                                       salt=FLASK_SECURITY_PASSWORD_SALT,
-                                       jwt_key=JWT_SECRET_KEY)
+                                       salt=FLASK_SECURITY_PASSWORD_SALT)
         with open(WEB_DOCKER_ENV_PROD, 'w') as f:
             f.write(output)