Merge pull request #561 from stratosphereips/develop

Slips v1.0.13

.dockerignore

@@ -1,26 +1,28 @@
-# slips generated alerts
-output/alerts.log
-output/alerts.json
-2021-*/
-2022-*/
+# Ignore all .log files in output/ dir
+output/*.log
+**/alerts.json
+**/.git
 
+#node modules
+**/node_modules
 
-# zeek log files
-zeek_files/*.log
 
-yara-python/*
+# files with API keys/tokens
+config/slack_bot_token_secret
+config/warden.conf
+config/RiskIQ_credentials
+
 
 Datasett/
+Dataset/
 
 # virtual env files
 ENV/*
+env/*
 
 # compiled yara rules
 modules/leak_detector/yara_rules/compiled/
 
-# api keys
-modules/RiskIQ/api_key_secret
-modules/virustotal/api_key_secret
 
 .git/
 .cache/
@@ -30,16 +32,12 @@ __pycache__/
 
 # CESNET module logs
 *.log
-modules/CESNET/*.log
 
 # redis backups
-redis_backups/*.rdb
+**/*.rdb
 
 # for IntelliJ IDE
 .idea/
 
 # for vscode
 .vscode/
-
-
-

.github/pull_request_template.md

@@ -18,6 +18,15 @@
 [ ] - Not correct; marked as **not** done
 -->
 
+
+## Steps you followed to test the changes purposed in this PR:
+
+
+* 
+*
+
+
+
 ## Check List (Check all the applicable boxes) <!-- Follow the above conventions to check the box -->
 
 - [ ] My code follows the code style of this project.
@@ -31,6 +40,8 @@
 
 <!-- Add all the screenshots which support your changes -->
 
+<!-- Add screenshots with the passing unit and integration tests locally -->
+
 ## Note to reviewers
 
 <!-- Add notes to reviewers if applicable -->

.github/workflows/CI-production-testing.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
 
-  test_slips_locally:
+  unit_and_integration_tests:
     # runs the tests on a GH VM
     runs-on: ubuntu-20.04
     # 2 hours timeout
@@ -39,9 +39,9 @@ jobs:
         python-version: "3.8"
 
     - name: Install Python dependencies
-      run:
+      run: |
         python -m pip install --upgrade pip
-        pip install -r install/requirements.txt
+        grep -v 'black' install/requirements.txt | xargs pip3 install  --no-cache-dir
 
     - name: Start redis server
       run: redis-server --daemonize yes
@@ -73,101 +73,3 @@ jobs:
         name: test_slips_locally-integration-tests-output
         path: |
           output/integration_tests
-
-  test_slips_in_docker:
-    # runs the tests in a docker(built by this job) on stop of a GH VM
-    runs-on: ubuntu-20.04
-    # 2 hours timeout
-    timeout-minutes: 7200
-
-    strategy:
-      matrix:
-        include:
-          - type: ubuntu-image
-            name: ubuntu-image
-            image_name: slips
-            path: ./docker/ubuntu-image/Dockerfile
-#
-#         #TODO our unit tests dont support macos yet
-#          - type: macosm1-image
-#            name: macosm1-image
-#            image_name: slips_macos_m1
-#            path: ./docker/macosm1-image/Dockerfile
-
-    steps:
-    # clone slips and checkout branch
-    # By default it checks out only one commit
-    - uses: actions/checkout@v3
-      with:
-        ref: 'develop'
-
-    # build slips from target dockerfile, but don't push
-    - name: Build our ${{ matrix.name }} from dockerfile
-      id: docker_build_slips
-      uses: docker/build-push-action@v2
-      with:
-        context: ./
-        file: ${{ matrix.path }}
-        tags: stratosphereips/${{ matrix.image_name }}:latest
-        push: false
-
-
-    # use the built image to run unit tests inside docker
-    # we ignore the blocking module unit tests because we cannot run
-    # docker with -cap-add=NET_ADMIN and use iptables in GH actions
-    - name: Run unit tests inside docker
-      uses: addnab/docker-run-action@v3
-      with:
-        image: stratosphereips/slips:latest
-        run: |
-          git reset --hard
-          git pull & git checkout -f origin/develop
-          redis-server --daemonize yes
-          python3  -m pytest tests/ --ignore="tests/test_database.py" --ignore="tests/integration_tests" -n 7 -p no:warnings -vv -s
-
-    - name: Run database tests inside docker
-      uses: addnab/docker-run-action@v3
-      with:
-        image: stratosphereips/slips:latest
-        run: |
-          git reset --hard
-          git pull & git checkout -f origin/develop
-          redis-server --daemonize yes
-          python3  -m pytest tests/test_database.py -p no:warnings -vv
-
-
-
-    - name: Run integration tests inside docker
-      uses: addnab/docker-run-action@v3
-      with:
-        image: stratosphereips/slips:latest
-        # mount the host's output/ dir to docker's output/ dir, the goal is to access integration
-        # tests output in docker from the host to upload it later
-        options: -v ${{ github.workspace }}/output:/StratosphereLinuxIPS/output
-        run: |
-          git reset --hard
-          git pull & git checkout -f origin/develop
-          redis-server --daemonize yes
-          python3 -m pytest -s tests/integration_tests/test_dataset.py -p no:warnings -vv
-
-    - name: Run config file integration tests inside docker
-      uses: addnab/docker-run-action@v3
-      with:
-        image: stratosphereips/slips:latest
-        # mount the host's output/ dir to docker's output/ dir, the goal is to access integration
-        # tests output in docker from the host to upload it later
-        options: -v ${{ github.workspace }}/output:/StratosphereLinuxIPS/output
-        run: |
-          git reset --hard
-          git pull & git checkout -f origin/develop
-          redis-server --daemonize yes
-          python3 -m pytest -s tests/integration_tests/test_config_files.py -p no:warnings -vv
-
-    - name: Upload Artifact
-      # run this job whether the above jobs failed or passed
-      if: success() || failure()
-      uses: actions/upload-artifact@v3
-      with:
-        name: test_slips_in_docker-integration-tests-output
-        path: |
-          ${{ github.workspace }}/output/integration_tests

.pre-commit-config.yaml

@@ -0,0 +1,47 @@
+repos:
+-   repo: https://github.com/Yelp/detect-secrets
+    rev: v1.4.0
+    hooks:
+        - id: detect-secrets
+          args: ['--baseline', '.secrets.baseline']
+          exclude: .*dataset/.*|
+                  (?x)(
+                         ^config/local_ti_files/own_malicious_JA3.csv$|
+                          .*test.* |
+                         .*\.md$
+                      )
+
+-     repo: https://github.com/pre-commit/pre-commit-hooks
+      rev: v3.1.0
+      hooks:
+        - id: trailing-whitespace
+        - id: check-added-large-files
+        - id: check-docstring-first
+        - id: check-merge-conflict
+        - id: end-of-file-fixer
+        - id: detect-private-key
+          exclude: .*dataset/.*|
+                (?x)(
+                     ^config/$|
+                      .*test.* |
+                     .*\.md$
+              )
+
+-     repo: https://github.com/astral-sh/ruff-pre-commit
+      # Ruff version.
+      rev: v0.3.4
+      hooks:
+        # Run the linter.
+        - id: ruff
+          args: [ --fix ]
+          # excludes formatting slips_files/common/imports.py
+          exclude: imports
+
+-     repo: https://github.com/psf/black-pre-commit-mirror
+      rev: 24.3.0
+      hooks:
+        - id: black
+          args: ['--line-length' , '79']
+          language_version: python3.8
+          # excludes formatting slips_files/common/imports.py
+          exclude: imports