Merge pull request #1405 from HXSecurity/beta

Release 1.10.0
HXSecurity · Apr 13, 2023 · 528eb62 · 528eb62
2 parents 5e332de + 635c0cb
commit 528eb62
Show file tree

Hide file tree

Showing 64 changed files with 14,954 additions and 1,034 deletions.
diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml
@@ -69,7 +69,7 @@ jobs:
           python3 manage.py updatedepartmenttoken
           python3 manage.py runserver 0.0.0.0:8000  &
           sleep 60
-          schemathesis   run --base-url "http://localhost:8000/"  -H 'Authorization: Token 67aebd78e700ad36a82a152276196b5f49fafeb0' http://localhost:8000/api/XZPcGFKoxYXScwGjQtJx8u/schema/  --hypothesis-suppress-health-check=data_too_large,filter_too_much,too_slow,return_value,large_base_example,not_a_test_method,function_scoped_fixture --validate-schema=false  --hypothesis-verbosity normal 
+          schemathesis   run --base-url "http://localhost:8000/"  -H 'Authorization: Token 67aebd78e700ad36a82a152276196b5f49fafeb0' http://localhost:8000/api/XZPcGFKoxYXScwGjQtJx8u/schema/  --hypothesis-suppress-health-check=data_too_large,filter_too_much,too_slow,large_base_example,function_scoped_fixture --validate-schema=false  --hypothesis-verbosity normal 
   
   Run-BuildCheck:
     if: ${{ github.repository_owner == 'HXSecurity' }}

diff --git a/.github/workflows/release_dongtai.yml b/.github/workflows/release_dongtai.yml
@@ -3,6 +3,14 @@ name: Release DongTai-Server
 on:
   release:
     types: [ created, edited ]
+  workflow_dispatch:
+    inputs:
+      agent_version:
+        required: true
+        type: string
+      server_version:
+        required: true
+        type: string
 
 jobs:
   build:
@@ -26,15 +34,22 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - id: release
+      - name: Set the value
+        id: release
         run: |
           TAG_NAME=${{ github.event.release.tag_name }}
           ID=`echo ${TAG_NAME##v}`
-          echo "::set-output name=VERSION::$ID"
+          if [ -z "${{ inputs.server_version }}" ]
+          then
+              echo "variable is empty"
+          else
+              ID=${{ inputs.server_version }}
+          fi
+          echo "iast_version=$ID" >> $GITHUB_ENV
 
       - name: Generate version file
         run: |
-          bash .github/workflows/version_update.sh "${{ steps.release.outputs.VERSION }}"
+          bash .github/workflows/version_update.sh "${{ env.iast_version }}"
 
       - name: Login to DockerHub
         uses: docker/login-action@v1
@@ -56,17 +71,17 @@ jobs:
           access-key-id: ${{ secrets.ALIYUN_OSS_KEY_ID }}
           access-key-secret: ${{ secrets.ALIYUN_OSS_KEY_SECRET }}
       - run: |
-          echo "${{ github.event.repository.name }},version,${{ steps.release.outputs.version }}" >> version.txt && \
+          echo "${{ github.event.repository.name }},version,${{ env.iast_version }}" >> version.txt && \
           echo "${{ github.event.repository.name }},commit_hash,${GITHUB_SHA}" >> version.txt && \
           cat version.txt
-          ossutil cp -rf version.txt oss://huoqi-public/iast/release-version/${{ github.event.repository.name }}/${{ steps.release.outputs.version }}/version.txt
-          ossutil cp oss://dongtai/agent/java/${{ steps.release.outputs.VERSION }}/ ./ --include "*.jar" -r
+          ossutil cp -rf version.txt oss://huoqi-public/iast/release-version/${{ github.event.repository.name }}/${{ env.iast_version }}/version.txt
+          ossutil cp oss://dongtai/agent/java/${{ inputs.agent_version }}/ ./ --include "*.jar" -r
           [ ! -f ./dongtai-agent.jar ] && echo "$FILE does not exist."
           [ ! -f ./dongtai-agent.jar ] && ossutil cp oss://dongtai/agent/java/latest/ ./ --include "*.jar" -r
           [ ! -f ./dongtai-agent.jar ] && echo "$FILE does not exist."
           ossutil cp oss://dongtai/agent/python/ ./  --include "*.tar.gz" -r
           ossutil cp oss://dongtai/agent/php/ ./  --include "*.tar.gz" -r
-          echo "REPLACE INTO project_version_control (version, component_name, component_version_hash) VALUES('${{ steps.release.outputs.version }}', '${{ github.event.repository.name }}', '${GITHUB_SHA}');" >> ./deploy/docker/version.sql
+          echo "REPLACE INTO project_version_control (version, component_name, component_version_hash) VALUES('${{ env.iast_version }}', '${{ github.event.repository.name }}', '${GITHUB_SHA}');" >> ./deploy/docker/version.sql
 
       - name: Setup QEMU
         uses: docker/setup-qemu-action@v2
@@ -81,7 +96,7 @@ jobs:
           push: true
           tags: | 
             dongtai/dongtai-server:latest
-            dongtai/dongtai-server:${{ steps.release.outputs.version }}
+            dongtai/dongtai-server:${{ env.iast_version }}
 
       - name: finish build
         uses: joelwmale/webhook-action@master
@@ -99,11 +114,18 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - id: release
+      - name: Set the value
+        id: release
         run: |
           TAG_NAME=${{ github.event.release.tag_name }}
           ID=`echo ${TAG_NAME##v}`
-          echo "::set-output name=VERSION::$ID"
+          if [ -z "${{ inputs.server_version }}" ]
+          then
+              echo "variable is empty"
+          else
+              ID=${{ inputs.server_version }}
+          fi
+          echo "iast_version=$ID" >> $GITHUB_ENV
       
       - uses: azure/setup-helm@v1
         with:
@@ -123,15 +145,15 @@ jobs:
 
       - name: Download existed repo files
         run: |
-          sed -i "s#tag: latest#tag: ${{ steps.release.outputs.VERSION }}#g" deploy/kubernetes/helm/values.yaml
+          sed -i "s#tag: latest#tag: ${{ env.iast_version }}#g" deploy/kubernetes/helm/values.yaml
           ossutil cp -rf  oss://dongtai-helm-charts/iast/ ~/helm/repo/  --include dongtai-iast-*.tgz --exclude "index.yaml"
 
       - name: Create helm package
         run: |
-          helm package deploy/kubernetes/helm  -d ~/helm/repo --app-version ${{ steps.get_version.outputs.VERSION }} --version ${{ steps.get_version.outputs.VERSION }}
+          helm package deploy/kubernetes/helm  -d ~/helm/repo --app-version ${{ env.iast_version }} --version ${{ env.iast_version }}
           helm repo index ~/helm/repo/ --url ${{ secrets.DONGTAI_IAST_CHART_REPO_URL }}
 
       - name: Push helm chart to repo
         run: |
-          ossutil cp -rf ~/helm/repo/dongtai-iast-${{ steps.get_version.outputs.VERSION }}.tgz  oss://dongtai-helm-charts/iast/
+          ossutil cp -rf ~/helm/repo/dongtai-iast-${{ env.iast_version }}.tgz  oss://dongtai-helm-charts/iast/
           ossutil cp -rf ~/helm/repo/index.yaml  oss://dongtai-helm-charts/iast/
diff --git a/.github/workflows/teststate.yml b/.github/workflows/teststate.yml
@@ -66,7 +66,7 @@ jobs:
           python3 manage.py updatedepartmenttoken
           python3 manage.py runserver 0.0.0.0:8000 > webapi.log &
           sleep 15
-          schemathesis run --base-url "http://localhost:8000/"  -H 'Authorization: Token 67aebd78e700ad36a82a152276196b5f49fafeb0' http://localhost:8000/api/XZPcGFKoxYXScwGjQtJx8u/schema/  --hypothesis-suppress-health-check=data_too_large,filter_too_much,too_slow,return_value,large_base_example,not_a_test_method,function_scoped_fixture --validate-schema=false  --hypothesis-verbosity normal 
+          schemathesis run --base-url "http://localhost:8000/"  -H 'Authorization: Token 67aebd78e700ad36a82a152276196b5f49fafeb0' http://localhost:8000/api/XZPcGFKoxYXScwGjQtJx8u/schema/  --hypothesis-suppress-health-check=data_too_large,filter_too_much,too_slow,large_base_example,function_scoped_fixture --validate-schema=false  --hypothesis-verbosity normal 
   
   Run-BuildCheck:
     runs-on: ubuntu-latest
@@ -235,14 +235,16 @@ jobs:
           sudo apt-get install build-essential cmake ninja-build python3-dev cython3 pybind11-dev libre2-dev -y
           python -m pip install --upgrade pip
           pip install wheel
+          pip install coverage
           pip install -r requirements.txt
           
       - name: Django Unit Testing
         run: |
           cp dongtai_conf/conf/config.ini.test dongtai_conf/conf/config.ini
           mkdir -p /tmp/logstash/report/{img,word,pdf,excel} && mkdir -p /tmp/iast_cache/package && mkdir -p /tmp/logstash/batchagent 
           python3 ./deploy/docker/version_update.py || true
-          python manage.py test
+          coverage run --source='.' manage.py test
+          coverage report
 
   Run-Unit-TEST-In-Cython:
     runs-on: ubuntu-latest
@@ -282,6 +284,7 @@ jobs:
           sudo apt-get install build-essential cmake ninja-build python3-dev cython3 pybind11-dev libre2-dev -y
           python -m pip install --upgrade pip
           pip install wheel
+          pip install coverage
           pip install -r requirements.txt
           
       - name: Django Unit Testing
@@ -292,7 +295,8 @@ jobs:
           pip install Cython==3.0.0a11
           python setup.py build_ext --inplace
           find . -name "*.so"  | grep test | xargs rm
-          python manage.py test
+          coverage run --source='.' manage.py test
+          coverage report
   
   Run-API-TEST-With-Cython:
     runs-on: ubuntu-latest
@@ -348,7 +352,7 @@ jobs:
           python3 manage.py updatedepartmenttoken
           python3 manage.py runserver 0.0.0.0:8000 > webapi.log &
           sleep 15
-          schemathesis run --base-url "http://localhost:8000/"  -H 'Authorization: Token 67aebd78e700ad36a82a152276196b5f49fafeb0' http://localhost:8000/api/XZPcGFKoxYXScwGjQtJx8u/schema/  --hypothesis-suppress-health-check=data_too_large,filter_too_much,too_slow,return_value,large_base_example,not_a_test_method,function_scoped_fixture --validate-schema=false  --hypothesis-verbosity normal 
+          schemathesis run --base-url "http://localhost:8000/"  -H 'Authorization: Token 67aebd78e700ad36a82a152276196b5f49fafeb0' http://localhost:8000/api/XZPcGFKoxYXScwGjQtJx8u/schema/  --hypothesis-suppress-health-check=data_too_large,filter_too_much,too_slow,large_base_example,function_scoped_fixture --validate-schema=false  --hypothesis-verbosity normal 
 
   analyze:
     name: Analyze

diff --git a/Pipfile b/Pipfile
@@ -5,19 +5,19 @@ name = "pypi"
 
 [packages]
 asgiref = "*"
-certifi = "==2022.12.7"
-cffi = "==1.15.1"
-chardet = "==3.0.4"
+certifi = ">=2022.12.7"
+cffi = ">=1.15.1"
+chardet = ">=3.0.4"
 django = "~=3.2.18"
-django-cors-headers = "==3.7.0"
-django-filter = "==2.4.0"
-django-ranged-response = "==0.2.0"
-django-xff = "==1.3.0"
+django-cors-headers = ">=3.7.0"
+django-filter = ">=2.4.0"
+django-ranged-response = ">=0.2.0"
+django-xff = ">=1.3.0"
 djangorestframework = "~=3.12.4"
-django-rest-framework-proxy = "==1.6.0"
+django-rest-framework-proxy = ">=1.6.0"
 django-utils = "==0.0.2"
 django-utils-six = "==2.0"
-django-simple-captcha = "==0.5.14"
+django-simple-captcha = "~=0.5.14"
 django-import-export = "==2.5.0"
 oss2 = "==2.13.1"
 idna = "==2.10"
@@ -30,7 +30,7 @@ urllib3 = "==1.26.5"
 xlwt = "==1.3.0"
 pyre2 = "~=0.3.6"
 celery = "==5.2.7"
-redis = "==3.5.3"
+redis = "==4.4.4"
 openpyxl = "==3.0.9"
 id-validator = "==1.0.20"
 jq = "==1.3.0"