Merge pull request #1744 from HXSecurity/beta

Beta

.dockerignore

@@ -3,7 +3,6 @@
 *.DS_Store
 /venv
 */__pycache__/*
-*/migrations/*
 *.pyc
 .idea/
 scaapi/cron/logs/sca.log
@@ -25,7 +24,6 @@ build_docker.sh
 *.DS_Store
 /venv
 */__pycache__/*
-*/migrations/*
 *.pyc
 .idea/
 scaapi/cron/logs/sca.log

.github/workflows/deploy-dev.yaml

@@ -80,20 +80,10 @@ jobs:
             registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server:${{ steps.version.outputs.GITHUB_REF }}-latest
             registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server:${{ steps.version.outputs.GITHUB_REF }}-1.0.${{github.run_number}}
 
-      - name: Set the value
-        id: release
-        run: |
-          if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then echo "helm_ns=test" >> $GITHUB_ENV; echo "helm_mysql=test" >> $GITHUB_ENV 
-          elif [ ${{ steps.version.outputs.GITHUB_REF }} = beta ] ; then echo "helm_ns=beta" >> $GITHUB_ENV; echo "helm_mysql=beta" >> $GITHUB_ENV 
-          else echo "helm_ns=main" >> $GITHUB_ENV ; echo "helm_mysql=temp" >> $GITHUB_ENV ;fi
-
       - name: deploy to cluster
-        uses: wahyd4/kubectl-helm-action@master
+        uses: tscuite/kubectl-helm-action@main
         env:
+          MAX: false
+          PROJECT: server
+          TOKEN_SCA: ${{ secrets.TOKEN_SCA }}
           KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_TEST_DATA }}
-        with:
-          args: |
-            git clone https://github.com/HXSecurity/DongTai.git
-            helm upgrade --install huoxian --create-namespace -n iast-${{ env.helm_ns }} ./DongTai/deploy/kubernetes/helm/ \
-            --set sca.sca_token=${{ secrets.TOKEN_SCA }} --set usb.usb_token=${{ secrets.TOKEN_SCA }} --set mysql.host=iast-mysql-${{ env.helm_mysql }}.huoxian.cn \
-            --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=iast${{github.run_number}} --set develop.agentZip=${{ env.helm_ns }} --values https://charts.dongtai.io/devops.yaml

.github/workflows/nightly.yaml

@@ -5,10 +5,10 @@ name: "Nightly"
 on:
   schedule:
     - cron: '0 0 * * *'
+  workflow_dispatch:
 
 jobs:
-  # This workflow contains a single job called "build"
-  Run-API-TEST:
+  Run-Unit-TEST-In-Cython:
     runs-on: ubuntu-latest
     services:
       mysql:
@@ -40,37 +40,87 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
-          ref: main
+          cache: "pip"
+
       - name: Install dependencies
         run: |
           sudo apt-get install build-essential cmake ninja-build python3-dev cython3 pybind11-dev libre2-dev -y
           python -m pip install --upgrade pip
           pip install wheel
+          pip install coverage
           pip install -r requirements.txt
-          pip install schemathesis
-          pip install httpx
-      
-      - name: Install dependencies
+          
+      - name: Django Unit Testing
         run: |
-          curl -X GET "https://iast.io/openapi/api/v1/agent/download?url=https://iast.io/openapi&language=python&projectName=WEBAPI${{ github.repository_owner }}.${{github.run_number}}" -H "Authorization: Token a303ab4bedc93f96808335d023d7ac4d2ba00773" -o dongtai-agent-python.tar.gz -k
-          pip install dongtai-agent-python.tar.gz
+          export CELERY_EAGER_TEST=TRUE
+          cp dongtai_conf/conf/config.ini.test dongtai_conf/conf/config.ini
+          mkdir -p /tmp/logstash/report/{img,word,pdf,excel} && mkdir -p /tmp/iast_cache/package && mkdir -p /tmp/logstash/batchagent 
+          python3 ./deploy/docker/version_update.py || true
+          python3 manage.py migrate
+          pip install Cython==3.0.0
+          python setup.py build_ext --inplace
+          find . -name "*.so"  | grep test | xargs rm
+          coverage run --source='.' manage.py test
+          coverage report
+
+  Run-API-TEST-With-Cython:
+    runs-on: ubuntu-latest
+    services:
+      mysql:
+        image: dongtai/dongtai-mysql-unittest:latest
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd "mysqladmin ping --silent"
+          --health-interval 10s
+          --health-timeout 180s
+          --health-retries 10
+      redis:
+        image: dongtai/dongtai-redis:latest
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: ["3.10"]
 
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
 
+      - name: Install dependencies
+        run: |
+          sudo apt-get install build-essential cmake ninja-build python3-dev cython3 pybind11-dev libre2-dev -y
+          python -m pip install --upgrade pip
+          pip install wheel
+          pip install -r requirements.txt
+          pip install schemathesis
+          pip install httpx
+          
       - name: Django Api Testing
         run: |
           mysql --host=127.0.0.1 -uroot -p'dongtai-iast' -D dongtai_webapi < /home/runner/work/DongTai/DongTai/test/init.sql
           cp dongtai_conf/conf/config.ini.test dongtai_conf/conf/config.ini
-          export PROJECT_NAME=DONGTAI-SERVER.${{ github.repository_owner }}
-          export PROJECT_VERSION=${{github.run_number}}
           export DOC=TRUE
-          export PYTHONAGENT=TRUE
           mkdir -p /tmp/logstash/report/{img,word,pdf,excel} && mkdir -p /tmp/iast_cache/package && mkdir -p /tmp/logstash/batchagent 
           python3 ./deploy/docker/version_update.py || true
-          python3 manage.py updatedepartmenttoken
-          python3 manage.py runserver 0.0.0.0:8000  &
-          sleep 60
-          schemathesis   run --base-url "http://localhost:8000/"  -H 'Authorization: Token 67aebd78e700ad36a82a152276196b5f49fafeb0' http://localhost:8000/api/XZPcGFKoxYXScwGjQtJx8u/schema/  --hypothesis-suppress-health-check=data_too_large,filter_too_much,too_slow,large_base_example,function_scoped_fixture --validate-schema=false  --hypothesis-verbosity normal 
-  
+          python3 manage.py migrate
+          pip install Cython==3.0.0
+          python setup.py build_ext --inplace
+          python setup.py build_ext --inplace
+          python3 manage.py runserver 0.0.0.0:8000 > webapi.log &
+          sleep 15
+          schemathesis run --rate-limit=28/s --base-url "http://localhost:8000/"  -H 'Authorization: Token 67aebd78e700ad36a82a152276196b5f49fafeb0' http://localhost:8000/api/XZPcGFKoxYXScwGjQtJx8u/schema/  --hypothesis-suppress-health-check=data_too_large,filter_too_much,too_slow,large_base_example,function_scoped_fixture --validate-schema=false  --hypothesis-verbosity normal 
+
   Run-BuildCheck:
     if: ${{ github.repository_owner == 'HXSecurity' }}
     runs-on: ubuntu-latest