ML expanded and retrained #2644
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow performs static analysis and checks coding style | |
| name: Static analysis and code style | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| jobs: | |
| # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # | |
| checkers: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # # # MUST be full history to check git workflow | |
| - name: Checkout | |
| id: code_checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| # # # line ending | |
| - name: Check for text file ending | |
| if: ${{ always() && steps.code_checkout.conclusion == 'success' }} | |
| run: | | |
| n=0 | |
| for f in $(find . -type f -not -wholename '*/.*' -a -not -wholename '*/tests/samples/*' -a -not -wholename '*/corpus/*'); do | |
| n=$(( 1 + ${n} )) | |
| filetype=$(file ${f}) | |
| if echo "${filetype}" | grep -q '.*text.*'; then | |
| echo "CHECK:'${filetype}'" | |
| lastbyte=$(hexdump -v -e '/1 "%02X\n"' ${f} | tail -1) | |
| echo "Last byte is '${lastbyte}'" | |
| if [ "0A" != "${lastbyte}" ]; then | |
| echo "File ${f} has inappropriate line ending" | |
| tail -1 ${f} | hexdump -C | |
| else | |
| n=$(( ${n} - 1 )) | |
| fi | |
| else | |
| echo "SKIP:'${filetype}'" | |
| n=$(( ${n} - 1 )) | |
| fi | |
| done | |
| exit ${n} | |
| # # # ml_model integrity | |
| - name: Check ml_model.onnx integrity | |
| if: ${{ always() && steps.code_checkout.conclusion == 'success' }} | |
| run: | | |
| md5sum --binary credsweeper/ml_model/ml_model.onnx | grep 819c91980420acc592a3aef679f6e326 | |
| # # # Python setup | |
| - name: Set up Python | |
| if: ${{ always() && steps.code_checkout.conclusion == 'success' }} | |
| id: setup_python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| - name: Install CredSweeper and auxiliary packages | |
| id: setup_credsweeper | |
| if: ${{ always() && steps.setup_python.conclusion == 'success' }} | |
| run: | | |
| python --version #dbg | |
| python -m pip install --upgrade pip | |
| pip install --requirement requirements.txt | |
| pip list #dbg | |
| # # # pylint | |
| - name: Analysing the code with pylint and minimum Python version 3.8 | |
| if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} | |
| run: pylint --py-version=3.8 --errors-only credsweeper | |
| - name: Analysing the code with pylint and minimum Python version 3.9 | |
| if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} | |
| run: pylint --py-version=3.9 --errors-only credsweeper | |
| - name: Analysing the code with pylint and minimum Python version 3.10 | |
| if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} | |
| run: pylint --py-version=3.10 --errors-only credsweeper | |
| - name: Analysing the code with pylint and minimum Python version 3.11 | |
| if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} | |
| run: pylint --py-version=3.11 --errors-only credsweeper | |
| # # # mypy | |
| - name: Analysing the code with mypy and minimum Python version 3.8 | |
| if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} | |
| run: | | |
| mypy --config-file .mypy.ini --python-version=3.8 credsweeper | |
| - name: Analysing the code with mypy and minimum Python version 3.9 | |
| if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} | |
| run: | | |
| mypy --config-file .mypy.ini --python-version=3.9 credsweeper | |
| - name: Analysing the code with mypy and minimum Python version 3.10 | |
| if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} | |
| run: | | |
| mypy --config-file .mypy.ini --python-version=3.10 credsweeper | |
| - name: Analysing the code with mypy and minimum Python version 3.11 | |
| if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} | |
| run: | | |
| mypy --config-file .mypy.ini --python-version=3.11 credsweeper | |
| # # # documentation | |
| - name: Analysing the code with pylint for NEW missed docstrings of classes or functions | |
| if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} | |
| run: | | |
| pylint --disable=E,R,W,C0114,C0103,C0412,C0413,C0415,C0200,C0201,C0325 --verbose credsweeper | |
| # # # Documentation check | |
| - name: Test for creation sphinx documentations | |
| if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} | |
| run: | | |
| cd docs | |
| pip install -r requirements.txt | |
| make html | |
| cd source | |
| python -m sphinx -T -E -b html -d _build/doctrees -D language=en . ./_html | |
| # # # yapf | |
| - name: Check project style | |
| if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} | |
| run: | | |
| for f in credsweeper tests docs experiment; do | |
| yapf --style .style.yapf --recursive --in-place --parallel $f | |
| done | |
| if [ 0 -ne $(git ls-files -m | wc -l) ]; then | |
| git diff | |
| echo "<- difference how to apply the style" | |
| exit 1 | |
| fi | |
| # # # flake8 | |
| - name: Analysing the code with flake8 | |
| id: test_flake8 | |
| if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} | |
| run: | | |
| ERRCNT=$(flake8 credsweeper --count --exit-zero --output-file=flake8.txt) | |
| if ! [ 0 -eq ${ERRCNT} ] ; then | |
| echo "flake8 found '${ERRCNT}' failures:" | |
| cat flake8.txt | |
| exit 1 | |
| fi | |
| - name: FLAKE 8 reports | |
| if: ${{ failure() && steps.test_flake8.conclusion == 'failure' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: flake8_report | |
| path: flake8.txt | |
| # # # Banner crc32 | |
| - name: Setup crc32 tool | |
| id: setup_crc32 | |
| if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} | |
| run: sudo apt-get update && sudo apt-get install libarchive-zip-perl && crc32 /etc/fstab | |
| - name: Banner and version check | |
| if: ${{ always() && steps.setup_crc32.conclusion == 'success' }} | |
| continue-on-error: true | |
| run: | | |
| crc32_int=0 | |
| for f in $(find credsweeper -iregex '.*\.\(py\|json\|yaml\|txt\|onnx\)$'); do | |
| file_crc32_hex=$(crc32 $f) | |
| file_crc32_int=$((16#${file_crc32_hex})) | |
| crc32_int=$(( ${crc32_int} ^ ${file_crc32_int} )) | |
| done | |
| version_with_crc="$(credsweeper --version | head -1) crc32:$(printf '%x' ${crc32_int})" | |
| echo "version_with_crc = '${version_with_crc}'" | |
| banner=$(credsweeper --banner --path requirements.txt | head -1) | |
| echo "banner = '${banner}'" | |
| if ! [ -n "${version_with_crc}" ] && [ -n "${banner}" ] && [ "${version_with_crc}" == "${banner}" ]; then | |
| echo "'${version_with_crc}' != '${banner}'" | |
| exit 1 | |
| fi | |
| # # # SECURITY.md check | |
| - name: SECURITY.md check | |
| if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} | |
| run: | | |
| # get actual version (major.minor) from credsweeper package | |
| V=$(python -c "from packaging.version import Version as V; import credsweeper; v=V(credsweeper.__version__); print(f'{v.major}.{v.minor}')") | |
| # check whether current version exists in the file | |
| grep $V SECURITY.md | |
| # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # |