diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 853a5cc4b..86e9026b5 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -7,7 +7,8 @@ on: pull_request_target: types: [opened, edited, synchronize] -permissions: read-all +permissions: + pull-requests: write jobs: check_changelog: @@ -19,7 +20,7 @@ jobs: steps: - name: Get changed files id: files - uses: Ana06/get-changed-files@e0c398b7065a8d84700c471b6afc4116d1ba4e96 # v2.2.0 + uses: Ana06/get-changed-files@25f79e676e7ea1868813e21465014798211fad8c # v2.3.0 - name: check changelog updated id: changelog_updated env: @@ -29,14 +30,14 @@ jobs: echo $FILES | grep -qF 'CHANGELOG.md' || echo $PR_BODY | grep -qiF "$NO_CHANGELOG" - name: Reject pull request if no CHANGELOG update if: ${{ always() && steps.changelog_updated.outcome == 'failure' }} - uses: Ana06/automatic-pull-request-review@0cf4e8a17ba79344ed3fdd7fed6dd0311d08a9d4 # v0.1.0 + uses: Ana06/automatic-pull-request-review@76aaf9b15b116a54e1da7a28a46f91fe089600bf # v0.2.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} event: REQUEST_CHANGES body: "Please add bug fixes, new features, breaking changes and anything else you think is worthwhile mentioning to the `master (unreleased)` section of CHANGELOG.md. If no CHANGELOG update is needed add the following to the PR description: `${{ env.NO_CHANGELOG }}`" allow_duplicate: false - name: Dismiss previous review if CHANGELOG update - uses: Ana06/automatic-pull-request-review@0cf4e8a17ba79344ed3fdd7fed6dd0311d08a9d4 # v0.1.0 + uses: Ana06/automatic-pull-request-review@76aaf9b15b116a54e1da7a28a46f91fe089600bf # v0.2.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} event: DISMISS diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 1844b881c..5485d0791 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -67,6 +67,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@807578363a7869ca324a79039e6db9c843e0e100 # v2.1.27 + uses: github/codeql-action/upload-sarif@592977e6ae857384aa79bb31e7a1d62d63449ec5 # v2.16.3 with: sarif_file: results.sarif diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index ea14817e6..34eabbedc 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -25,7 +25,7 @@ jobs: git tag $name -m "https://github.com/mandiant/capa/releases/$name" # TODO update branch name-major=${name%%.*} - name: Push tag to capa-rules - uses: ad-m/github-push-action@0fafdd62b84042d49ec0cb92d9cac7f7ce4ec79e # master + uses: ad-m/github-push-action@d91a481090679876dfc4178fef17f286781251df # v0.8.0 with: repository: mandiant/capa-rules github_token: ${{ secrets.CAPA_TOKEN }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 05d6414ad..9a96fbcd7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -153,11 +153,9 @@ jobs: matrix: python-version: ["3.8", "3.11"] java-version: ["17"] - gradle-version: ["7.3"] - ghidra-version: ["10.3"] - public-version: ["PUBLIC_20230510"] # for ghidra releases - jep-version: ["4.1.1"] - ghidrathon-version: ["3.0.0"] + ghidra-version: ["11.0.1"] + public-version: ["PUBLIC_20240130"] # for ghidra releases + ghidrathon-version: ["4.0.0"] steps: - name: Checkout capa with submodules uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -172,12 +170,6 @@ jobs: with: distribution: 'temurin' java-version: ${{ matrix.java-version }} - - name: Set up Gradle ${{ matrix.gradle-version }} - uses: gradle/gradle-build-action@40b6781dcdec2762ad36556682ac74e31030cfe2 # v2.5.1 - with: - gradle-version: ${{ matrix.gradle-version }} - - name: Install Jep ${{ matrix.jep-version }} - run : pip install jep==${{ matrix.jep-version }} - name: Install Ghidra ${{ matrix.ghidra-version }} run: | mkdir ./.github/ghidra @@ -186,10 +178,11 @@ jobs: - name: Install Ghidrathon run : | mkdir ./.github/ghidrathon - curl -o ./.github/ghidrathon/ghidrathon-${{ matrix.ghidrathon-version }}.zip "https://codeload.github.com/mandiant/Ghidrathon/zip/refs/tags/v${{ matrix.ghidrathon-version }}" - unzip .github/ghidrathon/ghidrathon-${{ matrix.ghidrathon-version }}.zip -d .github/ghidrathon/ - gradle -p ./.github/ghidrathon/Ghidrathon-${{ matrix.ghidrathon-version }}/ -PGHIDRA_INSTALL_DIR=$(pwd)/.github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC - unzip .github/ghidrathon/Ghidrathon-${{ matrix.ghidrathon-version }}/dist/*.zip -d .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC/Ghidra/Extensions + wget "https://github.com/mandiant/Ghidrathon/releases/download/v${{ matrix.ghidrathon-version }}/Ghidrathon-v${{ matrix.ghidrathon-version}}.zip" -O ./.github/ghidrathon/ghidrathon-v${{ matrix.ghidrathon-version }}.zip + unzip .github/ghidrathon/ghidrathon-v${{ matrix.ghidrathon-version }}.zip -d .github/ghidrathon/ + python -m pip install -r .github/ghidrathon/requirements.txt + python .github/ghidrathon/ghidrathon_configure.py $(pwd)/.github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC + unzip .github/ghidrathon/Ghidrathon-v${{ matrix.ghidrathon-version }}.zip -d .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC/Ghidra/Extensions - name: Install pyyaml run: sudo apt-get install -y libyaml-dev - name: Install capa @@ -201,4 +194,4 @@ jobs: cat ../output.log exit_code=$(cat ../output.log | grep exit | awk '{print $NF}') exit $exit_code - \ No newline at end of file + diff --git a/CHANGELOG.md b/CHANGELOG.md index e469a676b..ed44f85cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### New Features +- add function in capa/helpers to load plain and compressed JSON reports #1883 @Rohit1123 ### Breaking Changes @@ -27,9 +28,10 @@ ### Development -- ci: update github workflows to use latest version for depricated actions (checkout, setup-python, upload-artifact, download-artifact) #1967 @sjha2048 +- ci: Fix PR review in the changelog check GH action #2004 @Ana06 - ci: use rules number badge stored in our bot gist and generated using `schneegans/dynamic-badges-action` #2001 capa-rules#882 @Ana06 - +- ci: update github workflows to use latest version of actions that were using a deprecated version of node #1967 #2003 capa-rules#883 @sjha2048 @Ana06 +- ci: update github workflows to reflect the latest ghidrathon installation and bumped up jep, ghidra versions #2020 @psahithireddy ### Raw diffs - [capa v7.0.1...master](https://github.com/mandiant/capa/compare/v7.0.1...master) - [capa-rules v7.0.1...master](https://github.com/mandiant/capa-rules/compare/v7.0.1...master) diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..26cfe6359 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,8 @@ +cff-version: 1.2.0 +message: "If you use this software, please cite it as below." +authors: + - name: "The FLARE Team" +title: "capa, a tool to identify capabilities in programs and sandbox traces." +date-released: 2020-07-16 +url: "https://github.com/mandiant/capa" + diff --git a/capa/helpers.py b/capa/helpers.py index ad27f3903..ecf1b3200 100644 --- a/capa/helpers.py +++ b/capa/helpers.py @@ -6,6 +6,7 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. import sys +import gzip import json import inspect import logging @@ -30,7 +31,7 @@ EXTENSIONS_SHELLCODE_32 = ("sc32", "raw32") EXTENSIONS_SHELLCODE_64 = ("sc64", "raw64") -EXTENSIONS_DYNAMIC = ("json", "json_") +EXTENSIONS_DYNAMIC = ("json", "json_", "json.gz") EXTENSIONS_ELF = "elf_" EXTENSIONS_FREEZE = "frz" @@ -70,9 +71,19 @@ def assert_never(value) -> NoReturn: assert False, f"Unhandled value: {value} ({type(value).__name__})" # noqa: B011 -def get_format_from_report(sample: Path) -> str: - report = json.load(sample.open(encoding="utf-8")) +def load_json_from_path(json_path: Path): + with gzip.open(json_path, "r") as compressed_report: + try: + report_json = compressed_report.read() + except gzip.BadGzipFile: + report = json.load(json_path.open(encoding="utf-8")) + else: + report = json.loads(report_json) + return report + +def get_format_from_report(sample: Path) -> str: + report = load_json_from_path(sample) if "CAPE" in report: return FORMAT_CAPE diff --git a/capa/loader.py b/capa/loader.py index e4f0a5c92..024091e01 100644 --- a/capa/loader.py +++ b/capa/loader.py @@ -6,7 +6,6 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. import sys -import json import logging import datetime from typing import Set, Dict, List, Optional @@ -180,7 +179,7 @@ def get_extractor( if backend == BACKEND_CAPE: import capa.features.extractors.cape.extractor - report = json.loads(input_path.read_text(encoding="utf-8")) + report = capa.helpers.load_json_from_path(input_path) return capa.features.extractors.cape.extractor.CapeExtractor.from_report(report) elif backend == BACKEND_DOTNET: @@ -297,7 +296,7 @@ def get_file_extractors(input_file: Path, input_format: str) -> List[FeatureExtr elif input_format == FORMAT_CAPE: import capa.features.extractors.cape.extractor - report = json.loads(input_file.read_text(encoding="utf-8")) + report = capa.helpers.load_json_from_path(input_file) file_extractors.append(capa.features.extractors.cape.extractor.CapeExtractor.from_report(report)) return file_extractors diff --git a/rules b/rules index 34e375562..ec366561a 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 34e3755624530a6ed0da9942ad3c68ea8afa89d3 +Subproject commit ec366561ad16e139f442de0bf1a60fc3b01597e7 diff --git a/tests/data b/tests/data index dffd51c1b..9f7f3c5d8 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit dffd51c1b0d31f08cb896506e12bc0acd7b02db1 +Subproject commit 9f7f3c5d83b791a46db9f7cac2fdd972025edebb diff --git a/tests/fixtures.py b/tests/fixtures.py index ebfe557a5..ce21d7db1 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -191,14 +191,10 @@ def get_binja_extractor(path: Path): @lru_cache(maxsize=1) def get_cape_extractor(path): - import gzip - import json - + from capa.helpers import load_json_from_path from capa.features.extractors.cape.extractor import CapeExtractor - with gzip.open(path, "r") as compressed_report: - report_json = compressed_report.read() - report = json.loads(report_json) + report = load_json_from_path(path) return CapeExtractor.from_report(report) diff --git a/tests/test_main.py b/tests/test_main.py index 6d588dda1..2ee7e7da2 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -356,3 +356,9 @@ def test_main_cape1(tmp_path): assert capa.main.main([str(path), "-j", "-r", str(rules)]) == 0 assert capa.main.main([str(path), "-v", "-r", str(rules)]) == 0 assert capa.main.main([str(path), "-vv", "-r", str(rules)]) == 0 + + +def test_main_cape_gzip(): + # tests successful execution of .json.gz + path = str(fixtures.get_data_path_by_name("0000a657")) + assert capa.main.main([path]) == 0