Merge branch 'mandiant:master' into Aayush-Goel-04/Issue#520

.github/workflows/changelog.yml

@@ -7,7 +7,8 @@ on:
   pull_request_target:
     types: [opened, edited, synchronize]
 
-permissions: read-all
+permissions:
+  pull-requests: write
 
 jobs:
   check_changelog:
@@ -19,7 +20,7 @@ jobs:
     steps:
     - name: Get changed files
       id: files
-      uses: Ana06/get-changed-files@e0c398b7065a8d84700c471b6afc4116d1ba4e96 # v2.2.0
+      uses: Ana06/get-changed-files@25f79e676e7ea1868813e21465014798211fad8c # v2.3.0
     - name: check changelog updated
       id: changelog_updated
       env:
@@ -29,14 +30,14 @@ jobs:
         echo $FILES | grep -qF 'CHANGELOG.md' || echo $PR_BODY | grep -qiF "$NO_CHANGELOG"
     - name: Reject pull request if no CHANGELOG update
       if: ${{ always() && steps.changelog_updated.outcome == 'failure' }}
-      uses: Ana06/automatic-pull-request-review@0cf4e8a17ba79344ed3fdd7fed6dd0311d08a9d4 # v0.1.0
+      uses: Ana06/automatic-pull-request-review@76aaf9b15b116a54e1da7a28a46f91fe089600bf # v0.2.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         event: REQUEST_CHANGES
         body: "Please add bug fixes, new features, breaking changes and anything else you think is worthwhile mentioning to the `master (unreleased)` section of CHANGELOG.md. If no CHANGELOG update is needed add the following to the PR description: `${{ env.NO_CHANGELOG }}`"
         allow_duplicate: false
     - name: Dismiss previous review if CHANGELOG update
-      uses: Ana06/automatic-pull-request-review@0cf4e8a17ba79344ed3fdd7fed6dd0311d08a9d4 # v0.1.0
+      uses: Ana06/automatic-pull-request-review@76aaf9b15b116a54e1da7a28a46f91fe089600bf # v0.2.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         event: DISMISS

.github/workflows/scorecard.yml

@@ -67,6 +67,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@807578363a7869ca324a79039e6db9c843e0e100 # v2.1.27
+        uses: github/codeql-action/upload-sarif@592977e6ae857384aa79bb31e7a1d62d63449ec5 # v2.16.3
         with:
           sarif_file: results.sarif

.github/workflows/tag.yml

@@ -25,7 +25,7 @@ jobs:
         git tag $name -m "https://github.com/mandiant/capa/releases/$name"
         # TODO update branch name-major=${name%%.*}
     - name: Push tag to capa-rules
-      uses: ad-m/github-push-action@0fafdd62b84042d49ec0cb92d9cac7f7ce4ec79e # master
+      uses: ad-m/github-push-action@d91a481090679876dfc4178fef17f286781251df # v0.8.0
       with:
         repository: mandiant/capa-rules
         github_token: ${{ secrets.CAPA_TOKEN }}

.github/workflows/tests.yml

@@ -153,11 +153,9 @@ jobs:
       matrix:
         python-version: ["3.8", "3.11"]
         java-version: ["17"]
-        gradle-version: ["7.3"]
-        ghidra-version: ["10.3"]
-        public-version: ["PUBLIC_20230510"] # for ghidra releases
-        jep-version: ["4.1.1"]
-        ghidrathon-version: ["3.0.0"]
+        ghidra-version: ["11.0.1"]
+        public-version: ["PUBLIC_20240130"] # for ghidra releases
+        ghidrathon-version: ["4.0.0"] 
     steps:
     - name: Checkout capa with submodules
       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -172,12 +170,6 @@ jobs:
       with:
         distribution: 'temurin'
         java-version: ${{ matrix.java-version }}
-    - name: Set up Gradle ${{ matrix.gradle-version }} 
-      uses: gradle/gradle-build-action@40b6781dcdec2762ad36556682ac74e31030cfe2 # v2.5.1
-      with:
-        gradle-version: ${{ matrix.gradle-version }}
-    - name: Install Jep ${{ matrix.jep-version }} 
-      run : pip install jep==${{ matrix.jep-version }}
     - name: Install Ghidra ${{ matrix.ghidra-version }} 
       run: |
         mkdir ./.github/ghidra
@@ -186,10 +178,11 @@ jobs:
     - name: Install Ghidrathon
       run : |
         mkdir ./.github/ghidrathon
-        curl -o ./.github/ghidrathon/ghidrathon-${{ matrix.ghidrathon-version }}.zip "https://codeload.github.com/mandiant/Ghidrathon/zip/refs/tags/v${{ matrix.ghidrathon-version }}"
-        unzip .github/ghidrathon/ghidrathon-${{ matrix.ghidrathon-version }}.zip -d .github/ghidrathon/
-        gradle -p ./.github/ghidrathon/Ghidrathon-${{ matrix.ghidrathon-version }}/ -PGHIDRA_INSTALL_DIR=$(pwd)/.github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC
-        unzip .github/ghidrathon/Ghidrathon-${{ matrix.ghidrathon-version }}/dist/*.zip -d .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC/Ghidra/Extensions
+        wget "https://github.com/mandiant/Ghidrathon/releases/download/v${{ matrix.ghidrathon-version }}/Ghidrathon-v${{ matrix.ghidrathon-version}}.zip" -O ./.github/ghidrathon/ghidrathon-v${{ matrix.ghidrathon-version }}.zip
+        unzip .github/ghidrathon/ghidrathon-v${{ matrix.ghidrathon-version }}.zip -d .github/ghidrathon/
+        python -m pip install -r .github/ghidrathon/requirements.txt
+        python .github/ghidrathon/ghidrathon_configure.py $(pwd)/.github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC
+        unzip .github/ghidrathon/Ghidrathon-v${{ matrix.ghidrathon-version }}.zip -d .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC/Ghidra/Extensions
     - name: Install pyyaml
       run: sudo apt-get install -y libyaml-dev
     - name: Install capa
@@ -201,4 +194,4 @@ jobs:
         cat ../output.log
         exit_code=$(cat ../output.log | grep exit | awk '{print $NF}')
         exit $exit_code
- 
+ 

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### New Features
 
+- add function in capa/helpers to load plain and compressed JSON reports #1883 @Rohit1123
 
 ### Breaking Changes
 
@@ -27,9 +28,10 @@
 
 ### Development
 
-- ci: update github workflows to use latest version for depricated actions (checkout, setup-python, upload-artifact, download-artifact) #1967 @sjha2048
+- ci: Fix PR review in the changelog check GH action #2004 @Ana06
 - ci: use rules number badge stored in our bot gist and generated using `schneegans/dynamic-badges-action` #2001 capa-rules#882 @Ana06
-
+- ci: update github workflows to use latest version of actions that were using a deprecated version of node #1967 #2003 capa-rules#883 @sjha2048 @Ana06
+- ci: update github workflows to reflect the latest ghidrathon installation and bumped up jep, ghidra versions  #2020 @psahithireddy
 ### Raw diffs
 - [capa v7.0.1...master](https://github.com/mandiant/capa/compare/v7.0.1...master)
 - [capa-rules v7.0.1...master](https://github.com/mandiant/capa-rules/compare/v7.0.1...master)

CITATION.cff

@@ -0,0 +1,8 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+  - name: "The FLARE Team"
+title: "capa, a tool to identify capabilities in programs and sandbox traces."
+date-released: 2020-07-16
+url: "https://github.com/mandiant/capa"
+

capa/helpers.py

@@ -6,6 +6,7 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 import sys
+import gzip
 import json
 import inspect
 import logging
@@ -30,7 +31,7 @@
 
 EXTENSIONS_SHELLCODE_32 = ("sc32", "raw32")
 EXTENSIONS_SHELLCODE_64 = ("sc64", "raw64")
-EXTENSIONS_DYNAMIC = ("json", "json_")
+EXTENSIONS_DYNAMIC = ("json", "json_", "json.gz")
 EXTENSIONS_ELF = "elf_"
 EXTENSIONS_FREEZE = "frz"
 
@@ -70,9 +71,19 @@ def assert_never(value) -> NoReturn:
     assert False, f"Unhandled value: {value} ({type(value).__name__})"  # noqa: B011
 
 
-def get_format_from_report(sample: Path) -> str:
-    report = json.load(sample.open(encoding="utf-8"))
+def load_json_from_path(json_path: Path):
+    with gzip.open(json_path, "r") as compressed_report:
+        try:
+            report_json = compressed_report.read()
+        except gzip.BadGzipFile:
+            report = json.load(json_path.open(encoding="utf-8"))
+        else:
+            report = json.loads(report_json)
+    return report
 
+
+def get_format_from_report(sample: Path) -> str:
+    report = load_json_from_path(sample)
     if "CAPE" in report:
         return FORMAT_CAPE
 

capa/loader.py

@@ -6,7 +6,6 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 import sys
-import json
 import logging
 import datetime
 from typing import Set, Dict, List, Optional
@@ -180,7 +179,7 @@ def get_extractor(
     if backend == BACKEND_CAPE:
         import capa.features.extractors.cape.extractor
 
-        report = json.loads(input_path.read_text(encoding="utf-8"))
+        report = capa.helpers.load_json_from_path(input_path)
         return capa.features.extractors.cape.extractor.CapeExtractor.from_report(report)
 
     elif backend == BACKEND_DOTNET:
@@ -297,7 +296,7 @@ def get_file_extractors(input_file: Path, input_format: str) -> List[FeatureExtr
     elif input_format == FORMAT_CAPE:
         import capa.features.extractors.cape.extractor
 
-        report = json.loads(input_file.read_text(encoding="utf-8"))
+        report = capa.helpers.load_json_from_path(input_file)
         file_extractors.append(capa.features.extractors.cape.extractor.CapeExtractor.from_report(report))
 
     return file_extractors

tests/fixtures.py

@@ -191,14 +191,10 @@ def get_binja_extractor(path: Path):
 
 @lru_cache(maxsize=1)
 def get_cape_extractor(path):
-    import gzip
-    import json
-
+    from capa.helpers import load_json_from_path
     from capa.features.extractors.cape.extractor import CapeExtractor
 
-    with gzip.open(path, "r") as compressed_report:
-        report_json = compressed_report.read()
-        report = json.loads(report_json)
+    report = load_json_from_path(path)
 
     return CapeExtractor.from_report(report)
 

tests/test_main.py

@@ -356,3 +356,9 @@ def test_main_cape1(tmp_path):
     assert capa.main.main([str(path), "-j", "-r", str(rules)]) == 0
     assert capa.main.main([str(path), "-v", "-r", str(rules)]) == 0
     assert capa.main.main([str(path), "-vv", "-r", str(rules)]) == 0
+
+
+def test_main_cape_gzip():
+    # tests successful execution of .json.gz
+    path = str(fixtures.get_data_path_by_name("0000a657"))
+    assert capa.main.main([path]) == 0