Skip to content

Commit c8bfd70

Browse files
JarLobJarLob
authored
Merge branch 'main' into sockets
2 parents 40d2c51 + 5bcb9b2 commit c8bfd70

File tree

2,354 files changed

+172133
-111742
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

2,354 files changed

+172133
-111742
lines changed

‎.github/ISSUE_TEMPLATE/lgtm-com---false-positive.md

Lines changed: 0 additions & 24 deletions
This file was deleted.

‎.github/ISSUE_TEMPLATE/ql---general.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,5 @@ assignees: ''
1010
**Description of the issue**
1111

1212
<!-- Please explain briefly what is the problem.
13-
If it is about an LGTM project, please include its URL.-->
13+
If it is about a GitHub project, please include its URL. -->
1414

‎.github/ISSUE_TEMPLATE/ql--false-positive.md

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
---
2+
name: CodeQL false positive
3+
about: Report CodeQL alerts that you think should not have been detected (not applicable, not exploitable, etc.)
4+
title: False positive
5+
labels: false-positive
6+
assignees: ''
7+
8+
---
9+
10+
**Description of the false positive**
11+
12+
<!-- Please explain briefly why you think it shouldn't be included. -->
13+
14+
**Code samples or links to source code**
15+
16+
<!--
17+
For open source code: file links with line numbers on GitHub, for example:
18+
https://github.com/github/codeql/blob/dc440aaee6695deb0d9676b87e06ea984e1b4ae5/javascript/ql/test/query-tests/Security/CWE-078/CommandInjection/exec-sh2.js#L10
19+
20+
For closed source code: (redacted) code samples that illustrate the problem, for example:
21+
22+
```
23+
function execSh(command, options) {
24+
return cp.spawn(getShell(), ["-c", command], options) // <- command line injection
25+
};
26+
```
27+
-->
28+
29+
**URL to the alert on GitHub code scanning (optional)**
30+
31+
<!--
32+
1. Open the project on GitHub.com.
33+
2. Switch to the `Security` tab.
34+
3. Browse to the alert that you would like to report.
35+
4. Copy and paste the page URL here.
36+
-->

‎.github/actions/cache-query-compilation/action.yml

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
name: Cache query compilation
2+
description: Caches CodeQL compilation caches - should be run both on PRs and pushes to main.
3+
4+
inputs:
5+
key:
6+
description: 'The cache key to use - should be unique to the workflow'
7+
required: true
8+
9+
outputs:
10+
cache-dir:
11+
description: "The directory where the cache was stored"
12+
value: ${{ steps.fill-compilation-dir.outputs.compdir }}
13+
14+
runs:
15+
using: composite
16+
steps:
17+
- name: Cache the query compilation caches
18+
uses: ./.github/actions/incremental-cache
19+
with:
20+
path: '**/.cache'
21+
key: codeql-compile-${{ inputs.key }}
22+
- name: Fill compilation cache directory
23+
id: fill-compilation-dir
24+
shell: bash
25+
run: |
26+
# Move all the existing cache into another folder, so we only preserve the cache for the current queries.
27+
mkdir -p ${COMBINED_CACHE_DIR}
28+
rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
29+
# copy the contents of the .cache folders into the combined cache folder.
30+
cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
31+
# clean up the .cache folders
32+
rm -rf **/.cache/*
33+
34+
echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
35+
env:
36+
COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir

‎.github/actions/fetch-codeql/action.yml

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,22 @@
11
name: Fetch CodeQL
22
description: Fetches the latest version of CodeQL
3+
4+
inputs:
5+
channel:
6+
description: 'The CodeQL channel to use'
7+
required: false
8+
default: 'nightly'
9+
310
runs:
411
using: composite
512
steps:
613
- name: Fetch CodeQL
714
shell: bash
15+
env:
16+
GITHUB_TOKEN: ${{ github.token }}
17+
CHANNEL: ${{ inputs.channel }}
818
run: |
919
gh extension install github/gh-codeql
10-
gh codeql set-channel nightly
20+
gh codeql set-channel "$CHANNEL"
1121
gh codeql version
1222
gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_PATH}"
13-
env:
14-
GITHUB_TOKEN: ${{ github.token }}

‎.github/actions/incremental-cache/action.yml

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
name: Setup an incremental cache
2+
description: Special cache wrapper to be run on pull requests and pushes, that will try to restore
3+
a cache as close as possible to the merge base
4+
5+
inputs:
6+
path:
7+
description: 'The path to cache'
8+
required: true
9+
key:
10+
description: 'The cache key to use - should be unique to the workflow'
11+
required: true
12+
13+
runs:
14+
using: composite
15+
steps:
16+
# calculate the merge-base with main, in a way that works both on PRs and pushes to main.
17+
- name: Calculate merge-base
18+
shell: bash
19+
if: ${{ github.event_name == 'pull_request' }}
20+
env:
21+
BASE_BRANCH: ${{ github.base_ref }}
22+
run: |
23+
MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
24+
echo "merge_base=$MERGE_BASE" >> $GITHUB_ENV
25+
- name: Restore read-only cache (PR)
26+
if: ${{ github.event_name == 'pull_request' }}
27+
uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
28+
with:
29+
path: ${{ inputs.path }}
30+
read-only: true
31+
key: ${{ inputs.key }}-pr-${{ github.sha }}
32+
restore-keys: |
33+
${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
34+
${{ inputs.key }}-${{ github.base_ref }}-
35+
${{ inputs.key }}-main-
36+
- name: Fill cache (push)
37+
if: ${{ github.event_name != 'pull_request' }}
38+
uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
39+
with:
40+
path: ${{ inputs.path }}
41+
key: ${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
42+
restore-keys: | # restore the latest cache if the exact cache is unavailable, to speed up compilation.
43+
${{ inputs.key }}-${{ github.ref_name }}-
44+
${{ inputs.key }}-main-

‎.github/labeler.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,6 @@ documentation:
5151
- "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
5252
- "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll"
5353
- "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll"
54+
55+
"ATM":
56+
- javascript/ql/experimental/adaptivethreatmodeling/**/*

‎.github/workflows/atm-check-queries-run.yml

Lines changed: 0 additions & 13 deletions
This file was deleted.

‎.github/workflows/atm-check-query-suite.yml

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
name: "ATM - Check query suite"
2+
3+
env:
4+
QUERY_PACK: javascript/ql/experimental/adaptivethreatmodeling/src
5+
QUERY_SUITE: codeql-suites/javascript-atm-code-scanning.qls
6+
7+
on:
8+
pull_request:
9+
paths:
10+
- ".github/workflows/atm-check-query-suite.yml"
11+
- "javascript/ql/experimental/adaptivethreatmodeling/**"
12+
workflow_dispatch:
13+
14+
jobs:
15+
atm-check-query-suite:
16+
runs-on: ubuntu-latest
17+
18+
steps:
19+
- uses: actions/checkout@v3
20+
21+
- name: Setup CodeQL
22+
uses: ./.github/actions/fetch-codeql
23+
with:
24+
channel: release
25+
26+
- name: Install ATM model
27+
run: |
28+
set -exu
29+
30+
# Install dependencies of ATM query pack, i.e. the ATM model
31+
codeql pack install "${QUERY_PACK}"
32+
33+
# Retrieve model checksum
34+
model_checksum=$(codeql resolve extensions "${QUERY_PACK}/${QUERY_SUITE}" | jq -r '.models[0].checksum')
35+
36+
# Trust the model so that we can use it in the ATM boosted queries
37+
mkdir -p "$HOME/.config/codeql"
38+
echo "--insecurely-execute-ml-model-checksums ${model_checksum}" >> "$HOME/.config/codeql/config"
39+
40+
- name: Create test DB
41+
run: |
42+
DB_PATH="${RUNNER_TEMP}/db"
43+
echo "DB_PATH=${DB_PATH}" >> "${GITHUB_ENV}"
44+
45+
codeql database create "${DB_PATH}" --source-root config/atm --language javascript
46+
47+
- name: Run ATM query suite
48+
run: |
49+
SARIF_PATH="${RUNNER_TEMP}/sarif.json"
50+
echo "SARIF_PATH=${SARIF_PATH}" >> "${GITHUB_ENV}"
51+
52+
codeql database analyze \
53+
--format sarif-latest \
54+
--output "${SARIF_PATH}" \
55+
--sarif-group-rules-by-pack \
56+
-vv \
57+
-- \
58+
"${DB_PATH}" \
59+
"${QUERY_PACK}/${QUERY_SUITE}"
60+
61+
- name: Upload SARIF
62+
uses: actions/upload-artifact@v3
63+
with:
64+
name: javascript-ml-powered-queries.sarif
65+
path: "${{ env.SARIF_PATH }}"
66+
retention-days: 5
67+
68+
- name: Check results
69+
run: |
70+
# We should run at least the ML-powered queries in `expected_rules`.
71+
expected_rules="js/ml-powered/nosql-injection js/ml-powered/path-injection js/ml-powered/sql-injection js/ml-powered/xss"
72+
73+
for rule in ${expected_rules}; do
74+
found_rule=$(jq --arg rule "${rule}" '[.runs[0].tool.extensions[].rules | select(. != null) |
75+
flatten | .[].id] | any(. == $rule)' "${SARIF_PATH}")
76+
if [[ "${found_rule}" != "true" ]]; then
77+
echo "Expected SARIF output to contain rule '${rule}', but found no such rule."
78+
exit 1
79+
else
80+
echo "Found rule '${rule}'."
81+
fi
82+
done
83+
84+
# We should have at least one alert from an ML-powered query.
85+
num_alerts=$(jq '[.runs[0].results[] |
86+
select(.properties.score != null and (.rule.id | startswith("js/ml-powered/")))] | length' \
87+
"${SARIF_PATH}")
88+
if [[ "${num_alerts}" -eq 0 ]]; then
89+
echo "Expected to find at least one alert from an ML-powered query but found ${num_alerts}."
90+
exit 1
91+
else
92+
echo "Found ${num_alerts} alerts from ML-powered queries.";
93+
fi

‎.github/workflows/atm-model-integration-tests.yml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
name: ATM Model Integration Tests
2+
3+
on:
4+
workflow_dispatch:
5+
6+
jobs:
7+
hello-world:
8+
runs-on: ubuntu-latest
9+
10+
steps:
11+
- name: foo
12+
run: echo "Hello world"

‎.github/workflows/compile-queries.yml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: "Compile all queries using the latest stable CodeQL CLI"
2+
3+
on:
4+
push:
5+
branches: # makes sure the cache gets populated - running on the branches people tend to merge into.
6+
- main
7+
- "rc/*"
8+
- "codeql-cli-*"
9+
pull_request:
10+
11+
jobs:
12+
compile-queries:
13+
runs-on: ubuntu-latest-xl
14+
15+
steps:
16+
- uses: actions/checkout@v3
17+
- name: Setup CodeQL
18+
uses: ./.github/actions/fetch-codeql
19+
with:
20+
channel: 'release'
21+
- name: Cache compilation cache
22+
id: query-cache
23+
uses: ./.github/actions/cache-query-compilation
24+
with:
25+
key: all-queries
26+
- name: check formatting
27+
run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
28+
- name: compile queries - check-only
29+
# run with --check-only if running in a PR (github.sha != main)
30+
if : ${{ github.event_name == 'pull_request' }}
31+
shell: bash
32+
run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
33+
- name: compile queries - full
34+
# do full compile if running on main - this populates the cache
35+
if : ${{ github.event_name != 'pull_request' }}
36+
shell: bash
37+
run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
38+
env:
39+
COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir

0 commit comments

Comments
 (0)