Merge branch 'main' of https://github.com/github/codeql into python/a…

…dd-stdlib-models

.bazelrc

@@ -10,15 +10,16 @@ common --override_module=semmle_code=%workspace%/misc/bazel/semmle_code_stub
 
 build --repo_env=CC=clang --repo_env=CXX=clang++
 
-build:linux --cxxopt=-std=c++20
-# we currently cannot built the swift extractor for ARM
-build:macos --cxxopt=-std=c++20 --copt=-arch --copt=x86_64 --linkopt=-arch --linkopt=x86_64
-build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
+# we use transitions that break builds of `...`, so for `test` to work with that we need the following
+test --build_tests_only
 
 # this requires developer mode, but is required to have pack installer functioning
 startup --windows_enable_symlinks
 common --enable_runfiles
 
+# with the above, we can avoid building python zips which is the default on windows as that's expensive
+build --nobuild_python_zip
+
 common --registry=file:///%workspace%/misc/bazel/registry
 common --registry=https://bcr.bazel.build
 

.gitattributes

@@ -80,3 +80,9 @@ csharp/paket.lock linguist-generated=true
 csharp/.paket/Paket.Restore.targets linguist-generated=true eol=crlf
 csharp/paket.main.bzl linguist-generated=true
 csharp/paket.main_extension.bzl linguist-generated=true
+
+# ripunzip tool
+/misc/bazel/internal/ripunzip/ripunzip-* filter=lfs diff=lfs merge=lfs -text
+
+# swift prebuilt resources
+/swift/third_party/resource-dir/*.zip filter=lfs diff=lfs merge=lfs -text

.github/workflows/csharp-qltest.yml

@@ -65,7 +65,7 @@ jobs:
           key: csharp-qltest-${{ matrix.slice }}
       - name: Run QL tests
         run: |
-          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}
   unit-tests:
@@ -101,6 +101,6 @@ jobs:
           # Update existing stubs in the repo with the freshly generated ones
           mv "$STUBS_PATH/output/stubs/_frameworks" ql/test/resources/stubs/
           git status
-          codeql test run --threads=0 --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
+          codeql test run --threads=0 --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
         env:
           GITHUB_TOKEN: ${{ github.token }}

.github/workflows/go-tests-other-os.yml

@@ -7,6 +7,9 @@ on:
       - .github/workflows/go-tests-other-os.yml
       - .github/actions/**
       - codeql-workspace.yml
+      - MODULE.bazel
+      - .bazelrc
+      - misc/bazel/**
 
 permissions:
   contents: read

.github/workflows/go-tests.yml

@@ -15,6 +15,9 @@ on:
       - .github/workflows/go-tests.yml
       - .github/actions/**
       - codeql-workspace.yml
+      - MODULE.bazel
+      - .bazelrc
+      - misc/bazel/**
 
 permissions:
   contents: read

.github/workflows/ql-for-ql-build.yml

@@ -49,20 +49,20 @@ jobs:
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Release build
         if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd ql; ./scripts/create-extractor-pack.sh       
+        run: cd ql; ./scripts/create-extractor-pack.sh
         env:
-          GH_TOKEN: ${{ github.token }}   
+          GH_TOKEN: ${{ github.token }}
       - name: Cache compilation cache
         id: query-cache
         uses: ./.github/actions/cache-query-compilation
-        with: 
+        with:
           key: run-ql-for-ql
       - name: Make database and analyze
         run: |
           ./ql/target/release/buramu | tee deprecated.blame # Add a blame file for the extractor to parse.
-          ${CODEQL} database create -l=ql --search-path ql/extractor-pack ${DB}
+          ${CODEQL} database create -l=ql ${DB} --search-path "${{ github.workspace }}"
           ${CODEQL} database analyze -j0 --format=sarif-latest --output=ql-for-ql.sarif ${DB} ql/ql/src/codeql-suites/ql-code-scanning.qls  --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-        env: 
+        env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
           DB: ${{ runner.temp }}/DB
           LGTM_INDEX_FILTERS: |

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -53,8 +53,8 @@ jobs:
       - name: Create database
         run: |
           "${CODEQL}" database create \
-            --search-path "ql/extractor-pack" \
-            --threads 4 \
+          --search-path "${{ github.workspace }}"
+          --threads 4 \
             --language ql --source-root "${{ github.workspace }}/repo" \
             "${{ runner.temp }}/database"
         env:

.github/workflows/ql-for-ql-tests.yml

@@ -49,23 +49,23 @@ jobs:
       - name: Cache compilation cache
         id: query-cache
         uses: ./.github/actions/cache-query-compilation
-        with: 
+        with:
           key: ql-for-ql-tests
       - name: Run QL tests
         run: |
-          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" ql/ql/test
+          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}" --consistency-queries ql/ql/consistency-queries --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" ql/ql/test
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
 
-  other-os: 
+  other-os:
     strategy:
       matrix:
         os: [macos-latest, windows-latest]
     needs: [qltest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
-      - name: Install GNU tar 
+      - name: Install GNU tar
         if: runner.os == 'macOS'
         run: |
           brew install gnu-tar
@@ -100,13 +100,12 @@ jobs:
       - name: Run a single QL tests - Unix
         if: runner.os != 'Windows'
         run: |
-          "${CODEQL}" test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
+          "${CODEQL}" test run --check-databases --search-path "${{ github.workspace }}" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Run a single QL tests - Windows
         if: runner.os == 'Windows'
         shell: pwsh
         run: |
           $Env:PATH += ";$(dirname ${{ steps.find-codeql.outputs.codeql-path }})"
-          codeql test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
-      
+          codeql test run --check-databases --search-path "${{ github.workspace }}" ql/ql/test/queries/style/DeadCode/DeadCode.qlref

.github/workflows/ruby-dataset-measure.yml

@@ -44,7 +44,7 @@ jobs:
       - name: Create database
         run: |
           codeql database create \
-            --search-path "${{ github.workspace }}/ruby/extractor-pack" \
+            --search-path "${{ github.workspace }}" \
             --threads 4 \
             --language ruby --source-root "${{ github.workspace }}/repo" \
             "${{ runner.temp }}/database"

.github/workflows/ruby-qltest.yml

@@ -64,10 +64,10 @@ jobs:
       - name: Cache compilation cache
         id: query-cache
         uses: ./.github/actions/cache-query-compilation
-        with: 
+        with:
           key: ruby-qltest
       - name: Run QL tests
         run: |
-          codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}

.github/workflows/swift.yml

@@ -68,21 +68,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./swift/actions/run-ql-tests
-  integration-tests-linux:
-    if: github.repository_owner == 'github'
-    needs: build-and-test-linux
-    runs-on: ubuntu-latest-xl
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./swift/actions/run-integration-tests
-  integration-tests-macos:
-    if: ${{ github.repository_owner == 'github' && github.event_name == 'pull_request' }}
-    needs: build-and-test-macos
-    runs-on: macos-12-xl
-    timeout-minutes: 60
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./swift/actions/run-integration-tests
   clang-format:
     if : ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest

.github/workflows/zipmerge-test.yml

@@ -0,0 +1,23 @@
+name: "Test zipmerge code"
+
+on:
+  pull_request:
+    paths:
+      - "misc/bazel/internal/zipmerge/**"
+      - "MODULE.bazel"
+      - ".bazelrc*"
+    branches:
+      - main
+      - "rc/*"
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          bazel test //misc/bazel/internal/zipmerge:test --test_output=all

.gitignore

@@ -62,3 +62,6 @@ node_modules/
 
 # Temporary folders for working with generated models
 .model-temp
+
+# bazel-built in-tree extractor packs
+/*/extractor-pack

MODULE.bazel

@@ -24,6 +24,7 @@ bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
 bazel_dep(name = "fmt", version = "10.0.0")
 bazel_dep(name = "gazelle", version = "0.36.0")
 bazel_dep(name = "rules_dotnet", version = "0.15.1")
+bazel_dep(name = "googletest", version = "1.14.0.bcr.1")
 
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 
@@ -67,6 +68,36 @@ use_repo(node, "nodejs", "nodejs_toolchains")
 go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
 go_sdk.download(version = "1.22.2")
 
+lfs_files = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_files")
+
+lfs_files(
+    name = "ripunzip-linux",
+    srcs = ["//misc/bazel/internal/ripunzip:ripunzip-linux"],
+    executable = True,
+)
+
+lfs_files(
+    name = "ripunzip-windows",
+    srcs = ["//misc/bazel/internal/ripunzip:ripunzip-windows.exe"],
+    executable = True,
+)
+
+lfs_files(
+    name = "ripunzip-macos",
+    srcs = ["//misc/bazel/internal/ripunzip:ripunzip-macos"],
+    executable = True,
+)
+
+lfs_files(
+    name = "swift-resource-dir-linux",
+    srcs = ["//swift/third_party/resource-dir:resource-dir-linux.zip"],
+)
+
+lfs_files(
+    name = "swift-resource-dir-macos",
+    srcs = ["//swift/third_party/resource-dir:resource-dir-macos.zip"],
+)
+
 register_toolchains(
     "@nodejs_toolchains//:all",
 )

codeql-workspace.yml

@@ -6,19 +6,16 @@ provide:
   - "*/ql/consistency-queries/qlpack.yml"
   - "*/ql/automodel/src/qlpack.yml"
   - "*/ql/automodel/test/qlpack.yml"
+  - "*/extractor-pack/codeql-extractor.yml"
   - "python/extractor/qlpack.yml"
   - "shared/**/qlpack.yml"
   - "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml"
   - "go/ql/config/legacy-support/qlpack.yml"
-  - "go/build/codeql-extractor-go/codeql-extractor.yml"
   - "csharp/ql/campaigns/Solorigate/lib/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/src/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/test/qlpack.yml"
   - "misc/legacy-support/*/qlpack.yml"
   - "misc/suite-helpers/qlpack.yml"
-  - "ruby/extractor-pack/codeql-extractor.yml"
-  - "swift/extractor-pack/codeql-extractor.yml"
-  - "ql/extractor-pack/codeql-extractor.yml"
   - ".github/codeql/extensions/**/codeql-pack.yml"
 
 versionPolicies:

cpp/downgrades/BUILD.bazel

@@ -6,7 +6,7 @@ pkg_files(
         ["**"],
         exclude = ["BUILD.bazel"],
     ),
-    prefix = "cpp/downgrades",
+    prefix = "downgrades",
     strip_prefix = strip_prefix.from_pkg(),
     visibility = ["//cpp:__pkg__"],
 )

cpp/ql/lib/BUILD.bazel

@@ -5,11 +5,9 @@ package(default_visibility = ["//cpp:__pkg__"])
 pkg_files(
     name = "dbscheme",
     srcs = ["semmlecode.cpp.dbscheme"],
-    prefix = "cpp",
 )
 
 pkg_files(
     name = "dbscheme-stats",
     srcs = ["semmlecode.cpp.dbscheme.stats"],
-    prefix = "cpp",
 )

cpp/ql/lib/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 1.0.0
+
+### Breaking Changes
+
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.
+
 ## 0.13.1
 
 No user-facing changes.

cpp/ql/lib/change-notes/2024-05-23-Version1.md → cpp/ql/lib/change-notes/released/1.0.0.md

@@ -1,4 +1,5 @@
----
-category: breaking
----
+## 1.0.0
+
+### Breaking Changes
+
 * CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.13.1
+lastReleaseVersion: 1.0.0

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 1.0.0-dev
+version: 1.0.1-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/src/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 1.0.0
+
+### Breaking Changes
+
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.
+
+### Minor Analysis Improvements
+
+* The "Use of unique pointer after lifetime ends" query (`cpp/use-of-unique-pointer-after-lifetime-ends`) no longer reports an alert when the pointer is converted to a boolean
+* The "Variable not initialized before use" query (`cpp/not-initialised`) no longer reports an alert on static variables.
+
 ## 0.9.12
 
 ### New Queries

cpp/ql/src/Security/CWE/CWE-131/NoSpaceForZeroTerminator.qhelp

@@ -12,8 +12,8 @@ the required buffer size, but do not allocate space for the zero terminator.
 </overview>
 <recommendation>
 <p>
-The expression highlighted by this rule creates a buffer that is of insufficient size to contain
-the data being copied. This makes the code vulnerable to buffer overflow which can result in anything from a segmentation fault to a security vulnerability (particularly if the array is on stack-allocated memory).
+The highlighted code segment creates a buffer without ensuring it's large enough to accommodate the copied data.
+This leaves the code susceptible to a buffer overflow attack, which could lead to anything from program crashes to malicious code execution.
 </p>
 
 <p>

cpp/ql/src/change-notes/released/1.0.0.md

@@ -0,0 +1,10 @@
+## 1.0.0
+
+### Breaking Changes
+
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.
+
+### Minor Analysis Improvements
+
+* The "Use of unique pointer after lifetime ends" query (`cpp/use-of-unique-pointer-after-lifetime-ends`) no longer reports an alert when the pointer is converted to a boolean
+* The "Variable not initialized before use" query (`cpp/not-initialised`) no longer reports an alert on static variables.

cpp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.9.12
+lastReleaseVersion: 1.0.0