Merge pull request #16656 from github/criemen/pkg-javascript

Javascript: use `codeql_pack` for javascript extractor

csharp/.gitignore

@@ -14,5 +14,4 @@ csharp.log
 .vscode/launch.json
 
 extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json
-extractor-pack
 paket-files/

csharp/tools/BUILD.bazel

@@ -1,11 +1,11 @@
-load("@semmle_code//:dist.bzl", "pack_zip")
+load("//misc/bazel:pkg.bzl", "codeql_pkg_files")
 
-pack_zip(
+codeql_pkg_files(
     name = "tools",
-    srcs = glob(["**/*"]),
     excludes = [
         "BUILD.bazel",
     ],
+    exes = glob(["**/*"]),
     prefix = "tools",
     visibility = ["//csharp:__pkg__"],
 )

javascript/BUILD.bazel

@@ -1,6 +1,5 @@
 load("@rules_pkg//pkg:mappings.bzl", "pkg_files")
-load("@semmle_code//:dist.bzl", "dist")
-load("@semmle_code//buildutils-internal:zipmerge.bzl", "zipmerge")
+load("//misc/bazel:pkg.bzl", "codeql_pack")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -23,26 +22,25 @@ pkg_files(
     strip_prefix = None,
 )
 
-dist(
-    name = "javascript-extractor-pack",
+# We have to use a zip of the typescript parser wrapper, as it's generated by a genrule
+# and we don't know a list of its output files.
+codeql_pack(
+    name = "javascript",
     srcs = [
         ":dbscheme-group",
         "//javascript/downgrades",
         "//javascript/externs",
         "//javascript/extractor:tools-extractor",
         "@semmle_code//language-packs/javascript:resources",
     ],
-    prefix = "javascript",
+    visibility = ["//visibility:public"],
+    zips = {"//javascript/extractor/lib/typescript": "tools"},
 )
 
-# We have to zipmerge in the typescript parser wrapper, as it's generated by a genrule
-# and we don't know a list of its output files. Therefore, we sidestep the
-# rules_pkg tooling here, and generate the zip for the language pack manually.
-zipmerge(
-    name = "javascript",
-    srcs = [
-        ":javascript-extractor-pack.zip",
-        "//javascript/extractor/lib/typescript",
-    ],
-    out = "javascript.zip",
+# TODO copy for internal repository backward compatibility
+genrule(
+    name = "javascript.zip",
+    srcs = [":javascript-generic-zip"],
+    outs = ["javascript.zip"],
+    cmd = "cp $< $@",
 )

javascript/downgrades/BUILD.bazel

@@ -1,11 +1,12 @@
-load("@semmle_code//:dist.bzl", "pack_zip")
+load("//misc/bazel:pkg.bzl", "codeql_pkg_files", "strip_prefix")
 
-pack_zip(
+codeql_pkg_files(
     name = "downgrades",
     srcs = glob(
         ["**/*"],
         exclude = ["BUILD.bazel"],
     ),
     prefix = "downgrades",
-    visibility = ["//visibility:public"],
+    strip_prefix = strip_prefix.from_pkg(),
+    visibility = ["//javascript:__pkg__"],
 )

javascript/externs/BUILD.bazel

@@ -1,11 +1,12 @@
-load("@semmle_code//:dist.bzl", "pack_zip")
+load("//misc/bazel:pkg.bzl", "codeql_pkg_files", "strip_prefix")
 
-pack_zip(
+codeql_pkg_files(
     name = "externs",
     srcs = glob(
         ["**/*"],
         exclude = ["BUILD.bazel"],
     ),
     prefix = "tools/data/externs",
-    visibility = ["//visibility:public"],
+    strip_prefix = strip_prefix.from_pkg(),
+    visibility = ["//javascript:__pkg__"],
 )

javascript/extractor/lib/typescript/BUILD.bazel

@@ -1,5 +1,3 @@
-load("@semmle_code//:common.bzl", "on_windows")
-
 # Builds a zip file of the compiled typscript-parser-wrapper and its dependencies.
 genrule(
     name = "typescript",
@@ -33,19 +31,16 @@ genrule(
         # Install again with only runtime deps
         "$$NPM install --prod",
         "mv node_modules build/",
-        "mkdir -p javascript/tools/typescript-parser-wrapper",
-        "mv build/* javascript/tools/typescript-parser-wrapper",
+        "mkdir -p typescript-parser-wrapper",
+        "mv build/* typescript-parser-wrapper",
+        "OUT=$$BAZEL_ROOT/$@",
+        "case $$OSTYPE in",
+        "  cygwin|msys|win32) OUT=$$(cygpath -w $$OUT);;",
+        "esac",
         "",
-    ]) + on_windows(
-        " && ".join([
-            "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$(cygpath -w $$BAZEL_ROOT/$@) $$(find javascript -name '*' -print)",
-            "rm -rf $$TEMP",
-        ]),
-        " && ".join([
-            "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$BAZEL_ROOT/$@ $$(find javascript -name '*' -print)",
-            "rm -rf $$TEMP",
-        ]),
-    ),
+        "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$OUT $$(find typescript-parser-wrapper -name '*' -print)",
+        "rm -rf $$TEMP",
+    ]),
     tools = [
         "@bazel_tools//tools/zip:zipper",
         "@nodejs//:node_bin",

javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java

@@ -50,7 +50,7 @@ public static void setUp() throws Exception {
         entry = zis.getNextEntry();
       }
     }
-    Path tsWrapper = tempDir.resolve("javascript/tools/typescript-parser-wrapper/main.js");
+    Path tsWrapper = tempDir.resolve("typescript-parser-wrapper/main.js");
     if (!Files.exists(tsWrapper)) {
       throw new RuntimeException("Could not find ts-wrapper at " + tsWrapper);
     }

javascript/resources/BUILD.bazel

@@ -0,0 +1,15 @@
+load("//misc/bazel:pkg.bzl", "codeql_pkg_files")
+
+codeql_pkg_files(
+    name = "resources",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "tools/*.sh",
+            "BUILD.bazel",
+        ],
+    ),
+    exes = glob(["tools/*.sh"]),
+    strip_prefix = "",
+    visibility = ["//javascript:__pkg__"],
+)

javascript/resources/codeql-extractor.yml

@@ -0,0 +1,94 @@
+name: "javascript"
+aliases:
+  - javascript-typescript
+  - typescript
+display_name: "JavaScript/TypeScript"
+version: 1.22.1
+column_kind: "utf16"
+unicode_newlines: true
+build_modes:
+  - none
+file_coverage_languages:
+  - name: javascript
+    display_name: JavaScript
+    scc_languages:
+      - JavaScript
+  - name: typescript
+    display_name: TypeScript
+    scc_languages:
+      - TypeScript
+      - TypeScript Typings
+github_api_languages:
+  - JavaScript
+  - TypeScript
+scc_languages:
+  - JavaScript
+  - TypeScript
+  - TypeScript Typings
+file_types:
+  - name: javascript
+    display_name: JavaScript
+    extensions:
+      - .js
+      - .jsx
+  - name: ecmascript
+    display_name: ECMAScript
+    extensions:
+      - .es
+      - .es6
+      - .mjs
+  - name: typescript
+    display_name: TypeScript
+    extensions:
+      - .ts
+      - .tsx
+  - name: html
+    display_name: HTML
+    extensions:
+      - .html
+      - .htm
+      - .xhtm
+      - .xhtml
+  - name: vue
+    display_name: Vue.js component
+    extensions:
+      - .vue
+  - name: data
+    display_name: Data or configuration files
+    extensions:
+      - .json
+      - .yml
+      - .yaml
+      - .raml
+legacy_qltest_extraction: true
+options:
+  trap:
+    title: TRAP options
+    description: Options about how the extractor handles TRAP files
+    type: object
+    visibility: 3
+    properties:
+      cache:
+        title: TRAP cache options
+        description: Options about how the extractor handles its TRAP cache
+        type: object
+        properties:
+          dir:
+            title: TRAP cache directory
+            description: The directory of the TRAP cache to use
+            type: string
+          bound:
+            title: TRAP cache bound
+            description: A soft limit (in MB) on the size of the TRAP cache
+            type: string
+            pattern: "[0-9]+"
+          write:
+            title: TRAP cache writeable
+            description: Whether to write to the TRAP cache as well as reading it
+            type: string
+            pattern: "(true|TRUE|false|FALSE)"
+  skip_types:
+    title: Skip type extraction for TypeScript
+    description: Whether to skip the extraction of types in a TypeScript application
+    type: string
+    pattern: "^(false|true)$"

javascript/resources/tools/autobuild.cmd

@@ -0,0 +1,30 @@
+@echo off
+SETLOCAL EnableDelayedExpansion
+
+set jvm_args=-Xss16m
+
+rem If CODEQL_RAM is set, use half for Java and half for TS.
+if NOT [%CODEQL_RAM%] == [] (
+    set /a "half_ram=CODEQL_RAM/2"
+    set LGTM_TYPESCRIPT_RAM=%half_ram%
+    set jvm_args=!jvm_args! -Xmx!half_ram!m
+)
+
+rem If CODEQL_THREADS is set, propagate via LGTM_THREADS.
+if NOT [%CODEQL_THREADS%] == [] (
+    set LGTM_THREADS=%CODEQL_THREADS%
+)
+
+rem The JS autobuilder expects to find typescript modules under SEMMLE_DIST/tools.
+rem They are included in the pack, but we need to set SEMMLE_DIST appropriately.
+set SEMMLE_DIST=%CODEQL_EXTRACTOR_JAVASCRIPT_ROOT%
+
+rem The JS autobuilder expects LGTM_SRC to be set to the source root.
+set LGTM_SRC=%CD%
+
+type NUL && "%CODEQL_JAVA_HOME%\bin\java.exe" %jvm_args% ^
+    -cp "%CODEQL_EXTRACTOR_JAVASCRIPT_ROOT%\tools\extractor-javascript.jar" ^
+    com.semmle.js.extractor.AutoBuild
+exit /b %ERRORLEVEL%
+
+ENDLOCAL

javascript/resources/tools/autobuild.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+set -eu
+
+jvm_args=-Xss16m
+
+# If CODEQL_RAM is set, use half for Java and half for TS.
+if [ -n "${CODEQL_RAM:-}" ] ; then
+    half_ram="$(( CODEQL_RAM / 2 ))"
+    LGTM_TYPESCRIPT_RAM="$half_ram"
+    export LGTM_TYPESCRIPT_RAM
+    jvm_args="$jvm_args -Xmx${half_ram}m"
+fi
+
+# If CODEQL_THREADS is set, propagate via LGTM_THREADS.
+if [ -n "${CODEQL_THREADS:-}" ] ; then
+    LGTM_THREADS="$CODEQL_THREADS"
+    export LGTM_THREADS
+fi
+
+# The JS autobuilder expects to find typescript modules under SEMMLE_DIST/tools.
+# They are included in the pack, but we need to set SEMMLE_DIST appropriately.
+# We want to word-split $jvm_args, so disable the shellcheck warning.
+# shellcheck disable=SC2086
+env SEMMLE_DIST="$CODEQL_EXTRACTOR_JAVASCRIPT_ROOT" \
+    LGTM_SRC="$(pwd)" \
+    "${CODEQL_JAVA_HOME}/bin/java" $jvm_args \
+    -cp "$CODEQL_EXTRACTOR_JAVASCRIPT_ROOT/tools/extractor-javascript.jar" \
+    com.semmle.js.extractor.AutoBuild

javascript/resources/tools/baseline-config.json

@@ -0,0 +1,8 @@
+{
+    "paths-ignore": [
+        "**/node_modules/**",
+        "**/bower_components/**",
+        "**/*.min.js",
+        "**/*-min.js"
+    ]
+}

javascript/resources/tools/configure-baseline.cmd

@@ -0,0 +1,2 @@
+@echo off
+type "%CODEQL_EXTRACTOR_JAVASCRIPT_ROOT%\tools\baseline-config.json"

javascript/resources/tools/configure-baseline.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+cat "$CODEQL_EXTRACTOR_JAVASCRIPT_ROOT/tools/baseline-config.json"

javascript/resources/tools/index-dir.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+echo "Not implemented." 1>&2
+exit 1

ql/.gitignore

@@ -1,5 +1,4 @@
 target
-extractor-pack
 .vscode/launch.json
 .cache
 ql/test/**/*.testproj

ruby/.gitignore

@@ -1,5 +1,4 @@
 extractor/target
-extractor-pack
 .vscode/launch.json
 .cache
 ql/test/**/*.testproj