fix: support callv and proxies for lsp.js

js/language_server/Main.ml

@@ -67,7 +67,6 @@ let _ =
       RPC_server.io_ref := (module Io);
       Logs.set_level (Some Logs.Debug);
       Logs.set_reporter { Logs.report = Semgrep_js_shared.console_report };
-      Http_helpers.client_ref := Some (module Cohttp_lwt_jsoo.Client);
       Session.scan_config_parser_ref := scan_config_parser;
 
       let server =
@@ -78,6 +77,7 @@ let _ =
       Js.export_all
         (object%js
            method init wasm_module =
+             Semgrep_js_node_shared.init_cohttp ();
              init_jsoo wasm_module;
              set_parser_wasm_module wasm_module;
              Parsing_init.init ()

js/language_server/dune

@@ -19,13 +19,11 @@
   lwt_platform.js
 
    networking.http_helpers
-   cohttp-lwt-jsoo
   )
  (modes js)
  (js_of_ocaml
   (javascript_files
    ;; Custom XMLHttpRequest implementation that doesn't preset the User-Agent header
-   XMLHttpRequest.js
    semgrep.js
  ))
  (preprocess

js/node_shared/Semgrep_js_node_shared.ml

@@ -0,0 +1,45 @@
+(* Austin Theriault
+ *
+ * Copyright (C) Semgrep, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * version 2.1 as published by the Free Software Foundation, with the
+ * special exception on linking described in file LICENSE.
+ *
+ * This library is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the file
+ * LICENSE for more details.
+ *)
+
+(* Commentary *)
+(*  *)
+
+(*****************************************************************************)
+(* Prelude *)
+(*****************************************************************************)
+
+module Patched_cohttp_lwt_jsoo : Cohttp_lwt.S.Client = struct
+  include Cohttp_lwt_jsoo.Client
+
+  (* Callv isn't currently supported on the JS side so we have to provide it ourselves :( *)
+  let callv ?ctx (uri : Uri.t) stream =
+    (* Differs from request uri if HTTP(S)_PROXY is set. But we handle this in
+       the node_shared XHR*)
+    ignore uri;
+    Lwt.return
+      (Lwt_stream.map_s
+         (fun ((req, body) : Cohttp.Request.t * Cohttp_lwt.Body.t) ->
+           call ?ctx ~headers:req.headers ~body ~chunked:false req.meth
+             Cohttp.Request.(uri req))
+         stream)
+end
+
+(*****************************************************************************)
+(* Code *)
+(*****************************************************************************)
+
+let init_cohttp () =
+  Http_helpers.client_ref :=
+    Some (module Patched_cohttp_lwt_jsoo : Cohttp_lwt.S.Client)

js/language_server/XMLHttpRequest.js → js/node_shared/XMLHttpRequest.js

@@ -13,6 +13,26 @@
  * @author Dan DeFelippi <dan@driverdan.com>
  * @contributor David Ellis <d.f.ellis@ieee.org>
  * @license MIT
+ *
+ * Copyright (c) 2010 passive.ly LLC
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
 var Url = require("url");
@@ -385,6 +405,17 @@ XMLHttpRequest = function () {
       headers["Content-Length"] = 0;
     }
 
+    // get if HTTP_PROXY or HTTPS_PROXY is set
+    const proxy = process.env[ssl ? "HTTPS_PROXY" : "HTTP_PROXY"];
+    if (proxy) {
+      // set host to proxy
+      const proxyUrl = Url.parse(proxy);
+      host = proxyUrl.hostname;
+      port = proxyUrl.port;
+      // set path to full URL
+      uri = url.href;
+
+    }
     var options = {
       host: host,
       port: port,

js/node_shared/dune

@@ -9,10 +9,15 @@
 (library
  (public_name semgrep.semgrep_node_js_shared)
  (name semgrep_node_js_shared)
+ (libraries
+   cohttp-lwt-jsoo
+   networking.http_helpers
+ )
  (wrapped false)
  (js_of_ocaml
   (javascript_files
     unix.js
+    XMLHttpRequest.js ; for making HTTP requests
  ))
  (preprocess
   (pps js_of_ocaml-ppx lwt_ppx)))

js/shared/Semgrep_js_shared.ml

@@ -49,6 +49,7 @@ let init_jsoo yaml_wasm_module =
   Common.jsoo := true;
   Tree_sitter_run.Util_file.jsoo := true;
   Metrics_.g.payload.environment.isTranspiledJS <- true;
+  (* NOTE: HTTP stuff won't work on node unless node_shared is included *)
   (* Using semgrep.parsing_languages makes the JS goes
      from 16MB to 7MB (in release mode) and from 110MB to 50MB (in dev mode)
      TODO: we should add language parsers dynamically, loading language "bundles"

js/shared/dune

@@ -13,6 +13,7 @@
    semgrep.data
    semgrep.language_server
    lwt_platform.js
+   cohttp-lwt-jsoo
    tracing.js
  )
  (js_of_ocaml

js/tests/Test_jsoo.ml

@@ -76,7 +76,10 @@ let () =
       Logs.set_reporter { Logs.report = Semgrep_js_shared.console_report };
       Js.export_all
         (object%js
-           method init = Semgrep_js_shared.init_jsoo
+           method init =
+             Semgrep_js_node_shared.init_cohttp ();
+             Semgrep_js_shared.init_jsoo
+
            method getMountpoints = Semgrep_js_shared.get_jsoo_mountpoint ()
            method setParsePattern = Semgrep_js_shared.setParsePattern
            method setJustParseWithLang = Semgrep_js_shared.setJustParseWithLang

libs/networking/http_helpers/http_helpers.ml

@@ -57,6 +57,16 @@ struct
   (* Respect the proxy!*)
   (* Cohttp doesn't https://github.com/mirage/ocaml-cohttp/issues/459 *)
   (* https://github.com/0install/0install/blob/6c0f5c51bc099370a367102e48723a42cd352b3b/ocaml/zeroinstall/http.cohttp.ml#L232-L256 *)
+
+  (* What do we have to do to support a proxy? *)
+  (* On Native OCaml (not js), the certificate MUST be in a path listed in
+     https://github.com/mirage/ca-certs/blob/fa4ff942f1ac980e2502a0783ef10ade5ba497f2/lib/ca_certs.ml#L34-L50 *)
+  (* HTTP(s)_PROXY MUST have a scheme, http:// or https://, and must have a port
+     if it isn't on port 80/443 *)
+  (* So HTTP_PROXY=http://localhost:8080 or
+     HTTPS_PROXY=https://localhost:8080 *)
+  (* And note that HTTP_PROXY is only used for HTTP requests! We almost
+     exclusively use HTTPS urls so make sure both are set*)
   let get_proxy uri =
     let proxy_uri_env =
       match Uri.scheme uri with

src/osemgrep/language_server/Test_LS_e2e.ml

@@ -410,7 +410,12 @@ let send_initialize info ?(only_git_dirty = true) workspaceFolders =
             `Assoc
               [
                 ( "configuration",
-                  `List [ `String (rule_path |> Fpath.to_string) ] );
+                  `List
+                    [
+                      `String (rule_path |> Fpath.to_string);
+                      (* Make sure we can make network requests *)
+                      `String "p/ocaml";
+                    ] );
                 ("exclude", `List []);
                 ("include", `List []);
                 ("jobs", `Int 1);
@@ -664,6 +669,9 @@ let check_startup info folders (files : Fpath.t list) =
   let%lwt () = assert_progress info "Refreshing Rules" in
   let%lwt rulesRefreshedNotif = receive_notification info in
   let%lwt () = assert_notif rulesRefreshedNotif "semgrep/rulesRefreshed" in
+  let rules_length = List.length info.server.session.cached_session.rules in
+  Alcotest.(check bool)
+    "rules refreshed greater than 1" (rules_length > 20) true;
   let%lwt () = assert_progress info "Scanning Open Documents" in
 
   let%lwt scan_notifications =