Ported to WebExtensions

.eslintrc.json

@@ -0,0 +1,18 @@
+{
+  "env": {
+    "browser": true,
+    "es6": true,
+    "webextensions": true
+  },
+  "extends": "eslint:recommended",
+  "parserOptions": {
+    "ecmaVersion": 2017
+  },
+  "plugins": ["prettier"],
+  "rules": {
+    "no-shadow": ["error", { "allow": ["err"] }],
+    "prefer-arrow-callback": "error",
+    "curly": ["error", "multi", "consistent"],
+    "prettier/prettier": "error"
+  }
+}

.gitignore

@@ -0,0 +1,3 @@
+node_modules
+web-ext-artifacts
+package-lock.json

README.md

@@ -0,0 +1,18 @@
+# cliget
+
+Download login-protected files from the command line using curl, wget or aria2.
+
+This addon will generate commands that emulate the request as though it was
+coming from your browser by sending the same cookies, user agent string and
+referrer. With this addon you can download email attachments, purchased
+software/media, source code from a private repository to a remote server without
+having to download the files locally first. If come across a website where
+cliget doesn't work, please open an issue providing details to help reproduce
+the problem.
+
+*Windows users*: Enable the "Escape with double-quotes" option because Windows
+doesn't support single quotes. If you use cygwin, however, you don't need to
+enable this option.
+
+**Please be aware** of potential security and privacy implications from cookies
+being exposed in the download command.

aria2.js

@@ -0,0 +1,30 @@
+"use strict";
+
+window.aria2 = function(url, method, headers, payload, filename, options) {
+  if (method !== "GET") throw new Error("Unsupported HTTP method");
+
+  const esc = window.escapeShellArg;
+
+  let parts = ["aria2c"];
+
+  for (let header of headers) {
+    let headerName = header.name.toLowerCase();
+
+    if (headerName === "referer") {
+      parts.push(`--referer ${esc(header.value, options.doubleQuotes)}`);
+    } else if (headerName === "user-agent") {
+      parts.push(`--user-agent ${esc(header.value, options.doubleQuotes)}`);
+    } else {
+      let h = esc(`${header.name}: ${header.value}`, options.doubleQuotes);
+      parts.push(`--header ${h}`);
+    }
+  }
+
+  parts.push(esc(url, options.doubleQuotes));
+
+  if (filename) parts.push(`--out ${esc(filename, options.doubleQuotes)}`);
+
+  if (options.aria2Options) parts.push(options.aria2Options);
+
+  return parts.join(" ");
+};

background.js

@@ -0,0 +1,219 @@
+"use strict";
+
+const MAX_ITEMS = 10;
+
+const downloads = new Map();
+const currentRequests = new Map();
+
+const defaultOptions = {
+  doubleQuotes: false,
+  excludeHeaders: "Accept-Encoding Connection",
+  command: "curl",
+  curlOptions: "",
+  wgetOptions: "",
+  aria2Options: ""
+};
+
+function getOptions() {
+  return new Promise(resolve => {
+    browser.storage.local.get().then(res => {
+      res = Object.assign({}, defaultOptions, res);
+      resolve(res);
+    });
+  });
+}
+
+function setOptions(values) {
+  new Promise(resolve => {
+    browser.storage.local.set(values).then(() =>
+      getOptions().then(c => {
+        resolve(c);
+      })
+    );
+  });
+}
+
+function resetOptions() {
+  new Promise(resolve => {
+    browser.storage.local.clear().then(() =>
+      getOptions().then(c => {
+        resolve(c);
+      })
+    );
+  });
+}
+
+function clear() {
+  downloads.clear();
+}
+
+function getDownloadList() {
+  const list = [];
+  for (let [reqId, req] of downloads)
+    list.push({
+      id: reqId,
+      url: req.url,
+      filename: req.filename,
+      size: req.size
+    });
+
+  return list;
+}
+
+function generateCommand(reqId, options) {
+  const request = downloads.get(reqId);
+  if (!request) throw new Error("Request not found");
+
+  let excludeHeaders = options.excludeHeaders
+    .split(" ")
+    .map(h => h.toLowerCase());
+
+  let headers = request.headers.filter(
+    h => excludeHeaders.indexOf(h.name.toLowerCase()) === -1
+  );
+
+  const cmd = window[options.command](
+    request.url,
+    request.method,
+    headers,
+    request.payload,
+    request.filename,
+    options
+  );
+
+  return cmd;
+}
+
+function handleMessage(msg) {
+  const name = msg[0];
+  const args = msg.slice(1);
+
+  if (name === "getOptions") return getOptions();
+  else if (name === "setOptions") return setOptions(...args);
+  else if (name === "resetOptions") return resetOptions();
+  else if (name === "getDownloadList")
+    return new Promise(resolve => resolve(getDownloadList()));
+  else if (name === "clear") return clear(...args);
+  else if (name === "generateCommand")
+    return new Promise(resolve => {
+      try {
+        resolve(generateCommand(...args));
+      } catch (err) {
+        resolve(err.message);
+      }
+    });
+}
+
+browser.runtime.onMessage.addListener(handleMessage);
+
+function onBeforeRequest(details) {
+  if (
+    (details.type === "main_frame" || details.type === "sub_frame") &&
+    details.tabId >= 0
+  ) {
+    const now = Date.now();
+
+    // Just in case of a leak
+    currentRequests.forEach((req, reqId) => {
+      if (req.timestamp + 10000 < now) currentRequests.delete(reqId);
+    });
+
+    const req = {
+      id: details.requestId,
+      method: details.method,
+      url: details.url,
+      timestamp: now,
+      payload: details.requestBody
+    };
+    currentRequests.set(details.requestId, req);
+  }
+}
+
+function onSendHeaders(details) {
+  const req = currentRequests.get(details.requestId);
+  if (req) req.headers = details.requestHeaders;
+}
+
+function onResponseStarted(details) {
+  const request = currentRequests.get(details.requestId);
+
+  if (!request) return;
+
+  currentRequests.delete(details.requestId);
+
+  if (details.statusCode !== 200 || details.fromCache) return;
+
+  let contentType, contentDisposition;
+
+  for (let header of details.responseHeaders) {
+    let headerName = header.name.toLowerCase();
+    if (headerName === "content-type") {
+      contentType = header.value.toLowerCase();
+    } else if (headerName === "content-disposition") {
+      contentDisposition = header.value.toLowerCase();
+      request.filename = window.getFilenameFromContentDisposition(header.value);
+    } else if (headerName === "content-length") {
+      request.size = +header.value;
+    }
+  }
+
+  if (!contentDisposition || !contentDisposition.startsWith("attachment"))
+    if (
+      contentType.startsWith("text/html") ||
+      contentType.startsWith("text/plain") ||
+      contentType.startsWith("image/")
+    )
+      return;
+
+  if (!request.filename)
+    request.filename = window.getFilenameFromUrl(request.url);
+
+  downloads.set(details.requestId, request);
+
+  browser.browserAction.getBadgeText({}).then(txt => {
+    browser.browserAction.setBadgeText({ text: `${+txt + 1}` });
+  });
+
+  if (downloads.size > MAX_ITEMS) {
+    let keys = Array.from(downloads.keys());
+    keys.slice(0, keys.length - MAX_ITEMS).forEach(k => downloads.delete(k));
+  }
+}
+
+function onBeforeRedirect() {
+  // Need to listen to this event otherwise the new request will include
+  // the old URL. This is possibly a bug.
+}
+
+function onErrorOccurred(details) {
+  currentRequests.delete(details.requestId);
+}
+
+browser.webRequest.onBeforeRedirect.addListener(onBeforeRedirect, {
+  urls: ["<all_urls>"]
+});
+
+browser.webRequest.onErrorOccurred.addListener(onErrorOccurred, {
+  urls: ["<all_urls>"]
+});
+
+browser.webRequest.onBeforeRequest.addListener(
+  onBeforeRequest,
+  { urls: ["<all_urls>"] },
+  ["requestBody"]
+);
+browser.webRequest.onSendHeaders.addListener(
+  onSendHeaders,
+  { urls: ["<all_urls>"] },
+  ["requestHeaders"]
+);
+
+browser.webRequest.onResponseStarted.addListener(
+  onResponseStarted,
+  {
+    urls: ["<all_urls>"]
+  },
+  ["responseHeaders"]
+);
+
+browser.browserAction.setBadgeBackgroundColor({ color: "#4a90d9" });

curl.js

@@ -0,0 +1,71 @@
+"use strict";
+
+function escapeGlobbing(url) {
+  return url.replace(/[[\]{}]/g, m => `\\${m.slice(0, 1)}`);
+}
+
+window.curl = function(url, method, headers, payload, filename, options) {
+  const esc = window.escapeShellArg;
+
+  let contentType;
+  let parts = ["curl"];
+
+  for (let header of headers) {
+    let headerName = header.name.toLowerCase();
+
+    if (headerName === "content-type") {
+      contentType = header.value.toLowerCase();
+      let v = header.value;
+      if (v.startsWith("multipart/form-data;")) v = v.slice(0, 19);
+      let h = esc(`${header.name}: ${v}`, options.doubleQuotes);
+      parts.push(`--header ${h}`);
+    } else if (headerName === "content-length") {
+      // Implicitly added by curl
+    } else if (headerName === "referer") {
+      parts.push(`--referer ${esc(header.value, options.doubleQuotes)}`);
+    } else if (headerName === "cookie") {
+      parts.push(`--cookie ${esc(header.value, options.doubleQuotes)}`);
+    } else if (headerName === "user-agent") {
+      parts.push(`--user-agent ${esc(header.value, options.doubleQuotes)}`);
+    } else {
+      let h = esc(`${header.name}: ${header.value}`, options.doubleQuotes);
+      parts.push(`--header ${h}`);
+    }
+  }
+
+  if (method !== "GET" || payload) parts.push(`--request ${method}`);
+
+  if (payload)
+    if (payload.formData) {
+      if (contentType === "application/x-www-form-urlencoded")
+        for (let [key, values] of Object.entries(payload.formData))
+          for (let value of values) {
+            let v = esc(
+              `${encodeURIComponent(key)}=${value}`,
+              options.doubleQuotes
+            );
+            parts.push(`--data-urlencode ${v}`);
+          }
+      else if (contentType.startsWith("multipart/form-data;"))
+        // TODO comment about escaping of name value (e.g. = " ')
+        for (let [key, values] of Object.entries(payload.formData))
+          for (let value of values) {
+            let v = esc(
+              `${encodeURIComponent(key)}=${value}`,
+              options.doubleQuotes
+            );
+            parts.push(`--form-string ${v}`);
+          }
+    } else if (payload.raw) {
+      throw new Error("Unsupported upload data");
+    }
+
+  parts.push(esc(escapeGlobbing(url), options.doubleQuotes));
+
+  if (filename) parts.push(`--output ${esc(filename, options.doubleQuotes)}`);
+  else parts.push("--remote-name --remote-header-name");
+
+  if (options.curlOptions) parts.push(options.curlOptions);
+
+  return parts.join(" ");
+};