Adding more support for external publisher articles with a google search

manifest.json

@@ -2,7 +2,7 @@
   "name": "Medium Unlimited: Read paid content for free!",
   "description": "Unlocks medium.com for unlimited reads, no membership required",
   "short_name": "Medium Unlimited",
-  "version": "1.3.7",
+  "version": "1.4.0",
   "manifest_version": 2,
   "background": {
     "scripts": ["background.bundle.js"]
@@ -34,6 +34,7 @@
     "webRequest",
     "webRequestBlocking",
     "https://medium.com/*",
+    "https://www.google.com/*",
     "https://towardsdatascience.com/*",
     "https://hackernoon.com/*",
     "https://medium.freecodecamp.org/*",

package.json

@@ -18,8 +18,10 @@
   "devDependencies": {
     "babel-core": "^6.26.3",
     "babel-loader": "^7.1.4",
+    "babel-polyfill": "^6.26.0",
     "babel-preset-env": "^1.7.0",
     "babel-preset-react": "^6.24.1",
+    "babel-preset-stage-0": "^6.24.1",
     "babel-preset-stage-2": "^6.24.1",
     "copy-webpack-plugin": "^4.5.1",
     "css-loader": "^1.0.0",

src/background.js

@@ -26,36 +26,76 @@ function _processUserIdRequest(sendResponse) {
 
 function _processContentRequest(request, sendResponse) {
   log('Fetching content for', request.url);
-  inProgressUrls[request.url] = true;
   track('REQUESTED');
   _fetch(request.url)
-    .then(responseData => {
+    .then(async responseData => {
       const doc = document.createElement('html');
       doc.innerHTML = responseData.body;
       const hadMembershipPrompt = hasMembershipPrompt(doc);
       const content = extractArticleContent(doc);
       const counter = incrementReadCountAndGet();
-      const externalUrl = extractExternalUrlInContent(doc);
+      let externalUrl = extractExternalUrlInContent(doc);
+      let isGoogleResult = false;
+      if (hadMembershipPrompt && !externalUrl) {
+        externalUrl = await getUrlFromGoogleSearch(request.url)
+          .catch(err => log('Error while searching', err));
+        isGoogleResult = true;
+      }
       sendResponse({ status: 'SUCCESS', content, counter, hadMembershipPrompt, externalUrl });
       let trackStatus = 'SUCCESS';
       if (hadMembershipPrompt) {
-        if (externalUrl) {
+        if (externalUrl && !isGoogleResult) {
           trackStatus = 'PARTIAL-SUCCESS';
+        } else if (externalUrl && isGoogleResult) {
+          trackStatus = 'SERP-SUCCESS';
         } else {
           trackStatus = 'PARTIAL-FAILED';
         }
       }
       track(trackStatus);
-      delete inProgressUrls[request.url];
     })
     .catch(error => {
       sendResponse({status: 'ERROR', error: JSON.stringify(error)});
       track('FAILED');
-      delete inProgressUrls[request.url];
     });
   return true;
 }
 
+async function getUrlFromGoogleSearch(url) {
+  const slugs = url.split('/');
+  const lastSlug = slugs[slugs.length - 1];
+  const words = lastSlug.split('-');
+  const requiredSlug = words.splice(0, words.length - 1).join('-');
+  const response = await _fetch(`https://www.google.com/search?q=${requiredSlug}`);
+  const doc = document.createElement('html');
+  doc.innerHTML = response.body;
+  const searchElement = doc.querySelector('#search');
+  if (!searchElement) {
+    return;
+  }
+  const links = searchElement.getElementsByTagName('a');
+  let requiredLink = null;
+  for (const link of links) {
+    if (isValidArticleUrl(link.href)) {
+      requiredLink = link;
+      break;
+    }
+  }
+  if (!requiredLink) {
+    return;
+  }
+  return requiredLink.href;
+}
+
+function isValidArticleUrl(url) {
+  return (
+    url &&
+    url.indexOf('medium.com') === -1 &&
+    url.indexOf('webcache.googleusercontent.com') === -1 &&
+    url.indexOf('chrome-extension://') === -1
+  );
+}
+
 function extractArticleContent(doc) {
   const content = Array.from(
     doc.getElementsByClassName('postArticle-content')
@@ -79,9 +119,14 @@ function extractExternalUrlInContent(doc) {
 }
 
 function _fetch(url) {
-  return fetch(url, {credentials: 'include'}).then(response => {
+  inProgressUrls[url] = true;
+  return fetch(url, { credentials: 'include' }).then(response => {
     return response.text().then(body => {
-      return {status: response.status, body};
+      delete inProgressUrls[url];
+      return { status: response.status, body };
     });
+  }).catch(err => {
+    delete inProgressUrls[url];
+    throw err;
   });
 }

src/cookie_interceptors.js

@@ -2,6 +2,7 @@ import {urlWithoutQueryParams} from './utils';
 
 const urlsList = [
   'https://medium.com/*',
+  'https://www.google.com/*',
   'https://towardsdatascience.com/*',
   'https://hackernoon.com/*',
   'https://medium.freecodecamp.org/*',
@@ -12,7 +13,9 @@ const urlsList = [
 export default function intercept(inProgressUrls) {
   function onBeforeSendHeaders(details) {
     if (details.requestHeaders && shouldIntercept(details)) {
-      const newHeaders = removeHeader(details.requestHeaders, 'cookie');
+      let newHeaders = removeHeader(details.requestHeaders, 'cookie');
+      newHeaders = removeHeader(newHeaders, 'origin');
+
       return {requestHeaders: newHeaders};
     }
     return {requestHeaders: details.requestHeaders};
@@ -43,14 +46,7 @@ export default function intercept(inProgressUrls) {
   );
 
   function removeHeader(headers, headerToRemove) {
-    const newHeaders = [];
-    headers.forEach(({name, value}) => {
-      if (name.toLowerCase() === headerToRemove) {
-        return;
-      }
-      return newHeaders.push({name, value});
-    });
-    return newHeaders;
+    return headers.filter(({name}) => name.toLowerCase() != headerToRemove);
   }
 
   function shouldIntercept(details) {

webpack.config.js

@@ -2,7 +2,7 @@ const CopyWebpackPlugin = require('copy-webpack-plugin');
 
 module.exports = {
   entry: {
-    background: './src/background.js',
+    background: ['babel-polyfill', './src/background.js'],
     main: './src/medium-unlocker.js',
   },
   output: {