0016-Firefox-profile-import.patch

From: Krzysztof Jan Modras <chrmod@chrmod.net>
Date: Thu, 17 Sep 2020 15:36:45 +0200
Subject: Firefox profile import

---
 .../migration/FirefoxProfileMigrator.sys.mjs  | 342 +++++++++++++++++-
 browser/components/migration/moz.build        |   5 +-
 2 files changed, 344 insertions(+), 3 deletions(-)

diff --git a/browser/components/migration/FirefoxProfileMigrator.sys.mjs b/browser/components/migration/FirefoxProfileMigrator.sys.mjs
index 1bf7eb2c61..81b218cbc6 100644
--- a/browser/components/migration/FirefoxProfileMigrator.sys.mjs
+++ b/browser/components/migration/FirefoxProfileMigrator.sys.mjs
@@ -32,7 +32,7 @@ ChromeUtils.defineESModuleGetters(lazy, {
  *
  * This migrator is what powers the "Profile Refresh" mechanism.
  */
-export class FirefoxProfileMigrator extends MigratorBase {
+class FirefoxProfileMigratorOrig extends MigratorBase {
   static get key() {
     return "firefox";
   }
@@ -96,7 +96,7 @@ export class FirefoxProfileMigrator extends MigratorBase {
 
     // Being a startup-only migrator, we can rely on
     // MigrationUtils.profileStartup being set.
-    let currentProfileDir = MigrationUtils.profileStartup.directory;
+    let currentProfileDir = this.getCurrentProfileDir();
 
     // Surely data cannot be imported from the current profile.
     if (sourceProfileDir.equals(currentProfileDir)) {
@@ -159,6 +159,13 @@ export class FirefoxProfileMigrator extends MigratorBase {
       Services.prefs.savePrefFile(newPrefsFile);
     }
 
+    if (!this.startupOnlyMigrator && !MigrationUtils.isStartupMigration) {
+      let places = this.getHistoryAndBookmarksResource(sourceProfileDir, "places.sqlite");
+      let cookies = this.getCookiesResource(sourceProfileDir, "cookies.sqlite");
+      let formData = this.getFormDataResource(sourceProfileDir, "formhistory.sqlite");
+      return [places, cookies, formData].filter(r => r);
+    }
+
     let types = MigrationUtils.resourceTypes;
     let places = getFileResource(types.HISTORY, [
       "places.sqlite",
@@ -395,3 +402,334 @@ export class FirefoxProfileMigrator extends MigratorBase {
     return true;
   }
 }
+
+var { XPCOMUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+  NetUtil: "resource://gre/modules/NetUtil.jsm",
+});
+
+ChromeUtils.defineESModuleGetters(lazy, {
+  PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs",
+  FormHistory: "resource://gre/modules/FormHistory.sys.mjs",
+  Sqlite: "resource://gre/modules/Sqlite.sys.mjs",
+});
+
+
+let fxProductDir = lazy.FileUtils.getDir(
+  #if defined(XP_WIN)
+      "AppData", ["Mozilla", "Firefox"]
+  #elif defined(XP_MACOSX)
+      "ULibDir", ["Application Support", "Firefox"]
+  #else
+      "Home", [".mozilla", "firefox"]
+  #endif
+      , false);
+
+function isValidUrl(aUrl) {
+  let url = lazy.NetUtil.newURI(aUrl);
+  // Filter out the URLs with unsupported schemes.
+  const invalidSchemes = ["wyciwyg", "place", "about", "chrome"];
+  if (invalidSchemes.indexOf(url.scheme) >= 0) return false;
+  return true;
+}
+
+async function insertWholeBookmarkFolder(db, aId, aGuid) {
+  let query = `SELECT b.id, h.url, COALESCE(b.title, h.title) AS title,
+    b.type, k.keyword, b.dateAdded, b.lastModified
+    FROM moz_bookmarks b
+    LEFT JOIN moz_places h ON b.fk = h.id
+    LEFT JOIN moz_keywords k ON k.id = b.keyword_id
+    WHERE b.type IN (1,2) AND b.parent = ${aId}
+    ORDER BY b.position;`;
+  let rows = await db.execute(query);
+  let yieldCounter = 0;
+  for (let row of rows) {
+    let type = row.getResultByName("type");
+    let title = row.getResultByName("title");
+    let id = row.getResultByName("id");
+
+    switch (type) {
+      case lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK: // Bookmark Url - Handle keyword and favicon
+        let url = row.getResultByName("url");
+        if (isValidUrl(url)) {
+          await lazy.PlacesUtils.bookmarks.insert({ parentGuid: aGuid,
+                                                url,
+                                                title,
+                                              });
+        }
+        break;
+      case lazy.PlacesUtils.bookmarks.TYPE_FOLDER: // Bookmark Folder - Handle Tag and Livemark (later)
+        let newFolderGuid = (await lazy.PlacesUtils.bookmarks.insert({
+          parentGuid: aGuid,
+          type: lazy.PlacesUtils.bookmarks.TYPE_FOLDER,
+          title,
+        })).guid;
+        await insertWholeBookmarkFolder(db, id, newFolderGuid); // Recursive insert bookmarks
+        break;
+    }
+
+    // With many bookmarks we end up stealing the CPU - even with yielding!
+    // So we let everyone else have a go every few items (bug 1186714).
+    if (++yieldCounter % 50 == 0) {
+      await new Promise(resolve => {
+        Services.tm.currentThread.dispatch(resolve, Ci.nsIThread.DISPATCH_NORMAL);
+      });
+    }
+  }
+}
+
+export class FirefoxProfileMigrator extends FirefoxProfileMigratorOrig {
+  getHistoryAndBookmarksResource(sourceProfileDir, aFileName) {
+    let placesFile = this._getFileObject(sourceProfileDir, aFileName);
+    if (!placesFile)
+      return null;
+
+    return {
+      type: MigrationUtils.resourceTypes.HISTORY,
+      migrate: async (aCallback) => {
+        let db;
+        try {
+          db = await lazy.Sqlite.openConnection({
+            path: placesFile.path
+          });
+          // IMPORT BOOKMARKS
+          const topBookmarkFolderGuids = [
+                                          "menu________",
+                                          "toolbar_____",
+                                          "unfiled_____"
+                                          ];
+          let parentGuid = lazy.PlacesUtils.bookmarks.menuGuid;
+          // Create Firefox bookmarks folder on Bookmarks Menu
+          parentGuid = (await lazy.PlacesUtils.bookmarks.insert({
+            parentGuid,
+            type: lazy.PlacesUtils.bookmarks.TYPE_FOLDER,
+            title: "Firefox",
+          })).guid;
+          // Create top bookmarks folders on Firefox bookmarks folder and recursively insert child bookmarks
+          for (let guid of topBookmarkFolderGuids) {
+            let query = `SELECT b.id, b.title
+                        FROM moz_bookmarks b
+                        WHERE b.type = 2 AND b.guid = '${guid}'
+                        ORDER BY b.position`;
+            let rows = await db.execute(query);
+            if (rows.length > 0) {
+              let title = rows[0].getResultByName("title");
+              let id = rows[0].getResultByName("id");
+              let folderGuid = (await lazy.PlacesUtils.bookmarks.insert({
+                parentGuid,
+                type: lazy.PlacesUtils.bookmarks.TYPE_FOLDER,
+                title,
+              })).guid;
+              await insertWholeBookmarkFolder(db, id, folderGuid);
+            }
+          }
+
+          // IMPORT HISTORY
+          let rows = await db.execute(`SELECT h.url, h.title, v.visit_type, h.last_visit_date, h.typed
+                                      FROM moz_places h JOIN moz_historyvisits v
+                                      ON h.id = v.place_id
+                                      WHERE v.visit_type <= 3;`);
+          let pageInfos = [];
+          for (let row of rows) {
+            try {
+              // CLIQZ-SPECIAL: hack to transform date in required format when importing from FF
+              let timeStamp = row.getResultByName("last_visit_date");
+              try {
+                timeStamp = new Date(Math.floor(timeStamp/1000));
+              } catch(e) {
+                console.log('Erro in histry import')
+              }
+
+              // if having typed_count, we changes transition type to typed.
+              let transition = lazy.PlacesUtils.history.TRANSITIONS.LINK;
+              if (row.getResultByName("typed") > 0)
+                transition = lazy.PlacesUtils.history.TRANSITIONS.TYPED;
+              pageInfos.push({
+                title: row.getResultByName("title"),
+                isFPMigrator: true,
+                url: new URL(row.getResultByName("url")),
+                visits: [{
+                  transition,
+                  date: timeStamp,
+                }],
+              });
+            } catch (e) {
+              Cu.reportError(e);
+            }
+          }
+
+          if (pageInfos.length > 0) {
+            await MigrationUtils.insertVisitsWrapper(pageInfos);
+          }
+        } catch(e){
+          Cu.reportError(e);
+          aCallback(true);
+        } finally {
+          if(db) {
+            await db.close();
+          }
+          aCallback(true);
+          return;
+        }
+      }
+    };
+  }
+
+  getFormDataResource(sourceProfileDir, aFileName) {
+    let formDataFile = this._getFileObject(sourceProfileDir, aFileName);
+    if (!formDataFile)
+      return null;
+
+    return {
+      type: MigrationUtils.resourceTypes.FORMDATA,
+
+      migrate: async (aCallback) => {
+        let db;
+        try {
+          db = await lazy.Sqlite.openConnection({
+            path: formDataFile.path
+          });
+          let rows = await db.execute(`SELECT fieldname,
+                                              value,
+                                              timesUsed,
+                                              firstUsed,
+                                              lastUsed
+                                      FROM moz_formhistory`);
+          let changes = [];
+          for(let row of rows) {
+            changes.push({
+                          op: "add",
+                          fieldname: row.getResultByName("fieldname"),
+                          value:     row.getResultByName("value"),
+                          timesUsed: row.getResultByName("timesUsed"),
+                          firstUsed: row.getResultByName("firstUsed"),
+                          lastUsed:  row.getResultByName("lastUsed"),
+                        });
+          }
+          lazy.FormHistory.update(changes);
+        } catch(e) {
+          Cu.reportError(e);
+          aCallback(false);
+        } finally {
+          if (db) {
+            await db.close();
+          }
+          aCallback(true);
+        }
+      }
+    };
+  }
+
+  getCookiesResource(sourceProfileDir, aFileName) {
+    let cookiesFile = this._getFileObject(sourceProfileDir, aFileName);
+    if (!cookiesFile)
+      return null;
+
+    return {
+      type: MigrationUtils.resourceTypes.COOKIES,
+
+      migrate: async (aCallback) => {
+        let db;
+
+        try {
+          db = await lazy.Sqlite.openConnection({
+            path: cookiesFile.path
+          });
+          let rows = await db.execute(`SELECT name, value,
+                                              host, path,
+                                              expiry, isSecure,
+                                              isHttpOnly,
+                                              schemeMap
+                                        FROM moz_cookies`);
+          for(let row of rows) {
+            let schemeType = Ci.nsICookie.SCHEME_UNSET;
+            switch (row.getResultByName("schemeMap")) {
+              case 1:
+                schemeType = Ci.nsICookie.SCHEME_HTTP;
+                break;
+              case 2:
+                schemeType = Ci.nsICookie.SCHEME_HTTPS;
+                break;
+            }
+            Services.cookies.add(row.getResultByName("host"),
+                                  row.getResultByName("path"),
+                                  row.getResultByName("name"),
+                                  row.getResultByName("value"),
+                                  row.getResultByName("isSecure"),
+                                  row.getResultByName("isHttpOnly"),
+                                  false,
+                                  row.getResultByName("expiry"),
+                                  {},
+                                  Ci.nsICookie.SAMESITE_UNSET,
+                                  schemeType);
+          }
+        } catch(e) {
+          Cu.reportError(e);
+          aCallback(false);
+        } finally {
+          if (db) {
+            await db.close();
+          }
+          aCallback(true);
+        }
+      }
+    };
+  }
+
+  getCurrentProfileDir() {
+    let currentProfileDir = null;
+    if (!this.startupOnlyMigrator && !MigrationUtils.isStartupMigration) {
+      currentProfileDir = lazy.FileUtils.getDir("ProfD","");
+    } else {
+      currentProfileDir = MigrationUtils.profileStartup.directory;
+    }
+    return currentProfileDir;
+  }
+
+  _getAllProfiles() {
+    const profiles = new Map();
+    const profilesIni = fxProductDir.clone();
+    profilesIni.append("profiles.ini");
+    if (!(profilesIni.exists() &&
+          profilesIni.isFile() &&
+          profilesIni.isReadable()))
+      return profiles;
+    const factory = Cc["@mozilla.org/xpcom/ini-parser-factory;1"].
+                  getService(Ci.nsIINIParserFactory);
+    const iniParser = factory.createINIParser(profilesIni);
+
+    const sections = iniParser.getSections();
+    const profileSectionNameRE = /^Profile\d+$/;
+    while (sections.hasMore()) {
+      const section = sections.getNext();
+      if (!profileSectionNameRE.test(section))
+        continue;
+      try {
+        // The following code tries to replicate one in
+        // toolkit/profile/nsToolkitProfileService.cpp, Init() method.
+        const path = iniParser.getString(section, "Path");
+        const isRelative = iniParser.getString(section, "IsRelative") == "1";
+        let profileDir = fxProductDir.clone();
+        if (isRelative) {
+          profileDir.setRelativeDescriptor(fxProductDir, path);
+        }
+        else {
+          // TODO: Never saw absolute paths and never tested this.
+          profileDir.persistentDescriptor = path;
+        }
+
+        profiles.set(iniParser.getString(section, "Name"), profileDir);
+      } catch (e) {
+        dump("Profiles.ini section: '" + section + "', error: " + e + "\n");
+      }
+    }
+    return profiles;
+  }
+
+  get startupOnlyMigrator() {
+    return false;
+  }
+}
diff --git a/browser/components/migration/moz.build b/browser/components/migration/moz.build
index 020bf15802..50bb99b0f6 100644
--- a/browser/components/migration/moz.build
+++ b/browser/components/migration/moz.build
@@ -26,13 +26,16 @@ EXTRA_JS_MODULES += [
     "ChromeMigrationUtils.sys.mjs",
     "ChromeProfileMigrator.sys.mjs",
     "FileMigrators.sys.mjs",
-    "FirefoxProfileMigrator.sys.mjs",
     "InternalTestingProfileMigrator.sys.mjs",
     "MigrationUtils.sys.mjs",
     "MigratorBase.sys.mjs",
     "ProfileMigrator.sys.mjs",
 ]
 
+EXTRA_PP_JS_MODULES += [
+    "FirefoxProfileMigrator.sys.mjs",
+]
+
 FINAL_TARGET_FILES.actors = [
     "MigrationWizardChild.sys.mjs",
     "MigrationWizardParent.sys.mjs",
-- 
2.39.2 (Apple Git-143)