Merge pull request #1461 from mozilla/fennec-history-import

Fennec history import

components/places/android/src/main/java/mozilla/appservices/places/LibPlacesFFI.kt

@@ -37,6 +37,12 @@ internal interface LibPlacesFFI : Library {
         out_err: RustError.ByReference
     ): PlacesConnectionHandle
 
+    fun places_history_import_from_fennec(
+        handle: PlacesApiHandle,
+        db_path: String,
+        out_err: RustError.ByReference
+    )
+
     fun places_note_observation(
         handle: PlacesConnectionHandle,
         json_observation_data: String,

components/places/android/src/main/java/mozilla/appservices/places/PlacesConnection.kt

@@ -411,6 +411,13 @@ class PlacesWriterConnection internal constructor(connHandle: Long, api: PlacesA
         deleteVisitsBetween(since, Long.MAX_VALUE)
     }
 
+    override fun importVisitsFromFennec(path: String) {
+        rustCall { error ->
+            LibPlacesFFI.INSTANCE.places_history_import_from_fennec(
+                    this.handle.get(), path, error)
+        }
+    }
+
     override fun deleteVisitsBetween(startTime: Long, endTime: Long) {
         rustCall { error ->
             LibPlacesFFI.INSTANCE.places_delete_visits_between(
@@ -740,6 +747,16 @@ interface WritableHistoryConnection : ReadableHistoryConnection {
      */
     fun deleteVisitsSince(since: Long)
 
+    /**
+     * Imports visits from a Fennec `browser.db` database.
+     *
+     * It has been designed exclusively for non-sync users and should
+     * be called before bookmarks import.
+     *
+     * @param path Path to the `browser.db` file database.
+     */
+    fun importVisitsFromFennec(path: String)
+
     /**
      * Equivalent to deleteVisitsSince, but takes an `endTime` as well.
      *

components/places/ffi/src/lib.rs

@@ -75,6 +75,7 @@ pub extern "C" fn places_connection_new(
         Ok(CONNECTIONS.insert(api.open_connection(conn_type)?))
     })
 }
+
 #[no_mangle]
 pub extern "C" fn places_bookmarks_import_from_ios(
     api_handle: u64,
@@ -88,6 +89,19 @@ pub extern "C" fn places_bookmarks_import_from_ios(
     })
 }
 
+#[no_mangle]
+pub extern "C" fn places_history_import_from_fennec(
+    api_handle: u64,
+    db_path: FfiStr<'_>,
+    error: &mut ExternError,
+) {
+    log::debug!("places_history_import_from_fennec");
+    APIS.call_with_result(error, api_handle, |api| -> places::Result<_> {
+        places::import::import_fennec_history(api, db_path.as_str())?;
+        Ok(())
+    })
+}
+
 // Best effort, ignores failure.
 #[no_mangle]
 pub extern "C" fn places_api_return_write_conn(

components/places/src/db/db.rs

@@ -195,7 +195,7 @@ fn define_functions(c: &Connection) -> Result<()> {
     Ok(())
 }
 
-mod sql_fns {
+pub(crate) mod sql_fns {
     use crate::api::matcher::{split_after_host_and_port, split_after_prefix};
     use crate::hash;
     use crate::match_impl::{AutocompleteMatch, MatchBehavior, SearchBehavior};

components/places/src/error.rs

@@ -74,6 +74,9 @@ pub enum ErrorKind {
 
     #[fail(display = "Database cannot be upgraded")]
     DatabaseUpgradeError,
+
+    #[fail(display = "Database version {} is not supported", _0)]
+    UnsupportedDatabaseVersion(i64),
 }
 
 error_support::define_error! {

components/places/src/import/common.rs

@@ -0,0 +1,112 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use crate::api::places_api::SyncConn;
+use crate::error::*;
+use rusqlite::named_params;
+use url::Url;
+
+pub mod sql_fns {
+    use crate::storage::URL_LENGTH_MAX;
+    use crate::types::Timestamp;
+    use rusqlite::{functions::Context, types::ValueRef, Result};
+    use url::Url;
+
+    #[inline(never)]
+    pub fn sanitize_timestamp(ctx: &Context<'_>) -> Result<Timestamp> {
+        let now = Timestamp::now();
+        Ok(if let Ok(ts) = ctx.get::<Timestamp>(0) {
+            if Timestamp::EARLIEST < ts && ts < now {
+                ts
+            } else {
+                now
+            }
+        } else {
+            now
+        })
+    }
+
+    #[inline(never)]
+    pub fn validate_url(ctx: &Context<'_>) -> Result<Option<String>> {
+        let val = ctx.get_raw(0);
+        let href = if let ValueRef::Text(s) = val {
+            std::str::from_utf8(s)?
+        } else {
+            return Ok(None);
+        };
+        if href.len() > URL_LENGTH_MAX {
+            return Ok(None);
+        }
+        if let Ok(url) = Url::parse(href) {
+            Ok(Some(url.into_string()))
+        } else {
+            Ok(None)
+        }
+    }
+
+    #[inline(never)]
+    pub fn is_valid_url(ctx: &Context<'_>) -> Result<Option<bool>> {
+        Ok(match ctx.get_raw(0) {
+            ValueRef::Text(s) if s.len() <= URL_LENGTH_MAX => {
+                if let Ok(s) = std::str::from_utf8(s) {
+                    Some(Url::parse(s).is_ok())
+                } else {
+                    Some(false)
+                }
+            }
+            _ => Some(false),
+        })
+    }
+}
+
+pub fn attached_database<'a>(
+    conn: &'a SyncConn<'a>,
+    path: &Url,
+    db_alias: &'static str,
+) -> Result<ExecuteOnDrop<'a>> {
+    conn.execute_named(
+        "ATTACH DATABASE :path AS :db_alias",
+        named_params! {
+            ":path": path.as_str(),
+            ":db_alias": db_alias,
+        },
+    )?;
+    Ok(ExecuteOnDrop {
+        conn,
+        sql: format!("DETACH DATABASE {};", db_alias),
+    })
+}
+
+/// We use/abuse the mirror to perform our import, but need to clean it up
+/// afterwards. This is an RAII helper to do so.
+///
+/// Ideally, you should call `execute_now` rather than letting this drop
+/// automatically, as we can't report errors beyond logging when running
+/// Drop.
+pub struct ExecuteOnDrop<'a> {
+    conn: &'a SyncConn<'a>,
+    sql: String,
+}
+
+impl<'a> ExecuteOnDrop<'a> {
+    pub fn new(conn: &'a SyncConn<'a>, sql: String) -> Self {
+        Self { conn, sql }
+    }
+
+    pub fn execute_now(self) -> Result<()> {
+        self.conn.execute_batch(&self.sql)?;
+        // Don't run our `drop` function.
+        std::mem::forget(self);
+        Ok(())
+    }
+}
+
+impl Drop for ExecuteOnDrop<'_> {
+    fn drop(&mut self) {
+        if let Err(e) = self.conn.execute_batch(&self.sql) {
+            log::error!("Failed to clean up after import! {}", e);
+            log::debug!("  Failed query: {}", &self.sql);
+        }
+    }
+}

components/places/src/import/fennec.rs

@@ -0,0 +1,6 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+mod history;
+pub use history::import as import_history;

components/places/src/import/fennec/history.rs

@@ -0,0 +1,121 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use crate::api::places_api::PlacesApi;
+use crate::bookmark_sync::store::BookmarksStore;
+use crate::error::*;
+use crate::import::common::attached_database;
+use rusqlite::Connection;
+use sql_support::ConnExt;
+use url::Url;
+
+// From https://searchfox.org/mozilla-central/rev/597a69c70a5cce6f42f159eb54ad1ef6745f5432/mobile/android/base/java/org/mozilla/gecko/db/BrowserDatabaseHelper.java#73.
+const FENNEC_DB_VERSION: i64 = 39;
+
+pub fn import(places_api: &PlacesApi, path: impl AsRef<std::path::Path>) -> Result<()> {
+    let url = crate::util::ensure_url_path(path)?;
+    do_import(places_api, url)
+}
+
+fn do_import(places_api: &PlacesApi, android_db_file_url: Url) -> Result<()> {
+    let conn = places_api.open_sync_connection()?;
+
+    let scope = conn.begin_interrupt_scope();
+
+    define_sql_functions(&conn)?;
+
+    // Not sure why, but apparently beginning a transaction sometimes
+    // fails if we open the DB as read-only. Hopefully we don't
+    // unintentionally write to it anywhere...
+    // android_db_file_url.query_pairs_mut().append_pair("mode", "ro");
+
+    log::trace!("Attaching database {}", android_db_file_url);
+    let auto_detach = attached_database(&conn, &android_db_file_url, "fennec")?;
+
+    let db_version = conn.db.query_one::<i64>("PRAGMA fennec.user_version")?;
+    if db_version != FENNEC_DB_VERSION {
+        return Err(ErrorKind::UnsupportedDatabaseVersion(db_version).into());
+    }
+
+    let tx = conn.begin_transaction()?;
+
+    log::debug!("Populating missing entries in moz_places");
+    conn.execute_batch(&FILL_MOZ_PLACES)?;
+    scope.err_if_interrupted()?;
+
+    log::debug!("Inserting the history visits");
+    conn.execute_batch(&INSERT_HISTORY_VISITS)?;
+    scope.err_if_interrupted()?;
+
+    log::debug!("Committing...");
+    tx.commit()?;
+
+    // Note: update_frecencies manages its own transaction, which is fine,
+    // since nothing that bad will happen if it is aborted.
+    log::debug!("Updating frecencies");
+    let store = BookmarksStore::new(&conn, &scope);
+    store.update_frecencies()?;
+
+    log::info!("Successfully imported history visits!");
+
+    auto_detach.execute_now()?;
+
+    Ok(())
+}
+
+lazy_static::lazy_static! {
+    // Insert any missing entries into moz_places that we'll need for this.
+    static ref FILL_MOZ_PLACES: &'static str =
+        "INSERT OR IGNORE INTO main.moz_places(guid, url, url_hash, title, frecency, sync_change_counter)
+            SELECT
+                IFNULL(
+                    (SELECT p.guid FROM main.moz_places p WHERE p.url_hash = hash(h.url) AND p.url = h.url),
+                    generate_guid()
+                ),
+                h.url,
+                hash(h.url),
+                h.title,
+                -1,
+                1
+            FROM fennec.history h
+            WHERE is_valid_url(h.url)"
+    ;
+
+    // Insert history visits
+    static ref INSERT_HISTORY_VISITS: &'static str =
+        "INSERT OR IGNORE INTO main.moz_historyvisits(from_visit, place_id, visit_date, visit_type, is_local)
+            SELECT
+                NULL, -- Fenec does not store enough information to rebuild redirect chains.
+                (SELECT p.id FROM main.moz_places p WHERE p.url_hash = hash(h.url) AND p.url = h.url),
+                sanitize_timestamp(v.date),
+                v.visit_type, -- Fennec stores visit types maps 1:1 to ours.
+                v.is_local
+            FROM fennec.visits v
+            LEFT JOIN fennec.history h on v.history_guid = h.guid
+            WHERE is_valid_url(h.url)"
+    ;
+}
+
+pub(super) fn define_sql_functions(c: &Connection) -> Result<()> {
+    c.create_scalar_function(
+        "is_valid_url",
+        1,
+        true,
+        crate::import::common::sql_fns::is_valid_url,
+    )?;
+    c.create_scalar_function(
+        "sanitize_timestamp",
+        1,
+        true,
+        crate::import::common::sql_fns::sanitize_timestamp,
+    )?;
+    c.create_scalar_function("hash", -1, true, crate::db::db::sql_fns::hash)?;
+    c.create_scalar_function(
+        "generate_guid",
+        0,
+        false,
+        crate::db::db::sql_fns::generate_guid,
+    )?;
+    Ok(())
+}