Merge branch 'master' of github.com:mozilla-services/syncstorage-rs

.circleci/config.yml

@@ -28,8 +28,8 @@ commands:
       - run:
           name: Core Rust Checks
           command: |
-            cargo audit
             cargo fmt -- --check
+            cargo audit
   rust-clippy:
     steps:
       - run:

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "vendor/mozilla-rust-sdk/googleapis-raw/grpc"]
+	path = vendor/mozilla-rust-sdk/googleapis-raw/grpc
+	url = https://github.com/grpc/grpc.git

Cargo.toml

@@ -5,6 +5,7 @@ license = "MPL-2.0"
 authors = [
   "Ben Bangert <ben@groovie.org>",
   "Phil Jenvey <pjenvey@underboss.org>",
+  "Mozilla Services Engineering <services-engineering+code@mozilla.com>"
 ]
 edition = "2018"
 
@@ -17,58 +18,53 @@ actix-http = "1"
 actix-web = "2"
 actix-rt = "1"
 actix-cors = "0.2"
-base64 = "0.11.0"
+base64 = "0.12"
 bytes = "0.5"
 cadence = "0.19.1"
 chrono = "0.4"
-config = "0.9.3"
-diesel = { version = "1.4.3", features = ["mysql", "r2d2"] }
+config = "0.10"
+diesel = { version = "1.4.4", features = ["mysql", "r2d2"] }
 diesel_logger = "0.1.0"
 diesel_migrations = { version = "1.4.0", features = ["mysql"] }
 docopt = "1.1.0"
 env_logger = "0.7.1"
-failure = "0.1.6"
+failure = "0.1.7"
 futures = { version = "0.3", features = ["compat"] }
 googleapis-raw = { version = "0", path = "vendor/mozilla-rust-sdk/googleapis-raw" }
-grpcio = { version = "0.5.0-alpha.5" }
+grpcio = { version = "0.5.0" }
 lazy_static = "1.4.0"
-hawk = "3.0.0"
+hawk = "3.1.0"
 hkdf = "0.8.0"
 hmac = "0.7"
-itertools = "0.8.2"
 log = { version = "0.4.8", features = ["max_level_info", "release_max_level_info"] }
 mime = "0.3"
 mozsvc-common = "0.1"
-num_cpus = "1.11"
 # must match what's used by googleapis-raw
-protobuf = "2.7.0"
-openssl ="0.10"
+protobuf = "2.12"
 rand = "0.7"
 regex = "1.3"
-sentry = { version = "0.17.0", features = ["with_curl_transport"] }
+sentry = { version = "0.18", features = ["with_curl_transport"] }
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = { version = "1.0", features = ["arbitrary_precision"] }
-serde_urlencoded = "0.6.1"
 scheduled-thread-pool = "0.2"
-sha2 = "0.8.0"
-slog = { version = "2.5", features = ["max_level_trace", "release_max_level_error", "dynamic-keys"] }
-slog-async = "2.3"
+sha2 = "0.8"
+slog = { version = "2.5", features = ["max_level_info", "release_max_level_info", "dynamic-keys"] }
+slog-async = "2.5"
 slog-envlogger = "2.2.0"
 slog-mozlog-json = "0.1"
 slog-scope = "4.3"
 slog-stdlog = "4.0"
-slog-term = "2.4"
-time = "0.1.42"
-tokio = "0.2.9"
-url = "2.1.0"
+slog-term = "2.5"
+time = "0.2"
+url = "2.1"
 uuid = { version = "0.8.1", features = ["serde", "v4"] }
 validator = "0.10"
 validator_derive = "0.10"
-woothee = "0.10"
+woothee = "0.11"
 
 [dev-dependencies]
-codegen = { version = "0.1.0", path = "codegen" }
+futures-await-test = "0.3.0"
 
 [features]
 no_auth = []

Dockerfile

@@ -11,7 +11,8 @@ RUN apt-get -q update && \
 RUN \
     cargo --version && \
     rustc --version && \
-    cargo install --path . --locked --root /app
+    cargo install --path . --locked --root /app && \
+    cargo install --path tools/spanner/purge_ttl --locked --root /app
 
 FROM debian:buster-slim
 WORKDIR /app

src/db/mysql/models.rs

@@ -67,6 +67,7 @@ struct MysqlDbSession {
     coll_locks: HashMap<(u32, i32), CollectionLock>,
     /// Whether a transaction was started (begin() called)
     in_transaction: bool,
+    in_write_transaction: bool,
 }
 
 #[derive(Clone, Debug)]
@@ -162,7 +163,7 @@ impl MysqlDb {
         }
 
         // Lock the db
-        self.begin()?;
+        self.begin(false)?;
         let modified = user_collections::table
             .select(user_collections::modified)
             .filter(user_collections::user_id.eq(user_id as i32))
@@ -198,7 +199,7 @@ impl MysqlDb {
         }
 
         // Lock the db
-        self.begin()?;
+        self.begin(true)?;
         let modified = user_collections::table
             .select(user_collections::modified)
             .filter(user_collections::user_id.eq(user_id as i32))
@@ -224,11 +225,14 @@ impl MysqlDb {
         Ok(())
     }
 
-    pub(super) fn begin(&self) -> Result<()> {
+    pub(super) fn begin(&self, for_write: bool) -> Result<()> {
         self.conn
             .transaction_manager()
             .begin_transaction(&self.conn)?;
         self.session.borrow_mut().in_transaction = true;
+        if for_write {
+            self.session.borrow_mut().in_write_transaction = true;
+        }
         Ok(())
     }
 
@@ -269,7 +273,7 @@ impl MysqlDb {
 
     pub fn delete_storage_sync(&self, user_id: HawkIdentifier) -> Result<()> {
         let user_id = user_id.legacy_id as i32;
-        self.begin()?;
+        self.begin(true)?;
         // Delete user data.
         delete(bso::table)
             .filter(bso::user_id.eq(user_id))
@@ -317,7 +321,6 @@ impl MysqlDb {
             .execute(&self.conn)?;
             collections::table.select(last_insert_id).first(&self.conn)
         })?;
-        self.coll_cache.put(id, name.to_owned())?;
         Ok(id)
     }
 
@@ -343,7 +346,9 @@ impl MysqlDb {
         .optional()?
         .ok_or(DbErrorKind::CollectionNotFound)?
         .id;
-        self.coll_cache.put(id, name.to_owned())?;
+        if !self.session.borrow().in_write_transaction {
+            self.coll_cache.put(id, name.to_owned())?;
+        }
         Ok(id)
     }
 
@@ -775,7 +780,9 @@ impl MysqlDb {
 
             for (id, name) in result {
                 names.insert(id, name.clone());
-                self.coll_cache.put(id, name)?;
+                if !self.session.borrow().in_write_transaction {
+                    self.coll_cache.put(id, name)?;
+                }
             }
         }
 

src/db/spanner/models.rs

@@ -40,9 +40,6 @@ use googleapis_raw::spanner::v1::{
     type_pb::TypeCode,
 };
 
-#[allow(unused_imports)]
-use itertools::Itertools;
-
 #[allow(unused_imports)]
 use protobuf::{well_known_types::ListValue, Message, RepeatedField};
 
@@ -1567,7 +1564,7 @@ impl SpannerDb {
 
         for pbso in input.bsos {
             let id = pbso.id;
-            self.put_bso_async(params::PutBso {
+            self.put_bso_async_test(params::PutBso {
                 user_id: input.user_id.clone(),
                 collection: input.collection.clone(),
                 id: id.clone(),

src/db/tests/batch.rs

@@ -1,4 +1,4 @@
-use codegen::async_test;
+use futures_await_test::async_test;
 use log::debug;
 
 use super::support::{db, gbso, hid, postbso, Result};

src/db/tests/db.rs

@@ -4,7 +4,7 @@ use std::collections::HashMap;
 use lazy_static::lazy_static;
 use rand::{distributions::Alphanumeric, thread_rng, Rng};
 
-use codegen::async_test;
+use futures_await_test::async_test;
 
 use super::support::{db, dbso, dbsos, gbso, gbsos, hid, pbso, postbso, Result};
 use crate::db::{mysql::models::DEFAULT_BSO_TTL, params, util::SyncTimestamp, Sorting};

src/server/mod.rs

@@ -2,17 +2,16 @@
 
 use std::sync::Arc;
 
-use actix_cors::Cors;
-use actix_web::{
-    dev, http::StatusCode, middleware::errhandlers::ErrorHandlers, web, App, HttpRequest,
-    HttpResponse, HttpServer,
-};
-// use num_cpus;
 use crate::db::{pool_from_settings, DbPool};
 use crate::error::ApiError;
 use crate::server::metrics::Metrics;
 use crate::settings::{Secrets, ServerLimits, Settings};
 use crate::web::{handlers, middleware};
+use actix_cors::Cors;
+use actix_web::{
+    dev, http::StatusCode, middleware::errhandlers::ErrorHandlers, web, App, HttpRequest,
+    HttpResponse, HttpServer,
+};
 use cadence::StatsdClient;
 
 pub const BSO_ID_REGEX: &str = r"[ -~]{1,64}";

src/web/auth.rs

@@ -1,7 +1,12 @@
 //! Types for parsing and authenticating HAWK headers.
 //! Matches the [Python logic](https://github.com/mozilla-services/tokenlib).
 //! We may want to extract this to its own repo/crate in due course.
-#![cfg_attr(feature = "no_auth", allow(dead_code, unused_imports, unused_variables))]
+#![cfg_attr(
+    feature = "no_auth",
+    allow(dead_code, unused_imports, unused_variables)
+)]
+
+use std::convert::TryInto;
 
 use base64;
 use chrono::offset::Utc;
@@ -95,8 +100,8 @@ impl HawkPayload {
 
         #[cfg(not(feature = "no_auth"))]
         {
-            let mut duration = Duration::weeks(52)
-                .to_std()
+            let mut duration: std::time::Duration = Duration::weeks(52)
+                .try_into()
                 .map_err(|_| ApiErrorKind::Internal("Duration::weeks".to_owned()))?;
             if cfg!(test) {
                 // test cases are valid until 3018. Add millenia as required.