Merge pull request #7 from mozilla/master

move sentence migration after migration from S3
common-voice · Jan 21, 2018 · ae30a0d · ae30a0d
2 parents 9c6ce16 + ae200a6
commit ae30a0d
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 7 deletions.
diff --git a/server/src/lib/model/db/migrate-data/migrate-clip.ts b/server/src/lib/model/db/migrate-data/migrate-clip.ts
@@ -22,10 +22,11 @@ export async function migrateClip(
       [clip.client_id, clip.original_sentence_id, clip.path, sentenceText]
     );
 
-  const [[sentence]] = (await connection.query(
-    'SELECT * FROM sentences WHERE id = ?',
-    [clip.original_sentence_id]
-  )) as any;
+  const [
+    [sentence],
+  ] = (await connection.query('SELECT * FROM sentences WHERE id = ?', [
+    clip.original_sentence_id,
+  ])) as any;
 
   if (sentence) {
     await insertClip(sentence.text);
@@ -41,7 +42,7 @@ export async function migrateClip(
 
     if (sentenceText) {
       await connection.execute(
-        'INSERT INTO sentences (id, text) VALUES (?, ?) ON DUPLICATE KEY UPDATE id = id',
+        'INSERT INTO sentences (id, text, is_used) VALUES (?, ?, FALSE) ON DUPLICATE KEY UPDATE is_used = FALSE',
         [clip.original_sentence_id, sentenceText]
       );
       await insertClip(sentenceText);

diff --git a/server/src/lib/model/db/migrate-data/migrate.ts b/server/src/lib/model/db/migrate-data/migrate.ts
@@ -13,12 +13,17 @@ function print(...args: any[]) {
   console.log.apply(console, args);
 }
 
+/**
+ * Migrates all the data from S3 into the database
+ * Most sentences are retrieved from disk (inside the data dir), but some might be missing and will be fetched from S3.
+ * Initially that sentence migration happened first, to reduce the number of roundtrips. But since we don't get whether
+ * the sentence is_used from S3, we have to migrate the sentences from S3 first, and then set the is_used flag based
+ * on whether the sentence sits in the "not-used" dir or not.
+ */
 export async function migrate(connection: IConnection) {
   print('starting');
 
   try {
-    await migrateSentences(connection, print);
-
     const gen = fetchS3Data(print);
 
     const votesWithUnknownClips = [];
@@ -53,6 +58,9 @@ export async function migrate(connection: IConnection) {
         print('processed', processedObjectsCount, 'objects from S3');
       }
     }
+
+    await migrateSentences(connection, print);
+
     const [[row]] = await connection.execute(
       'SELECT ' +
         '(SELECT COUNT(*) FROM sentences) AS sentencesCount,' +