Merge branch 'stage' into production

bundler/src/core/clips.ts

@@ -2,9 +2,12 @@ import fs from 'node:fs'
 import path from 'node:path'
 import { Transform } from 'node:stream'
 import { streamingQuery } from '../infrastructure/database'
-import { streamDownloadFileFromBucket } from '../infrastructure/storage'
+import {
+  doesFileExistInBucket,
+  streamDownloadFileFromBucket,
+} from '../infrastructure/storage'
 import { AppEnv, ClipRow } from '../types'
-import { taskEither as TE } from 'fp-ts'
+import { task as T, taskEither as TE } from 'fp-ts'
 import { hashClientId } from './clients'
 import { getClipsBucketName, getQueriesDir } from '../config/config'
 import { pipe } from 'fp-ts/lib/function'
@@ -107,6 +110,25 @@ const downloadClips = (releaseDirPath: string) =>
     objectMode: true,
   })
 
+const checkClipForExistence = () =>
+  new Transform({
+    transform(chunk: ClipRow, encoding, callback) {
+      pipe(
+        doesFileExistInBucket(CLIPS_BUCKET)(chunk.path),
+        TE.getOrElse(() => T.of(false)),
+      )().then(doesExist => {
+        if (doesExist) {
+          this.push(chunk, encoding)
+          callback()
+        } else {
+          console.log(`Skipping file ${chunk.path}`)
+          callback()
+        }
+      })
+    },
+    objectMode: true,
+  })
+
 const fetchAllClipsForLocale = (
   locale: string,
   includeClipsFrom: string,
@@ -127,6 +149,7 @@ const fetchAllClipsForLocale = (
         )
         console.log('Start Stream Processing')
         stream
+          .pipe(checkClipForExistence())
           .pipe(downloadClips(releaseDirPath))
           .pipe(transformClips(isMinorityLanguage))
           .pipe(writeFileStreamToTsv(locale, releaseDirPath))

bundler/src/infrastructure/storage.ts

@@ -37,14 +37,6 @@ const streamUpload =
     )
   }
 
-const crc32 =
-  (storage: Storage) =>
-  (data: Buffer): string => {
-    const crc23c = storage.crc32cGenerator()
-    crc23c.update(data)
-    return crc23c.toString()
-  }
-
 const upload =
   (storage: Storage) =>
   (bucket: string) =>
@@ -93,9 +85,22 @@ const streamDownloadFile =
     return storage.bucket(bucket).file(path).createReadStream()
   }
 
+const fileExists =
+  (storage: Storage) =>
+  (bucket: string) =>
+  (path: string): TE.TaskEither<Error, boolean> => {
+    return TE.tryCatch(
+      async () => {
+        const [exists] = await storage.bucket(bucket).file(path).exists()
+        return exists
+      },
+      (err: unknown) => Error(String(err)),
+    )
+  }
+
 export const streamUploadToBucket = streamUpload(storage)
 export const uploadToBucket = upload(storage)
 export const getMetadataFromFile = getMetadata(storage)
 export const downloadFileFromBucket = downloadFile(storage)
 export const streamDownloadFileFromBucket = streamDownloadFile(storage)
-export const crc23cStorage = crc32(storage)
+export const doesFileExistInBucket = fileExists(storage)

server/src/application/bulk-submissions/repository/bulk-submission-repository.ts

@@ -36,16 +36,17 @@ type BulkSubmissionStatusId = {
 
 const createInsertQuery = () =>
   `
-    INSERT IGNORE INTO bulk_submissions (status, locale_id, size, path, name, submitter, import_status) 
+    INSERT INTO bulk_submissions (status, locale_id, size, path, name, submitter, import_status) 
     VALUES (?, ?, ?, ?, ?, ?, ?)
+    ON DUPLICATE KEY UPDATE updated_at = NOW()
   `
 const insertBulkSubmission =
   (db: Mysql) => (bulkSubmission: BulkSubmission) => {
     return TE.tryCatch(
       async () => {
-
+        const [[statusNew]] = await db.query('SELECT id from bulk_submission_status WHERE status = \'new\'')
         await db.query(createInsertQuery(), [
-          `(SELECT id from bulk_submission_status WHERE status = 'open')`,
+          statusNew.id,
           bulkSubmission.localeId,
           bulkSubmission.size,
           bulkSubmission.path,
@@ -56,7 +57,10 @@ const insertBulkSubmission =
 
         return true
       },
-      (err: Error) => err
+      (err: unknown) => {
+        console.log(err)
+        return Error(String(err))
+      }
     )
   }
 

server/src/infrastructure/queues/bulkSubmissionQueue.ts

@@ -16,7 +16,7 @@ const BULK_SUBMISSION_UPLOAD_JOB_NAME = 'bulk-submission-upload-job'
 export const setupBulkSubmissionQueue = () => {
   return pipe(
     getQueue<BulkSubmissionUploadJob>(BULK_SUBMISSION_UPLOAD_QUEUE_NAME),
-    IO.chainFirst(addProcessorToQueue(bulkSubmissionUploadProcessor)),
+    IO.chainFirst(addProcessorToQueue(bulkSubmissionUploadProcessor)(BULK_SUBMISSION_UPLOAD_JOB_NAME)),
     IO.chainFirst(attachEventHandlerToQueue('error')(console.error)),
     IO.chainFirst(attachEventHandlerToQueue('failure')(console.error))
   )()

server/src/infrastructure/queues/queues.ts

@@ -31,8 +31,9 @@ export const getQueue = createQueueWithOptions(getRedisConfig(getConfig()))
 
 export const addProcessorToQueue =
   <T>(processor: Queue.ProcessPromiseFunction<T>) =>
+  (processorName: string) =>
   (q: Queue.Queue<T>): IO.IO<void> => {
-    q.process(processor)
+    q.process(processorName, processor)
     return IO.of(constVoid())
   }
 
@@ -45,9 +46,9 @@ export const addSandboxedProcessorToQueue =
 
 export const attachEventHandlerToQueue =
   (event: string) =>
-  <T>(errorHandler: (job: Queue.Job<T>) => any) =>
+  <T>(eventHandler: (job: Queue.Job<T>) => any) =>
   (q: Queue.Queue<T>): IO.IO<void> => {
-    return IO.of(q.on(event, errorHandler))
+    return IO.of(q.on(event, eventHandler))
   }
 
 export const addJobToQueue =

web/cypress/e2e/home-page.cy.ts

@@ -1,6 +1,6 @@
 describe('The Home Page', () => {
   it('successfully loads', () => {
-    cy.visit('http://localhost:9000')
+    cy.visit('/')
 
     cy.contains('Common Voice')
     cy.contains('Speak')

web/cypress/e2e/listen-page.cy.ts

@@ -0,0 +1,115 @@
+describe('The Listen Page', () => {
+  it('successfully loads', () => {
+    cy.visit('/listen')
+
+    cy.get('[data-testid=listen]').should('exist')
+  })
+
+  it('can listen to a clip', () => {
+    cy.visit('/listen')
+
+    const playButton = cy.get('[data-testid=play-button]')
+
+    playButton.should('exist')
+
+    // assert video is paused
+    cy.get('audio').should('have.prop', 'paused', true)
+
+    playButton.click()
+
+    // assert video is playing
+    cy.get('audio').should('have.prop', 'paused', false)
+  })
+
+  it('votes yes', () => {
+    cy.visit('/listen')
+
+    const yesButton = cy.get('[data-testid=vote-yes-button]')
+
+    // assert yes and no buttons should be disabled before listening to a clip
+    yesButton.should('be.disabled')
+
+    const playButton = cy.get('[data-testid=play-button]')
+
+    playButton.click()
+
+    // vote yes
+    yesButton.click()
+
+    cy.get('[data-testid=instruction]').should(
+      'have.text',
+      "Great work! Listen again when you're ready"
+    )
+  })
+
+  it('votes no', () => {
+    cy.visit('/listen')
+
+    const playButton = cy.get('[data-testid=play-button]')
+    const noButton = cy.get('[data-testid=vote-no-button]')
+
+    // play the next clip
+    playButton.click()
+
+    // vote no
+    noButton.click()
+
+    cy.get('[data-testid=instruction]').should(
+      'have.text',
+      "Great work! Listen again when you're ready"
+    )
+  })
+
+  it('skips a clip', () => {
+    cy.visit('/listen')
+
+    const skipButton = cy.get('[data-testid=skip-button]')
+
+    cy.get('[data-testid=card-1]')
+      .invoke('text')
+      .then(text1 => {
+        skipButton.click()
+
+        cy.get('[data-testid=card-1]')
+          .invoke('text')
+          .should(text2 => {
+            // when skipped the contents of the card should be different
+            expect(text1).not.eq(text2)
+          })
+      })
+  })
+
+  it('listens to 5 clips', () => {
+    cy.visit('/listen')
+
+    const playButton = cy.get('[data-testid=play-button]')
+
+    const yesButton = cy.get('[data-testid=vote-yes-button]')
+    const noButton = cy.get('[data-testid=vote-no-button]')
+
+    playButton.click()
+    noButton.click()
+
+    playButton.click()
+    yesButton.click()
+
+    playButton.click()
+    noButton.click()
+
+    playButton.click()
+    noButton.click()
+
+    playButton.click()
+    yesButton.click()
+
+    const successPage = cy.get('[data-testid=contribution-success]')
+    successPage.should('exist')
+
+    const contributeMoreButton = cy.get('[data-testid=contribute-more-button]')
+
+    contributeMoreButton.click()
+
+    // expect card to be visible
+    cy.get('[data-testid=card-1]').should('exist')
+  })
+})

web/locales/bg/messages.ftl

@@ -84,7 +84,9 @@ ja = Японски
 ka = Грузински
 kaa = Каракалпакски
 kab = Кабилски
+kbd = Кабардински
 kk = Казахкски
+kmr = Курманджи кюрдски
 ko = Корейски
 kpv = Коми-зирянски
 kw = Корнуолски
@@ -100,6 +102,7 @@ mhr = Мари
 mk = Македонски
 ml = Малаялам
 mn = Монголски
+mos = Море
 mr = Марати
 mrj = Уралски
 ms = Малайски
@@ -966,3 +969,10 @@ contribution-criteria-page-description = Разберете на какво да
 
 # [/SentenceCollector]
 
+sentence-collection = Списък с изречения
+
+## REVIEW PAGE
+
+
+## BULK SUBMISSION 
+

web/locales/cy/messages.ftl

@@ -882,6 +882,8 @@ language-section-in-progress = Ar y Ffordd
 language-section-in-progress-new-description = Mae'r ieithoedd hyn ar hyn o bryd o dan ddatblygiad cymunedol. Mae'r bariau cynnydd yn nodi pa mor bell y mae pob iaith yn y broses o <localizationGlossaryLink>leoleiddio'u gwefan</localizationGlossaryLink> a <sentenceCollectionGlossaryLink>chasglu brawddegau</sentenceCollectionGlossaryLink>.
 language-section-launched = Ar Gael
 language-section-launched-description = Ar gyfer yr ieithoedd sydd wedi'u lansio, mae'r wefan wedi'i <localizationGlossaryLink> leoleiddio'n </localizationGlossaryLink>llwyddiannus, ac mae ganddo ddigon o <sentenceCollectionGlossaryLink>frawddegau wedi'u casglu</sentenceCollectionGlossaryLink> i ganiatáu cyfraniadau <speakLink>Siarad</speakLink> a <listenLink>Gwrando</ listenLink>.
+# lastUpdatedTimeStamp is a timestamp that indicates when the language stats was last updated
+language-section-last-updated = Diweddariad Diwethaf: { $lastUpdate }
 languages-show-more = Gweld Mwy
 languages-show-less = Gweld Llai
 language-meter-in-progress = Cynnydd

web/locales/de/messages.ftl

@@ -863,6 +863,8 @@ language-section-in-progress = Vorbereitungsphase
 language-section-in-progress-new-description = Diese Sprachen werden derzeit durch die Gemeinschaft entwickelt. Die Fortschrittsbalken zeigen an, wie weit bei den einzelnen Sprachen die <localizationGlossaryLink>Website-Lokalisierung</localizationGlossaryLink> und die <sentenceCollectionGlossaryLink>Sammlung von Sätzen</sentenceCollectionGlossaryLink> umgesetzt wurde.
 language-section-launched = Aufnahmephase
 language-section-launched-description = Für diese aktiven Sprachen wurde die Website erfolgreich <localizationGlossaryLink>lokalisiert</localizationGlossaryLink>, und es wurden ausreichend <satzCollectionGlossaryLink>Sätze gesammelt</satzCollectionGlossaryLink>, damit weiterhin Beiträge <speakLink>gesprochen</speakLink> und <listenLink>angehört</listenLink> werden können.
+# lastUpdatedTimeStamp is a timestamp that indicates when the language stats was last updated
+language-section-last-updated = Letzte Aktualisierung: { $lastUpdatedTimeStamp }
 languages-show-more = Mehr anzeigen
 languages-show-less = Weniger anzeigen
 language-meter-in-progress = Fortschritt

web/locales/el/messages.ftl

@@ -866,6 +866,8 @@ language-section-in-progress = Σε εξέλιξη
 language-section-in-progress-new-description = Αυτές οι γλώσσες αναπτύσσονται προς το παρόν από την κοινότητα. Οι γραμμές προόδου δείχνουν σε ποιο στάδιο έχει φτάσει κάθε γλώσσα, όσον αφορά τη <localizationGlossaryLink>μετάφραση του ιστοτόπου</localizationGlossaryLink> και τη <sentenceCollectionGlossaryLink>συλλογή προτάσεων</sentenceCollectionGlossaryLink>.
 language-section-launched = Σε λειτουργία
 language-section-launched-description = Για αυτές τις εν λειτουργία γλώσσες, ο ιστότοπος έχει <localizationGlossaryLink>μεταφραστεί</localizationGlossaryLink> επιτυχώς και έχει αρκετές <sentenceCollectionGlossaryLink>συλλεγμένες προτάσεις</sentenceCollectionGlossaryLink> για να είναι η δυνατή η συνεχής συνεισφορά <speakLink>ομιλίας</speakLink> και <listenLink>ακρόασης</listenLink>.
+# lastUpdatedTimeStamp is a timestamp that indicates when the language stats was last updated
+language-section-last-updated = Τελευταία ενημέρωση: { $lastUpdatedTimeStamp }
 languages-show-more = Προβολή περισσότερων
 languages-show-less = Προβολή λιγότερων
 language-meter-in-progress = Πρόοδος

web/locales/fy-NL/messages.ftl

@@ -873,6 +873,8 @@ language-section-in-progress = Yn útfiering
 language-section-in-progress-new-description = Dizze talen wurde op dit stuit troch de mienskip ûntwikkele. De foartgongsbalken jouwe oan hoe fier elke taal is mei de <localizationGlossaryLink>websiteoersetting</localizationGlossaryLink> en it <sentenceCollectionGlossaryLink>sammeljen fan sinnen</sentenceCollectionGlossaryLink>.
 language-section-launched = Lansearre
 language-section-launched-description = Foar dizze beskikbere talen is de website mei sukses <localizationGlossaryLink>oerset</localizationGlossaryLink>, en binne der genôch <sentenceCollectionGlossaryLink>sinnen sammele</sentenceCollectionGlossaryLink> foar trochgeande bydragen oan it <speakLink>sprekken</speakLink> en <listenLink>harkjen</listenLink>.
+# lastUpdatedTimeStamp is a timestamp that indicates when the language stats was last updated
+language-section-last-updated = Lêst bywurke: { $lastUpdatedTimeStamp }
 languages-show-more = Mear besjen
 languages-show-less = Minder besjen
 language-meter-in-progress = Yn útfiering