diff --git a/.github/workflows/api-deploy.yml b/.github/workflows/api-deploy.yml index b8d5b5a4472..3d73f9d1286 100644 --- a/.github/workflows/api-deploy.yml +++ b/.github/workflows/api-deploy.yml @@ -33,9 +33,37 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_DATA }} + api-languages: + uses: JesusFilm/core/.github/workflows/api-deploy-worker.yml@main + with: + name: api-languages + repository: jfp-api-languages + branch: ${{ github.ref_name }} + secrets: + ARANGODB_URL: ${{ secrets.ARANGODB_URL }} + ARANGODB_USER: ${{ secrets.ARANGODB_USER }} + ARANGODB_PASS: ${{ secrets.ARANGODB_PASS }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_DATA }} + api-videos: + uses: JesusFilm/core/.github/workflows/api-deploy-worker.yml@main + with: + name: api-videos + repository: jfp-api-videos + branch: ${{ github.ref_name }} + secrets: + ARANGODB_URL: ${{ secrets.ARANGODB_URL }} + ARANGODB_USER: ${{ secrets.ARANGODB_USER }} + ARANGODB_PASS: ${{ secrets.ARANGODB_PASS }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_DATA }} api-gateway: - needs: [api-journeys, api-users] + needs: [api-journeys, api-users, api-languages, api-videos] uses: JesusFilm/core/.github/workflows/api-gateway-worker.yml@main with: name: api-gateway diff --git a/.github/workflows/api-gateway-worker.yml b/.github/workflows/api-gateway-worker.yml index a19bb18a6c0..d52d405976b 100644 --- a/.github/workflows/api-gateway-worker.yml +++ b/.github/workflows/api-gateway-worker.yml @@ -82,6 +82,22 @@ jobs: if: ${{ inputs.branch }} == 'main' run: rover subgraph introspect http://ad67299f5087f4d1b845926ce1832198-64788321.us-east-2.elb.amazonaws.com:4002/graphql > dist/apps/api-gateway/api-users-schema.graphql + - name: Build api-languages-stage schema + if: ${{ inputs.branch }} == 'stage' + run: rover subgraph introspect https://api-languages-stage.core.jesusfilm.org/graphql > dist/apps/api-gateway/api-languages-schema.graphql + + - name: Build api-languages-main schema + if: ${{ inputs.branch }} == 'main' + run: rover subgraph introspect https://api-languages.core.jesusfilm.org/graphql > dist/apps/api-gateway/api-languages-schema.graphql + + - name: Build api-videos-stage schema + if: ${{ inputs.branch }} == 'stage' + run: rover subgraph introspect https://api-videos-stage.core.jesusfilm.org/graphql > dist/apps/api-gateway/api-videos-schema.graphql + + - name: Build api-videos-main schema + if: ${{ inputs.branch }} == 'main' + run: rover subgraph introspect https://api-videos.core.jesusfilm.org/graphql > dist/apps/api-gateway/api-videos-schema.graphql + - name: Build api-gateway schema run: rover supergraph compose --config apps/api-gateway/supergraph-$ENV_SUFFIX.yml > dist/apps/api-gateway/schema.graphql diff --git a/apps/api-gateway/project.json b/apps/api-gateway/project.json index f0e0b4a1e15..42d71d631f2 100644 --- a/apps/api-gateway/project.json +++ b/apps/api-gateway/project.json @@ -42,6 +42,12 @@ { "command": "nx serve api-users" }, + { + "command": "nx serve api-languages" + }, + { + "command": "nx serve api-videos" + }, { "command": "nx serve api-gateway" }, diff --git a/apps/api-gateway/schema.graphql b/apps/api-gateway/schema.graphql index 782552edb3c..3c11e74a474 100644 --- a/apps/api-gateway/schema.graphql +++ b/apps/api-gateway/schema.graphql @@ -291,6 +291,7 @@ enum join__Graph { JOURNEYS @join__graph(name: "journeys" url: "http://127.0.0.1:4001/graphql") LANGUAGES @join__graph(name: "languages" url: "http://127.0.0.1:4003/graphql") USERS @join__graph(name: "users" url: "http://127.0.0.1:4002/graphql") + VIDEOS @join__graph(name: "videos" url: "http://127.0.0.1:4004/graphql") } type Journey @@ -353,11 +354,12 @@ input JourneyUpdateInput { type Language @join__owner(graph: LANGUAGES) @join__type(graph: LANGUAGES, key: "id") + @join__type(graph: VIDEOS, key: "id") { bcp47: String @join__field(graph: LANGUAGES) id: ID! @join__field(graph: LANGUAGES) iso3: String @join__field(graph: LANGUAGES) - name(languageId: ID, primary: Boolean): [Translation]! @join__field(graph: LANGUAGES) + name(languageId: ID, primary: Boolean): [Translation!]! @join__field(graph: LANGUAGES) } type LinkAction implements Action { @@ -458,6 +460,7 @@ type Query { language(id: ID!): Language @join__field(graph: LANGUAGES) languages(limit: Int, page: Int): [Language!]! @join__field(graph: LANGUAGES) me: User @join__field(graph: USERS) + videos(limit: Int, page: Int, where: VideosFilter): [Video!]! @join__field(graph: VIDEOS) } type RadioOptionBlock implements Block { @@ -701,6 +704,20 @@ enum UserJourneyRole { owner } +type Video + @join__owner(graph: VIDEOS) + @join__type(graph: VIDEOS, key: "id") +{ + description(languageId: ID, primary: Boolean): [Translation!]! @join__field(graph: VIDEOS) + id: ID! @join__field(graph: VIDEOS) + image: String @join__field(graph: VIDEOS) + snippet(languageId: ID, primary: Boolean): [Translation!]! @join__field(graph: VIDEOS) + studyQuestions(languageId: ID, primary: Boolean): [Translation!]! @join__field(graph: VIDEOS) + title(languageId: ID, primary: Boolean): [Translation!]! @join__field(graph: VIDEOS) + variant(languageId: ID): VideoVariant @join__field(graph: VIDEOS) + variantLanguages: [Language!]! @join__field(graph: VIDEOS) +} + type VideoArclight implements VideoContent { languageId: String! mediaComponentId: String! @@ -820,6 +837,11 @@ enum VideoResponseStateEnum { PLAYING } +input VideosFilter { + availableVariantLanguageIds: [ID!] + title: String +} + """ VideoTriggerBlock is a block that indicates the video to navigate to the next block at the designated time. @@ -837,3 +859,22 @@ type VideoTriggerBlock implements Block { """ triggerStart: Int! } + +type VideoVariant { + downloads: [VideoVariantDownload!]! + duration: Int! + hls: String! + language: Language! + subtitle(languageId: ID, primary: Boolean): [Translation!]! +} + +type VideoVariantDownload { + quality: VideoVariantDownloadQuality! + size: Float! + url: String! +} + +enum VideoVariantDownloadQuality { + high + low +} diff --git a/apps/api-gateway/supergraph-main.yml b/apps/api-gateway/supergraph-main.yml index 65557de80ff..1dcd8d48d52 100644 --- a/apps/api-gateway/supergraph-main.yml +++ b/apps/api-gateway/supergraph-main.yml @@ -7,3 +7,11 @@ subgraphs: routing_url: http://api-users-main:4002/graphql schema: file: ../../dist/apps/api-gateway/api-users-schema.graphql + languages: + routing_url: http://api-languages-main:4003/graphql + schema: + file: ../../dist/apps/api-gateway/api-languages-schema.graphql + videos: + routing_url: http://api-videos-main:4004/graphql + schema: + file: ../../dist/apps/api-gateway/api-videos-schema.graphql diff --git a/apps/api-gateway/supergraph-stage.yml b/apps/api-gateway/supergraph-stage.yml index 92a911f2111..0564843d460 100644 --- a/apps/api-gateway/supergraph-stage.yml +++ b/apps/api-gateway/supergraph-stage.yml @@ -7,3 +7,11 @@ subgraphs: routing_url: http://api-users-stage:4002/graphql schema: file: ../../dist/apps/api-gateway/api-users-schema.graphql + languages: + routing_url: http://api-languages-stage:4003/graphql + schema: + file: ../../dist/apps/api-gateway/api-languages-schema.graphql + videos: + routing_url: http://api-videos-stage:4004/graphql + schema: + file: ../../dist/apps/api-gateway/api-videos-schema.graphql diff --git a/apps/api-gateway/supergraph.yml b/apps/api-gateway/supergraph.yml index b729d0e0efd..cf1f667e765 100644 --- a/apps/api-gateway/supergraph.yml +++ b/apps/api-gateway/supergraph.yml @@ -11,3 +11,7 @@ subgraphs: routing_url: http://127.0.0.1:4003/graphql schema: file: ../api-languages/schema.graphql + videos: + routing_url: http://127.0.0.1:4004/graphql + schema: + file: ../api-videos/schema.graphql diff --git a/apps/api-languages/schema.graphql b/apps/api-languages/schema.graphql index f61eaa8cb5d..22c6fb8aeaf 100644 --- a/apps/api-languages/schema.graphql +++ b/apps/api-languages/schema.graphql @@ -2,7 +2,7 @@ type Language @key(fields: "id") { id: ID! bcp47: String iso3: String - name(languageId: ID, primary: Boolean): [Translation]! + name(languageId: ID, primary: Boolean): [Translation!]! } type Translation { diff --git a/apps/api-languages/src/app/__generated__/graphql.ts b/apps/api-languages/src/app/__generated__/graphql.ts index f7392a1fc28..002c1396d76 100644 --- a/apps/api-languages/src/app/__generated__/graphql.ts +++ b/apps/api-languages/src/app/__generated__/graphql.ts @@ -12,7 +12,7 @@ export class Language { id: string; bcp47?: Nullable; iso3?: Nullable; - name: Nullable[]; + name: Translation[]; } export class Translation { diff --git a/apps/api-languages/src/app/modules/language/language.graphql b/apps/api-languages/src/app/modules/language/language.graphql index 17e32e68ca9..af52327c776 100644 --- a/apps/api-languages/src/app/modules/language/language.graphql +++ b/apps/api-languages/src/app/modules/language/language.graphql @@ -2,7 +2,7 @@ type Language @key(fields: "id") { id: ID! bcp47: String iso3: String - name(languageId: ID, primary: Boolean): [Translation]! + name(languageId: ID, primary: Boolean): [Translation!]! } extend type Query { diff --git a/apps/api-languages/src/main.ts b/apps/api-languages/src/main.ts index f48950768fa..34e51a52bf9 100644 --- a/apps/api-languages/src/main.ts +++ b/apps/api-languages/src/main.ts @@ -1,10 +1,12 @@ import { Logger } from '@nestjs/common' import { NestFactory } from '@nestjs/core' +import { json } from 'body-parser' import { AppModule } from './app/app.module' async function bootstrap(): Promise { const port = process.env.PORT ?? '4003' const app = await NestFactory.create(AppModule) + await app.use(json({ limit: '50mb' })) await app.listen(port, () => { Logger.log('Listening at http://localhost:' + port + '/graphql') }) diff --git a/apps/api-videos/.env.example b/apps/api-videos/.env.example new file mode 100644 index 00000000000..e0df6388eda --- /dev/null +++ b/apps/api-videos/.env.example @@ -0,0 +1,2 @@ +DATABASE_URL="arangodb://arangodb:8529" +ARCLIGHT_API_KEY= \ No newline at end of file diff --git a/apps/api-videos/.eslintrc.json b/apps/api-videos/.eslintrc.json new file mode 100644 index 00000000000..a7eb98f6587 --- /dev/null +++ b/apps/api-videos/.eslintrc.json @@ -0,0 +1,21 @@ +{ + "extends": ["../../.eslintrc.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "parserOptions": { + "project": ["apps/api-videos/tsconfig.*?.json"] + }, + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/apps/api-videos/Dockerfile b/apps/api-videos/Dockerfile new file mode 100644 index 00000000000..b3a389f2c97 --- /dev/null +++ b/apps/api-videos/Dockerfile @@ -0,0 +1,8 @@ +FROM node:14-alpine +WORKDIR /app +COPY ./dist/apps/api-videos . +EXPOSE 4004 +# dependencies that nestjs needs +RUN npm install --production --silent +RUN npm install tslib apollo-server-express @nestjs/mapped-types +CMD node ./main.js \ No newline at end of file diff --git a/apps/api-videos/db/db.ts b/apps/api-videos/db/db.ts new file mode 100644 index 00000000000..4a12769acc5 --- /dev/null +++ b/apps/api-videos/db/db.ts @@ -0,0 +1,13 @@ +import { Database } from 'arangojs' + +export function ArangoDB(): Database { + let db: Database + if (process.env.DATABASE_DB != null) + db = new Database({ + url: process.env.DATABASE_URL, + databaseName: process.env.DATABASE_DB + }) + else db = new Database({ url: process.env.DATABASE_URL }) + db.useBasicAuth(process.env.DATABASE_USER, process.env.DATABASE_PASS) + return db +} diff --git a/apps/api-videos/db/seed.ts b/apps/api-videos/db/seed.ts new file mode 100644 index 00000000000..ee9a10b8cc8 --- /dev/null +++ b/apps/api-videos/db/seed.ts @@ -0,0 +1,317 @@ +import { aql } from 'arangojs' +import fetch from 'node-fetch' +import { ArangoDB } from './db' + +interface Video { + _key: string +} + +const db = ArangoDB() + +interface MediaComponent { + mediaComponentId: string + primaryLanguageId: number + title: string + shortDescription: string + longDescription: string + metadataLanguageTag: string + imageUrls: { + mobileCinematicHigh: string + } + studyQuestions: string[] +} + +interface MediaComponentLanguage { + refId: string + languageId: number + lengthInMilliseconds: number + subtitleUrls: { + vtt?: Array<{ + languageId: number + url: string + }> + } + streamingUrls: { + hls: Array<{ + url: string + }> + } + downloadUrls: { + low?: { + url: string + sizeInBytes: number + } + high?: { + url: string + sizeInBytes: number + } + } +} + +interface Language { + languageId: number + bcp47: string +} + +interface Translation { + value: string + languageId: string + primary: boolean +} + +interface Download { + quality: string + size: number + url: string +} + +interface VideoVariant { + subtitle: Translation[] + hls: string + languageId: string + duration: number + downloads: Download[] +} + +interface Video { + title: Translation[] + snippet: Translation[] + description: Translation[] + studyQuestions: Translation[][] + image: string + variants: VideoVariant[] + tagIds: string[] +} + +async function getLanguages(): Promise { + const response: { + _embedded: { mediaLanguages: Language[] } + } = await ( + await fetch( + `https://api.arclight.org/v2/media-languages?limit=5000&filter=default&apiKey=${ + process.env.ARCLIGHT_API_KEY ?? '' + }` + ) + ).json() + return response._embedded.mediaLanguages +} + +async function getMediaComponents( + type: 'content' | 'container' +): Promise { + const response: { + _embedded: { mediaComponents: MediaComponent[] } + } = await ( + await fetch( + `https://api.arclight.org/v2/media-components?limit=5000&isDeprecated=false&type=${type}&contentTypes=video&apiKey=${ + process.env.ARCLIGHT_API_KEY ?? '' + }` + ) + ).json() + return response._embedded.mediaComponents +} + +async function getMediaComponentLanguage( + mediaComponentId: string +): Promise { + const response: { + _embedded: { mediaComponentLanguage: MediaComponentLanguage[] } + } = await ( + await fetch( + `https://api.arclight.org/v2/media-components/${mediaComponentId}/languages?apiKey=${ + process.env.ARCLIGHT_API_KEY ?? '' + }` + ) + ).json() + return response._embedded.mediaComponentLanguage +} + +async function digestContent( + languages: Language[], + mediaComponent: MediaComponent +): Promise { + const metadataLanguageId = + languages + .find(({ bcp47 }) => bcp47 === mediaComponent.metadataLanguageTag) + ?.languageId.toString() ?? '529' // english by default + + console.log('content:', mediaComponent.mediaComponentId) + + const variants: VideoVariant[] = [] + for (const mediaComponentLanguage of await getMediaComponentLanguage( + mediaComponent.mediaComponentId + )) { + variants.push(await digestMediaComponentLanguage(mediaComponentLanguage)) + } + + const body = { + primaryLanguageId: mediaComponent.primaryLanguageId.toString(), + title: [ + { + value: mediaComponent.title, + languageId: metadataLanguageId, + primary: true + } + ], + snippet: [ + { + value: mediaComponent.shortDescription, + languageId: metadataLanguageId, + primary: true + } + ], + description: [ + { + value: mediaComponent.longDescription, + languageId: metadataLanguageId, + primary: true + } + ], + studyQuestions: mediaComponent.studyQuestions.map((studyQuestion) => [ + { + languageId: metadataLanguageId, + value: studyQuestion, + primary: true + } + ]), + image: mediaComponent.imageUrls.mobileCinematicHigh, + tagIds: [], + variants + } + + const video = await getVideo(mediaComponent.mediaComponentId) + if (video != null) { + await db.collection('videos').update(mediaComponent.mediaComponentId, body) + } else { + await db + .collection('videos') + .save({ _key: mediaComponent.mediaComponentId, ...body }) + } +} + +async function digestMediaComponentLanguage( + mediaComponentLanguage: MediaComponentLanguage +): Promise { + const downloads: Download[] = [] + for (const [key, value] of Object.entries( + mediaComponentLanguage.downloadUrls + )) { + downloads.push({ + quality: key, + size: value.sizeInBytes, + url: value.url + }) + } + return { + subtitle: + mediaComponentLanguage.subtitleUrls.vtt?.map(({ languageId, url }) => ({ + languageId: languageId.toString(), + value: url, + primary: languageId === mediaComponentLanguage.languageId + })) ?? [], + hls: mediaComponentLanguage.streamingUrls.hls[0].url, + languageId: mediaComponentLanguage.languageId.toString(), + duration: Math.round(mediaComponentLanguage.lengthInMilliseconds * 0.001), + downloads + } +} + +async function getMediaComponentLinks( + mediaComponentId: string +): Promise { + const response: { + linkedMediaComponentIds: { contains: string[] } + } = await ( + await fetch( + `https://api.arclight.org/v2/media-component-links/${mediaComponentId}?apiKey=${ + process.env.ARCLIGHT_API_KEY ?? '' + }` + ) + ).json() + return response.linkedMediaComponentIds.contains +} + +async function digestContainer( + languages: Language[], + mediaComponent: MediaComponent +): Promise { + console.log('container:', mediaComponent.mediaComponentId) + for (const videoId of await getMediaComponentLinks( + mediaComponent.mediaComponentId + )) { + const video = await getVideo(videoId) + if (video == null) continue + + if (video.tagIds.includes(mediaComponent.mediaComponentId)) continue + await db.collection('videos').update(videoId, { + tagIds: [...video.tagIds, mediaComponent.mediaComponentId] + }) + } +} + +async function getVideo(videoId: string): Promise