YPP: whitelist channels (Release v1.4.0)

CHANGELOG.md

@@ -1,3 +1,7 @@
+### 1.4.0
+
+- Adds support for whitelisting a channel such that a whitelisted channel will be exempted from requirements check when signing up for the YPP program. Adds `POST /channels/whitelist` endpoint to whitelist a channel/s & `DELETE /channels/whitelist/{ytChannelHandle}` endpoint to remove a channel from whitelist.
+
 ### 1.3.0
 
 - Integrates ElasticSearch alerting feature based on the filtration criteria set on the ingested logs.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "youtube-sync",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "license": "MIT",
   "scripts": {
     "postpack": "rm -f oclif.manifest.json",

src/infrastructure/index.ts

@@ -1,5 +1,5 @@
 import * as aws from '@pulumi/aws'
-import { resourcePrefix, Stats, YtChannel, YtUser, YtVideo } from '../types/youtube'
+import { resourcePrefix, Stats, WhitelistChannel, YtChannel, YtUser, YtVideo } from '../types/youtube'
 
 const nameof = <T>(name: keyof T) => <string>name
 
@@ -107,7 +107,20 @@ const statsTable = new aws.dynamodb.Table('stats', {
   billingMode: 'PAY_PER_REQUEST',
 })
 
+const whitelistChannelsTable = new aws.dynamodb.Table('whitelistChannels', {
+  name: `${resourcePrefix}whitelistChannels`,
+  hashKey: nameof<WhitelistChannel>('channelHandle'),
+  attributes: [
+    {
+      name: nameof<WhitelistChannel>('channelHandle'),
+      type: 'S',
+    },
+  ],
+  billingMode: 'PAY_PER_REQUEST',
+})
+
 export const usersTableArn = userTable.arn
 export const channelsTableArn = channelsTable.arn
 export const videosTableArn = videosTable.arn
 export const statsTableArn = statsTable.arn
+export const whitelistChannelsTableArn = whitelistChannelsTable.arn

src/repository/DynamodbService.ts

@@ -5,12 +5,14 @@ import { ChannelsRepository, ChannelsService } from './channel'
 import { StatsRepository } from './stats'
 import { UsersRepository, UsersService } from './user'
 import { VideosRepository, VideosService } from './video'
+import { WhitelistChannelsRepository } from './whitelistChannels'
 
 interface IDynamodbClient {
   channels: ChannelsRepository
   users: UsersRepository
   videos: VideosRepository
   stats: StatsRepository
+  whitelistChannels: WhitelistChannelsRepository
 }
 
 const DynamodbClient = {
@@ -20,6 +22,7 @@ const DynamodbClient = {
       users: new UsersRepository(tablePrefix),
       videos: new VideosRepository(tablePrefix),
       stats: new StatsRepository(tablePrefix),
+      whitelistChannels: new WhitelistChannelsRepository(tablePrefix),
     }
   },
 }

src/repository/whitelistChannels.ts

@@ -0,0 +1,103 @@
+import AsyncLock from 'async-lock'
+import * as dynamoose from 'dynamoose'
+import { ConditionInitializer } from 'dynamoose/dist/Condition'
+import { AnyItem } from 'dynamoose/dist/Item'
+import { Query, QueryResponse, Scan, ScanResponse } from 'dynamoose/dist/ItemRetriever'
+import { DYNAMO_MODEL_OPTIONS, IRepository, mapTo } from '.'
+import { ResourcePrefix, WhitelistChannel } from '../types/youtube'
+
+function whitelistChannelsModel(tablePrefix: ResourcePrefix) {
+  const schema = new dynamoose.Schema(
+    {
+      channelHandle: {
+        type: String,
+        hashKey: true,
+      },
+    },
+    {
+      saveUnknown: false,
+      timestamps: {
+        createdAt: {
+          createdAt: {
+            type: {
+              value: Date,
+              settings: {
+                storage: 'iso',
+              },
+            },
+          },
+        },
+      },
+    }
+  )
+  return dynamoose.model(`${tablePrefix}whitelistChannels`, schema, DYNAMO_MODEL_OPTIONS)
+}
+
+export class WhitelistChannelsRepository implements IRepository<WhitelistChannel> {
+  private model
+
+  // lock any updates on whitelistChannels table
+  private readonly ASYNC_LOCK_ID = 'whitelistChannels'
+  private asyncLock: AsyncLock = new AsyncLock()
+
+  constructor(tablePrefix: ResourcePrefix) {
+    this.model = whitelistChannelsModel(tablePrefix)
+  }
+
+  async upsertAll(): Promise<WhitelistChannel[]> {
+    throw new Error('Not implemented')
+  }
+
+  async scan(init: ConditionInitializer, f: (q: Scan<AnyItem>) => Scan<AnyItem>): Promise<WhitelistChannel[]> {
+    return this.asyncLock.acquire(this.ASYNC_LOCK_ID, async () => {
+      let lastKey = undefined
+      const results = []
+      do {
+        let scannedBatch: ScanResponse<AnyItem> = await f(this.model.scan(init))
+          .startAt(lastKey as any)
+          .exec()
+        let batchResult = scannedBatch.map((b) => mapTo<WhitelistChannel>(b))
+        results.push(...batchResult)
+        lastKey = scannedBatch.lastKey
+      } while (lastKey)
+      return results
+    })
+  }
+
+  async get(channelId: string): Promise<WhitelistChannel | undefined> {
+    return this.asyncLock.acquire(this.ASYNC_LOCK_ID, async () => {
+      const result = await this.model.get(channelId)
+      return result ? mapTo<WhitelistChannel>(result) : undefined
+    })
+  }
+
+  async save(model: WhitelistChannel): Promise<WhitelistChannel> {
+    return this.asyncLock.acquire(this.ASYNC_LOCK_ID, async () => {
+      const result = await this.model.update(model)
+      return mapTo<WhitelistChannel>(result)
+    })
+  }
+
+  async delete(channelId: string): Promise<void> {
+    return this.asyncLock.acquire(this.ASYNC_LOCK_ID, async () => {
+      await this.model.delete(channelId)
+      return
+    })
+  }
+
+  async query(init: ConditionInitializer, f: (q: Query<AnyItem>) => Query<AnyItem>) {
+    return this.asyncLock.acquire(this.ASYNC_LOCK_ID, async () => {
+      let lastKey = undefined
+      const results = []
+      do {
+        let queriedBatch: QueryResponse<AnyItem> = await f(this.model.query(init))
+          .startAt(lastKey as any)
+          .exec()
+        let batchResult = queriedBatch.map((b) => mapTo<WhitelistChannel>(b))
+        results.push(...batchResult)
+        lastKey = queriedBatch.lastKey
+      } while (lastKey)
+      return results
+    })
+  }
+}

src/services/httpApi/api-spec.json

@@ -357,6 +357,94 @@
         ]
       }
     },
+    "/channels/whitelist": {
+      "post": {
+        "operationId": "ChannelsController_addWhitelistChannels",
+        "summary": "",
+        "description": "Whitelist a given youtube channel/s by it's channel handle",
+        "parameters": [
+          {
+            "name": "authorization",
+            "required": true,
+            "in": "header",
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "requestBody": {
+          "required": true,
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              }
+            }
+          }
+        },
+        "responses": {
+          "default": {
+            "description": "",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "array",
+                  "items": {
+                    "$ref": "#/components/schemas/WhitelistChannelDto"
+                  }
+                }
+              }
+            }
+          }
+        },
+        "tags": [
+          "channels"
+        ]
+      }
+    },
+    "/channels/whitelist/{channelHandle}": {
+      "delete": {
+        "operationId": "ChannelsController_deleteWhitelistedChannel",
+        "summary": "",
+        "description": "Remove a whitelisted channel by it's channel handle",
+        "parameters": [
+          {
+            "name": "authorization",
+            "required": true,
+            "in": "header",
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "channelHandle",
+            "required": true,
+            "in": "path",
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "default": {
+            "description": "",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/WhitelistChannelDto"
+                }
+              }
+            }
+          }
+        },
+        "tags": [
+          "channels"
+        ]
+      }
+    },
     "/channels/induction/requirements": {
       "get": {
         "operationId": "ChannelsController_inductionRequirements",
@@ -882,6 +970,17 @@
           "joystreamVideo"
         ]
       },
+      "WhitelistChannelDto": {
+        "type": "object",
+        "properties": {
+          "channelHandle": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "channelHandle"
+        ]
+      },
       "ChannelInductionRequirementsDto": {
         "type": "object",
         "properties": {

src/services/httpApi/controllers/channels.ts

@@ -2,6 +2,7 @@ import {
   BadRequestException,
   Body,
   Controller,
+  Delete,
   Get,
   Headers,
   Inject,
@@ -31,6 +32,7 @@ import {
   UserDto,
   VerifyChannelDto,
   VideoDto,
+  WhitelistChannelDto,
 } from '../dtos'
 
 @Controller('channels')
@@ -310,6 +312,56 @@ export class ChannelsController {
     return result
   }
 
+  @Post('/whitelist')
+  @ApiResponse({ type: WhitelistChannelDto, isArray: true })
+  @ApiOperation({ description: `Whitelist a given youtube channel/s by it's channel handle` })
+  async addWhitelistChannels(
+    @Headers('authorization') authorizationHeader: string,
+    @Body(new ParseArrayPipe({ items: WhitelistChannelDto, whitelist: true })) channels: WhitelistChannelDto[]
+  ) {
+    const yppOwnerKey = authorizationHeader ? authorizationHeader.split(' ')[1] : ''
+    // TODO: fix this YT_SYNCH__HTTP_API__OWNER_KEY config value
+    if (yppOwnerKey !== process.env.YT_SYNCH__HTTP_API__OWNER_KEY) {
+      throw new UnauthorizedException('Invalid YPP owner key')
+    }
+
+    try {
+      for (const { channelHandle } of channels) {
+        await this.dynamodbService.repo.whitelistChannels.save({ channelHandle, createdAt: new Date() })
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : error
+      throw new NotFoundException(message)
+    }
+  }
+
+  @Delete('/whitelist/:channelHandle')
+  @ApiResponse({ type: WhitelistChannelDto })
+  @ApiOperation({ description: `Remove a whitelisted channel by it's channel handle` })
+  async deleteWhitelistedChannel(
+    @Headers('authorization') authorizationHeader: string,
+    @Param('channelHandle') channelHandle: string
+  ) {
+    const yppOwnerKey = authorizationHeader ? authorizationHeader.split(' ')[1] : ''
+    // TODO: fix this YT_SYNCH__HTTP_API__OWNER_KEY config value
+    if (yppOwnerKey !== process.env.YT_SYNCH__HTTP_API__OWNER_KEY) {
+      throw new UnauthorizedException('Invalid YPP owner key')
+    }
+
+    try {
+      const whitelistChannel = await this.dynamodbService.repo.whitelistChannels.get(channelHandle)
+
+      if (!whitelistChannel) {
+        throw new NotFoundException(`Channel with handle ${channelHandle} is not whitelisted`)
+      }
+
+      await this.dynamodbService.repo.whitelistChannels.delete(channelHandle)
+    } catch (error) {
+      const message = error instanceof Error ? error.message : error
+      throw new NotFoundException(message)
+    }
+  }
+
   @Get('/induction/requirements')
   @ApiResponse({ type: ChannelInductionRequirementsDto })
   @ApiOperation({ description: 'Retrieves Youtube Partner program induction requirements' })

src/services/httpApi/controllers/users.ts

@@ -51,8 +51,16 @@ export class UsersController {
         )
       }
 
-      // get verified channel from user
-      await this.youtube.getVerifiedChannel(user)
+      // get channel from user
+      const channel = await this.youtube.getChannel(user)
+
+      // check if the channel is whitelisted
+      const whitelistedChannel = await this.dynamodbService.repo.whitelistChannels.get(channel.customUrl)
+
+      if (!whitelistedChannel) {
+        // verify given channel
+        await this.youtube.verifyChannel(channel)
+      }
 
       // save user
       await this.dynamodbService.users.save(user)

src/services/httpApi/dtos.ts

@@ -8,6 +8,7 @@ import {
   IsOptional,
   IsString,
   IsUrl,
+  Matches,
   ValidateIf,
   ValidateNested,
 } from 'class-validator'
@@ -199,3 +200,9 @@ export class VerifyChannelDto {
   @IsNumber() @ApiProperty({ required: true }) joystreamChannelId: number
   @IsBoolean() @ApiProperty({ required: true }) isVerified: boolean
 }
+
+export class WhitelistChannelDto {
+  @Matches(/^@/, { message: 'The channel handle should start with a "@"' })
+  @ApiProperty({ required: true })
+  channelHandle: string
+}