fix: use newer soundcloud apis (#69)

* feat: add helper fns to call additional soundcloud apis

* feat: add tests for newer fns

package.json

@@ -13,7 +13,7 @@
     "dist"
   ],
   "scripts": {
-    "dev": "sls invoke local -f dat-song-bot",
+    "dev": "sls invoke local -f dat-song-bot -s development",
     "test": "tsdx test",
     "release": "semantic-release",
     "deploy": "sls deploy"

serverless.yml

@@ -6,11 +6,11 @@ plugins:
 
 provider:
   name: aws
-  runtime: nodejs8.10
+  runtime: nodejs12
   stage: ${opt:stage, 'production'}
   region: ap-south-1
   environment:
-    NODE_ENV: 'production'
+    NODE_ENV: ${opt:stage, 'production'}
     YOUTUBE_CLIENT_ID: ${ssm:YOUTUBE_CLIENT_ID}
     YOUTUBE_CLIENT_SECRET: ${ssm:YOUTUBE_CLIENT_SECRET~true}
     YOUTUBE_REFRESH_TOKEN: ${ssm:YOUTUBE_REFRESH_TOKEN~true}
@@ -22,6 +22,7 @@ provider:
 package:
   exclude:
     - node_modules/puppeteer/.local-chromium/** # exclude puppeteer chrome if exists
+    - test
 
 custom:
   webpack:

src/audio.ts

@@ -1,4 +1,5 @@
 import axios from 'axios';
+import url from 'url';
 import { pick, chain, sum } from 'lodash';
 import config from './config';
 import qs from 'qs';
@@ -11,19 +12,27 @@ interface SCUser {
   kind: 'user';
   permalink_url: string;
   uri: string;
+  first_name?: string;
+  last_name?: string;
+  full_name?: string;
   username: string;
   permalink: string;
   last_modified: string;
+  urn?: string;
+  verified?: boolean;
+  city?: string | null;
+  country_code?: string | null;
 }
 
 export interface Track {
+  full_duration?: number;
   comment_count: number;
   release: string | null;
   original_content_size: number;
   track_type: string | null;
   original_format: string;
   streamable: boolean;
-  download_url: string;
+  download_url: string | null;
   id: number;
   state: 'processing' | 'failed' | 'finished';
   last_modified: string;
@@ -76,6 +85,21 @@ export interface Track {
   user_playback_count: number;
   stream_url: string;
   label_id: number | null;
+  publisher_metadata?: {
+    urn: string;
+    contains_music: boolean;
+    id: number;
+  };
+  has_downloads_left?: boolean;
+  public?: boolean;
+  visuals?: string | null;
+  secret_token?: string | null;
+  urn?: string;
+  display_date?: string;
+  release_date?: string | null;
+
+  // custom
+  media_url?: string;
 }
 
 export type PickedTrack = Pick<
@@ -92,6 +116,7 @@ export type PickedTrack = Pick<
   | 'id'
   | 'duration'
   | 'uri'
+  | 'media_url'
 >;
 interface SCUserWebProfile {
   kind: 'web-profile';
@@ -103,6 +128,126 @@ interface SCUserWebProfile {
   created_at: string;
 }
 
+export interface Transcoding {
+  url: string;
+  preset: string;
+  duration: number;
+  snipped: boolean;
+  format: {
+    protocol: 'hls' | 'progressive';
+    mime_type: 'audio/mpeg' | 'audio/ogg; codecs="opus"';
+  };
+  quality: 'sq';
+}
+
+export interface TrackMedia {
+  transcodings: Array<Transcoding>;
+}
+
+export const getTranscodingForTrack = async (
+  track_id: Track['id']
+): Promise<Transcoding> => {
+  try {
+    audioLogger('Fetching media information for track');
+    const response = await axios.get<Array<Track & { media: TrackMedia }>>(
+      `https://api-v2.soundcloud.com/tracks`,
+      {
+        params: {
+          ids: [track_id],
+          client_id: config.SOUNDCLOUD_CLIENT_ID,
+        },
+      }
+    );
+
+    if (!response.data.length) {
+      return Promise.reject(
+        `v2 api responded with 0 tracks for track id - ${track_id}`
+      );
+    }
+
+    if (!response.data[0].media.transcodings.length) {
+      return Promise.reject(
+        `No transcodings found for this track (track id - ${track_id})`
+      );
+    }
+
+    audioLogger('Picking a suitable transcoding');
+
+    const transcoding = response.data[0].media.transcodings.find(
+      transcoding => {
+        return (
+          transcoding.format.protocol === 'progressive' &&
+          transcoding.format.mime_type.includes('audio/mpeg')
+        );
+      }
+    );
+
+    if (!transcoding) {
+      return Promise.reject(
+        `No valid transcoding found for track (track id - ${track_id})`
+      );
+    }
+
+    if (!transcoding.url) {
+      return Promise.reject(
+        `URL property is undefined for the selected transcoding ${JSON.stringify(
+          transcoding,
+          null,
+          2
+        )} (track id - ${track_id})`
+      );
+    }
+
+    if (url.parse(transcoding.url).host !== 'api-v2.soundcloud.com') {
+      return Promise.reject(
+        `Media URL is not a soundcloud url for track (track id - ${track_id})`
+      );
+    }
+
+    audioLogger('Found a sutiable transcoding for this track');
+
+    return transcoding;
+  } catch (error) {
+    audioLogger(
+      `Something went wrong while fetching transcodings for the track - ${track_id}`
+    );
+    audioLogger(error);
+    return Promise.reject(error);
+  }
+};
+
+export const getStreamUrlFromTranscoding = async (
+  transcoding: Transcoding,
+  track_id: Track['id']
+): Promise<string> => {
+  try {
+    audioLogger(`Fetching a streamable media url for this track - ${track_id}`);
+
+    const { url } = transcoding;
+    const { data } = await axios.get<{ url: string }>(url, {
+      params: {
+        client_id: config.SOUNDCLOUD_CLIENT_ID,
+      },
+    });
+
+    if (!data.url) {
+      return Promise.reject(
+        `No stream url found in the response of the transcoding API (track id - ${track_id}, media url - ${transcoding.url})`
+      );
+    }
+
+    audioLogger(`Fetched the streamable media url`);
+
+    return data.url;
+  } catch (error) {
+    audioLogger(
+      `Something went wrong while fetching stream url for the track - ${track_id}`
+    );
+    audioLogger(error);
+    return Promise.reject(error);
+  }
+};
+
 export const getTracksFromSoundcloud = async () => {
   try {
     audioLogger('fetching tracks');
@@ -159,6 +304,7 @@ export const getTracksFromSoundcloud = async () => {
       'id',
       'duration',
       'uri',
+      'media_url',
     ]);
   } catch (e) {
     audioLogger(`Something went wrong while fetching / picking track`);

src/index.ts

@@ -5,9 +5,14 @@ import {
   closePage,
   processVideo,
 } from './video';
-import { getTracksFromSoundcloud } from './audio';
+import {
+  getTracksFromSoundcloud,
+  getTranscodingForTrack,
+  getStreamUrlFromTranscoding,
+  Transcoding,
+} from './audio';
 import { getUnsplashPhoto } from './image';
-import { videoLogger } from './lib/utils';
+import { videoLogger, pipe } from './lib/utils';
 import { uploadVideo, connectToYoutube } from './upload';
 import { init as initSentry } from '@sentry/node';
 import config from './config';
@@ -31,12 +36,23 @@ export const main: TReturnValue = async () => {
       getTracksFromSoundcloud(),
       launchPage(),
     ]);
+
+    const streamUrl: string = await pipe<number, string>([
+      getTranscodingForTrack,
+      (transcoding: Transcoding): Promise<string> =>
+        getStreamUrlFromTranscoding(transcoding, song.id),
+    ])(song.id);
+
+    // Define the property `media_url` on the track object with the value of `streamUrl`
+    song.media_url = streamUrl;
+
     const image = await getUnsplashPhoto(song.tag_list);
     const svgContent = prepareSvg(
       image.urls.custom,
       song.title.replace(/(")|(')|(\.)/g, ''),
       song.user.username
     );
+
     await generateImage(IMAGE_OUTPUT, svgContent);
     await processVideo(VIDEO_OUTPUT, song, IMAGE_OUTPUT);
     const response = await uploadVideo(

src/lib/utils.ts

@@ -12,3 +12,19 @@ export const durationToSeconds = (d: string) =>
     .reduce(
       (prev, curr, index) => (prev = prev + (index === 2 ? curr : curr * 60))
     );
+
+export const pipe = <
+  InitialValue = any,
+  ReturnValue = any,
+  Action extends Function = any
+>(
+  fns: Action[]
+) => async (initialValue: InitialValue): Promise<ReturnValue> => {
+  let result = initialValue;
+
+  for await (let fn of fns) {
+    result = await fn(result);
+  }
+
+  return (<unknown>result) as ReturnValue;
+};

src/video.ts

@@ -71,7 +71,7 @@ export const generateImage = async (outputPath: string, content: string) => {
 
 export const processVideo = (
   outputPath: string,
-  song: Pick<PickedTrack, 'duration' | 'stream_url' | 'download_url'>,
+  song: Pick<PickedTrack, 'duration' | 'media_url'>,
   image: string
 ): Promise<void> => {
   videoLogger('Starting to process video');
@@ -81,11 +81,7 @@ export const processVideo = (
     .inputFPS(30)
     .loop()
     .withSize('1920x1080')
-    .input(
-      `${song.download_url || song.stream_url}?client_id=${
-        config.SOUNDCLOUD_CLIENT_ID
-      }`
-    )
+    .input(`${song.media_url}?client_id=${config.SOUNDCLOUD_CLIENT_ID}`)
     .outputOption('-shortest')
     .videoCodec('libx264')
     .videoBitrate(10000, true)