feat: Ikonki kanałów YouTube są teraz pobierane z API (#159)

* feat: Ikonki kanałów YouTube są teraz pobierane z API

* Fix tests

* Fix ts error

.env-sample

@@ -3,6 +3,7 @@ DATABASE_POOL_URL="postgres://postgres:postgres@localhost:5438/postgres"
 NEXT_PUBLIC_SUPABASE_URL="http://localhost:8765"
 NEXT_PUBLIC_SUPABASE_ANON_KEY="eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJzdXBhYmFzZSIsImlhdCI6MTYwMzk2ODgzNCwiZXhwIjoyNTUwNjUzNjM0LCJyb2xlIjoiYW5vbiJ9.36fUebxgx1mcBo4s19v0SzqmzunP--hm_hep0uLX0ew"
 
+YOUTUBE_API_KEY=""
 
 # Supabase Key (service_role, private): eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJzdXBhYmFzZSIsImlhdCI6MTYwMzk2ODgzNCwiZXhwIjoyNTUwNjUzNjM0LCJyb2xlIjoic2VydmljZV9yb2xlIn0.necIJaiP7X2T2QjGeV-FhpkizcNTX8HjDDBAxpgQTEI
 # Email testing interface URL: http://localhost:9000

api-helpers/config.ts

@@ -9,6 +9,7 @@ type NameToType = {
   readonly NODE_ENV: 'production' | 'development';
   readonly NEXT_PUBLIC_SUPABASE_URL: string;
   readonly NEXT_PUBLIC_SUPABASE_ANON_KEY: string;
+  readonly YOUTUBE_API_KEY: string;
 
   readonly NEXT_PUBLIC_ALGOLIA_APP_ID: string;
   readonly ALGOLIA_API_SECRET: string;

api-helpers/contentCreatorFunctions.test.ts

@@ -146,8 +146,8 @@ describe('getYoutubeRss', () => {
     );
   });
   it('return undefined when given url without channelId or userId', () => {
-    expect(
+    expect(() =>
       getYouTubeRss('https://www.youtube.com/watch?v=pHlqEvAwdVc&list=RDpHlqEvAwdVc&start_radio=1'),
-    ).toBe(undefined);
+    ).toThrowError(/Bad Request/);
   });
 });

api-helpers/contentCreatorFunctions.ts

@@ -5,6 +5,8 @@ import type { PrismaClient } from '@prisma/client';
 import Cheerio from 'cheerio';
 import Slugify from 'slugify';
 
+import { getYouTubeChannelFavicon } from './youtube';
+
 const NEVER = new Date(0);
 const YOUTUBE_REGEX = /^(http(s)?:\/\/)?((w){3}.)?youtu(be|.be)?(\.com)?\/.+/;
 const YOUTUBE_CHANNEL_ID_REGEX = /channel\/(.*?)(\/|$)/;
@@ -43,9 +45,9 @@ type BlogType = 'youtube' | 'other';
 const getBlogData = async (
   url: string,
 ): Promise<{ readonly data: BlogData; readonly type: BlogType }> => {
-  const youtubeRss = getYouTubeRss(url);
-  if (youtubeRss) {
-    return { data: await getBlogDataForYouTubeRss(url, youtubeRss), type: 'youtube' };
+  const youtubeRssUrl = getYouTubeRss(url);
+  if (youtubeRssUrl) {
+    return { data: await getBlogDataForYouTubeRss(url, youtubeRssUrl), type: 'youtube' };
   }
   return { data: await getBlogDataForUrl(url), type: 'other' };
 };
@@ -67,29 +69,36 @@ const getYouTubeChannelFeedUrl = (url: string) => {
   if (getYouTubeUserFromUrl(url)) {
     return `https://www.youtube.com/feeds/videos.xml?user=${getYouTubeUserFromUrl(url) as string}`;
   }
-  return undefined;
+  throw Boom.badRequest();
 };
 
-const getYouTubeChannelIdFromUrl = (url: string) => {
+export const getYouTubeChannelIdFromUrl = (url: string) => {
   const channelId = YOUTUBE_CHANNEL_ID_REGEX.exec(url)?.[1];
   return channelId;
 };
 
-const getYouTubeUserFromUrl = (url: string) => {
+export const getYouTubeUserFromUrl = (url: string) => {
   const user = YOUTUBE_USER_REGEX.exec(url)?.[1];
   return user;
 };
 
-const getBlogDataForYouTubeRss = async (url: string, youtubeRss: string) => {
-  const response = await fetch(youtubeRss);
-  const xmlText = await response.text();
+const getBlogDataForYouTubeRss = async (url: string, youtubeRssUrl: string) => {
+  const channelId = getYouTubeChannelIdFromUrl(url);
+  const username = getYouTubeUserFromUrl(url);
+
+  const [favicon, rssResponse] = await Promise.all([
+    getYouTubeChannelFavicon({ channelId, username }),
+    fetch(youtubeRssUrl),
+  ]);
+
+  const xmlText = await rssResponse.text();
   if (xmlText) {
     const $ = Cheerio.load(xmlText, { xmlMode: true, decodeEntities: true });
     return {
       name: getBlogName($),
       href: url,
-      favicon: 'https://www.youtube.com/s/desktop/d743f786/img/favicon_48.png',
-      rss: youtubeRss,
+      favicon: favicon,
+      rss: youtubeRssUrl,
     };
   }
   throw Boom.badData();

api-helpers/feedFunctions.ts

@@ -7,9 +7,15 @@ import { EMPTY, from, of } from 'rxjs';
 import { catchError, map, mergeMap, groupBy, last, timeout, filter } from 'rxjs/operators';
 import Slugify from 'slugify';
 
-import { getBlogName, getFavicon } from './contentCreatorFunctions';
+import {
+  getBlogName,
+  getFavicon,
+  getYouTubeChannelIdFromUrl,
+  getYouTubeUserFromUrl,
+} from './contentCreatorFunctions';
 import { logger } from './logger';
 import { streamToRx } from './rxjs-utils';
+import { getYouTubeChannelFavicon } from './youtube';
 
 const MAX_CONCURRENCY = 5;
 const MAX_FETCHING_TIME = ms('6 s');
@@ -41,10 +47,7 @@ function getFeedStreamFor(blog: Blog) {
     mergeMap((res) => {
       logger.debug(`Got stream for blog ${blog.name}`);
       const charset = getContentTypeParams(res.headers.get('content-type') || '').charset;
-      const responseStream = maybeTranslate(
-        (res.body as unknown) as NodeJS.ReadableStream,
-        charset,
-      );
+      const responseStream = maybeTranslate(res.body as unknown as NodeJS.ReadableStream, charset);
       logger.debug(`Translated ${blog.name}`);
       const feedparser = new FeedParser({});
       return streamToRx<FeedParser>(responseStream.pipe(feedparser)).pipe(
@@ -78,35 +81,36 @@ function maybeTranslate(res: NodeJS.ReadableStream, charset: string | undefined)
   return res;
 }
 
-const feedParserItemToArticle = (now: Date) => (blogId: Blog['id']) => (
-  item: FeedParser.Item,
-): Prisma.ArticleCreateInput => {
-  logger.debug(`Mapping articles for blog ID ${blogId}`);
-
-  const description =
-    item.summary ||
-    item.description ||
-    item.meta.description ||
-    // @todo legacy ?
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    ((item as any)?.['media:group']?.['media:description']?.['#'] as string | undefined) ||
-    null;
-
-  const article: Prisma.ArticleCreateInput = {
-    title: item.title,
-    href: item.link,
-    description,
-    publishedAt: item.pubdate || now,
-    slug: Slugify(item.title, { lower: true }),
-    blog: {
-      connect: {
-        id: blogId,
+const feedParserItemToArticle =
+  (now: Date) =>
+  (blogId: Blog['id']) =>
+  (item: FeedParser.Item): Prisma.ArticleCreateInput => {
+    logger.debug(`Mapping articles for blog ID ${blogId}`);
+
+    const description =
+      item.summary ||
+      item.description ||
+      item.meta.description ||
+      // @todo legacy ?
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+      ((item as any)?.['media:group']?.['media:description']?.['#'] as string | undefined) ||
+      null;
+
+    const article: Prisma.ArticleCreateInput = {
+      title: item.title,
+      href: item.link,
+      description,
+      publishedAt: item.pubdate || now,
+      slug: Slugify(item.title, { lower: true }),
+      blog: {
+        connect: {
+          id: blogId,
+        },
       },
-    },
+    };
+    logger.debug(article, 'Mapping done!');
+    return article;
   };
-  logger.debug(article, 'Mapping done!');
-  return article;
-};
 
 const getNewArticlesForBlog = (now: Date) => (blog: Blog) => {
   return getFeedStreamFor(blog).pipe(
@@ -166,7 +170,7 @@ const getUpdatedInfoFor = (blog: Blog) => {
       return EMPTY;
     }),
     mergeMap((res) => from(res.text())),
-    map((text) => getBlogInfoFromRss(text, blog)),
+    mergeMap((text) => from(getBlogInfoFromRss(text, blog))),
     map((updatedInfo) => {
       logger.debug(`Got updated info for blog: ${updatedInfo.name || blog.name}`);
       return {
@@ -177,12 +181,18 @@ const getUpdatedInfoFor = (blog: Blog) => {
   );
 };
 
-const getBlogInfoFromRss = (text: string, blog: Blog) => {
-  const $ = Cheerio.load(text, { xmlMode: true, decodeEntities: true });
+const getBlogInfoFromRss = async (rssContent: string, blog: Blog) => {
+  const channelId = getYouTubeChannelIdFromUrl(blog.href);
+  const username = getYouTubeUserFromUrl(blog.href);
+
+  const $ = Cheerio.load(rssContent, { xmlMode: true, decodeEntities: true });
   const type = blog.rss.includes('youtube.com') ? 'youtube' : 'other';
 
   const blogName = getBlogName($) || undefined;
-  const favicon = getFavicon($) || undefined;
+  const favicon =
+    (type === 'youtube' ? await getYouTubeChannelFavicon({ channelId, username }) : '') ||
+    getFavicon($) ||
+    undefined;
 
   const name = blogName ? (type === 'youtube' ? `${blogName} YouTube` : blogName) : undefined;
   const slug = name ? Slugify(name, { lower: true }) : undefined;

api-helpers/youtube.ts

@@ -0,0 +1,29 @@
+import * as Googleapis from 'googleapis';
+
+import { getConfig } from './config';
+
+const DEFAULT_AVATAR = 'https://www.youtube.com/s/desktop/d743f786/img/favicon_48.png';
+
+export const getYouTubeChannelFavicon = async ({
+  channelId,
+  username,
+}: {
+  readonly channelId?: string;
+  readonly username?: string;
+}) => {
+  if (!channelId && !username) {
+    return DEFAULT_AVATAR;
+  }
+
+  const yt = Googleapis.google.youtube('v3');
+
+  const where = channelId ? { id: [channelId] } : { forUsername: username };
+
+  const result = await yt.channels.list({
+    ...where,
+    key: getConfig('YOUTUBE_API_KEY'),
+    part: ['snippet'],
+  });
+
+  return result.data.items?.[0].snippet?.thumbnails?.default?.url || DEFAULT_AVATAR;
+};

components/AddContentCreatorForm/FormStatus.tsx

@@ -11,12 +11,13 @@ type Props = {
 };
 
 const errorToMessage: Record<number, string> = {
-  409: 'Blog o podanym adresie został już dodany do naszej bazy danych. Jeżeli nie pojawiają się najnowsze wpisy, lub masz inne zastrzeżenia - proszę skontaktuj się z administratorem.',
-  422: 'Nie udało się odnaleźć pliku RSS na twojej stronie, jeśli ten błąd się powtarza, proszę skontaktuj się z administratorem.',
+  409: 'Blog o podanym adresie został już dodany do naszej bazy danych. Jeżeli nie pojawiają się najnowsze wpisy lub masz inne zastrzeżenia – proszę skontaktuj się z administratorem.',
+  400: `Nie udało się wyciągnąć ID kanału z podanego adresu YouTube. Postaraj się znaleźć adres w postaci https://youtube.com/channel/{ID_KANAŁU} lub https://youtube.com/user/{ID_UŻYTKOWNIKA}`,
+  422: 'Nie udało się odnaleźć pliku RSS na twojej stronie. Upewnij się, że strona posiada RSS/ATOM i link do niego jest poprawnie dodany w sekcji <head> na stronie.',
 };
 
 const defaultErrorMessage =
-  'Wystąpił błąd podczas dodawania nowego serwisu, sprawdź poprawność danych i spróbuj ponownie';
+  'Wystąpił błąd podczas dodawania nowego serwisu. Sprawdź poprawność danych i spróbuj ponownie.';
 
 function getStatusMessage({ status, errorCode }: Props) {
   switch (status) {

components/MainTiles/MainTiles.tsx

@@ -41,7 +41,7 @@ export const MainTiles = memo<MainTilesProps>((props) => {
       <div className={styles.buttons}>
         <Link href="/zglos-serwis" passHref>
           <Button as="a" icon="icon-plus">
-            Zgłoś serwis
+            Dodaj serwis
           </Button>
         </Link>
         <DisplayStyleSwitch value={props.displayStyle} onChange={changeDisplayStyle} />

package.json

@@ -50,6 +50,7 @@
     "clsx": "1.1.1",
     "feed": "4.2.2",
     "feedparser": "2.2.10",
+    "googleapis": "81.0.0",
     "hcaptcha": "0.0.2",
     "html-entities": "2.3.2",
     "iconv-lite": "0.6.3",

pages/zglos-serwis.tsx

@@ -3,7 +3,7 @@ import { Layout } from '../components/Layout';
 
 export default function AddContentCreatorPage() {
   return (
-    <Layout title="Zgłoś serwis">
+    <Layout title="Dodaj serwis">
       <AddContentCreatorSection />
     </Layout>
   );