[docs infra] no s3 when pulling versioned content (#6705)

docs/next/package.json

@@ -26,6 +26,7 @@
     "@tailwindcss/typography": "^0.4.0",
     "add": "^2.0.6",
     "autoprefixer": "^10.1.0",
+    "axios": "^0.26.0",
     "classnames": "^2.2.6",
     "dotenv": "^8.2.0",
     "fast-glob": "^3.2.5",

docs/next/pages/[...page].tsx

@@ -6,6 +6,7 @@ import Pagination from "../components/Pagination";
 import { SphinxPrefix, sphinxPrefixFromPage } from "../util/useSphinx";
 import { useVersion, versionFromPage } from "../util/useVersion";
 
+import axios from "axios";
 import { GetStaticProps } from "next";
 import Link from "../components/Link";
 import { MdxRemote } from "next-mdx-remote/types";
@@ -22,8 +23,7 @@ import rehypePlugins from "components/mdx/rehypePlugins";
 import remark from "remark";
 import renderToString from "next-mdx-remote/render-to-string";
 import { useRouter } from "next/router";
-import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3";
-import { Readable } from "stream";
+
 import { Shimmer } from "components/Shimmer";
 
 const components: MdxRemote.Components = MDXComponents;
@@ -243,40 +243,16 @@ export default function MdxPage(props: Props) {
   }
 }
 
-async function streamToString(stream: Readable): Promise<string> {
-  return await new Promise((resolve, reject) => {
-    const chunks: Uint8Array[] = [];
-    stream.on("data", (chunk) => chunks.push(chunk));
-    stream.on("error", reject);
-    stream.on("end", () => resolve(Buffer.concat(chunks).toString("utf-8")));
-  });
-}
-const useS3Client = () => {
-  if (process.env.VERCEL) {
-    // If the app is running on Vercel, AWS creds need to be set via env vars
-    return new S3Client({
-      credentials: {
-        accessKeyId: process.env.AWS_ACCESS_KEY_ID_DOCS,
-        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY_DOCS,
-      },
-      region: "us-west-1",
-    });
-  } else {
-    return new S3Client({ region: "us-west-1" });
-  }
-};
-
-const s3Client = useS3Client();
-
-async function getVersionedContent(version: string, asPath: string) {
-  // get versioned content from remote public s3 bucket
-  const command = new GetObjectCommand({
-    Bucket: "dagster-docs-versioned-content",
-    Key: path.join("versioned_content", version, asPath),
-  });
-  const { Body } = await s3Client.send(command);
-  const content = await streamToString(Body as Readable);
-  return content;
+async function getVersionedContent(
+  version: string,
+  asPath: string
+): Promise<string> {
+  const bucket = "dagster-docs-versioned-content";
+  const region = "us-west-1";
+  const folder = "versioned_content";
+  const url = `https://${bucket}.s3.${region}.amazonaws.com/${folder}/${version}${asPath}`;
+  const response = await axios.get(url, { transformResponse: (x) => x });
+  return response.data;
 }
 
 async function getContent(version: string, asPath: string) {
@@ -300,7 +276,7 @@ async function getSphinxData(
   page: string[]
 ) {
   if (sphinxPrefix === SphinxPrefix.API_DOCS) {
-    const content = await getContent(version, "api/sections.json");
+    const content = await getContent(version, "/api/sections.json");
     const {
       api: { apidocs: data },
     } = JSON.parse(content);
@@ -316,7 +292,7 @@ async function getSphinxData(
       props: { type: PageType.HTML, data: { body, toc } },
     };
   } else {
-    const content = await getContent(version, "api/modules.json");
+    const content = await getContent(version, "/api/modules.json");
     const data = JSON.parse(content);
     let curr = data;
     for (const part of page) {
@@ -357,7 +333,7 @@ export const getStaticProps: GetStaticProps = async ({ params }) => {
 
   try {
     // 1. Read and parse versioned search
-    const searchContent = await getContent(version, "api/searchindex.json");
+    const searchContent = await getContent(version, "/api/searchindex.json");
     const searchIndex = JSON.parse(searchContent);
 
     // 2. Read and parse versioned MDX content

docs/next/yarn.lock

@@ -3129,6 +3129,13 @@ axios@^0.21.1:
   dependencies:
     follow-redirects "^1.10.0"
 
+axios@^0.26.0:
+  version "0.26.0"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-0.26.0.tgz#9a318f1c69ec108f8cd5f3c3d390366635e13928"
+  integrity sha512-lKoGLMYtHvFrPVt3r+RBMp9nh34N0M8zEfCWqdWZx6phynIEhQqAdydpyBAAG211zlhX9Rgu08cOamy6XjE5Og==
+  dependencies:
+    follow-redirects "^1.14.8"
+
 babel-jest@^27.5.1:
   version "27.5.1"
   resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-27.5.1.tgz#a1bf8d61928edfefd21da27eb86a695bfd691444"
@@ -4207,6 +4214,11 @@ follow-redirects@^1.10.0:
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.13.3.tgz#e5598ad50174c1bc4e872301e82ac2cd97f90267"
   integrity sha512-DUgl6+HDzB0iEptNQEXLx/KhTmDb8tZUHSeLqpnjpknR70H0nC2t9N73BK6fN4hOvJ84pKlIQVQ4k5FFlBedKA==
 
+follow-redirects@^1.14.8:
+  version "1.14.9"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.9.tgz#dd4ea157de7bfaf9ea9b3fbd85aa16951f78d8d7"
+  integrity sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==
+
 form-data@^2.5.0:
   version "2.5.1"
   resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.5.1.tgz#f2cbec57b5e59e23716e128fe44d4e5dd23895f4"