Scaffold external templates (#1867)

* Refactor * Support downloading external templates * Change error message * Fix types * Fix demo-store match and force rmDir * Remove https path segment in downloaded templates * Changesets * Fix tests
Shopify · Mar 18, 2024 · da95bb1 · da95bb1
1 parent 5f1295f
commit da95bb1
Show file tree

Hide file tree

Showing 5 changed files with 138 additions and 52 deletions.
diff --git a/.changeset/poor-boats-cheer.md b/.changeset/poor-boats-cheer.md
@@ -0,0 +1,13 @@
+---
+'@shopify/cli-hydrogen': minor
+---
+
+Support scaffolding projects from external repositories using the `--template` flag.
+
+The following examples are equivalent:
+
+```sh
+npm create @shopify/hydrogen -- --template shopify/hydrogen-demo-store
+npm create @shopify/hydrogen -- --template github.com/shopify/hydrogen-demo-store
+npm create @shopify/hydrogen -- --template https://github.com/shopify/hydrogen-demo-store
+```
diff --git a/packages/cli/src/commands/hydrogen/init.test.ts b/packages/cli/src/commands/hydrogen/init.test.ts
@@ -29,7 +29,7 @@ const {renderTasksHook} = vi.hoisted(() => ({renderTasksHook: vi.fn()}));
 vi.mock('../../lib/check-version.js');
 
 vi.mock('../../lib/template-downloader.js', async () => ({
-  getLatestTemplates: () =>
+  downloadMonorepoTemplates: () =>
     Promise.resolve({
       version: '',
       templatesDir: fileURLToPath(
@@ -39,6 +39,12 @@ vi.mock('../../lib/template-downloader.js', async () => ({
         new URL('../../../../../examples', import.meta.url),
       ),
     }),
+  downloadExternalRepo: () =>
+    Promise.resolve({
+      templateDir: fileURLToPath(
+        new URL('../../../../../templates/skeleton', import.meta.url),
+      ),
+    }),
 }));
 
 vi.mock('@shopify/cli-kit/node/ui', async () => {
@@ -146,7 +152,7 @@ describe('init', () => {
             path: tmpDir,
             git: false,
             language: 'ts',
-            template: 'https://github.com/some/repo',
+            template: 'missing-template',
           }),
         ).resolves.ok;
       });

diff --git a/packages/cli/src/commands/hydrogen/init.ts b/packages/cli/src/commands/hydrogen/init.ts
@@ -145,8 +145,10 @@ export async function runInit(
   const controller = new AbortController();
 
   try {
-    return options.template
-      ? await setupRemoteTemplate(options, controller)
+    const template = options.template;
+
+    return template
+      ? await setupRemoteTemplate({...options, template}, controller)
       : await setupLocalStarterTemplate(options, controller);
   } catch (error) {
     controller.abort();

diff --git a/packages/cli/src/lib/onboarding/remote.ts b/packages/cli/src/lib/onboarding/remote.ts
@@ -1,11 +1,14 @@
 import {readdir} from 'node:fs/promises';
 import {AbortError} from '@shopify/cli-kit/node/error';
-import {AbortController} from '@shopify/cli-kit/node/abort';
+import {AbortController, AbortSignal} from '@shopify/cli-kit/node/abort';
 import {copyFile, fileExists} from '@shopify/cli-kit/node/fs';
 import {readAndParsePackageJson} from '@shopify/cli-kit/node/node-package-manager';
 import {joinPath} from '@shopify/cli-kit/node/path';
 import {renderInfo, renderTasks} from '@shopify/cli-kit/node/ui';
-import {getLatestTemplates} from '../template-downloader.js';
+import {
+  downloadExternalRepo,
+  downloadMonorepoTemplates,
+} from '../template-downloader.js';
 import {applyTemplateDiff} from '../template-diff.js';
 import {getCliCommand} from '../shell.js';
 import {
@@ -20,50 +23,24 @@ import {
   type InitOptions,
 } from './common.js';
 
+const DEMO_STORE_REPO = 'shopify/hydrogen-demo-store';
+
 /**
  * Flow for creating a project starting from a remote template (e.g. demo-store).
  */
 export async function setupRemoteTemplate(
-  options: InitOptions,
+  options: InitOptions & Required<Pick<InitOptions, 'template'>>,
   controller: AbortController,
 ) {
-  // TODO: support GitHub repos as templates
-  const appTemplate = options.template!;
+  const appTemplate =
+    options.template === 'demo-store' ? DEMO_STORE_REPO : options.template;
+
   let abort = createAbortHandler(controller);
 
   // Start downloading templates early.
-  const backgroundDownloadPromise = getLatestTemplates({
-    signal: controller.signal,
-  })
-    .then(async ({templatesDir, examplesDir}) => {
-      const templatePath = joinPath(templatesDir, appTemplate);
-      const examplePath = joinPath(examplesDir, appTemplate);
-
-      if (await fileExists(templatePath)) {
-        return {templatesDir, sourcePath: templatePath};
-      }
-
-      if (await fileExists(examplePath)) {
-        return {templatesDir, sourcePath: examplePath};
-      }
-
-      const availableTemplates = (
-        await Promise.all([readdir(examplesDir), readdir(templatesDir)]).catch(
-          () => [],
-        )
-      )
-        .flat()
-        .filter((name) => name !== 'skeleton' && !name.endsWith('.md'))
-        .sort();
-
-      throw new AbortError(
-        `Unknown value in \`--template\` flag "${appTemplate}".\nSkip the flag or provide the name of a template or example in the Hydrogen repository.`,
-        availableTemplates.length === 0
-          ? ''
-          : {list: {title: 'Available templates:', items: availableTemplates}},
-      );
-    })
-    .catch(abort);
+  const backgroundDownloadPromise = appTemplate.includes('/')
+    ? getExternalTemplate(appTemplate, controller.signal).catch(abort)
+    : getMonorepoTemplate(appTemplate, controller.signal).catch(abort);
 
   const project = await handleProjectLocation({...options, controller});
 
@@ -80,18 +57,14 @@ export async function setupRemoteTemplate(
       // do not continue if it's already aborted
       if (controller.signal.aborted) return;
 
-      const {sourcePath, templatesDir} = downloaded;
+      const {sourcePath, skeletonPath} = downloaded;
 
       const pkgJson = await readAndParsePackageJson(
         joinPath(sourcePath, 'package.json'),
       );
 
       if (pkgJson.scripts?.dev?.includes('--diff')) {
-        return applyTemplateDiff(
-          project.directory,
-          sourcePath,
-          joinPath(templatesDir, 'skeleton'),
-        );
+        return applyTemplateDiff(project.directory, sourcePath, skeletonPath);
       }
 
       return copyFile(sourcePath, project.directory);
@@ -167,7 +140,7 @@ export async function setupRemoteTemplate(
 
   renderInfo({
     headline: `Your project will display inventory from ${
-      options.template === 'demo-store'
+      options.template.endsWith(DEMO_STORE_REPO)
         ? 'the Hydrogen Demo Store'
         : 'Mock.shop'
     }.`,
@@ -179,3 +152,54 @@ export async function setupRemoteTemplate(
     ...setupSummary,
   };
 }
+
+type DownloadedTemplate = {
+  sourcePath: string;
+  skeletonPath?: string;
+};
+
+async function getExternalTemplate(
+  appTemplate: string,
+  signal: AbortSignal,
+): Promise<DownloadedTemplate> {
+  const {templateDir} = await downloadExternalRepo(appTemplate, signal);
+  return {sourcePath: templateDir};
+}
+
+async function getMonorepoTemplate(
+  appTemplate: string,
+  signal: AbortSignal,
+): Promise<DownloadedTemplate> {
+  const {templatesDir, examplesDir} = await downloadMonorepoTemplates({
+    signal,
+  });
+
+  const skeletonPath = joinPath(templatesDir, 'skeleton');
+  const templatePath = joinPath(templatesDir, appTemplate);
+  const examplePath = joinPath(examplesDir, appTemplate);
+
+  if (await fileExists(templatePath)) {
+    return {skeletonPath, sourcePath: templatePath};
+  }
+
+  if (await fileExists(examplePath)) {
+    return {skeletonPath, sourcePath: examplePath};
+  }
+
+  const availableTemplates = (
+    await Promise.all([readdir(examplesDir), readdir(templatesDir)]).catch(
+      () => [],
+    )
+  )
+    .flat()
+    .filter((name) => name !== 'skeleton' && !name.endsWith('.md'))
+    .concat('demo-store') // Note: demo-store is handled as an external template
+    .sort();
+
+  throw new AbortError(
+    `Unknown value in \`--template\` flag "${appTemplate}".\nSkip the flag or provide the name of a template or example in the Hydrogen repository or a URL to a git repository.`,
+    availableTemplates.length === 0
+      ? ''
+      : {list: {title: 'Available templates:', items: availableTemplates}},
+  );
+}
diff --git a/packages/cli/src/lib/template-downloader.ts b/packages/cli/src/lib/template-downloader.ts
@@ -4,10 +4,13 @@ import {pipeline} from 'stream/promises';
 import gunzipMaybe from 'gunzip-maybe';
 import {extract} from 'tar-fs';
 import {fetch} from '@shopify/cli-kit/node/http';
-import {mkdir, fileExists} from '@shopify/cli-kit/node/fs';
+import {parseGitHubRepositoryURL} from '@shopify/cli-kit/node/github';
+import {mkdir, fileExists, rmdir} from '@shopify/cli-kit/node/fs';
 import {AbortError} from '@shopify/cli-kit/node/error';
 import {AbortSignal} from '@shopify/cli-kit/node/abort';
 import {getSkeletonSourceDir} from './build.js';
+import {joinPath} from '@shopify/cli-kit/node/path';
+import {downloadGitRepository} from '@shopify/cli-kit/node/git';
 
 // Note: this skips pre-releases
 const REPO_RELEASES_URL = `https://api.github.com/repos/shopify/hydrogen/releases/latest`;
@@ -40,7 +43,7 @@ async function getLatestReleaseDownloadUrl(signal?: AbortSignal) {
   };
 }
 
-async function downloadTarball(
+async function downloadMonorepoTarball(
   url: string,
   storageDir: string,
   signal?: AbortSignal,
@@ -72,7 +75,7 @@ async function downloadTarball(
   );
 }
 
-export async function getLatestTemplates({
+export async function downloadMonorepoTemplates({
   signal,
 }: {signal?: AbortSignal} = {}) {
   if (process.env.LOCAL_DEV) {
@@ -96,7 +99,7 @@ export async function getLatestTemplates({
 
     const templateStorageVersionPath = path.join(templateStoragePath, version);
     if (!(await fileExists(templateStorageVersionPath))) {
-      await downloadTarball(url, templateStorageVersionPath, signal);
+      await downloadMonorepoTarball(url, templateStorageVersionPath, signal);
     }
 
     return {
@@ -113,3 +116,41 @@ export async function getLatestTemplates({
     );
   }
 }
+
+export async function downloadExternalRepo(
+  appTemplate: string,
+  signal: AbortSignal,
+) {
+  const parsed = parseGitHubRepositoryURL(appTemplate);
+  if (parsed.isErr()) {
+    throw new AbortError(parsed.error.message);
+  }
+
+  const externalTemplates = fileURLToPath(
+    new URL('../external-templates', import.meta.url),
+  );
+  if (!(await fileExists(externalTemplates))) {
+    await mkdir(externalTemplates);
+  }
+
+  const result = parsed.value;
+  const templateDir = joinPath(
+    externalTemplates,
+    result.full.replace(/^https?:\/\//, '').replace(/[^\w]+/, '_'),
+  );
+
+  if (await fileExists(templateDir)) {
+    await rmdir(templateDir, {force: true});
+  }
+
+  // TODO use AbortSignal?
+  await downloadGitRepository({
+    repoUrl: result.full,
+    destination: templateDir,
+    shallow: true,
+  });
+
+  await rmdir(joinPath(templateDir, '.git'), {force: true});
+
+  return {templateDir};
+}