Skip to content

Commit

Permalink
[static-build] Don't rely on hugo binary existing in build-container (
Browse files Browse the repository at this point in the history
#11455)

* Makes `hugo` framework preset work when Node.js v20.x is selected in
Project Settings.
* Stores the downloaded `hugo` binary in the `.vercel/cache` directory,
so that it does not need to be re-downloaded upon every deployment.
* Makes `vc build` work when run locally for Hugo projects - tested on
macOS arm64.
  • Loading branch information
TooTallNate committed Apr 18, 2024
1 parent fc7a8bc commit f4c181a
Show file tree
Hide file tree
Showing 7 changed files with 178 additions and 22 deletions.
5 changes: 5 additions & 0 deletions .changeset/odd-rivers-jog.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@vercel/static-build': minor
---

Don't rely on `hugo` binary existing in build-container
4 changes: 0 additions & 4 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,6 @@ jobs:
with:
node-version: ${{ matrix.nodeVersion || env.NODE_VERSION }}

- name: Install Hugo
if: matrix.runner == 'macos-14'
run: curl -L -O https://github.com/gohugoio/hugo/releases/download/v0.56.0/hugo_0.56.0_macOS-64bit.tar.gz && tar -xzf hugo_0.56.0_macOS-64bit.tar.gz && mv ./hugo packages/cli/test/dev/fixtures/08-hugo/

# yarn 1.22.21 introduced a Corepack bug when running tests.
# this can be removed once https://github.com/yarnpkg/yarn/issues/9015 is resolved
- name: install yarn@1.22.19
Expand Down
15 changes: 11 additions & 4 deletions packages/cli/test/dev/integration-3.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { spawnAsync } from '@vercel/build-utils';
import { resolve, delimiter } from 'path';

const {
Expand Down Expand Up @@ -97,10 +98,16 @@ test('[vercel dev] 08-hugo', async () => {
new Error('Dev server timed out while waiting to be ready')
);

// 2. Update PATH to find the Hugo executable installed via GH Actions
process.env.PATH = `${resolve(fixture('08-hugo'))}${delimiter}${
process.env.PATH
}`;
// 2. Download `hugo` and update PATH
const hugoFixture = resolve(fixture('08-hugo'));
await spawnAsync(
`curl -sSL https://github.com/gohugoio/hugo/releases/download/v0.56.0/hugo_0.56.0_macOS-64bit.tar.gz | tar -xz -C "${hugoFixture}"`,
[],
{
shell: true,
}
);
process.env.PATH = `${hugoFixture}${delimiter}${process.env.PATH}`;

// 3. Rerun the test now that Hugo is in the PATH
tester = testFixtureStdio(
Expand Down
2 changes: 1 addition & 1 deletion packages/static-build/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"scripts": {
"build": "node ../../utils/build-builder.mjs",
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --bail --runInBand",
"test-unit": "pnpm test test/build.test.ts test/gatsby.test.ts test/prepare-cache.test.ts",
"test-unit": "pnpm test test/build.test.ts test/hugo.test.ts test/gatsby.test.ts test/prepare-cache.test.ts",
"test-e2e": "pnpm test test/integration-*.test.js",
"type-check": "tsc --noEmit"
},
Expand Down
45 changes: 32 additions & 13 deletions packages/static-build/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ import fetch from 'node-fetch';
import getPort from 'get-port';
import isPortReachable from 'is-port-reachable';
import frameworks, { Framework } from '@vercel/frameworks';
import type { ChildProcess, SpawnOptions } from 'child_process';
import { existsSync, readFileSync, statSync, readdirSync } from 'fs';
import { spawn, type ChildProcess, type SpawnOptions } from 'child_process';
import { existsSync, readFileSync, statSync, readdirSync, mkdirSync } from 'fs';
import { cpus } from 'os';
import {
BuildV2,
Expand All @@ -15,7 +15,6 @@ import {
PrepareCache,
glob,
download,
spawnAsync,
execCommand,
spawnCommand,
runNpmInstall,
Expand Down Expand Up @@ -46,6 +45,8 @@ import {
detectFrameworkRecord,
LocalFileSystemDetector,
} from '@vercel/fs-detectors';
import { getHugoUrl } from './utils/hugo';
import { once } from 'events';

const SUPPORTED_RUBY_VERSION = '3.2.0';
const sleep = (n: number) => new Promise(resolve => setTimeout(resolve, n));
Expand Down Expand Up @@ -262,7 +263,12 @@ function getFramework(
return framework;
}

async function fetchBinary(url: string, framework: string, version: string) {
async function fetchBinary(
url: string,
framework: string,
version: string,
dest = '/usr/local/bin'
) {
const res = await fetch(url);
if (res.status === 404) {
throw new NowBuildError({
Expand All @@ -271,9 +277,16 @@ async function fetchBinary(url: string, framework: string, version: string) {
link: 'https://vercel.link/framework-versioning',
});
}
await spawnAsync(`curl -sSL ${url} | tar -zx -C /usr/local/bin`, [], {
shell: true,
const cp = spawn('tar', ['-zx', '-C', dest], {
stdio: ['pipe', 'ignore', 'ignore'],
});
res.body.pipe(cp.stdin);
const [exitCode] = await once(cp, 'exit');
if (exitCode !== 0) {
throw new Error(
`Extraction of ${framework} failed (exit code ${exitCode})`
);
}
}

async function getUpdatedDistPath(
Expand Down Expand Up @@ -352,13 +365,19 @@ export const build: BuildV2 = async ({
if (config.zeroConfig) {
const { HUGO_VERSION, ZOLA_VERSION, GUTENBERG_VERSION } = process.env;

if (HUGO_VERSION && !meta.isDev) {
console.log('Installing Hugo version ' + HUGO_VERSION);
const [major, minor] = HUGO_VERSION.split('.').map(Number);
const isOldVersion = major === 0 && minor < 43;
const prefix = isOldVersion ? `hugo_` : `hugo_extended_`;
const url = `https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/${prefix}${HUGO_VERSION}_Linux-64bit.tar.gz`;
await fetchBinary(url, 'Hugo', HUGO_VERSION);
if ((HUGO_VERSION || framework?.slug === 'hugo') && !meta.isDev) {
const hugoVersion = HUGO_VERSION || '0.58.2';
const hugoDir = path.join(
workPath,
`.vercel/cache/hugo-v${hugoVersion}-${process.platform}-${process.arch}`
);
if (!existsSync(hugoDir)) {
console.log('Installing Hugo version ' + hugoVersion);
const url = await getHugoUrl(hugoVersion);
mkdirSync(hugoDir, { recursive: true });
await fetchBinary(url, 'Hugo', hugoVersion, hugoDir);
}
process.env.PATH = `${hugoDir}${path.delimiter}${process.env.PATH}`;
}

if (ZOLA_VERSION && !meta.isDev) {
Expand Down
84 changes: 84 additions & 0 deletions packages/static-build/src/utils/hugo.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
import fetch from 'node-fetch';
import { NowBuildError } from '@vercel/build-utils';

export async function getHugoUrl(
version: string,
platform = process.platform,
arch = process.arch
): Promise<string> {
const oses = (
{
linux: ['linux'],
darwin: ['darwin', 'macos'],
win32: ['windows'],
} as Record<string, string[]>
)[platform];
if (!oses) {
throw new Error(`Unsupported platform: ${platform}`);
}
const arches = (
{
arm64: ['arm64'],
x64: ['amd64', '64bit'],
} as Record<string, string[]>
)[arch];
if (!arches) {
throw new Error(`Unsupported arch: ${arch}`);
}
if (platform === 'darwin') {
arches.push('universal');
if (arch === 'arm64') {
// On Mac ARM64, assume Rosetta is available to execute 64-bit binaries
arches.push('64bit');
}
}

const checksumsUrl = `https://github.com/gohugoio/hugo/releases/download/v${version}/hugo_${version}_checksums.txt`;
const checksumsRes = await fetch(checksumsUrl);
if (checksumsRes.status === 404) {
throw new NowBuildError({
code: 'STATIC_BUILD_BINARY_NOT_FOUND',
message: `Version ${version} of Hugo does not exist. Please specify a different one.`,
link: 'https://vercel.link/framework-versioning',
});
}
const checksumsBody = await checksumsRes.text();
const checksums = new Map<string, string>();
for (const line of checksumsBody.trim().split('\n')) {
const [sha, name] = line.split(/\s+/);
checksums.set(name, sha);
}

const file =
findFile(checksums.keys(), oses, arches, true) ||
findFile(checksums.keys(), oses, arches, false);
if (!file) {
throw new Error(
`Could not determine filename for Hugo v${version} for ${platform} / ${arch}`
);
}

return `https://github.com/gohugoio/hugo/releases/download/v${version}/${file}`;
}

function findFile(
names: Iterable<string>,
oses: string[],
arches: string[],
extended: boolean
): string | null {
const prefix = `hugo_${extended ? 'extended_' : ''}`;
for (const name of names) {
if (!name.startsWith(prefix) || !name.endsWith('.tar.gz')) continue;
const suffix = name.split('_').pop();
if (!suffix) continue;
const [os, arch] = suffix
.replace(/\.(.*)$/, '')
.toLowerCase()
.split('-');
if (oses.includes(os) && arches.includes(arch)) {
return name;
}
}
return null;
}
45 changes: 45 additions & 0 deletions packages/static-build/test/hugo.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import { getHugoUrl } from '../src/utils/hugo';

describe('getHugoUrl()', () => {
it('should return URL for v0.42.2 on macOS arm64', async () => {
const url = await getHugoUrl('0.42.2', 'darwin', 'arm64');
expect(url).toEqual(
'https://github.com/gohugoio/hugo/releases/download/v0.42.2/hugo_0.42.2_macOS-64bit.tar.gz'
);
});

it('should return URL for v0.42.2 on Linux x64', async () => {
const url = await getHugoUrl('0.42.2', 'linux', 'x64');
expect(url).toEqual(
'https://github.com/gohugoio/hugo/releases/download/v0.42.2/hugo_0.42.2_Linux-64bit.tar.gz'
);
});

it('should return URL for v0.58.2 on macOS arm64', async () => {
const url = await getHugoUrl('0.58.2', 'darwin', 'arm64');
expect(url).toEqual(
'https://github.com/gohugoio/hugo/releases/download/v0.58.2/hugo_extended_0.58.2_macOS-64bit.tar.gz'
);
});

it('should return URL for v0.58.2 on Linux x64', async () => {
const url = await getHugoUrl('0.58.2', 'linux', 'x64');
expect(url).toEqual(
'https://github.com/gohugoio/hugo/releases/download/v0.58.2/hugo_extended_0.58.2_Linux-64bit.tar.gz'
);
});

it('should return URL for v0.125.0 on macOS arm64', async () => {
const url = await getHugoUrl('0.125.0', 'darwin', 'arm64');
expect(url).toEqual(
'https://github.com/gohugoio/hugo/releases/download/v0.125.0/hugo_extended_0.125.0_darwin-universal.tar.gz'
);
});

it('should return URL for v0.125.0 on Linux x64', async () => {
const url = await getHugoUrl('0.125.0', 'linux', 'x64');
expect(url).toEqual(
'https://github.com/gohugoio/hugo/releases/download/v0.125.0/hugo_extended_0.125.0_Linux-64bit.tar.gz'
);
});
});

0 comments on commit f4c181a

Please sign in to comment.