lastModified

docs/javascript/files.md

@@ -10,7 +10,7 @@ Load files — whether static or generated dynamically by a [data loader](../loa
 import {FileAttachment} from "npm:@observablehq/stdlib";
 ```
 
-The `FileAttachment` function takes a path and returns a file handle. This handle exposes the file’s name and [MIME type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types).
+The `FileAttachment` function takes a path and returns a file handle. This handle exposes the file’s name, [MIME type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types), and last modification date as the number of milliseconds since UNIX epoch.
 
 ```js echo
 FileAttachment("volcano.json")
@@ -32,7 +32,7 @@ volcano
 
 ### Static analysis
 
-The `FileAttachment` function can _only_ be passed a static string literal; constructing a dynamic path such as `FileAttachment("my" + "file.csv")` is invalid syntax. Static analysis is used to invoke [data loaders](../loaders) at build time, and ensures that only referenced files are included in the generated output during build. In the future [#260](https://github.com/observablehq/framework/issues/260), it will also allow content hashes for cache breaking during deploy.
+The `FileAttachment` function can _only_ be passed a static string literal; constructing a dynamic path such as `FileAttachment("my" + "file.csv")` is invalid syntax. Static analysis is used to invoke [data loaders](../loaders) at build time, and ensures that only referenced files are included in the generated output during build. This also allows to include a content hash in the file name for cache breaking during deploy.
 
 If you have multiple files, you can enumerate them explicitly like so:
 
@@ -52,6 +52,10 @@ const frames = [
 
 None of the files in `frames` above are loaded until a [content method](#supported-formats) is invoked, for example by saying `frames[0].image()`.
 
+### Edge cases
+
+Missing files don’t exhibit a lastModified property. To determine the file’s MIME type, its extension is checked against the [mime](https://www.npmjs.com/package/mime) database; it defaults to `application/octet-stream`.
+
 ## Supported formats
 
 `FileAttachment` supports a variety of methods for loading file contents:

src/client/stdlib/fileAttachment.js

@@ -11,8 +11,8 @@ export function FileAttachment(name, base = location.href) {
   const url = new URL(name, base).href;
   const file = files.get(url);
   if (!file) throw new Error(`File not found: ${name}`);
-  const {path, mimeType} = file;
-  return new FileAttachmentImpl(new URL(path, location).href, name.split("/").pop(), mimeType);
+  const {path, mimeType, lastModified} = file;
+  return new FileAttachmentImpl(new URL(path, location).href, name.split("/").pop(), mimeType, lastModified);
 }
 
 async function remote_fetch(file) {
@@ -28,9 +28,10 @@ async function dsv(file, delimiter, {array = false, typed = false} = {}) {
 }
 
 export class AbstractFile {
-  constructor(name, mimeType = "application/octet-stream") {
-    Object.defineProperty(this, "name", {value: `${name}`, enumerable: true});
+  constructor(name, mimeType, lastModified) {
     Object.defineProperty(this, "mimeType", {value: `${mimeType}`, enumerable: true});
+    Object.defineProperty(this, "name", {value: `${name}`, enumerable: true});
+    if (lastModified !== undefined) Object.defineProperty(this, "lastModified", {value: Number(lastModified), enumerable: true}); // prettier-ignore
   }
   async blob() {
     return (await remote_fetch(this)).blob();
@@ -95,8 +96,8 @@ export class AbstractFile {
 }
 
 class FileAttachmentImpl extends AbstractFile {
-  constructor(url, name, mimeType) {
-    super(name, mimeType);
+  constructor(url, name, mimeType, lastModified) {
+    super(name, mimeType, lastModified);
     Object.defineProperty(this, "_url", {value: url});
   }
   async url() {

src/dataloader.ts

@@ -5,10 +5,12 @@ import {createGunzip} from "node:zlib";
 import {spawn} from "cross-spawn";
 import JSZip from "jszip";
 import {extract} from "tar-stream";
+import {isEnoent} from "./error.js";
 import {maybeStat, prepareOutput} from "./files.js";
 import {FileWatchers} from "./fileWatchers.js";
 import {getFileHash} from "./javascript/module.js";
 import type {Logger, Writer} from "./logger.js";
+import {resolvePath} from "./path.js";
 import {cyan, faint, green, red, yellow} from "./tty.js";
 
 const runningCommands = new Map<string, Promise<string>>();
@@ -48,6 +50,7 @@ export interface LoaderOptions {
 export class LoaderResolver {
   private readonly root: string;
   private readonly interpreters: Map<string, string[]>;
+  lastModified: Map<string, number>;
 
   constructor({root, interpreters}: {root: string; interpreters?: Record<string, string[] | null>}) {
     this.root = root;
@@ -56,6 +59,7 @@ export class LoaderResolver {
         (entry): entry is [string, string[]] => entry[1] != null
       )
     );
+    this.lastModified = new Map();
   }
 
   /**
@@ -129,11 +133,33 @@ export class LoaderResolver {
     return FileWatchers.of(this, path, watchPaths, callback);
   }
 
-  getFileHash(path: string): string {
+  // Compute file Hash and update the lastModified map. For data loaders, use
+  // the output if it is already available (cached). In build this is always the
+  // case (unless the data loaders fail). However in preview we return the page
+  // before running the data loaders (which will run on demand from the page),
+  // so there might be a temporary discrepancy when a cache is stale.
+  getFileHash(name: string): string {
+    let path = name;
     if (!existsSync(join(this.root, path))) {
       const loader = this.find(path);
       if (loader) path = relative(this.root, loader.path);
     }
+    try {
+      const ts = Math.floor(statSync(join(this.root, path)).mtimeMs);
+      const key = resolvePath(this.root, name);
+      this.lastModified.set(key, ts);
+      if (name !== path) {
+        try {
+          const cachePath = join(".observablehq", "cache", name);
+          this.lastModified.set(key, Math.floor(statSync(join(this.root, cachePath)).mtimeMs));
+          path = cachePath; // on success
+        } catch (error) {
+          if (!isEnoent(error)) throw error;
+        }
+      }
+    } catch (error) {
+      if (!isEnoent(error)) throw error;
+    }
     return getFileHash(this.root, path);
   }
 

src/render.ts

@@ -8,7 +8,7 @@ import {transpileJavaScript} from "./javascript/transpile.js";
 import type {MarkdownPage} from "./markdown.js";
 import type {PageLink} from "./pager.js";
 import {findLink, normalizePath} from "./pager.js";
-import {relativePath} from "./path.js";
+import {relativePath, resolvePath} from "./path.js";
 import type {Resolvers} from "./resolvers.js";
 import {getResolvers} from "./resolvers.js";
 import {rollupClient} from "./rollup.js";
@@ -30,7 +30,8 @@ export async function renderPage(page: MarkdownPage, options: RenderOptions & Re
   const sidebar = data?.sidebar !== undefined ? Boolean(data.sidebar) : options.sidebar;
   const toc = mergeToc(data?.toc, options.toc);
   const draft = Boolean(data?.draft);
-  const {files, resolveFile, resolveImport} = resolvers;
+  const {files, resolveFile, resolveImport, lastModified} = resolvers;
+
   return String(html`<!DOCTYPE html>
 <meta charset="utf-8">${path === "/404" ? html`\n<base href="${preview ? "/" : base}">` : ""}
 <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
@@ -63,7 +64,9 @@ import ${preview || page.code.length ? `{${preview ? "open, " : ""}define} from
         )};`
       : ""
   }${data?.sql ? `\nimport {registerTable} from ${JSON.stringify(resolveImport("npm:@observablehq/duckdb"))};` : ""}${
-    files.size ? `\n${renderFiles(files, resolveFile)}` : ""
+    files.size
+      ? `\n${renderFiles(files, resolveFile, (name: string) => lastModified.get(resolvePath(path, name)))}`
+      : ""
   }${
     data?.sql
       ? `\n${Object.entries<string>(data.sql)
@@ -84,18 +87,27 @@ ${html.unsafe(rewriteHtml(page.html, resolvers.resolveFile))}</main>${renderFoot
 `);
 }
 
-function renderFiles(files: Iterable<string>, resolve: (name: string) => string): string {
+function renderFiles(
+  files: Iterable<string>,
+  resolve: (name: string) => string,
+  getLastModified: (name: string) => number | undefined
+): string {
   return Array.from(files)
     .sort()
-    .map((f) => renderFile(f, resolve))
+    .map((f) => renderFile(f, resolve, getLastModified))
     .join("");
 }
 
-function renderFile(name: string, resolve: (name: string) => string): string {
+function renderFile(
+  name: string,
+  resolve: (name: string) => string,
+  getLastModified: (name: string) => number | undefined
+): string {
   return `\nregisterFile(${JSON.stringify(name)}, ${JSON.stringify({
     name,
     mimeType: mime.getType(name) ?? undefined,
-    path: resolve(name)
+    path: resolve(name),
+    lastModified: getLastModified(name)
   })});`;
 }
 

src/resolvers.ts

@@ -22,6 +22,7 @@ export interface Resolvers {
   resolveFile(specifier: string): string;
   resolveImport(specifier: string): string;
   resolveStylesheet(specifier: string): string;
+  lastModified: Map<string, number>;
 }
 
 const defaultImports = [
@@ -81,6 +82,7 @@ export async function getResolvers(
   const staticImports = new Set<string>(defaultImports);
   const stylesheets = new Set<string>();
   const resolutions = new Map<string, string>();
+  const lastModified = loaders.lastModified;
 
   // Add stylesheets. TODO Instead of hard-coding Source Serif Pro, parse the
   // page’s stylesheet to look for external imports.
@@ -106,8 +108,7 @@ export async function getResolvers(
     }
   }
 
-  // Compute the content hash. TODO In build, this needs to consider the output
-  // of data loaders, rather than the source of data loaders.
+  // Compute the content hash.
   for (const f of assets) hash.update(loaders.getFileHash(resolvePath(path, f)));
   for (const f of files) hash.update(loaders.getFileHash(resolvePath(path, f)));
   for (const i of localImports) hash.update(getModuleHash(root, resolvePath(path, i)));
@@ -252,7 +253,8 @@ export async function getResolvers(
     stylesheets,
     resolveFile,
     resolveImport,
-    resolveStylesheet
+    resolveStylesheet,
+    lastModified
   };
 }
 

test/build-test.ts

@@ -101,6 +101,7 @@ class TestEffects extends FileBuildEffects {
   async writeFile(outputPath: string, contents: string | Buffer): Promise<void> {
     if (typeof contents === "string" && outputPath.endsWith(".html")) {
       contents = contents.replace(/^(\s*<script>\{).*(\}<\/script>)$/gm, "$1/* redacted init script */$2");
+      contents = contents.replace(/^(registerFile\(.*,"lastModified":)\d+(\}\);)$/gm, "$1/* ts */1706742000000$2");
     }
     return super.writeFile(outputPath, contents);
   }

test/output/build/archives.posix/tar.html

@@ -15,13 +15,13 @@
 import {define} from "./_observablehq/client.js";
 import {registerFile} from "./_observablehq/stdlib.js";
 
-registerFile("./dynamic-tar-gz/does-not-exist.txt", {"name":"./dynamic-tar-gz/does-not-exist.txt","mimeType":"text/plain","path":"./dynamic-tar-gz/does-not-exist.txt"});
-registerFile("./dynamic-tar-gz/file.txt", {"name":"./dynamic-tar-gz/file.txt","mimeType":"text/plain","path":"./_file/dynamic-tar-gz/file.c93138d8.txt"});
-registerFile("./dynamic-tar/does-not-exist.txt", {"name":"./dynamic-tar/does-not-exist.txt","mimeType":"text/plain","path":"./dynamic-tar/does-not-exist.txt"});
-registerFile("./dynamic-tar/file.txt", {"name":"./dynamic-tar/file.txt","mimeType":"text/plain","path":"./_file/dynamic-tar/file.c93138d8.txt"});
-registerFile("./static-tar/does-not-exist.txt", {"name":"./static-tar/does-not-exist.txt","mimeType":"text/plain","path":"./static-tar/does-not-exist.txt"});
-registerFile("./static-tar/file.txt", {"name":"./static-tar/file.txt","mimeType":"text/plain","path":"./_file/static-tar/file.c93138d8.txt"});
-registerFile("./static-tgz/file.txt", {"name":"./static-tgz/file.txt","mimeType":"text/plain","path":"./_file/static-tgz/file.c93138d8.txt"});
+registerFile("./dynamic-tar-gz/does-not-exist.txt", {"name":"./dynamic-tar-gz/does-not-exist.txt","mimeType":"text/plain","path":"./dynamic-tar-gz/does-not-exist.txt","lastModified":/* ts */1706742000000});
+registerFile("./dynamic-tar-gz/file.txt", {"name":"./dynamic-tar-gz/file.txt","mimeType":"text/plain","path":"./_file/dynamic-tar-gz/file.c93138d8.txt","lastModified":/* ts */1706742000000});
+registerFile("./dynamic-tar/does-not-exist.txt", {"name":"./dynamic-tar/does-not-exist.txt","mimeType":"text/plain","path":"./dynamic-tar/does-not-exist.txt","lastModified":/* ts */1706742000000});
+registerFile("./dynamic-tar/file.txt", {"name":"./dynamic-tar/file.txt","mimeType":"text/plain","path":"./_file/dynamic-tar/file.c93138d8.txt","lastModified":/* ts */1706742000000});
+registerFile("./static-tar/does-not-exist.txt", {"name":"./static-tar/does-not-exist.txt","mimeType":"text/plain","path":"./static-tar/does-not-exist.txt","lastModified":/* ts */1706742000000});
+registerFile("./static-tar/file.txt", {"name":"./static-tar/file.txt","mimeType":"text/plain","path":"./_file/static-tar/file.c93138d8.txt","lastModified":/* ts */1706742000000});
+registerFile("./static-tgz/file.txt", {"name":"./static-tgz/file.txt","mimeType":"text/plain","path":"./_file/static-tgz/file.c93138d8.txt","lastModified":/* ts */1706742000000});
 
 define({id: "d5134368", inputs: ["FileAttachment","display"], body: async (FileAttachment,display) => {
 display(await(

test/output/build/archives.posix/zip.html

@@ -15,10 +15,10 @@
 import {define} from "./_observablehq/client.js";
 import {registerFile} from "./_observablehq/stdlib.js";
 
-registerFile("./dynamic/file.txt", {"name":"./dynamic/file.txt","mimeType":"text/plain","path":"./_file/dynamic/file.c93138d8.txt"});
-registerFile("./dynamic/not-found.txt", {"name":"./dynamic/not-found.txt","mimeType":"text/plain","path":"./dynamic/not-found.txt"});
-registerFile("./static/file.txt", {"name":"./static/file.txt","mimeType":"text/plain","path":"./_file/static/file.d9014c46.txt"});
-registerFile("./static/not-found.txt", {"name":"./static/not-found.txt","mimeType":"text/plain","path":"./static/not-found.txt"});
+registerFile("./dynamic/file.txt", {"name":"./dynamic/file.txt","mimeType":"text/plain","path":"./_file/dynamic/file.c93138d8.txt","lastModified":/* ts */1706742000000});
+registerFile("./dynamic/not-found.txt", {"name":"./dynamic/not-found.txt","mimeType":"text/plain","path":"./dynamic/not-found.txt","lastModified":/* ts */1706742000000});
+registerFile("./static/file.txt", {"name":"./static/file.txt","mimeType":"text/plain","path":"./_file/static/file.d9014c46.txt","lastModified":/* ts */1706742000000});
+registerFile("./static/not-found.txt", {"name":"./static/not-found.txt","mimeType":"text/plain","path":"./static/not-found.txt","lastModified":/* ts */1706742000000});
 
 define({id: "d3b9d0ee", inputs: ["FileAttachment","display"], body: async (FileAttachment,display) => {
 display(await(

test/output/build/archives.win32/tar.html

@@ -15,9 +15,9 @@
 import {define} from "./_observablehq/client.js";
 import {registerFile} from "./_observablehq/stdlib.js";
 
-registerFile("./static-tar/does-not-exist.txt", {"name":"./static-tar/does-not-exist.txt","mimeType":"text/plain","path":"./static-tar/does-not-exist.txt"});
-registerFile("./static-tar/file.txt", {"name":"./static-tar/file.txt","mimeType":"text/plain","path":"./_file/static-tar/file.c93138d8.txt"});
-registerFile("./static-tgz/file.txt", {"name":"./static-tgz/file.txt","mimeType":"text/plain","path":"./_file/static-tgz/file.c93138d8.txt"});
+registerFile("./static-tar/does-not-exist.txt", {"name":"./static-tar/does-not-exist.txt","mimeType":"text/plain","path":"./static-tar/does-not-exist.txt","lastModified":/* ts */1706742000000});
+registerFile("./static-tar/file.txt", {"name":"./static-tar/file.txt","mimeType":"text/plain","path":"./_file/static-tar/file.c93138d8.txt","lastModified":/* ts */1706742000000});
+registerFile("./static-tgz/file.txt", {"name":"./static-tgz/file.txt","mimeType":"text/plain","path":"./_file/static-tgz/file.c93138d8.txt","lastModified":/* ts */1706742000000});
 
 define({id: "d5134368", inputs: ["FileAttachment","display"], body: async (FileAttachment,display) => {
 display(await(

test/output/build/archives.win32/zip.html

@@ -15,8 +15,8 @@
 import {define} from "./_observablehq/client.js";
 import {registerFile} from "./_observablehq/stdlib.js";
 
-registerFile("./static/file.txt", {"name":"./static/file.txt","mimeType":"text/plain","path":"./_file/static/file.d9014c46.txt"});
-registerFile("./static/not-found.txt", {"name":"./static/not-found.txt","mimeType":"text/plain","path":"./static/not-found.txt"});
+registerFile("./static/file.txt", {"name":"./static/file.txt","mimeType":"text/plain","path":"./_file/static/file.d9014c46.txt","lastModified":/* ts */1706742000000});
+registerFile("./static/not-found.txt", {"name":"./static/not-found.txt","mimeType":"text/plain","path":"./static/not-found.txt","lastModified":/* ts */1706742000000});
 
 define({id: "d3b9d0ee", inputs: ["FileAttachment","display"], body: async (FileAttachment,display) => {
 display(await(

test/output/build/fetches/foo.html

@@ -16,8 +16,8 @@
 import {define} from "./_observablehq/client.js";
 import {registerFile} from "./_observablehq/stdlib.js";
 
-registerFile("./foo/foo-data.csv", {"name":"./foo/foo-data.csv","mimeType":"text/csv","path":"./_file/foo/foo-data.24ef4634.csv"});
-registerFile("./foo/foo-data.json", {"name":"./foo/foo-data.json","mimeType":"application/json","path":"./_file/foo/foo-data.67358ed8.json"});
+registerFile("./foo/foo-data.csv", {"name":"./foo/foo-data.csv","mimeType":"text/csv","path":"./_file/foo/foo-data.24ef4634.csv","lastModified":/* ts */1706742000000});
+registerFile("./foo/foo-data.json", {"name":"./foo/foo-data.json","mimeType":"application/json","path":"./_file/foo/foo-data.67358ed8.json","lastModified":/* ts */1706742000000});
 
 define({id: "47a695da", inputs: ["display"], outputs: ["fooJsonData","fooCsvData"], body: async (display) => {
 const {fooJsonData, fooCsvData} = await import("./_import/foo/foo.6fd063d5.js");

test/output/build/fetches/top.html

@@ -17,10 +17,10 @@
 import {define} from "./_observablehq/client.js";
 import {registerFile} from "./_observablehq/stdlib.js";
 
-registerFile("./foo/foo-data.csv", {"name":"./foo/foo-data.csv","mimeType":"text/csv","path":"./_file/foo/foo-data.24ef4634.csv"});
-registerFile("./foo/foo-data.json", {"name":"./foo/foo-data.json","mimeType":"application/json","path":"./_file/foo/foo-data.67358ed8.json"});
-registerFile("./top-data.csv", {"name":"./top-data.csv","mimeType":"text/csv","path":"./_file/top-data.24ef4634.csv"});
-registerFile("./top-data.json", {"name":"./top-data.json","mimeType":"application/json","path":"./_file/top-data.67358ed8.json"});
+registerFile("./foo/foo-data.csv", {"name":"./foo/foo-data.csv","mimeType":"text/csv","path":"./_file/foo/foo-data.24ef4634.csv","lastModified":/* ts */1706742000000});
+registerFile("./foo/foo-data.json", {"name":"./foo/foo-data.json","mimeType":"application/json","path":"./_file/foo/foo-data.67358ed8.json","lastModified":/* ts */1706742000000});
+registerFile("./top-data.csv", {"name":"./top-data.csv","mimeType":"text/csv","path":"./_file/top-data.24ef4634.csv","lastModified":/* ts */1706742000000});
+registerFile("./top-data.json", {"name":"./top-data.json","mimeType":"application/json","path":"./_file/top-data.67358ed8.json","lastModified":/* ts */1706742000000});
 
 define({id: "cb908c08", inputs: ["display"], outputs: ["fooCsvData","fooJsonData","topCsvData","topJsonData"], body: async (display) => {
 const {fooCsvData, fooJsonData, topCsvData, topJsonData} = await import("./_import/top.d8f5cc36.js");