-
Notifications
You must be signed in to change notification settings - Fork 12
/
file_fetcher.ts
256 lines (235 loc) · 7.43 KB
/
file_fetcher.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
import { AuthTokens } from "./auth_tokens.ts";
import { colors, fromFileUrl, readAll } from "./deps.ts";
import type { LoadResponse } from "./deps.ts";
import type { HttpCache } from "./http_cache.ts";
/** A setting that determines how the cache is handled for remote dependencies.
*
* The default is `"use"`.
*
* - `"only"` - only the cache will be re-used, and any remote modules not in
* the cache will error.
* - `"use"` - the cache will be used, meaning existing remote files will not be
* reloaded.
* - `"reloadAll"` - any cached modules will be ignored and their values will be
* fetched.
* - `string[]` - an array of string specifiers, that if they match the start of
* the requested specifier, will be reloaded.
*/
export type CacheSetting = "only" | "reloadAll" | "use" | string[];
function shouldUseCache(cacheSetting: CacheSetting, specifier: URL): boolean {
switch (cacheSetting) {
case "only":
case "use":
return true;
case "reloadAll":
return false;
default: {
const specifierStr = specifier.toString();
for (const value of cacheSetting) {
if (specifierStr.startsWith(value)) {
return false;
}
}
return true;
}
}
}
const SUPPORTED_SCHEMES = [
"data:",
"blob:",
"file:",
"http:",
"https:",
] as const;
type SupportedSchemes = typeof SUPPORTED_SCHEMES[number];
function getValidatedScheme(specifier: URL) {
const scheme = specifier.protocol;
// deno-lint-ignore no-explicit-any
if (!SUPPORTED_SCHEMES.includes(scheme as any)) {
throw new TypeError(
`Unsupported scheme "${scheme}" for module "${specifier.toString()}". Supported schemes: ${
JSON.stringify(SUPPORTED_SCHEMES)
}.`,
);
}
return scheme as SupportedSchemes;
}
export function stripHashbang(value: string): string {
return value.startsWith("#!") ? value.slice(value.indexOf("\n")) : value;
}
async function fetchLocal(specifier: URL): Promise<LoadResponse | undefined> {
const local = fromFileUrl(specifier);
if (!local) {
throw new TypeError(
`Invalid file path.\n Specifier: ${specifier.toString()}`,
);
}
try {
const source = await Deno.readTextFile(local);
const content = stripHashbang(source);
return {
kind: "module",
content,
specifier: specifier.toString(),
};
} catch {
// ignoring errors, we will just return undefined
}
}
const decoder = new TextDecoder();
export class FileFetcher {
#allowRemote: boolean;
#authTokens: AuthTokens;
#cache = new Map<string, LoadResponse>();
#cacheSetting: CacheSetting;
#httpCache: HttpCache;
constructor(
httpCache: HttpCache,
cacheSetting: CacheSetting = "use",
allowRemote = true,
) {
Deno.permissions.request({ name: "env", variable: "DENO_AUTH_TOKENS" });
this.#authTokens = new AuthTokens(Deno.env.get("DENO_AUTH_TOKENS"));
this.#allowRemote = allowRemote;
this.#cacheSetting = cacheSetting;
this.#httpCache = httpCache;
}
async #fetchBlobDataUrl(specifier: URL): Promise<LoadResponse> {
const cached = await this.#fetchCached(specifier, 0);
if (cached) {
return cached;
}
if (this.#cacheSetting === "only") {
throw new Deno.errors.NotFound(
`Specifier not found in cache: "${specifier.toString()}", --cached-only is specified.`,
);
}
const response = await fetch(specifier.toString());
const content = await response.text();
const headers: Record<string, string> = {};
for (const [key, value] of response.headers) {
headers[key.toLowerCase()] = value;
}
await this.#httpCache.set(specifier, headers, content);
return {
kind: "module",
specifier: specifier.toString(),
headers,
content,
};
}
async #fetchCached(
specifier: URL,
redirectLimit: number,
): Promise<LoadResponse | undefined> {
if (redirectLimit < 0) {
throw new Deno.errors.Http("Too many redirects");
}
const cached = await this.#httpCache.get(specifier);
if (!cached) {
return undefined;
}
const [file, headers] = cached;
const location = headers["location"];
if (location) {
const redirect = new URL(location, specifier);
file.close();
return this.#fetchCached(redirect, redirectLimit - 1);
}
const bytes = await readAll(file);
file.close();
const content = decoder.decode(bytes);
return {
kind: "module",
specifier: specifier.toString(),
headers,
content,
};
}
async #fetchRemote(
specifier: URL,
redirectLimit: number,
): Promise<LoadResponse | undefined> {
if (redirectLimit < 0) {
throw new Deno.errors.Http("Too many redirects.");
}
if (shouldUseCache(this.#cacheSetting, specifier)) {
const response = await this.#fetchCached(specifier, redirectLimit);
if (response) {
return response;
}
}
if (this.#cacheSetting === "only") {
throw new Deno.errors.NotFound(
`Specifier not found in cache: "${specifier.toString()}", --cached-only is specified.`,
);
}
const requestHeaders = new Headers();
const cached = await this.#httpCache.get(specifier);
if (cached) {
const [file, cachedHeaders] = cached;
file.close();
if (cachedHeaders["etag"]) {
requestHeaders.append("if-none-match", cachedHeaders["etag"]);
}
}
const authToken = this.#authTokens.get(specifier);
if (authToken) {
requestHeaders.append("authorization", authToken);
}
console.log(`${colors.green("Download")} ${specifier.toString()}`);
const response = await fetch(specifier.toString(), {
headers: requestHeaders,
});
if (!response.ok) {
if (response.status === 404) {
return undefined;
} else {
throw new Deno.errors.Http(`${response.status} ${response.statusText}`);
}
}
// WHATWG fetch follows redirects automatically, so we will try to
// determine if that ocurred and cache the value.
if (specifier.toString() !== response.url) {
const headers = { "location": response.url };
await this.#httpCache.set(specifier, headers, "");
}
const url = new URL(response.url);
const content = await response.text();
const headers: Record<string, string> = {};
for (const [key, value] of response.headers) {
headers[key.toLowerCase()] = value;
}
await this.#httpCache.set(url, headers, content);
return {
kind: "module",
specifier: response.url,
headers,
content,
};
}
async fetch(specifier: URL): Promise<LoadResponse | undefined> {
const scheme = getValidatedScheme(specifier);
const response = this.#cache.get(specifier.toString());
if (response) {
return response;
} else if (scheme === "file:") {
return fetchLocal(specifier);
} else if (scheme === "data:" || scheme === "blob:") {
const response = await this.#fetchBlobDataUrl(specifier);
this.#cache.set(specifier.toString(), response);
return response;
} else if (!this.#allowRemote) {
throw new Deno.errors.PermissionDenied(
`A remote specifier was requested: "${specifier.toString()}", but --no-remote is specifier`,
);
} else {
const response = await this.#fetchRemote(specifier, 10);
if (response) {
this.#cache.set(specifier.toString(), response);
}
return response;
}
}
}