Merge pull request #340 from crazy-max/oci

oci: loadArchive to import an index from a tar archive image bundle

.gitattributes

@@ -1,2 +1,3 @@
 /.yarn/releases/** binary
 /.yarn/plugins/** binary
+/__tests__/fixtures/oci-archive/** binary

__tests__/fixtures/hello.Dockerfile

@@ -14,7 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM busybox
+FROM busybox AS build
 ARG NAME=foo
 ARG TARGETPLATFORM
-RUN echo "Hello $NAME from $TARGETPLATFORM"
+RUN echo "Hello $NAME from $TARGETPLATFORM" > foo
+
+FROM scratch
+COPY --from=build /foo /

__tests__/oci/oci.test.ts

@@ -0,0 +1,50 @@
+/**
+ * Copyright 2024 actions-toolkit authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {afterEach, beforeEach, describe, expect, jest, test} from '@jest/globals';
+import * as fs from 'fs';
+import path from 'path';
+import * as rimraf from 'rimraf';
+
+import {OCI} from '../../src/oci/oci';
+
+const fixturesDir = path.join(__dirname, '..', 'fixtures');
+
+// prettier-ignore
+const tmpDir = path.join(process.env.TEMP || '/tmp', 'docker-jest');
+
+beforeEach(() => {
+  jest.clearAllMocks();
+});
+
+afterEach(function () {
+  rimraf.sync(tmpDir);
+});
+
+describe('loadArchive', () => {
+  // prettier-ignore
+  test.each(fs.readdirSync(path.join(fixturesDir, 'oci-archive')).filter(file => {
+      return fs.statSync(path.join(path.join(fixturesDir, 'oci-archive'), file)).isFile();
+    }).map(filename => [filename]))('extracting %p', async (filename) => {
+    const res = await OCI.loadArchive({
+      file: path.join(fixturesDir, 'oci-archive', filename)
+    });
+    expect(res).toBeDefined();
+    expect(res?.root.index).toBeDefined();
+    expect(res?.root.layout).toBeDefined();
+    // console.log(JSON.stringify(res, null, 2));
+  });
+});

package.json

@@ -58,17 +58,21 @@
     "@octokit/plugin-rest-endpoint-methods": "^10.4.0",
     "async-retry": "^1.3.3",
     "csv-parse": "^5.5.6",
+    "gunzip-maybe": "^1.4.2",
     "handlebars": "^4.7.8",
     "js-yaml": "^4.1.0",
     "jwt-decode": "^4.0.0",
     "semver": "^7.6.2",
+    "tar-stream": "^3.1.7",
     "tmp": "^0.2.3"
   },
   "devDependencies": {
     "@types/csv-parse": "^1.2.2",
+    "@types/gunzip-maybe": "^1.4.2",
     "@types/js-yaml": "^4.0.9",
     "@types/node": "^20.12.10",
     "@types/semver": "^7.5.8",
+    "@types/tar-stream": "^3.1.3",
     "@types/tmp": "^0.2.6",
     "@typescript-eslint/eslint-plugin": "^7.8.0",
     "@typescript-eslint/parser": "^7.8.0",

src/oci/oci.ts

@@ -0,0 +1,163 @@
+/**
+ * Copyright 2024 actions-toolkit authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import fs from 'fs';
+import gunzip from 'gunzip-maybe';
+import * as path from 'path';
+import {Readable} from 'stream';
+import * as tar from 'tar-stream';
+
+import {Archive, LoadArchiveOpts} from '../types/oci/oci';
+import {Index} from '../types/oci';
+import {Manifest} from '../types/oci/manifest';
+import {Image} from '../types/oci/config';
+import {IMAGE_BLOBS_DIR_V1, IMAGE_INDEX_FILE_V1, IMAGE_LAYOUT_FILE_V1, ImageLayout} from '../types/oci/layout';
+import {MEDIATYPE_IMAGE_INDEX_V1, MEDIATYPE_IMAGE_MANIFEST_V1} from '../types/oci/mediatype';
+
+export class OCI {
+  public static loadArchive(opts: LoadArchiveOpts): Promise<Archive> {
+    return new Promise<Archive>((resolve, reject) => {
+      const tarex: tar.Extract = tar.extract();
+
+      let rootIndex: Index;
+      let rootLayout: ImageLayout;
+      const indexes: Record<string, Index> = {};
+      const manifests: Record<string, Manifest> = {};
+      const images: Record<string, Image> = {};
+      const blobs: Record<string, unknown> = {};
+
+      tarex.on('entry', async (header, stream, next) => {
+        if (header.type === 'file') {
+          const filename = path.normalize(header.name);
+          if (filename === IMAGE_INDEX_FILE_V1) {
+            rootIndex = await OCI.streamToJson<Index>(stream);
+          } else if (filename === IMAGE_LAYOUT_FILE_V1) {
+            rootLayout = await OCI.streamToJson<ImageLayout>(stream);
+          } else if (filename.startsWith(path.join(IMAGE_BLOBS_DIR_V1, path.sep))) {
+            const blob = await OCI.extractBlob(stream);
+            const digest = `${filename.split(path.sep)[1]}:${filename.split(path.sep)[filename.split(path.sep).length - 1]}`;
+            if (OCI.isIndex(blob)) {
+              indexes[digest] = <Index>JSON.parse(blob);
+            } else if (OCI.isManifest(blob)) {
+              manifests[digest] = <Manifest>JSON.parse(blob);
+            } else if (OCI.isImage(blob)) {
+              images[digest] = <Image>JSON.parse(blob);
+            } else {
+              blobs[digest] = blob;
+            }
+          } else {
+            reject(new Error(`Invalid OCI archive: unexpected file ${filename}`));
+          }
+        }
+        stream.resume();
+        next();
+      });
+
+      tarex.on('finish', () => {
+        if (!rootIndex || !rootLayout) {
+          reject(new Error('Invalid OCI archive: missing index or layout'));
+        }
+        resolve({
+          root: {
+            index: rootIndex,
+            layout: rootLayout
+          },
+          indexes: indexes,
+          manifests: manifests,
+          images: images,
+          blobs: blobs
+        } as Archive);
+      });
+
+      tarex.on('error', error => {
+        reject(error);
+      });
+
+      fs.createReadStream(opts.file).pipe(gunzip()).pipe(tarex);
+    });
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private static isIndex(blob: any): boolean {
+    try {
+      const index = <Index>JSON.parse(blob);
+      return index.mediaType === MEDIATYPE_IMAGE_INDEX_V1;
+    } catch {
+      return false;
+    }
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private static isManifest(blob: any): boolean {
+    try {
+      const manifest = <Manifest>JSON.parse(blob);
+      return manifest.mediaType === MEDIATYPE_IMAGE_MANIFEST_V1 && manifest.layers.length > 0;
+    } catch {
+      return false;
+    }
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private static isImage(blob: any): boolean {
+    try {
+      const image = <Image>JSON.parse(blob);
+      return image.rootfs.type !== '';
+    } catch {
+      return false;
+    }
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private static extractBlob(stream: Readable): Promise<any> {
+    return new Promise<unknown>((resolve, reject) => {
+      const chunks: Buffer[] = [];
+      const dstream = stream.pipe(gunzip());
+      dstream.on('data', chunk => {
+        chunks.push(chunk);
+      });
+      dstream.on('end', () => {
+        resolve(Buffer.concat(chunks).toString('utf8'));
+      });
+      dstream.on('error', async error => {
+        reject(error);
+      });
+    });
+  }
+
+  private static async streamToJson<T>(stream: Readable): Promise<T> {
+    return new Promise<T>((resolve, reject) => {
+      const chunks: string[] = [];
+      let bytes = 0;
+      stream.on('data', chunk => {
+        bytes += chunk.length;
+        if (bytes <= 2 * 1024 * 1024) {
+          chunks.push(chunk.toString('utf8'));
+        } else {
+          reject(new Error('The data stream exceeds the size limit for JSON parsing.'));
+        }
+      });
+      stream.on('end', () => {
+        try {
+          resolve(JSON.parse(chunks.join('')));
+        } catch (error) {
+          reject(error);
+        }
+      });
+      stream.on('error', async error => {
+        reject(error);
+      });
+    });
+  }
+}

src/types/oci/config.ts

@@ -0,0 +1,52 @@
+/**
+ * Copyright 2024 actions-toolkit authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {Digest} from './digest';
+import {Platform} from './descriptor';
+
+export interface ImageConfig {
+  User?: string;
+  ExposedPorts?: Record<string, unknown>;
+  Env?: string[];
+  Entrypoint?: string[];
+  Cmd?: string[];
+  Volumes?: Record<string, unknown>;
+  WorkingDir?: string;
+  Labels?: Record<string, string>;
+  StopSignal?: string;
+  ArgsEscaped?: boolean;
+}
+
+export interface RootFS {
+  type: string;
+  diff_ids: Digest[];
+}
+
+export interface History {
+  created?: string; // assuming RFC 3339 formatted string
+  created_by?: string;
+  author?: string;
+  comment?: string;
+  empty_layer?: boolean;
+}
+
+export interface Image extends Platform {
+  created?: string; // assuming RFC 3339 formatted string
+  author?: string;
+  config?: ImageConfig;
+  rootfs: RootFS;
+  history?: History[];
+}

src/types/oci/descriptor.ts

@@ -0,0 +1,45 @@
+/**
+ * Copyright 2024 actions-toolkit authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {Digest} from './digest';
+
+import {MEDIATYPE_EMPTY_JSON_V1} from './mediatype';
+
+export interface Descriptor {
+  mediaType: string;
+  digest: Digest;
+  size: number;
+  urls?: string[];
+  annotations?: Record<string, string>;
+  data?: string;
+  platform?: Platform;
+  artifactType?: string;
+}
+
+export interface Platform {
+  architecture: string;
+  os: string;
+  'os.version'?: string;
+  'os.features'?: string[];
+  variant?: string;
+}
+
+export const DescriptorEmptyJSON: Descriptor = {
+  mediaType: MEDIATYPE_EMPTY_JSON_V1,
+  digest: 'sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a',
+  size: 2,
+  data: '{}'
+};

src/types/oci/digest.ts

@@ -0,0 +1,17 @@
+/**
+ * Copyright 2024 actions-toolkit authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type Digest = string;

src/types/oci/index.ts

@@ -0,0 +1,26 @@
+/**
+ * Copyright 2024 actions-toolkit authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {Versioned} from './versioned';
+import {Descriptor} from './descriptor';
+
+export interface Index extends Versioned {
+  mediaType?: string;
+  artifactType?: string;
+  manifests: Descriptor[];
+  subject?: Descriptor;
+  annotations?: Record<string, string>;
+}