Skip to content

Commit

Permalink
feat(datasource/docker): Support OCI image manifests (#14480)
Browse files Browse the repository at this point in the history 
Co-authored-by: Rhys Arkins <rhys@arkins.net>
Co-authored-by: Jamie Magee <jamie.magee@gmail.com>
  • Loading branch information
@bpfoster @rarkins @JamieMagee
3 people committed Mar 14, 2022
1 parent edd5f1a commit 24fa081
Show file tree
Hide file tree
Showing 4 changed files with 266 additions and 17 deletions.
38 changes: 38 additions & 0 deletions lib/modules/datasource/docker/__snapshots__/index.spec.ts.snap
View file
@@ -1,5 +1,19 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`modules/datasource/docker/index getReleases ignores empty OCI manifest indexes 1`] = `
Object {
"registryUrl": "https://registry.company.com",
"releases": Array [],
}
`;

exports[`modules/datasource/docker/index getReleases ignores empty manifest lists 1`] = `
Object {
"registryUrl": "https://registry.company.com",
"releases": Array [],
}
`;

exports[`modules/datasource/docker/index getReleases ignores unsupported manifest 1`] = `
Object {
"registryUrl": "https://registry.company.com",
Expand All @@ -14,6 +28,30 @@ Object {
}
`;

exports[`modules/datasource/docker/index getReleases supports OCI manifests with media type 1`] = `
Object {
"registryUrl": "https://registry.company.com",
"releases": Array [
Object {
"version": "1",
},
],
"sourceUrl": "https://github.com/renovatebot/renovate",
}
`;

exports[`modules/datasource/docker/index getReleases supports OCI manifests without media type 1`] = `
Object {
"registryUrl": "https://registry.company.com",
"releases": Array [
Object {
"version": "1",
},
],
"sourceUrl": "https://github.com/renovatebot/renovate",
}
`;

exports[`modules/datasource/docker/index getReleases supports labels 1`] = `
Object {
"registryUrl": "https://registry.company.com",
Expand Down
113 changes: 113 additions & 0 deletions lib/modules/datasource/docker/index.spec.ts
View file
Expand Up @@ -1140,6 +1140,27 @@ Object {
expect(res).toMatchSnapshot();
});

it('ignores empty manifest lists', async () => {
httpMock
.scope('https://registry.company.com/v2')
.get('/')
.times(2)
.reply(200)
.get('/node/tags/list?n=10000')
.reply(200, { tags: ['latest'] })
.get('/node/manifests/latest')
.reply(200, {
schemaVersion: 2,
mediaType: MediaType.manifestListV2,
manifests: [],
});
const res = await getPkgReleases({
datasource: DockerDatasource.id,
depName: 'registry.company.com/node',
});
expect(res).toMatchSnapshot();
});

it('ignores unsupported manifest', async () => {
httpMock
.scope('https://registry.company.com/v2')
Expand Down Expand Up @@ -1177,6 +1198,98 @@ Object {
expect(res).toMatchSnapshot();
});

it('supports OCI manifests with media type', async () => {
httpMock
.scope('https://registry.company.com/v2')
.get('/')
.times(4)
.reply(200)
.get('/node/tags/list?n=10000')
.reply(200, { tags: ['1'] })
.get('/node/manifests/1')
.reply(200, {
schemaVersion: 2,
mediaType: MediaType.ociManifestIndexV1,
manifests: [{ digest: 'some-image-digest' }],
})
.get('/node/manifests/some-image-digest')
.reply(200, {
schemaVersion: 2,
mediaType: MediaType.ociManifestV1,
config: { digest: 'some-config-digest' },
})
.get('/node/blobs/some-config-digest')
.reply(200, {
config: {
Labels: {
'org.opencontainers.image.source':
'https://github.com/renovatebot/renovate',
},
},
});
const res = await getPkgReleases({
datasource: DockerDatasource.id,
depName: 'registry.company.com/node',
});
expect(res).toMatchSnapshot();
});

it('supports OCI manifests without media type', async () => {
httpMock
.scope('https://registry.company.com/v2')
.get('/')
.times(4)
.reply(200)
.get('/node/tags/list?n=10000')
.reply(200, { tags: ['1'] })
.get('/node/manifests/1')
.reply(200, {
schemaVersion: 2,
mediaType: MediaType.ociManifestIndexV1,
manifests: [{ digest: 'some-image-digest' }],
})
.get('/node/manifests/some-image-digest')
.reply(200, {
schemaVersion: 2,
config: { digest: 'some-config-digest' },
})
.get('/node/blobs/some-config-digest')
.reply(200, {
config: {
Labels: {
'org.opencontainers.image.source':
'https://github.com/renovatebot/renovate',
},
},
});
const res = await getPkgReleases({
datasource: DockerDatasource.id,
depName: 'registry.company.com/node',
});
expect(res).toMatchSnapshot();
});

it('ignores empty OCI manifest indexes', async () => {
httpMock
.scope('https://registry.company.com/v2')
.get('/')
.times(2)
.reply(200)
.get('/node/tags/list?n=10000')
.reply(200, { tags: ['latest'] })
.get('/node/manifests/latest')
.reply(200, {
schemaVersion: 2,
mediaType: MediaType.ociManifestIndexV1,
manifests: [],
});
const res = await getPkgReleases({
datasource: DockerDatasource.id,
depName: 'registry.company.com/node',
});
expect(res).toMatchSnapshot();
});

it('supports redirect', async () => {
httpMock
.scope('https://registry.company.com/v2', {
Expand Down
86 changes: 70 additions & 16 deletions lib/modules/datasource/docker/index.ts
View file
Expand Up @@ -32,7 +32,14 @@ import {
import { Datasource } from '../datasource';
import type { GetReleasesConfig, ReleaseResult } from '../types';
import { sourceLabels } from './common';
import { Image, ImageList, MediaType, RegistryRepository } from './types';
import {
Image,
ImageList,
MediaType,
OciImage,
OciImageList,
RegistryRepository,
} from './types';

export const DOCKER_HUB = 'https://index.docker.io';

Expand Down Expand Up @@ -368,8 +375,12 @@ export class DockerDatasource extends Datasource {
logger.debug('No docker auth found - returning');
return null;
}
headers.accept =
'application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json';
headers.accept = [
MediaType.manifestListV2,
MediaType.manifestV2,
MediaType.ociManifestV1,
MediaType.ociManifestIndexV1,
].join(', ');
const url = `${registryHost}/v2/${dockerRepository}/manifests/${tag}`;
const manifestResponse = await this.http[mode](url, {
headers,
Expand Down Expand Up @@ -444,7 +455,11 @@ export class DockerDatasource extends Datasource {
if (!manifestResponse) {
return null;
}
const manifest = JSON.parse(manifestResponse.body) as ImageList | Image;
const manifest = JSON.parse(manifestResponse.body) as
| ImageList
| Image
| OciImageList
| OciImage;
if (manifest.schemaVersion !== 2) {
logger.debug(
{ registry, dockerRepository, tag },
Expand All @@ -453,23 +468,62 @@ export class DockerDatasource extends Datasource {
return null;
}

if (manifest.mediaType === MediaType.manifestListV2) {
if (manifest.manifests.length) {
logger.trace(
{ registry, dockerRepository, tag },
'Found manifest list, using first image'
);
return this.getConfigDigest(
registry,
dockerRepository,
manifest.manifests[0].digest
);
} else {
logger.debug(
{ manifest },
'Invalid manifest list with no manifests - returning'
);
return null;
}
}

if (
manifest.mediaType === MediaType.manifestListV2 &&
manifest.manifests.length
manifest.mediaType === MediaType.manifestV2 &&
is.string(manifest.config?.digest)
) {
logger.trace(
{ registry, dockerRepository, tag },
'Found manifest list, using first image'
);
return this.getConfigDigest(
registry,
dockerRepository,
manifest.manifests[0].digest
);
return manifest.config?.digest;
}

// OCI image lists are not required to specify a mediaType
if (
manifest.mediaType === MediaType.manifestV2 &&
manifest.mediaType === MediaType.ociManifestIndexV1 ||
(!manifest.mediaType && hasKey('manifests', manifest))
) {
const imageList = manifest as OciImageList;
if (imageList.manifests.length) {
logger.trace(
{ registry, dockerRepository, tag },
'Found manifest index, using first image'
);
return this.getConfigDigest(
registry,
dockerRepository,
manifest.manifests[0].digest
);
} else {
logger.debug(
{ manifest },
'Invalid manifest index with no manifests - returning'
);
return null;
}
}

// OCI manifests are not required to specify a mediaType
if (
(manifest.mediaType === MediaType.ociManifestV1 ||
(!manifest.mediaType && hasKey('config', manifest))) &&
is.string(manifest.config?.digest)
) {
return manifest.config?.digest;
Expand Down
46 changes: 45 additions & 1 deletion lib/modules/datasource/docker/types.ts
View file
Expand Up @@ -3,19 +3,22 @@
/**
* Media Types
* https://docs.docker.com/registry/spec/manifest-v2-2/#media-types
* https://github.com/opencontainers/image-spec/blob/main/media-types.md
*/
// eslint-disable-next-line typescript-enum/no-enum
export enum MediaType {
manifestV1 = 'application/vnd.docker.distribution.manifest.v1+json',
manifestV2 = 'application/vnd.docker.distribution.manifest.v2+json',
manifestListV2 = 'application/vnd.docker.distribution.manifest.list.v2+json',
ociManifestV1 = 'application/vnd.oci.image.manifest.v1+json',
ociManifestIndexV1 = 'application/vnd.oci.image.index.v1+json',
}
/* eslint-enable @typescript-eslint/naming-convention */

export interface MediaObject {
readonly digest: string;
readonly mediaType: MediaType;
readonly site: number;
readonly size: number;
}

export interface ImageListImage extends MediaObject {
Expand Down Expand Up @@ -45,6 +48,47 @@ export interface Image extends MediaObject {
readonly config: MediaObject;
}

/**
* OCI content descriptor
* https://github.com/opencontainers/image-spec/blob/main/descriptor.md
*/
export interface OciDescriptor {
readonly mediaType?: MediaType;
readonly digest: string;
readonly size: number;
readonly urls: string[];
readonly annotations: Record<string, string>;
}

/**
* OCI Image Manifest
* The same structure as docker image manifest, but mediaType is not required and is not present in the wild.
* https://github.com/opencontainers/image-spec/blob/main/manifest.md
*/
export interface OciImage {
readonly schemaVersion: 2;
readonly mediaType?: MediaType.ociManifestV1;
readonly config: OciDescriptor;
readonly layers: OciDescriptor[];
readonly annotations: Record<string, string>;
}

export interface OciImageListManifest extends OciDescriptor {
readonly mediaType?: MediaType.ociManifestV1 | MediaType.ociManifestIndexV1;
readonly platform: Record<string, unknown>;
}

/**
* OCI Image List
* mediaType is not required.
* https://github.com/opencontainers/image-spec/blob/main/image-index.md
*/
export interface OciImageList {
readonly schemaVersion: 2;
readonly mediaType?: MediaType.ociManifestIndexV1;
readonly manifests: OciImageListManifest[];
}

export interface RegistryRepository {
registryHost: string;
dockerRepository: string;
Expand Down

0 comments on commit 24fa081

Please sign in to comment.