Skip to content

Commit

Permalink
feat: parse child_type resource (#176)
Browse files Browse the repository at this point in the history
* feat: parse child_type resource

* fix: YAGNI: we will not need to get resource by name

* fix: no need to have this.names

* fix: typo in comments
  • Loading branch information
alexander-fenster committed Dec 17, 2019
1 parent 25871fb commit ef7482f
Show file tree
Hide file tree
Showing 4 changed files with 371 additions and 87 deletions.
63 changes: 11 additions & 52 deletions typescript/src/schema/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ import * as fs from 'fs';
import * as path from 'path';

import { Naming } from './naming';
import { Proto, MessagesMap, ResourceDescriptor, ResourceMap } from './proto';
import { Proto, MessagesMap } from './proto';
import { ResourceDatabase, ResourceDescriptor } from './resourceDatabase';

const googleGaxLocation = path.dirname(require.resolve('google-gax'));
const gaxProtosLocation = path.join(googleGaxLocation, '..', '..', 'protos');
Expand Down Expand Up @@ -51,7 +52,7 @@ export class API {
// users specify the actual package name, if not, set it to product name.
this.publishName = publishName || this.naming.productName.toKebabCase();
// construct resource map
const resourceMap = getResourceMap(fileDescriptors);
const resourceMap = getResourceDatabase(fileDescriptors);
// parse resource map to Proto constructor
this.protos = fileDescriptors
.filter(fd => fd.name)
Expand Down Expand Up @@ -112,58 +113,17 @@ export class API {
}
}

function processOneResource(
option: ResourceDescriptor | undefined,
fileAndMessageNames: string,
resourceMap: ResourceMap
): void {
if (!option) {
return;
}
if (!option.type) {
console.warn(
`Warning: in ${fileAndMessageNames} refers to a resource which does not have a type: ${option}`
);
return;
}

const arr = option.type.match(/\/([^.]+)$/);
if (!arr?.[1]) {
console.warn(
`Warning: in ${fileAndMessageNames} refers to a resource which does not have a proper name: ${option}`
);
return;
}
option.name = arr[1];

const pattern = option.pattern;
if (!pattern?.[0]) {
console.warn(
`Warning: in ${fileAndMessageNames} refers to a resource which does not have a proper pattern: ${option}`
);
return;
}
const params = pattern[0].match(/{[a-zA-Z]+}/g) || [];
for (let i = 0; i < params.length; i++) {
params[i] = params[i].replace('{', '').replace('}', '');
}
option.params = params;

resourceMap[option.type!] = option;
}

function getResourceMap(
function getResourceDatabase(
fileDescriptors: plugin.google.protobuf.IFileDescriptorProto[]
): ResourceMap {
const resourceMap: ResourceMap = {};
): ResourceDatabase {
const resourceDatabase = new ResourceDatabase();
for (const fd of fileDescriptors.filter(fd => fd)) {
// process file-level options
for (const resource of fd.options?.['.google.api.resourceDefinition'] ??
[]) {
processOneResource(
resourceDatabase.registerResource(
resource as ResourceDescriptor,
`file ${fd.name} resource_definition option`,
resourceMap
`file ${fd.name} resource_definition option`
);
}

Expand All @@ -176,12 +136,11 @@ function getResourceMap(

for (const property of Object.keys(messages)) {
const m = messages[property];
processOneResource(
resourceDatabase.registerResource(
m?.options?.['.google.api.resource'] as ResourceDescriptor | undefined,
`file ${fd.name} message ${property}`,
resourceMap
`file ${fd.name} message ${property}`
);
}
}
return resourceMap;
return resourceDatabase;
}
67 changes: 32 additions & 35 deletions typescript/src/schema/proto.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import * as plugin from '../../../pbjs-genfiles/plugin';
import { CommentsMap, Comment } from './comments';
import * as objectHash from 'object-hash';
import { milliseconds } from '../util';
import { ResourceDescriptor, ResourceDatabase } from './resourceDatabase';

const defaultNonIdempotentRetryCodesName = 'non_idempotent';
const defaultNonIdempotentCodes: plugin.google.rpc.Code[] = [];
Expand Down Expand Up @@ -184,22 +185,14 @@ interface ServiceDescriptorProto
grpcServiceConfig: plugin.grpc.service_config.ServiceConfig;
}

export interface ResourceDescriptor
extends plugin.google.api.IResourceDescriptor {
name: string;
params: string[];
}

export interface ResourceMap {
[name: string]: ResourceDescriptor;
}

export interface ServicesMap {
[name: string]: ServiceDescriptorProto;
}

export interface MessagesMap {
[name: string]: plugin.google.protobuf.IDescriptorProto;
}

export interface EnumsMap {
[name: string]: plugin.google.protobuf.IEnumDescriptorProto;
}
Expand Down Expand Up @@ -479,7 +472,7 @@ function augmentService(
service: plugin.google.protobuf.IServiceDescriptorProto,
commentsMap: CommentsMap,
grpcServiceConfig: plugin.grpc.service_config.ServiceConfig,
resourceMap: ResourceMap
resourceDatabase: ResourceDatabase
) {
const augmentedService = service as ServiceDescriptorProto;
augmentedService.packageName = packageName;
Expand Down Expand Up @@ -530,32 +523,36 @@ function augmentService(
'.google.api.oauthScopes'
].split(',');
}
augmentedService.pathTemplates = [];

// Build a list of resources referenced by this service
const uniqueResources: { [name: string]: ResourceDescriptor } = {};
for (const property of Object.keys(messages)) {
const m = messages[property];
if (m?.field) {
const fields = m.field;
for (const fieldDescriptor of fields) {
if (fieldDescriptor?.options) {
const option = fieldDescriptor.options;
if (option?.['.google.api.resourceReference']) {
const resourceReference = option['.google.api.resourceReference'];
const type = resourceReference.type;
if (!type || !resourceMap[type.toString()]) {
const resourceJson = JSON.stringify(resourceReference);
console.warn(
`Warning: in service proto ${service.name} message ${property} refers to an unknown resource: ${resourceJson}`
);
continue;
}
const resource = resourceMap[resourceReference.type!.toString()];
if (augmentedService.pathTemplates.includes(resource)) continue;
augmentedService.pathTemplates.push(resource);
}
}
const errorLocation = `service ${service.name} message ${property}`;
for (const fieldDescriptor of messages[property].field ?? []) {
// note: ResourceDatabase can accept `undefined` values, so we happily use optional chaining here.
const resourceReference =
fieldDescriptor.options?.['.google.api.resourceReference'];

// 1. If this resource reference has .child_type, figure out if we have any known parent resources.
const parentResources = resourceDatabase.getParentResourcesByChildType(
resourceReference?.childType,
errorLocation
);
parentResources.map(
resource => (uniqueResources[resource.name] = resource)
);

// 2. If this resource reference has .type, we should have a known resource with this type.
const resource = resourceDatabase.getResourceByType(
resourceReference?.type,
errorLocation
);
if (resource) {
uniqueResources[resource.name] = resource;
}
}
}
augmentedService.pathTemplates = Object.values(uniqueResources);
return augmentedService;
}

Expand All @@ -571,7 +568,7 @@ export class Proto {
fd: plugin.google.protobuf.IFileDescriptorProto,
packageName: string,
grpcServiceConfig: plugin.grpc.service_config.ServiceConfig,
resourceMap: ResourceMap
resourceDatabase: ResourceDatabase
) {
fd.enumType = fd.enumType || [];
fd.messageType = fd.messageType || [];
Expand Down Expand Up @@ -605,7 +602,7 @@ export class Proto {
service,
commentsMap,
grpcServiceConfig,
resourceMap
resourceDatabase
)
)
.reduce((map, service) => {
Expand Down
158 changes: 158 additions & 0 deletions typescript/src/schema/resourceDatabase.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import * as plugin from '../../../pbjs-genfiles/plugin';

export interface ResourceDescriptor
extends plugin.google.api.IResourceDescriptor {
name: string;
params: string[];
}

export class ResourceDatabase {
private patterns: { [pattern: string]: ResourceDescriptor };
private types: { [type: string]: ResourceDescriptor };

constructor() {
this.patterns = {};
this.types = {};
}

registerResource(
resource: plugin.google.api.IResourceDescriptor | undefined,
errorLocation?: string
) {
if (!resource) {
return;
}

if (!resource.type) {
if (errorLocation) {
console.warn(
`Warning: ${errorLocation} refers to a resource which does not have a type: ${resource}`
);
}
return;
}

const arr = resource.type.match(/\/([^.]+)$/);
if (!arr?.[1]) {
if (errorLocation) {
console.warn(
`Warning: ${errorLocation} refers to a resource which does not have a proper name: ${resource}`
);
}
return;
}
const name = arr[1];

const pattern = resource.pattern;
if (!pattern?.[0]) {
if (errorLocation) {
console.warn(
`Warning: ${errorLocation} refers to a resource which does not have a proper pattern: ${resource}`
);
}
return;
}
const params = pattern[0].match(/{[a-zA-Z]+}/g) || [];
for (let i = 0; i < params.length; i++) {
params[i] = params[i].replace('{', '').replace('}', '');
}

const resourceDescriptor: ResourceDescriptor = Object.assign(
{
name,
params,
},
resource
);

this.patterns[pattern?.[0]] = resourceDescriptor;
this.types[resourceDescriptor.type!] = resourceDescriptor;
}

getResourceByType(
type: string | null | undefined,
errorLocation?: string
): ResourceDescriptor | undefined {
if (!type) {
return undefined;
}
if (!this.types[type]) {
if (errorLocation) {
console.warn(
`Warning: ${errorLocation} refers to an unknown resource: ${type}`
);
}
return undefined;
}
return this.types[type];
}

getResourceByPattern(
pattern: string | null | undefined,
errorLocation?: string
): ResourceDescriptor | undefined {
if (!pattern) {
return undefined;
}
if (!this.patterns[pattern]) {
if (errorLocation) {
console.warn(
`Warning: ${errorLocation} refers to an unknown resource: ${pattern}`
);
}
return undefined;
}
return this.patterns[pattern];
}

getParentResourcesByChildType(
childType: string | null | undefined,
errorLocation?: string
): ResourceDescriptor[] {
// childType looks like "datacatalog.googleapis.com/EntryGroup"
// its pattern would be like "projects/{project}/locations/{location}/entryGroups/{entry_group}"
const result: ResourceDescriptor[] = [];

if (!childType) {
return result;
}

const childResource = this.getResourceByType(childType, errorLocation);
if (!childResource) {
return result;
}

const childPattern = childResource.pattern?.[0];
if (!childPattern) {
return result;
}

let pattern = '';
for (const segment of childPattern.split('/')) {
if (pattern !== '') {
pattern += '/';
}
pattern += segment;
const parent = this.getResourceByPattern(pattern);
if (parent) {
result.push(parent);
}
}

return result;
}
}

0 comments on commit ef7482f

Please sign in to comment.