Skip to content

Commit

Permalink
Add tests for deserializing LitTypes from LitMetadata.
Browse files Browse the repository at this point in the history 
PiperOrigin-RevId: 464626730
  • Loading branch information
@cjqian
cjqian authored and LIT team committed Aug 1, 2022
1 parent 40d14e5 commit ccbb72c
Show file tree
Hide file tree
Showing 4 changed files with 305 additions and 11 deletions.
278 changes: 277 additions & 1 deletion lit_nlp/client/lib/testing_utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

import 'jasmine';

import {LitMetadata} from './types';
import {LitMetadata, SerializedLitMetadata} from './types';
import {createLitType} from './utils';

/**
Expand Down Expand Up @@ -187,3 +187,279 @@ export const mockMetadata: LitMetadata = {
'defaultLayout': 'default',
'canonicalURL': undefined
};

/**
* Mock serialized metadata describing a set of models, datasets, generators,
* and intepretators. Corresponds to the mockMetadata above.
*/
export const mockSerializedMetadata: SerializedLitMetadata = {
'models': {
'sst_0_micro': {
'spec': {
'input': {
'passage': {
'__class__': 'LitType',
'__name__': 'TextSegment',
'required': true
},
'passage_tokens': {
'__class__': 'LitType',
'__name__': 'Tokens',
'required': false,
'parent': 'passage'
}
},
'output': {
'probabilities': {
'__class__': 'LitType',
'__name__': 'MulticlassPreds',
'required': true,
'vocab': ['0', '1'],
'null_idx': 0,
'parent': 'label'
},
'pooled_embs': {
'__class__': 'LitType',
'__name__': 'Embeddings',
'required': true
},
'mean_word_embs': {
'__class__': 'LitType',
'__name__': 'Embeddings',
'required': true
},
'tokens': {
'__class__': 'LitType',
'__name__': 'Tokens',
'required': true,
'parent': undefined
},
'passage_tokens': {
'__class__': 'LitType',
'__name__': 'Tokens',
'required': true,
'parent': 'passage'
},
'passage_grad': {
'__class__': 'LitType',
'__name__': 'TokenGradients',
'required': true,
'align': 'passage_tokens'
},
'layer_0/attention': {
'__class__': 'LitType',
'__name__': 'AttentionHeads',
'required': true,
'align_in': 'tokens',
'align_out': 'tokens',
},
'layer_1/attention': {
'__class__': 'LitType',
'__name__': 'AttentionHeads',
'required': true,
'align_in': 'tokens',
'align_out': 'tokens',
}
}
},
'datasets': ['sst_dev'],
'generators':
['word_replacer', 'scrambler', 'backtranslation', 'hotflip'],
'interpreters':
['grad_norm', 'grad_sum', 'lime', 'metrics', 'pca', 'umap']
},
'sst_1_micro': {
'spec': {
'input': {
'passage': {
'__class__': 'LitType',
'__name__': 'TextSegment',
'required': true
},
'passage_tokens': {
'__class__': 'LitType',
'__name__': 'Tokens',
'required': false,
'parent': 'passage'
}
},
'output': {
'probabilities': {
'__class__': 'LitType',
'__name__': 'MulticlassPreds',
'required': true,
'vocab': ['0', '1'],
'null_idx': 0,
'parent': 'label'
},
'pooled_embs': {
'__class__': 'LitType',
'__name__': 'Embeddings',
'required': true
},
'mean_word_embs': {
'__class__': 'LitType',
'__name__': 'Embeddings',
'required': true
},
'tokens': {
'__class__': 'LitType',
'__name__': 'Tokens',
'required': true,
'parent': undefined
},
'passage_tokens': {
'__class__': 'LitType',
'__name__': 'Tokens',
'required': true,
'parent': 'passage'
},
'passage_grad': {
'__class__': 'LitType',
'__name__': 'TokenGradients',
'required': true,
'align': 'passage_tokens'
},
'layer_0/attention': {
'__class__': 'LitType',
'__name__': 'AttentionHeads',
'required': true,
'align_in': 'tokens',
'align_out': 'tokens',
},
'layer_1/attention': {
'__class__': 'LitType',
'__name__': 'AttentionHeads',
'required': true,
'align_in': 'tokens',
'align_out': 'tokens',
}
}
},
'datasets': ['sst_dev'],
'generators':
['word_replacer', 'scrambler', 'backtranslation', 'hotflip'],
'interpreters':
['grad_norm', 'grad_sum', 'lime', 'metrics', 'pca', 'umap']
}
},
'datasets': {
'sst_dev': {
'size': 872,
'spec': {
'passage': {
'__class__': 'LitType',
'__name__': 'TextSegment',
'required': true
},
'label': {
'__class__': 'LitType',
'__name__': 'CategoryLabel',
'required': true,
'vocab': ['0', '1']
}
}
},
'color_test': {
'size': 2,
'spec': {
'testNumFeat0':
{'__class__': 'LitType', '__name__': 'Scalar', 'required': true},
'testNumFeat1':
{'__class__': 'LitType', '__name__': 'Scalar', 'required': true},
'testFeat0': {
'__class__': 'LitType',
'__name__': 'CategoryLabel',
'required': true,
'vocab': ['0', '1']
},
'testFeat1': {
'__class__': 'LitType',
'__name__': 'CategoryLabel',
'required': true,
'vocab': ['a', 'b', 'c']
}
}
},
'penguin_dev': {
'size': 10,
'spec': {
'body_mass_g': {
'__class__': 'LitType',
'__name__': 'Scalar',
'step': 1,
'required': true
},
'culmen_depth_mm': {
'__class__': 'LitType',
'__name__': 'Scalar',
'step': 1,
'required': true
},
'culmen_length_mm': {
'__class__': 'LitType',
'__name__': 'Scalar',
'step': 1,
'required': true
},
'flipper_length_mm': {
'__class__': 'LitType',
'__name__': 'Scalar',
'step': 1,
'required': true
},
'island': {
'__class__': 'LitType',
'__name__': 'CategoryLabel',
'required': true,
'vocab': ['Biscoe', 'Dream', 'Torgersen']
},
'sex': {
'__class__': 'LitType',
'__name__': 'CategoryLabel',
'required': true,
'vocab': ['female', 'male']
},
'species': {
'__class__': 'LitType',
'__name__': 'CategoryLabel',
'required': true,
'vocab': ['Adelie', 'Chinstrap', 'Gentoo']
},
'isAlive': {
'__class__': 'LitType',
'__name__': 'BooleanLitType',
'required': false
}
}
}
},
'generators': {
'word_replacer': {
'configSpec': {
'Substitutions': {
'__class__': 'LitType',
'__name__': 'TextSegment',
'required': true,
'default': 'great -> terrible'
}
},
'metaSpec': {}
},
'scrambler': emptySpec(),
'backtranslation': emptySpec(),
'hotflip': emptySpec(),
},
'interpreters': {
'grad_norm': emptySpec(),
'grad_sum': emptySpec(),
'lime': emptySpec(),
'metrics': emptySpec(),
'pca': emptySpec(),
'umap': emptySpec(),
},
'layouts': {},
'demoMode': false,
'defaultLayout': 'default',
'canonicalURL': undefined
};
15 changes: 15 additions & 0 deletions lit_nlp/client/lib/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,13 @@ export interface Spec {
[key: string]: LitType;
}

/** Serialized Spec data returned from the backend. */
export interface SerializedSpec {
// All LitTypes have a `__name__` field; we deserialize and cast the
// LitType before accessing additional fields.
[key: string]: {__name__: string};
}

export interface ComponentInfo {
configSpec: Spec;
metaSpec: Spec;
Expand Down Expand Up @@ -86,6 +93,14 @@ export interface LitMetadata {
onboardEndDoc?: string;
}

/**
* Serialized LitMetadata returned by the backend.
*/
export type SerializedLitMetadata = {
// tslint:disable-next-line:no-any
[K in keyof LitMetadata]: any;
};

export interface Input {
// tslint:disable-next-line:no-any
[key: string]: any;
Expand Down
12 changes: 3 additions & 9 deletions lit_nlp/client/lib/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import {unsafeHTML} from 'lit/directives/unsafe-html.js';

import {marked} from 'marked';
import {LitName, LitType, LitTypeWithParent, MulticlassPreds, REGISTRY} from './lit_types';
import {FacetMap, LitMetadata, ModelInfoMap, Spec} from './types';
import {FacetMap, LitMetadata, ModelInfoMap, SerializedLitMetadata, SerializedSpec, Spec} from './types';

/** Calculates the mean for a list of numbers */
export function mean(values: number[]): number {
Expand Down Expand Up @@ -116,18 +116,13 @@ export function createLitType(
newType[key] = constructorParams[key];
} else {
throw new Error(
`Attempted to set unrecognized property ${key} on ${newType}.`);
`Attempted to set unrecognized property ${key} on ${typeName}.`);
}
}

return newType;
}


interface SerializedSpec {
[key: string]: {__name__: string};
}

/**
* Converts serialized LitTypes within a Spec into LitType instances.
*/
Expand All @@ -140,11 +135,10 @@ export function deserializeLitTypesInSpec(serializedSpec: SerializedSpec): Spec
return typedSpec;
}


/**
* Converts serialized LitTypes within the LitMetadata into LitType instances.
*/
export function deserializeLitTypesInLitMetadata(metadata: LitMetadata):
export function deserializeLitTypesInLitMetadata(metadata: SerializedLitMetadata) :
LitMetadata {
for (const model of Object.keys(metadata.models)) {
metadata.models[model].spec.input =
Expand Down
11 changes: 10 additions & 1 deletion lit_nlp/client/lib/utils_test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import 'jasmine';

import * as litTypes from '../lib/lit_types';
import {mockMetadata, mockSerializedMetadata} from './testing_utils';
import {Spec} from '../lib/types';

import * as utils from './utils';
Expand Down Expand Up @@ -194,7 +195,6 @@ describe('createLitType test', () => {
});

describe('deserializeLitTypesInSpec test', () => {
// TODO(b/162269499): Add test for deserializeLitTypesInLitMetadata.
const testSpec = {
'probabilities': {
'__class__': 'LitType',
Expand Down Expand Up @@ -225,6 +225,15 @@ describe('deserializeLitTypesInSpec test', () => {
.toBe(true);
});
});

describe('deserializeLitTypesInLitMetadata test', () => {
it('deserializes lit metadata', () => {
const result =
utils.deserializeLitTypesInLitMetadata(mockSerializedMetadata);
expect(result).toEqual(mockMetadata);
});
});

describe('isLitSubtype test', () => {
it('checks is lit subtype', () => {
const testType = utils.createLitType('StringLitType');
Expand Down

0 comments on commit ccbb72c

Please sign in to comment.