Skip to content

Commit

Permalink
Transcribe data model update to support new features including PII co…
Browse files Browse the repository at this point in the history 
…ntent identification and redaction, partial results stabilization, and custom language models for Amazon Transcribe and PHI content identification for Amazon Transcribe Medical
  • Loading branch information
@akashuc
akashuc committed Nov 18, 2021
1 parent c6fd2e3 commit f59d894
Show file tree
Hide file tree
Showing 26 changed files with 4,260 additions and 2,965 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added

- Adds support to live transcription for new features including PII content identification and redaction, partial results stabilization, and custom language models for Amazon Transcribe and PHI content identification for Amazon Transcribe Medical.

### Removed

### Fixed
Expand Down
52 changes: 52 additions & 0 deletions demos/browser/app/meetingV2/meetingV2.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,58 @@ <h1 class="h3 mb-3 font-weight-normal text-center">Select devices</h1>
</select>
</div>
</div>
<div id="engine-transcribe-custom-language-model">
<div id="engine-transcribe-language-model" class="custom-control custom-checkbox" style="text-align: left;">
<input type="checkbox" id="custom-language-model-checkbox" class="custom-control-input">
<label for="custom-language-model-checkbox" class="custom-control-label">Enable Custom Language Model</label>
</div>
<div id="language-model" class="hidden">
<input type="text" id="language-model-input-text" style="text-align: left; width:100%">
</div>
</div>
<div id="engine-transcribe-stability">
<div id="engine-transcribe-partial-stabilization" class="custom-control custom-checkbox" style="text-align: left;">
<input type="checkbox" id="partial-stabilization-checkbox" class="custom-control-input">
<label for="partial-stabilization-checkbox" class="custom-control-label">Enable Partial Results Stabilization</label>
</div>
<div id="transcribe-partial-stability" class="hidden">
<label for="partial-stability" class="sr-only">Transcribe Entities</label>
<select id="partial-stability" class="custom-select" style="width:100%">
<option value="" selected> -- DEFAULT (HIGH) -- </option>
<option value="low">LOW</option>
<option value="medium">MEDIUM</option>
<option value="high">HIGH</option>
</select>
</div>
</div>
<div id="engine-transcribe-content-identification" class="custom-control custom-checkbox" style="text-align: left;">
<input type="checkbox" id="content-identification-checkbox" class="custom-control-input">
<label for="content-identification-checkbox" class="custom-control-label">Enable PII Content Identification</label>
</div>
<div id="engine-transcribe-medical-content-identification" class="custom-control custom-checkbox hidden" style="text-align: left;">
<input type="checkbox" id="medical-content-identification-checkbox" class="custom-control-input">
<label for="medical-content-identification-checkbox" class="custom-control-label">Enable PHI Content Identification</label>
</div>
<div id="engine-transcribe-redaction" class="custom-control custom-checkbox" style="text-align: left;">
<input type="checkbox" id="content-redaction-checkbox" class="custom-control-input">
<label for="content-redaction-checkbox" class="custom-control-label">Enable PII Content Redaction</label>
</div>
<div id="transcribe-entity-types" class="hidden">
<label for="transcribe-entity" class="sr-only">Transcribe Entities</label>
<select id="transcribe-entity" class="custom-select" style="width:100%" multiple>
<option selected value> -- DEFAULT (ALL) -- </option>
<option value="BANK_ROUTING">BANK ROUTING</option>
<option value="CREDIT_DEBIT_NUMBER">CREDIT/DEBIT NUMBER</option>
<option value="CREDIT_DEBIT_CVV">CREDIT/DEBIT CVV</option>
<option value="CREDIT_DEBIT_EXPIRY">CREDIT/DEBIT EXPIRTY</option>
<option value="PIN">PIN</option>
<option value="EMAIL">EMAIL</option>
<option value="ADDRESS">ADDRESS</option>
<option value="NAME">NAME</option>
<option value="PHONE">PHONE</option>
<option value="SSN">SSN</option>
</select>
</div>
<button id="button-start-transcription" type="button" class="btn btn-success mx-1 mx-xl-2 my-2" title="Start" data-toggle="button" aria-pressed="false" autocomplete="off">
<%= require('../../node_modules/open-iconic/svg/caret-right.svg') %>
</button>
Expand Down
159 changes: 138 additions & 21 deletions demos/browser/app/meetingV2/meetingV2.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,12 +232,21 @@ interface Toggle {
}

interface TranscriptSegment {
content: string;
contentSpan: HTMLSpanElement,
attendee: Attendee;
startTimeMs: number;
endTimeMs: number;
}

interface TranscriptionStreamParams {
contentIdentificationType?: 'PII' | 'PHI';
contentRedactionType?: 'PII';
enablePartialResultsStability?: boolean;
partialResultsStability?: string;
piiEntityTypes?: string;
languageModelName?: string;
}

export class DemoMeetingApp
implements AudioVideoObserver, DeviceChangeObserver, ContentShareObserver, VideoDownlinkObserver {
static readonly DID: string = '+17035550122';
Expand Down Expand Up @@ -356,6 +365,7 @@ export class DemoMeetingApp
partialTranscriptDiv: HTMLDivElement | undefined;
partialTranscriptResultTimeMap = new Map<string, number>();
partialTranscriptResultMap = new Map<string, TranscriptResult>();
transcriptEntitySet = new Set<string>();

addFatalHandlers(): void {
fatal = this.fatal.bind(this);
Expand Down Expand Up @@ -979,41 +989,117 @@ export class DemoMeetingApp
document.getElementsByName('transcription-engine').forEach(e => {
e.addEventListener('change', () => {
const engineTranscribeChecked = (document.getElementById('engine-transcribe') as HTMLInputElement).checked;
const contentIdentificationChecked = (document.getElementById('content-identification-checkbox') as HTMLInputElement).checked;
const contentRedactionChecked = (document.getElementById('content-redaction-checkbox') as HTMLInputElement).checked;
document.getElementById('engine-transcribe-language').classList.toggle('hidden', !engineTranscribeChecked);
document.getElementById('engine-transcribe-medical-language').classList.toggle('hidden', engineTranscribeChecked);
document.getElementById('engine-transcribe-region').classList.toggle('hidden', !engineTranscribeChecked);
document.getElementById('engine-transcribe-medical-region').classList.toggle('hidden', engineTranscribeChecked);
document.getElementById('engine-transcribe-medical-content-identification').classList.toggle('hidden', engineTranscribeChecked);
document.getElementById('engine-transcribe-content-identification').classList.toggle('hidden', !engineTranscribeChecked);
document.getElementById('engine-transcribe-redaction').classList.toggle('hidden', !engineTranscribeChecked);
document.getElementById('engine-transcribe-partial-stabilization').classList.toggle('hidden', !engineTranscribeChecked);
document.getElementById('engine-transcribe-custom-language-model').classList.toggle('hidden', !engineTranscribeChecked);
if (!engineTranscribeChecked) {
document.getElementById('transcribe-entity-types').classList.toggle('hidden', true);
} else if (engineTranscribeChecked && (contentIdentificationChecked || contentRedactionChecked)) {
document.getElementById('transcribe-entity-types').classList.toggle('hidden', false);
}
});
});

const contentIdentificationCb = document.getElementById('content-identification-checkbox') as HTMLInputElement;
contentIdentificationCb.addEventListener('click', () => {
(document.getElementById('content-redaction-checkbox') as HTMLInputElement).disabled = contentIdentificationCb.checked;
(document.getElementById('transcribe-entity-types') as HTMLInputElement).classList.toggle('hidden', !contentIdentificationCb.checked);
});

const contentRedactionCb = document.getElementById('content-redaction-checkbox') as HTMLInputElement;
contentRedactionCb.addEventListener('click', () => {
(document.getElementById('content-identification-checkbox') as HTMLInputElement).disabled = contentRedactionCb.checked;
(document.getElementById('transcribe-entity-types') as HTMLInputElement).classList.toggle('hidden', !contentRedactionCb.checked);
});

const partialResultsStabilityCb = document.getElementById('partial-stabilization-checkbox') as HTMLInputElement;
partialResultsStabilityCb.addEventListener('click', () => {
(document.getElementById('transcribe-partial-stability').classList.toggle('hidden', !partialResultsStabilityCb.checked));
});

const languageModelCb = document.getElementById('custom-language-model-checkbox') as HTMLInputElement;
languageModelCb.addEventListener('click', () => {
(document.getElementById('language-model').classList.toggle('hidden', !languageModelCb.checked));
});

const buttonStartTranscription = document.getElementById('button-start-transcription');
buttonStartTranscription.addEventListener('click', async () => {
let engine = '';
let languageCode = '';
let region = '';
const transcriptionStreamParams: TranscriptionStreamParams = {};
if ((document.getElementById('engine-transcribe') as HTMLInputElement).checked) {
engine = 'transcribe';
languageCode = (document.getElementById('transcribe-language') as HTMLInputElement).value;
region = (document.getElementById('transcribe-region') as HTMLInputElement).value;

if (isChecked('content-identification-checkbox')) {
transcriptionStreamParams.contentIdentificationType = 'PII';
}

if (isChecked('content-redaction-checkbox')) {
transcriptionStreamParams.contentRedactionType = 'PII';
}

if (isChecked('partial-stabilization-checkbox')) {
transcriptionStreamParams.enablePartialResultsStability = true;
}

let partialResultsStability = (document.getElementById('partial-stability') as HTMLInputElement).value;
if (partialResultsStability) {
transcriptionStreamParams.partialResultsStability = partialResultsStability;
}
if (isChecked('content-identification-checkbox') || isChecked('content-redaction-checkbox')) {
const selected = document.querySelectorAll('#transcribe-entity option:checked');
let values = '';
if (selected.length > 0) {
values = Array.from(selected).filter(node => (node as HTMLInputElement).value !== '').map(el => (el as HTMLInputElement).value).join(',');
}
if (values !== '') {
transcriptionStreamParams.piiEntityTypes = values;
}
}

if (isChecked('custom-language-model-checkbox')) {
let languageModelName = (document.getElementById('language-model-input-text') as HTMLInputElement).value;
if (languageModelName) {
transcriptionStreamParams.languageModelName = languageModelName;
}
}
} else if ((document.getElementById('engine-transcribe-medical') as HTMLInputElement).checked) {
engine = 'transcribe_medical';
languageCode = (document.getElementById('transcribe-medical-language') as HTMLInputElement).value;
region = (document.getElementById('transcribe-medical-region') as HTMLInputElement).value;
if (isChecked('medical-content-identification-checkbox')) {
transcriptionStreamParams.contentIdentificationType = 'PHI';
}
} else {
throw new Error('Unknown transcription engine');
}
await startLiveTranscription(engine, languageCode, region);
await startLiveTranscription(engine, languageCode, region, transcriptionStreamParams);
});

const startLiveTranscription = async (engine: string, languageCode: string, region: string) => {
const response = await fetch(`${DemoMeetingApp.BASE_URL}start_transcription?title=${encodeURIComponent(this.meeting)}&engine=${encodeURIComponent(engine)}&language=${encodeURIComponent(languageCode)}&region=${encodeURIComponent(region)}`, {
function isChecked(id: string): boolean {
return (document.getElementById(id) as HTMLInputElement).checked;
}

const startLiveTranscription = async (engine: string, languageCode: string, region: string, transcriptionStreamParams: TranscriptionStreamParams) => {
const transcriptionAdditionalParams = JSON.stringify(transcriptionStreamParams);
const response = await fetch(`${DemoMeetingApp.BASE_URL}start_transcription?title=${encodeURIComponent(this.meeting)}&engine=${encodeURIComponent(engine)}&language=${encodeURIComponent(languageCode)}&region=${encodeURIComponent(region)}&transcriptionStreamParams=${encodeURIComponent(transcriptionAdditionalParams)}`, {
method: 'POST',
});
const json = await response.json();
if (json.error) {
throw new Error(`Server error: ${json.error}`);
}

document.getElementById('live-transcription-modal').style.display = 'none';
};

Expand Down Expand Up @@ -1826,7 +1912,7 @@ export class DemoMeetingApp
if (languageCode && LANGUAGES_NO_WORD_SEPARATOR.has(languageCode)) {
this.noWordSeparatorForTranscription = true;
}
} else if (transcriptEvent.type === TranscriptionStatusType.STOPPED && this.enableLiveTranscription) {
} else if ((transcriptEvent.type === TranscriptionStatusType.STOPPED || transcriptEvent.type === TranscriptionStatusType.FAILED) && this.enableLiveTranscription) {
// When we receive a STOPPED status event:
// 1. toggle enabled 'Live Transcription' button to disabled
this.enableLiveTranscription = false;
Expand All @@ -1842,7 +1928,17 @@ export class DemoMeetingApp
for (const result of transcriptEvent.results) {
const resultId = result.resultId;
const isPartial = result.isPartial;

if (!isPartial) {
if (result.alternatives[0].entities?.length > 0) {
for (const entity of result.alternatives[0].entities) {
//split the entity based on space
let contentArray = entity.content.split(' ');
for (const content of contentArray) {
this.transcriptEntitySet.add(content);
}
}
}
}
this.partialTranscriptResultMap.set(resultId, result);
this.partialTranscriptResultTimeMap.set(resultId, result.endTimeMs);
this.renderPartialTranscriptResults();
Expand All @@ -1861,6 +1957,7 @@ export class DemoMeetingApp
}

this.partialTranscriptResultTimeMap.delete(resultId);
this.transcriptEntitySet.clear();

if (this.partialTranscriptResultTimeMap.size === 0) {
// No more partial results in current batch, reset current batch
Expand Down Expand Up @@ -1907,52 +2004,75 @@ export class DemoMeetingApp
this.appendNewSpeakerTranscriptDiv(segment, speakerToTranscriptSpanMap);
} else {
const transcriptSpan = speakerToTranscriptSpanMap.get(newSpeakerId);
transcriptSpan.innerText = transcriptSpan.innerText + '\u00a0' + segment.content;
transcriptSpan.appendChild(this.createSpaceSpan());
transcriptSpan.appendChild(segment.contentSpan);
}
}
}
};

populatePartialTranscriptSegmentsFromResult = (segments: TranscriptSegment[], result: TranscriptResult) => {
let startTimeMs: number = null;
let content = '';
let attendee: Attendee = null;
let contentSpan;
for (const item of result.alternatives[0].items) {
const itemContentSpan = document.createElement('span') as HTMLSpanElement;
itemContentSpan.innerText = item.content;
itemContentSpan.classList.add('transcript-content');
// underline the word with red to show confidence level of predicted word being less than 0.3
// for redaction, words are represented as '[Name]' and has a confidence of 0. Redacted words are only shown with highlighting.
if (item.hasOwnProperty('confidence') && !item.content.startsWith("[") && item.confidence < 0.3) {
itemContentSpan.classList.add('confidence-style');
}

// highlight the word in green to show the predicted word is a PII/PHI entity
if (this.transcriptEntitySet.size > 0 && this.transcriptEntitySet.has(item.content)) {
itemContentSpan.classList.add('entity-color');
}

if (!startTimeMs) {
content = item.content;
contentSpan = document.createElement('span') as HTMLSpanElement;
contentSpan.appendChild(itemContentSpan);
attendee = item.attendee;
startTimeMs = item.startTimeMs;
} else if (item.type === TranscriptItemType.PUNCTUATION) {
content = content + item.content;
contentSpan.appendChild(itemContentSpan);
segments.push({
content: content,
contentSpan,
attendee: attendee,
startTimeMs: startTimeMs,
endTimeMs: item.endTimeMs
});
content = '';
startTimeMs = null;
attendee = null;
} else {
if (this.noWordSeparatorForTranscription) {
content = content + item.content;
contentSpan.appendChild(itemContentSpan);
} else {
content = content + ' ' + item.content;
contentSpan.appendChild(this.createSpaceSpan());
contentSpan.appendChild(itemContentSpan);
}
}
}

// Reached end of the result but there is no closing punctuation
if (startTimeMs) {
segments.push({
content: content,
contentSpan: contentSpan,
attendee: attendee,
startTimeMs: startTimeMs,
endTimeMs: result.endTimeMs,
});
}
};

createSpaceSpan(): HTMLSpanElement {
const spaceSpan = document.createElement('span') as HTMLSpanElement;
spaceSpan.classList.add('transcript-content');
spaceSpan.innerText = '\u00a0';
return spaceSpan;
};

appendNewSpeakerTranscriptDiv = (
segment: TranscriptSegment,
speakerToTranscriptSpanMap: Map<string, HTMLSpanElement>) =>
Expand All @@ -1965,14 +2085,11 @@ export class DemoMeetingApp
speakerSpan.innerText = segment.attendee.externalUserId.split('#').slice(-1)[0] + ': ';
speakerTranscriptDiv.appendChild(speakerSpan);

const transcriptSpan = document.createElement('span') as HTMLSpanElement;
transcriptSpan.classList.add('transcript-content');
transcriptSpan.innerText = segment.content;
speakerTranscriptDiv.appendChild(transcriptSpan);
speakerTranscriptDiv.appendChild(segment.contentSpan);

this.partialTranscriptDiv.appendChild(speakerTranscriptDiv);

speakerToTranscriptSpanMap.set(segment.attendee.attendeeId, transcriptSpan);
speakerToTranscriptSpanMap.set(segment.attendee.attendeeId, segment.contentSpan);
};

appendStatusDiv = (status: TranscriptionStatus) => {
Expand Down
10 changes: 10 additions & 0 deletions demos/browser/app/meetingV2/styleV2.scss
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1001,6 +1001,16 @@ a.markdown:active {
cursor: pointer;
}

.confidence-style {
text-decoration: underline;
text-decoration-color: red;
}

.entity-color {
color: green;
font-weight: bold;
}

#live-transcription-modal-content > * {
margin: 8px 0;
}
2 changes: 1 addition & 1 deletion demos/browser/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
},
"dependencies": {
"amazon-chime-sdk-js": "file:../..",
"aws-sdk": "^2.1006.0",
"aws-sdk": "^2.1032.0",
"bootstrap": "^4.5.2",
"compression": "^1.7.4",
"jquery": "^3.5.1",
Expand Down

0 comments on commit f59d894

Please sign in to comment.