Add some UI enhancements

- consistently use the term "voice-over" throughout the UI and README (was sometimes referred to as speech or audio) - add a tooltip to the "analyse voice-over" checkbox - Fix Combiner Service's stringent property expectations for VideoVariant - add notice texts to the render queue and below the generation prompt - change the layout of the generation prompt to bundle the prompt, button and duration settings together in one "row" - clear renderQueue upon uploading a new video or running a previous analysis - remove unnecessary Output event on segments-list component Change-Id: I304860d7ef7a9161a5b6e3a311547c1fc10a867e
google-marketing-solutions · May 2, 2024 · 5d22142 · 5d22142
1 parent 4887416
commit 5d22142
Show file tree

Hide file tree

Showing 10 changed files with 114 additions and 66 deletions.
diff --git a/README.md b/README.md
@@ -123,7 +123,7 @@ The diagram below shows how Vigenair's components interact and communicate with
 
 Vigenair supports different rendering settings for the audio of the generated videos. The image below describes the supported options and how they differ:
 
-<center><img src='./img/audio.png' width='640px' alt="Vigenair's audio rendering options" /></center>
+<center><img src='./img/audio.png' alt="Vigenair's audio rendering options" /></center>
 
 ## How to Contribute
 

diff --git a/img/audio.png b/img/audio.png
diff --git a/service/combiner/combiner.py b/service/combiner/combiner.py
@@ -35,7 +35,7 @@
 from vertexai.preview.generative_models import GenerativeModel
 
 
-@dataclasses.dataclass(frozen=True)
+@dataclasses.dataclass(init=False)
 class VideoVariantRenderSettings:
   """Represents the settings for a video variant.
 
@@ -58,6 +58,12 @@ class VideoVariantRenderSettings:
   use_music_overlay: bool = False
   use_continuous_audio: bool = False
 
+  def __init__(self, **kwargs):
+    field_names = set([f.name for f in dataclasses.fields(self)])
+    for k, v in kwargs.items():
+      if k in field_names:
+        setattr(self, k, v)
+
   def __str__(self):
     return (
         f'VideoVariantRenderSettings(generate_image_assets={self.generate_image_assets}, '
@@ -68,7 +74,7 @@ def __str__(self):
     )
 
 
-@dataclasses.dataclass(frozen=True)
+@dataclasses.dataclass(init=False)
 class VideoVariantSegment:
   """Represents a segment of a video variant.
 
@@ -82,6 +88,12 @@ class VideoVariantSegment:
   start_s: float
   end_s: float
 
+  def __init__(self, **kwargs):
+    field_names = set([f.name for f in dataclasses.fields(self)])
+    for k, v in kwargs.items():
+      if k in field_names:
+        setattr(self, k, v)
+
   def __str__(self):
     return (
         f'VideoVariantSegment(av_segment_id={self.av_segment_id}, '
@@ -90,7 +102,7 @@ def __str__(self):
     )
 
 
-@dataclasses.dataclass(frozen=True)
+@dataclasses.dataclass(init=False)
 class VideoVariant:
   """Represents a video variant.
 
@@ -112,6 +124,12 @@ class VideoVariant:
   score_reasoning: str
   render_settings: VideoVariantRenderSettings
 
+  def __init__(self, **kwargs):
+    field_names = set([f.name for f in dataclasses.fields(self)])
+    for k, v in kwargs.items():
+      if k in field_names:
+        setattr(self, k, v)
+
   def __str__(self):
     return (
         f'VideoVariant(variant_id={self.variant_id}, '

diff --git a/ui/src/ui/src/app/api-calls/api-calls.service.interface.ts b/ui/src/ui/src/app/api-calls/api-calls.service.interface.ts
@@ -65,11 +65,6 @@ export interface RenderQueueVariant {
   scenes: string;
 }
 
-export interface SelectedSegmentEventParams {
-  segmentId: number;
-  selected: boolean;
-}
-
 export interface ApiCalls {
   uploadVideo(file: Blob, analyseAudio: boolean): Observable<string>;
   deleteGcsFolder(folder: string): void;

diff --git a/ui/src/ui/src/app/app.component.css b/ui/src/ui/src/app/app.component.css
@@ -149,6 +149,10 @@ mat-divider {
   color: #33691e;
 }
 
+.variant-score-modified {
+  color: #aaaaaa;
+}
+
 ::ng-deep #mat-badge-content-0 {
   right: 3px;
   top: 3px;
@@ -237,3 +241,8 @@ mat-divider {
   border-left: solid 1px var(--mat-standard-button-toggle-divider-color);
   padding: 4px;
 }
+
+.notice-text {
+  font-size: 14px;
+  color: #aaaaaa;
+}
diff --git a/ui/src/ui/src/app/app.component.html b/ui/src/ui/src/app/app.component.html
@@ -17,7 +17,9 @@
 <mat-toolbar color="primary" class="sticky">
   <span style="font-size: 1.5em; margin-right: auto"> ViGenAiR </span>
   <span class="header-text">
-    <a href="https://github.com/google-marketing-solutions/vigenair"
+    <a
+      target="_blank"
+      href="https://github.com/google-marketing-solutions/vigenair"
       >Recrafting Video Ads with Generative AI</a
     ></span
   >
@@ -55,6 +57,7 @@
         (file)="onFileSelected($event)"
       ></file-chooser>
       <mat-checkbox
+        matTooltip="Uncheck for videos where there is no voice-over"
         style="
           justify-content: center;
           display: flex;
@@ -65,7 +68,7 @@
         [(ngModel)]="analyseAudio"
         [disabled]="!selectedFile || loading"
       >
-        Analyse audio
+        Analyse voice-over
       </mat-checkbox>
       <button
         type="button"
@@ -87,6 +90,7 @@
         >My videos only</mat-slide-toggle
       >
       <mat-form-field
+        style="width: 300px"
         subscriptSizing="dynamic"
         *ngIf="previousRuns && previousRuns.length > 0"
       >
@@ -97,9 +101,14 @@
           (click)="getPreviousRuns()"
         >
           @for (run of previousRuns; track $index) {
-            <mat-option *ngIf="isCurrentUserRun(run)" [value]="run">{{
-              run.split('--')[0]
-            }}</mat-option>
+            <mat-option *ngIf="isCurrentUserRun(run)" [value]="run"
+              >{{ run.split('--')[0]
+              }}{{
+                run.split('--').length === 4 && run.split('--')[1] === 'n'
+                  ? ' (no voice-over)'
+                  : ''
+              }}</mat-option
+            >
           }
         </mat-select>
       </mat-form-field>
@@ -141,7 +150,6 @@
           [segmentMode]="segmentModeToggle.value"
           [allowSelection]="variants !== undefined"
           [currentSegmentId]="currentSegmentId || 0"
-          (toggleSelectedSegmentEvent)="toggleSelectedSegment($event)"
         ></segments-list>
         <div
           [hidden]="segmentModeToggle.value !== 'preview'"
@@ -194,7 +202,7 @@
                 {{ variant.description }}
               </div>
               <div
-                style="font-size: 3em"
+                style="font-size: 2.5em"
                 [ngClass]="'score-' + variant.score"
                 class="row"
               >
@@ -223,16 +231,23 @@
       <ng-container *ngIf="avSegments">
         <mat-divider></mat-divider>
         <div *ngIf="variants?.length">
-          <div style="margin-top: 8px; font-weight: 500">
-            <u>Variant rendering settings</u>
-          </div>
+          <br />
+          <span style="font-size: 18px">Variant rendering settings</span>
+          <br /><br />
           <div style="margin-top: 8px">
             <mat-form-field subscriptSizing="dynamic">
               <mat-label>Variant audio:</mat-label>
               <mat-select [(ngModel)]="audioSettings">
                 <mat-option value="segment">Individual segments</mat-option>
-                <mat-option value="music">Overlay music only</mat-option>
-                <mat-option value="continuous">Overlay all</mat-option>
+                <span
+                  matTooltip="Only available for videos with voice-over"
+                  [matTooltipDisabled]="analyseAudio"
+                >
+                  <mat-option value="music" [disabled]="!analyseAudio"
+                    >Overlay music only</mat-option
+                  >
+                </span>
+                <mat-option value="continuous">Overlay all audio</mat-option>
               </mat-select>
             </mat-form-field>
           </div>
@@ -258,7 +273,7 @@
             </span>
           </div>
         </div>
-        <div class="row" style="gap: 16px">
+        <div class="row" style="gap: 16px; margin-top: 12px">
           <mat-form-field style="flex: 1" subscriptSizing="dynamic">
             <mat-label>What should your video variants be about?</mat-label>
             <textarea
@@ -267,33 +282,38 @@
               placeholder="e.g. 'Discovering new experiences'"
             ></textarea>
           </mat-form-field>
-          <button
-            type="button"
-            mat-raised-button
-            color="primary"
-            (click)="generateVariants()"
-            [disabled]="loading"
-          >
-            Generate variants
-          </button>
-        </div>
-        <div
-          class="row"
-          style="justify-content: start; margin-top: 16px; display: block"
-        >
-          <div style="margin-top: 8px">
-            <mat-label>Target duration:</mat-label>
-            <mat-slider
-              [max]="math.round(this.previewVideoElem.duration)"
-              [min]="10"
-              [step]="this.step"
-              [discrete]="true"
-              [showTickMarks]="true"
+          <div style="display: flex; flex-direction: column; margin-top: 16px">
+            <button
+              type="button"
+              mat-raised-button
+              color="primary"
+              (click)="generateVariants()"
+              [disabled]="loading"
             >
-              <input matSliderThumb [(ngModel)]="duration" #slider />
-            </mat-slider>
+              Generate variants
+            </button>
+            <div style="margin-top: 4px">
+              <mat-label>Target duration:</mat-label>
+              <mat-slider
+                [max]="math.round(previewVideoElem.duration)"
+                [min]="10"
+                [step]="step"
+                [discrete]="true"
+                [showTickMarks]="true"
+              >
+                <input matSliderThumb [(ngModel)]="duration" #slider />
+              </mat-slider>
+            </div>
           </div>
         </div>
+        <div class="row" style="margin-top: 0px">
+          <span class="notice-text"
+            >Segments selected by the LLM might not follow prompt instructions,
+            and the overall video variant might not follow the desired target
+            duration. It is recommended to review and potentially modify the
+            selection. Navigating between variants will clear any changes.
+          </span>
+        </div>
       </ng-container>
     </mat-expansion-panel>
 
@@ -351,7 +371,14 @@
             </div>
           </div>
           <div>
-            <div class="variant-score" matTooltip="Score: {{ variant.score }}">
+            <div
+              class="variant-score{{
+                variant.userSelection ? '-modified' : ''
+              }}"
+              matTooltip="Score: {{ variant.score }}{{
+                variant.userSelection ? ' (modified)' : ''
+              }}"
+            >
               @for (_ of stars; track _; let i = $index) {
                 <mat-icon
                   fontIcon="{{
@@ -402,7 +429,7 @@
         </div>
       }
     </div>
-    <div class="row">
+    <div class="row" style="flex-direction: column">
       <button
         type="button"
         mat-raised-button
@@ -411,6 +438,11 @@
       >
         Render
       </button>
+      <br />
+      <span class="notice-text"
+        >Uploading a new video or selecting a previous one will clear the render
+        queue.
+      </span>
     </div>
   </mat-sidenav>
 </mat-sidenav-container>
diff --git a/ui/src/ui/src/app/app.component.ts b/ui/src/ui/src/app/app.component.ts
@@ -62,7 +62,6 @@ import {
   GenerateVariantsResponse,
   RenderQueueVariant,
   RenderSettings,
-  SelectedSegmentEventParams,
 } from './api-calls/api-calls.service.interface';
 
 type ProcessStatus = 'hourglass_top' | 'pending' | 'check_circle';
@@ -405,6 +404,8 @@ export class AppComponent {
     this.combinationStatus = 'hourglass_top';
     this.segmentsStatus = 'hourglass_top';
     this.previewTrackElem.nativeElement.src = '';
+    this.renderQueue = [];
+    this.renderQueueJsonArray = [];
   }
 
   processVideo(folder: string) {
@@ -441,8 +442,8 @@ export class AppComponent {
   }
 
   calculateVideoDefaultDuration(duration: number) {
-    const fullDuration = Math.round(duration) % 10;
-    const step = fullDuration ? Math.min(10, fullDuration) : 10;
+    const durationInt = Math.round(duration) % 10;
+    const step = durationInt ? Math.min(10, durationInt) : 10;
     const halfDuration = Math.round(duration / 2);
 
     this.step = step;
@@ -476,11 +477,6 @@ export class AppComponent {
     }
   }
 
-  toggleSelectedSegment(selectedSegment: SelectedSegmentEventParams) {
-    this.avSegments[selectedSegment.segmentId].selected =
-      selectedSegment.selected;
-  }
-
   variantChanged() {
     this.setSelectedSegments();
   }

diff --git a/ui/src/ui/src/app/segments-list/segments-list.component.html b/ui/src/ui/src/app/segments-list/segments-list.component.html
@@ -22,7 +22,10 @@
     [ngClass]="segmentMode === 'preview' ? 'filmstrip-row' : 'row'"
     *ngFor="let segment of segmentList; index as i"
   >
-    <div style="display: flex; position: relative" (click)="toggleSegment(i, !segment.selected)">
+    <div
+      style="display: flex; position: relative"
+      (click)="toggleSegmentSelection(i)"
+    >
       <img
         attr.id="segment-{{ i }}"
         [class.filmstrip-img]="segmentMode === 'preview'"