From 5239891a4ee9ad3a9598a0cc35967443df0db64f Mon Sep 17 00:00:00 2001
From: Hristo Iankov <35353708+hristoiankov@users.noreply.github.com>
Date: Fri, 21 Jul 2023 21:06:16 -0400
Subject: [PATCH 1/5] table stats

---
 cider-app/src/app/entity-table/entity-table.component.html | 4 ++++
 cider-app/src/app/entity-table/entity-table.component.ts   | 1 +
 2 files changed, 5 insertions(+)

diff --git a/cider-app/src/app/entity-table/entity-table.component.html b/cider-app/src/app/entity-table/entity-table.component.html
index f7bdf80..1c55712 100644
--- a/cider-app/src/app/entity-table/entity-table.component.html
+++ b/cider-app/src/app/entity-table/entity-table.component.html
@@ -9,6 +9,10 @@
             <button pButton pRipple label="Clear" icon="pi pi-filter-slash" class="p-button p-mr-2" (click)="clear(dt)"></button>
             <button *ngIf="allowEditing && showActions" pButton pRipple label="New" icon="pi pi-plus" class="p-button p-mr-2" (click)="openCreateNew()"></button>
             <div class="p-ml-auto">
+                <ng-container *ngIf="showStats">
+                    <button pButton pRipple label="" icon="pi pi-upload" class="p-button-success p-mr-2" 
+                        pTooltip="Show stats" (click)="openStatsDialog()"></button>
+                </ng-container>
                 <ng-container *ngIf="allowImportExport">
                     <button pButton pRipple label="" icon="pi pi-upload" class="p-button-success p-mr-2" 
                         pTooltip="Import data" (click)="openImportDialog()"></button>
diff --git a/cider-app/src/app/entity-table/entity-table.component.ts b/cider-app/src/app/entity-table/entity-table.component.ts
index bcf7822..b431e23 100644
--- a/cider-app/src/app/entity-table/entity-table.component.ts
+++ b/cider-app/src/app/entity-table/entity-table.component.ts
@@ -27,6 +27,7 @@ export class EntityTableComponent<Entity, Identifier extends string | number> im
   @Input() lazy: boolean = true;
   @Input() showActions: boolean = true;
   @Input() showInlineEditor: boolean = true;
+  @Input() showStats: boolean = false;
   @Input() saveToService: boolean = true;
   @Output() selectionChange: EventEmitter<Entity | Entity[] | undefined> = new EventEmitter<Entity | Entity[] | undefined>();
   FieldType = FieldType;

From 30ed17d988aab3f624da643eded941273c4a2df0 Mon Sep 17 00:00:00 2001
From: Hristo Iankov <35353708+hristoiankov@users.noreply.github.com>
Date: Sat, 22 Jul 2023 00:54:41 -0400
Subject: [PATCH 2/5] token stats all

---
 cider-app/src/app/cards/cards.component.html  |  2 +-
 .../entity-table/entity-table.component.html  | 35 +++++++++++-
 .../entity-table/entity-table.component.scss  | 14 +++++
 .../entity-table/entity-table.component.ts    | 54 +++++++++++++++++++
 .../src/app/entity-table/table-stat.type.ts   | 19 +++++++
 5 files changed, 121 insertions(+), 3 deletions(-)
 create mode 100644 cider-app/src/app/entity-table/table-stat.type.ts

diff --git a/cider-app/src/app/cards/cards.component.html b/cider-app/src/app/cards/cards.component.html
index 4a21d22..1791218 100644
--- a/cider-app/src/app/cards/cards.component.html
+++ b/cider-app/src/app/cards/cards.component.html
@@ -1,5 +1,5 @@
 <app-page-header header="Cards" subheader="List of all the cards in the deck"></app-page-header>
 <app-cards-tab-menu></app-cards-tab-menu>
 <p-card styleClass="page-card tabbed-card">
-    <app-entity-table [service]="cardsService" [allowImportExport]="true"></app-entity-table>
+    <app-entity-table [service]="cardsService" [allowImportExport]="true" [showStats]="true"></app-entity-table>
 </p-card>
\ No newline at end of file
diff --git a/cider-app/src/app/entity-table/entity-table.component.html b/cider-app/src/app/entity-table/entity-table.component.html
index 1c55712..c7eb759 100644
--- a/cider-app/src/app/entity-table/entity-table.component.html
+++ b/cider-app/src/app/entity-table/entity-table.component.html
@@ -10,8 +10,8 @@
             <button *ngIf="allowEditing && showActions" pButton pRipple label="New" icon="pi pi-plus" class="p-button p-mr-2" (click)="openCreateNew()"></button>
             <div class="p-ml-auto">
                 <ng-container *ngIf="showStats">
-                    <button pButton pRipple label="" icon="pi pi-upload" class="p-button-success p-mr-2" 
-                        pTooltip="Show stats" (click)="openStatsDialog()"></button>
+                    <button pButton pRipple label="" icon="pi pi-percentage" class="p-button-info p-mr-2" 
+                        pTooltip="Show token stats" (click)="openStatsDialog()"></button>
                 </ng-container>
                 <ng-container *ngIf="allowImportExport">
                     <button pButton pRipple label="" icon="pi pi-upload" class="p-button-success p-mr-2" 
@@ -194,3 +194,34 @@
         <button pButton pRipple label="Import" icon="pi pi-check" class="p-button-text" (click)="confirmImport(dt)"></button>
     </ng-template>
 </p-dialog>
+<p-dialog header="Token Stats" [(visible)]="statsVisible" [style]="{width: '450px'}" [modal]="true">
+    <ng-template pTemplate="content">
+        <div class="token-stats">
+            <div class="token-row token-header">
+                <div>Field</div>
+                <div>Token</div>
+                <div>Count</div>
+                <div>Copy Count</div>
+            </div>
+            <ng-container *ngFor="let stat of stats">
+                <div class="token-row">
+                    <div><h3>{{stat.header}}</h3></div>
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                </div>
+                <ng-container *ngFor="let token of stat.tokens">
+                    <div class="token-row">
+                        <div></div>
+                        <div>{{token.token}}</div>
+                        <div>{{token.count}}</div>
+                        <div>{{token.copiesCount}}</div>
+                    </div>
+                </ng-container>
+            </ng-container>
+        </div>
+    </ng-template>
+    <ng-template pTemplate="footer">
+        <button pButton pRipple label="Cancel" icon="pi pi-times" class="p-button-text" (click)="closeStatsDialog()"></button>
+    </ng-template>
+</p-dialog>
diff --git a/cider-app/src/app/entity-table/entity-table.component.scss b/cider-app/src/app/entity-table/entity-table.component.scss
index ddc3ddd..3b787df 100644
--- a/cider-app/src/app/entity-table/entity-table.component.scss
+++ b/cider-app/src/app/entity-table/entity-table.component.scss
@@ -139,4 +139,18 @@
 :host ::ng-deep .p-datatable .p-datatable-tbody > tr.p-highlight {
     background: var(--surface-d);
     color: var(--text-color);
+}
+
+.token-stats {
+    .token-row {
+        display: table-row;
+        &.token-header {
+            color: #5c7089;
+        }
+        & > div {
+            display: table-cell;
+            padding-left: 5px;
+            padding-right: 5px;
+        }
+    }
 }
\ No newline at end of file
diff --git a/cider-app/src/app/entity-table/entity-table.component.ts b/cider-app/src/app/entity-table/entity-table.component.ts
index b431e23..206b9d5 100644
--- a/cider-app/src/app/entity-table/entity-table.component.ts
+++ b/cider-app/src/app/entity-table/entity-table.component.ts
@@ -7,6 +7,7 @@ import { FieldType } from '../data-services/types/field-type.type';
 import { SortDirection } from '../data-services/types/search-sort.type';
 import XlsxUtils from '../shared/utils/xlsx-utils';
 import { Subject, debounceTime } from 'rxjs';
+import { TableStat, TokenStat } from './table-stat.type';
 
 @Component({
   selector: 'app-entity-table',
@@ -38,6 +39,8 @@ export class EntityTableComponent<Entity, Identifier extends string | number> im
   idField?: string;
   lookups: Map<EntityService<any, string | number>, Map<string | number, string>> = new Map();
   importVisible: boolean = false;
+  statsVisible: boolean = false;
+  stats: TableStat[] = [];
   importFile: File | undefined = undefined;
   saveSubject: Subject<Entity> = new Subject();
   optionsCache: Map<EntityService<any, string | number>, any[]>;
@@ -151,6 +154,57 @@ export class EntityTableComponent<Entity, Identifier extends string | number> im
     });
   }
 
+  public openStatsDialog() {
+    // calculate the stats
+    //
+    // field-name:
+    //   token-name   token-count    token-copies-count
+    //   token-name-2 token-count-2  token-copies-count-2
+    //   ...
+    // ...
+    // 
+    // text = 'text',
+    // textArea = 'text-area',
+    // number = 'number',
+    // file = 'file',
+    // option = 'option',
+    // optionList = 'optionList'
+    //
+    const stats = this.columns.filter(column => !column.hidden).map(column => {
+      const tokenStats = new Map<string, TokenStat>();
+      this.records.forEach(record => {
+        const copies = (record as any)['count'] || 1;
+        const value = record[column.field];
+        if (value) {
+          const strValue = '' + value;
+          strValue.replace(/[<][^>]*[>]/, '').split(/ /).filter(str => str).forEach(token => {
+            const tokenStat = tokenStats.get(token);
+            if (tokenStat) {
+              tokenStats.set(token, 
+                {token: token, count: tokenStat.count + 1, 
+                  copiesCount: tokenStat.copiesCount + copies} as TokenStat);
+            } else {
+              tokenStats.set(token, 
+                {token: token, count: 1, copiesCount: copies} as TokenStat);
+            }
+          });
+        }
+      });
+      const tokens = Array.from(tokenStats.values()).sort((a, b) => b.count - a.count);
+      console.log('tokens:', tokens);
+      return {
+        header: column.header,
+        tokens: tokens
+      } as TableStat;
+    });
+    this.stats = stats;
+    this.statsVisible = true;
+  }
+
+  public closeStatsDialog() {
+    this.statsVisible = false;
+  }
+
   public openCreateNew() {
     this.entity = {} as Entity;
     this.dialogVisible = true;
diff --git a/cider-app/src/app/entity-table/table-stat.type.ts b/cider-app/src/app/entity-table/table-stat.type.ts
new file mode 100644
index 0000000..e99bc6c
--- /dev/null
+++ b/cider-app/src/app/entity-table/table-stat.type.ts
@@ -0,0 +1,19 @@
+
+    // TableStat
+    //
+    // field-name:
+    //   token-name   token-count    token-copies-count
+    //   token-name-2 token-count-2  token-copies-count-2
+    //   ...
+    // ...
+
+    export interface TableStat {
+        header: string;
+        tokens: TokenStat[];
+    }
+
+    export interface TokenStat {
+        token: string;
+        count: number;
+        copiesCount: number;
+    }
\ No newline at end of file

From 4739cfac13dbb1d904682ac531a696c988730886 Mon Sep 17 00:00:00 2001
From: Hristo Iankov <35353708+hristoiankov@users.noreply.github.com>
Date: Sat, 22 Jul 2023 01:02:38 -0400
Subject: [PATCH 3/5] top 20 stats

---
 cider-app/src/app/entity-table/entity-table.component.html | 2 +-
 cider-app/src/app/entity-table/entity-table.component.ts   | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/cider-app/src/app/entity-table/entity-table.component.html b/cider-app/src/app/entity-table/entity-table.component.html
index c7eb759..aedb5c4 100644
--- a/cider-app/src/app/entity-table/entity-table.component.html
+++ b/cider-app/src/app/entity-table/entity-table.component.html
@@ -194,7 +194,7 @@
         <button pButton pRipple label="Import" icon="pi pi-check" class="p-button-text" (click)="confirmImport(dt)"></button>
     </ng-template>
 </p-dialog>
-<p-dialog header="Token Stats" [(visible)]="statsVisible" [style]="{width: '450px'}" [modal]="true">
+<p-dialog [header]="'Token Stats (Top ' + statsTopX + ')'" [(visible)]="statsVisible" [style]="{width: '450px'}" [modal]="true">
     <ng-template pTemplate="content">
         <div class="token-stats">
             <div class="token-row token-header">
diff --git a/cider-app/src/app/entity-table/entity-table.component.ts b/cider-app/src/app/entity-table/entity-table.component.ts
index 206b9d5..8cc5929 100644
--- a/cider-app/src/app/entity-table/entity-table.component.ts
+++ b/cider-app/src/app/entity-table/entity-table.component.ts
@@ -41,6 +41,7 @@ export class EntityTableComponent<Entity, Identifier extends string | number> im
   importVisible: boolean = false;
   statsVisible: boolean = false;
   stats: TableStat[] = [];
+  statsTopX: number = 20;
   importFile: File | undefined = undefined;
   saveSubject: Subject<Entity> = new Subject();
   optionsCache: Map<EntityService<any, string | number>, any[]>;
@@ -183,7 +184,7 @@ export class EntityTableComponent<Entity, Identifier extends string | number> im
               tokenStats.set(token, 
                 {token: token, count: tokenStat.count + 1, 
                   copiesCount: tokenStat.copiesCount + copies} as TokenStat);
-            } else {
+            } else if (tokenStats.size < this.statsTopX) {
               tokenStats.set(token, 
                 {token: token, count: 1, copiesCount: copies} as TokenStat);
             }

From 6882c0c7c024fc9d036418c5b444dd3c02493627 Mon Sep 17 00:00:00 2001
From: Hristo Iankov <35353708+hristoiankov@users.noreply.github.com>
Date: Sat, 22 Jul 2023 01:16:58 -0400
Subject: [PATCH 4/5] clean stats

---
 cider-app/src/app/entity-table/entity-table.component.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cider-app/src/app/entity-table/entity-table.component.ts b/cider-app/src/app/entity-table/entity-table.component.ts
index 8cc5929..84492c3 100644
--- a/cider-app/src/app/entity-table/entity-table.component.ts
+++ b/cider-app/src/app/entity-table/entity-table.component.ts
@@ -178,7 +178,7 @@ export class EntityTableComponent<Entity, Identifier extends string | number> im
         const value = record[column.field];
         if (value) {
           const strValue = '' + value;
-          strValue.replace(/[<][^>]*[>]/, '').split(/ /).filter(str => str).forEach(token => {
+          strValue.replace(/[<][^>]*[>]/g, '').split(/ |\n|\r/).filter(str => str).forEach(token => {
             const tokenStat = tokenStats.get(token);
             if (tokenStat) {
               tokenStats.set(token, 

From 8939a4cd2dce7dbb7bb418d517f74b3089ac8eaf Mon Sep 17 00:00:00 2001
From: Hristo Iankov <35353708+hristoiankov@users.noreply.github.com>
Date: Sat, 22 Jul 2023 09:15:05 -0400
Subject: [PATCH 5/5] stats styling fixes

---
 .../entity-table/entity-table.component.html  | 15 +++++-----
 .../entity-table/entity-table.component.scss  | 29 ++++++++++++++++++-
 .../entity-table/entity-table.component.ts    |  3 +-
 3 files changed, 37 insertions(+), 10 deletions(-)

diff --git a/cider-app/src/app/entity-table/entity-table.component.html b/cider-app/src/app/entity-table/entity-table.component.html
index aedb5c4..4db4018 100644
--- a/cider-app/src/app/entity-table/entity-table.component.html
+++ b/cider-app/src/app/entity-table/entity-table.component.html
@@ -194,26 +194,27 @@
         <button pButton pRipple label="Import" icon="pi pi-check" class="p-button-text" (click)="confirmImport(dt)"></button>
     </ng-template>
 </p-dialog>
-<p-dialog [header]="'Token Stats (Top ' + statsTopX + ')'" [(visible)]="statsVisible" [style]="{width: '450px'}" [modal]="true">
+<p-dialog [header]="'Token Stats (Top ' + statsTopX + ')'" [(visible)]="statsVisible" [style]="{width: '50%'}" 
+    [modal]="true" class="stats-dialog">
     <ng-template pTemplate="content">
-        <div class="token-stats">
+        <div class="token-stats token-table">
             <div class="token-row token-header">
-                <div>Field</div>
+                <div class="field">Field</div>
                 <div>Token</div>
                 <div>Count</div>
                 <div>Copy Count</div>
             </div>
             <ng-container *ngFor="let stat of stats">
                 <div class="token-row">
-                    <div><h3>{{stat.header}}</h3></div>
-                    <div></div>
+                    <div class="field"><h3>{{stat.header}}</h3></div>
+                    <div class="token"></div>
                     <div></div>
                     <div></div>
                 </div>
                 <ng-container *ngFor="let token of stat.tokens">
                     <div class="token-row">
-                        <div></div>
-                        <div>{{token.token}}</div>
+                        <div class="field"></div>
+                        <div class="token"><span>{{token.token}}</span></div>
                         <div>{{token.count}}</div>
                         <div>{{token.copiesCount}}</div>
                     </div>
diff --git a/cider-app/src/app/entity-table/entity-table.component.scss b/cider-app/src/app/entity-table/entity-table.component.scss
index 3b787df..d903a22 100644
--- a/cider-app/src/app/entity-table/entity-table.component.scss
+++ b/cider-app/src/app/entity-table/entity-table.component.scss
@@ -141,16 +141,43 @@
     color: var(--text-color);
 }
 
-.token-stats {
+:host ::ng-deep .stats-dialog .p-dialog-content {
+    padding-top: 0;
+
+}
+
+.token-stats.token-table {
+    display: table;
+    width: 100%;
     .token-row {
         display: table-row;
         &.token-header {
             color: #5c7089;
+            top: 0;
+            position: sticky;
+            background-color: #2b313d;
+            & > div {
+                padding-top: 1rem;
+                padding-bottom: 1rem;
+            }
         }
         & > div {
             display: table-cell;
             padding-left: 5px;
             padding-right: 5px;
+            max-width: 200px;
+            h3 {
+                color: #5c7089;
+            }
+            span {
+                background-color: #333e4d;
+                border-radius: 10px;
+                padding: 2px;
+                padding-left: 7px;
+                padding-right: 7px;
+                margin: 1px;
+                display: inline-block;
+            }
         }
     }
 }
\ No newline at end of file
diff --git a/cider-app/src/app/entity-table/entity-table.component.ts b/cider-app/src/app/entity-table/entity-table.component.ts
index 84492c3..796a2d7 100644
--- a/cider-app/src/app/entity-table/entity-table.component.ts
+++ b/cider-app/src/app/entity-table/entity-table.component.ts
@@ -178,7 +178,7 @@ export class EntityTableComponent<Entity, Identifier extends string | number> im
         const value = record[column.field];
         if (value) {
           const strValue = '' + value;
-          strValue.replace(/[<][^>]*[>]/g, '').split(/ |\n|\r/).filter(str => str).forEach(token => {
+          strValue.replace(/[<][^>]*[>]|["'.,]/g, '').split(/ |\n|\r/).filter(str => str).forEach(token => {
             const tokenStat = tokenStats.get(token);
             if (tokenStat) {
               tokenStats.set(token, 
@@ -192,7 +192,6 @@ export class EntityTableComponent<Entity, Identifier extends string | number> im
         }
       });
       const tokens = Array.from(tokenStats.values()).sort((a, b) => b.count - a.count);
-      console.log('tokens:', tokens);
       return {
         header: column.header,
         tokens: tokens