Release/2.5.0

README.md

@@ -9,7 +9,7 @@
 
 ## Developer Setup
 
-**This repository requires `Python 3.10` at minimum -- prefer the latest `3.10.X`.**
+**This repository requires `Python 3.12` at minimum -- prefer the latest `3.12.X`.**
 
 A local Kubernetes cluster requires
 
@@ -28,7 +28,7 @@ it for you.
 
 Example install
 ```bash
-python -m virtualenv -p /usr/bin/python3.10 venv
+python3.12 -m venv venv
 source venv/bin/activate
 # Install platform and developer extra packages
 pip install --editable '.[dev]'

common/api/flask_ext/authentication/jwt_plugin.py

@@ -44,7 +44,7 @@ class JWTAuth(BaseAuthPlugin):
     logout_callbacks: dict[str, Callable] = {}
     """Callback functions to be called when the user logs out."""
 
-    secret_key: str = NotImplemented
+    secret_key: bytes | str = NotImplemented
     """
     Secret key utilized to encode/decode the JTW tokens.
     It is automatically set after flask's namesake setting upon initialization.

common/tests/integration/entity_services/test_event_service.py

@@ -2,6 +2,7 @@
 from unittest.mock import patch
 
 import pytest
+from werkzeug.datastructures import MultiDict
 
 from common.entities import DB
 from common.entity_services import EventService
@@ -22,14 +23,14 @@ def test_get_events_with_rules_filters(
 ):
     filter_cases = (
         ({}, (event_entity, event_entity_2)),
-        ({"component_id": [str(pipeline.id)]}, (event_entity,)),
-        ({"instance_id": [str(instance_instance_set.instance_id)]}, (event_entity,)),
-        ({"journey_id": [str(instance_instance_set.instance.journey_id)]}, (event_entity,)),
-        ({"event_type": ["BATCH_PIPELINE_STATUS"]}, (event_entity,)),
-        ({"event_type": ["DATASET_OPERATION"]}, (event_entity_2,)),
-        ({"event_id": [str(event_entity_2.id)]}, (event_entity_2,)),
-        ({"run_id": [str(run.id)]}, (event_entity,)),
-        ({"task_id": [str(task.id)]}, (event_entity,)),
+        ({"component_id": str(pipeline.id)}, (event_entity,)),
+        ({"instance_id": str(instance_instance_set.instance_id)}, (event_entity,)),
+        ({"journey_id": str(instance_instance_set.instance.journey_id)}, (event_entity,)),
+        ({"event_type": "BATCH_PIPELINE_STATUS"}, (event_entity,)),
+        ({"event_type": "DATASET_OPERATION"}, (event_entity_2,)),
+        ({"event_id": str(event_entity_2.id)}, (event_entity_2,)),
+        ({"run_id": str(run.id)}, (event_entity,)),
+        ({"task_id": str(task.id)}, (event_entity,)),
         ({"date_range_start": "2024-01-20T09:56:00"}, (event_entity,)),
         ({"date_range_end": "2024-01-20T09:59:10"}, (event_entity, event_entity_2)),
         (
@@ -42,7 +43,7 @@ def test_get_events_with_rules_filters(
 
     rules = ListRules()
     for filter_params, expected_result in filter_cases:
-        filters = ProjectEventFilters.from_params(params=filter_params, project_ids=[project.id])
+        filters = ProjectEventFilters.from_params(params=MultiDict(filter_params.items()), project_ids=[project.id])
         page = EventService.get_events_with_rules(rules=rules, filters=filters)
         assert set(page.results) == set(expected_result), filter_params
 
@@ -51,7 +52,7 @@ def test_get_events_with_rules_filters(
 @pytest.mark.parametrize("sort_order,reverse", ((SortOrder.ASC, False), (SortOrder.DESC, True)))
 def test_get_events_with_rules_sort(sort_order, reverse, event_entity, event_entity_2, project):
     rules = ListRules(sort=sort_order)
-    filters = ProjectEventFilters.from_params(params={}, project_ids=[project.id])
+    filters = ProjectEventFilters.from_params(params=MultiDict({}), project_ids=[project.id])
 
     page = EventService.get_events_with_rules(rules=rules, filters=filters)
 
@@ -62,7 +63,7 @@ def test_get_events_with_rules_sort(sort_order, reverse, event_entity, event_ent
 @pytest.mark.integration
 def test_get_events_with_rules_prefetch(event_entity, project, instance):
     rules = ListRules()
-    filters = ProjectEventFilters.from_params(params={}, project_ids=[project.id])
+    filters = ProjectEventFilters.from_params(params=MultiDict({}), project_ids=[project.id])
 
     with assert_num_queries(3):
         (result_event,) = EventService.get_events_with_rules(rules=rules, filters=filters)

common/tests/unit/test_decorators.py

@@ -136,11 +136,11 @@ def test_class_mangled_auto_name():
 @pytest.mark.unit
 def test_no_duplicate_names():
     """The same decorator function cannot be applied to two names on the same class."""
-    with pytest.raises(RuntimeError):
+    with pytest.raises(TypeError):
 
         class DuplicateCheck:
             @cached_property
-            def x() -> int:
+            def x(self) -> int:
                 return 3
 
             y = x

deploy/charts/observability-app/values.yaml

@@ -9,7 +9,7 @@ observability:
   flask_debug: "false"
   services_secrets_name: external-service-keys
   keys_secrets_name: internal-keys
-  pythonpath: /dk/lib/python3.10/site-packages
+  pythonpath: /dk/lib/python3.12/site-packages
   image:
     repository: docker.io/datakitchen
     tag: "v2"

deploy/docker/observability-be.dockerfile

@@ -1,8 +1,17 @@
 # DEV NOTE:  YOU MUST RUN `docker build` FROM THE TOP-LEVEL OF `observability-be` AND POINT TO THIS FILE.
 ARG BASE_IMAGE_URL
-FROM ${BASE_IMAGE_URL}python:3.10-slim-bullseye AS build-image
+FROM ${BASE_IMAGE_URL}python:3.12.7-alpine3.20 AS build-image
 LABEL maintainer="DataKitchen"
 
+RUN apk update && apk upgrade && apk add --no-cache \
+    # Tools needed for building the python wheels
+    gcc \
+    g++ \
+    make \
+    cmake \
+    musl-dev \
+    librdkafka-dev=2.4.0-r0
+
 COPY pyproject.toml /tmp/dk/
 #           -O: Strips asserts from the code which removes some unnecessary codepaths resulting in a small
 #               performance improvement
@@ -12,7 +21,7 @@ RUN python3 -O -m pip install /tmp/dk --prefix=/dk
 
 # Copy and build the actual application
 COPY . /tmp/dk/
-ENV PYTHONPATH ${PYTHONPATH}:/dk/lib/python3.10/site-packages
+ENV PYTHONPATH ${PYTHONPATH}:/dk/lib/python3.12/site-packages
 #    --no-deps: The previous pip layer will have already installed the dependencies. This
 #               will disable doing a second dependency resolution check.
 #           -O: Strips asserts from the code which removes some unnecessary codepaths resulting in a small
@@ -21,7 +30,9 @@ ENV PYTHONPATH ${PYTHONPATH}:/dk/lib/python3.10/site-packages
 # --prefix=/dk: The destination installation environment folder
 RUN python3 -O -m pip install --no-deps /tmp/dk --prefix=/dk
 
-FROM ${BASE_IMAGE_URL}python:3.10-slim-bullseye AS runtime-image
+FROM ${BASE_IMAGE_URL}python:3.12.7-alpine3.20 AS runtime-image
+
+RUN apk update && apk upgrade && apk add --no-cache librdkafka=2.4.0-r0
 
 # Grab the pre-built app from the build-image. This way we don't have
 # excess laying around in the final image.
@@ -30,5 +41,5 @@ COPY --from=build-image /dk /dk
 COPY --from=build-image /tmp/dk/deploy/conf/gunicorn.conf.py /tmp/dk/deploy/conf/yoyo.ini /dk/
 COPY --from=build-image /tmp/dk/deploy/migrations/ /dk/lib/migrations/
 
-ENV PYTHONPATH ${PYTHONPATH}:/dk/lib/python3.10/site-packages
+ENV PYTHONPATH ${PYTHONPATH}:/dk/lib/python3.12/site-packages
 ENV PATH ${PATH}:/dk/bin

deploy/docker/observability-ui.dockerfile

@@ -1,24 +1,14 @@
 ARG BASE_IMAGE_URL
 
-FROM --platform=${BUILDPLATFORM} ${BASE_IMAGE_URL}debian:bookworm-slim AS build-image
+FROM --platform=${BUILDPLATFORM} ${BASE_IMAGE_URL}node:23.10-alpine3.21 AS build-image
 
 WORKDIR /observability_ui
 COPY observability_ui/ /observability_ui
 
-SHELL ["/bin/bash", "--login", "-c"]
-
-RUN apt-get update -y && \
-  apt-get upgrade -y && \
-  apt-get install curl -y && \
-  apt-get install jq -y
-RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
-RUN nvm install $(jq -r .engines.node package.json)
-RUN npm install --global yarn
 RUN yarn
 RUN yarn build:ci
 
-
-FROM ${BASE_IMAGE_URL}nginxinc/nginx-unprivileged:1.25
+FROM ${BASE_IMAGE_URL}nginxinc/nginx-unprivileged:alpine3.21
 
 WORKDIR /observability_ui
 

observability_ui/apps/shell/src/app/projects/events/event-list/event-list.component.ts

@@ -83,7 +83,7 @@ export class EventListComponent extends CoreComponent implements OnInit, HasSear
       const filters: EventSearchFields = {
         event_type: event_type?.split(',').filter(e => e) ?? [] as any,
         component_id: component_id?.split(',').filter(e => e) ?? [] as any,
-      }
+      };
 
       if (date_range_start) {
         const startDate = beginningOfDay(new Date(date_range_start));

observability_ui/apps/shell/src/app/projects/instances/alerts-dialog/alerts-dialog.component.scss

@@ -2,7 +2,7 @@
 @use '@observability/ui/styles/mixins' as mixins;
 
 .table-wrapper-container {
-  height: 400px;
+  height: 500px;
   display: flex;
 
   table-wrapper {

observability_ui/apps/shell/src/app/projects/instances/alerts-dialog/alerts-dialog.component.ts

@@ -79,7 +79,7 @@ export class AlertsDialogComponent {
     instance: Instance
   }, private store: InstancesStore, private router: Router) {
     let urlTree = this.router.parseUrl(this.router.url);
-    urlTree.queryParams['_pageSize'] = '5';
+    urlTree.queryParams['_pageSize'] = '25';
     urlTree.queryParams['_pageIndex'] = 0;
 
     void this.router.navigateByUrl(urlTree);

observability_ui/apps/shell/src/app/projects/instances/instance-timeline/instance-timeline.component.scss

@@ -31,6 +31,10 @@ mat-card-content {
   }
 }
 
+gantt-chart {
+  max-height: 100%;
+}
+
 .run-label {
   @include mixins.flex-row($justify: flex-start);
 

observability_ui/apps/shell/src/app/projects/integrations/integrations.component.html

@@ -130,5 +130,5 @@ <h1>{{'integrations' | translate | titlecase}}</h1>
   [class.hidden]="(total$ | async) === 0"
   [length]="(total$ | async)"
   [pageSize]="pageSize"
-  [pageSizeOptions]="[5, 10, 25, 50, 100]"
+  [pageSizeOptions]="[25, 50, 100, 200]"
   [showFirstLastButtons]="true"></mat-paginator>

observability_ui/apps/shell/src/app/projects/integrations/integrations.component.ts

@@ -26,7 +26,7 @@ export class IntegrationsComponent extends CoreComponent implements OnInit, HasP
 
   agents$ = this.agentStore.list$;
   total$ = this.agentStore.total$;
-  pageSize = 10;
+  pageSize = 25;
 
   @BindToQueryParams()
   @PersistOnLocalStorage({ namespace: Prop('storageKey') })

observability_ui/apps/shell/src/app/projects/integrations/tools/dbt-core-tool/dbt-core-tool.component.ts

@@ -43,7 +43,7 @@ export class DBTCoreToolComponent extends AbstractTool {
   static override _image = 'docker.io/datakitchen/dbt-core-connector';
 
   override readonly envList = [
-    { name: 'EVENTS_API_HOST', placeholder: '', required: true },
+    { name: 'EVENTS_API_HOST', placeholder: '# the base API URL for Observability', required: true },
     { name: 'EVENTS_API_KEY', placeholder: '# an API key for the Observability project', required: true },
     { name: 'PIPELINE_NAME', placeholder: '# a unique name for this dbt job', required: true },
     { name: 'PIPELINE_KEY', placeholder: '# a unique key for this dbt job', required: true },
@@ -61,8 +61,4 @@ export class DBTCoreToolComponent extends AbstractTool {
     DOCKER_TAG: new FormControl('', [ Validators.required ]),
     DEFAULT_DEPLOYMENT_MODE: new FormControl('docker', [ Validators.required, CustomValidators.oneOf([ 'docker', 'kubernetes' ]) ]),
   });
-
-  protected override disableOnStart: FormControl<any>[] = [
-    this.envListForm.controls['EVENTS_API_HOST'],
-  ];
 }

observability_ui/apps/shell/src/app/projects/journeys/journeys-list/journeys-list.component.html

@@ -96,6 +96,6 @@ <h1>Journeys</h1>
   <mat-paginator [class.hidden]="(total$ | async) === 0"
     [length]="(total$ | async)"
     [pageSize]="pageSize"
-    [pageSizeOptions]="[5, 10, 25, 50, 100]"
+    [pageSizeOptions]="[25, 50, 100, 200]"
     [showFirstLastButtons]="true"></mat-paginator>
 </ng-container>

observability_ui/apps/shell/src/app/projects/journeys/journeys-list/journeys-list.component.ts

@@ -36,7 +36,7 @@ export class JourneysListComponent extends CoreComponent implements OnInit, HasP
   search$: BehaviorSubject<SearchFields> = new BehaviorSubject<SearchFields>({ search: '', labels: '' });
 
   parentId: string;
-  pageSize: number = 10;
+  pageSize: number = 25;
 
   @ViewChild(MatPaginator) paginator!: MatPaginator;
 

observability_ui/apps/shell/src/app/projects/runs/events-table/events-table.component.html

@@ -190,6 +190,14 @@
             class="text--link-with-icon fx-row">
             <mat-icon class="icon-size-16 mr-1">data_object</mat-icon>
           </a>
+
+          <a dkTooltip="Go to batch run"
+            *ngIf="event.run?.id"
+            [routerLink]="[ '/projects', projectId(), 'events', 'runs', 'details', event.run.id ]"
+            class="text--link-with-icon"
+            target="_blank">
+            <mat-icon class="icon-size-16 mr-1">share</mat-icon>
+          </a>
         </div>
       </div>
 

observability_ui/apps/shell/src/app/projects/runs/events-table/events-table.component.spec.ts

@@ -4,7 +4,8 @@ import { MockModule } from 'ng-mocks';
 import { MatLegacyDialog as MatDialog, MatLegacyDialogModule as MatDialogModule } from '@angular/material/legacy-dialog';
 import { MetadataViewerComponent, TableWrapperModule } from '@observability-ui/ui';
 import { Mocked, MockProvider } from '@datakitchen/ngx-toolkit';
-import { EventType, EventTypes, TestStatus } from '@observability-ui/core';
+import { EventType, EventTypes, ProjectStore, TestStatus } from '@observability-ui/core';
+import { of } from 'rxjs';
 
 describe('EventsTableComponent', () => {
   let component: EventsTableComponent;
@@ -29,7 +30,12 @@ describe('EventsTableComponent', () => {
   beforeEach(async () => {
     await TestBed.configureTestingModule({
       declarations: [ EventsTableComponent ],
-      providers: [ MockProvider(MatDialog) ],
+      providers: [
+        MockProvider(MatDialog),
+        MockProvider(ProjectStore, class {
+          current$ = of({});
+        }),
+      ],
       imports: [ MockModule(MatDialogModule), MockModule(TableWrapperModule) ]
     }).compileComponents();
 

observability_ui/apps/shell/src/app/projects/runs/events-table/events-table.component.ts

@@ -1,10 +1,11 @@
 import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core';
-import { DatasetOperationEventData, EventType, EventTypes, MessageLogEventData, RunProcessedStatus, RunStatusEventData, TestOutcomesEventData, TestStatus } from '@observability-ui/core';
+import { DatasetOperationEventData, EventType, EventTypes, MessageLogEventData, ProjectStore, RunProcessedStatus, RunStatusEventData, TestOutcomesEventData, TestStatus } from '@observability-ui/core';
 import { MetadataViewerComponent, MetadataViewerData, TableChangeEvent } from '@observability-ui/ui';
 import { MatLegacyDialog as MatDialog } from '@angular/material/legacy-dialog';
 import { FormGroup } from '@angular/forms';
-import { BehaviorSubject } from 'rxjs';
+import { BehaviorSubject, map } from 'rxjs';
 import { CoreComponent, omit } from '@datakitchen/ngx-toolkit';
+import { toSignal } from '@angular/core/rxjs-interop';
 
 @Component({
   selector: 'shell-events-table',
@@ -15,6 +16,7 @@ export class EventsTableComponent extends CoreComponent implements OnInit {
   protected readonly RunProcessedStatus = RunProcessedStatus;
   protected readonly EventTypes = EventTypes;
 
+  projectId = toSignal(this.projectStore.current$.pipe(map(({ id }) => id)));
 
   @Input() set items(events: EventType[]) {
     this.items$.next(this.getEventsWithTestSummary(this.addComponentKeys(events)));
@@ -38,7 +40,10 @@ export class EventsTableComponent extends CoreComponent implements OnInit {
   public items$: BehaviorSubject<EventType[]> = new BehaviorSubject<EventType[]>([]);
 
 
-  constructor(private matDialog: MatDialog) {
+  constructor(
+    private matDialog: MatDialog,
+    private projectStore: ProjectStore,
+  ) {
     super();
   }
 

observability_ui/apps/shell/src/app/projects/runs/run-timeline/run-timeline.component.scss

@@ -11,3 +11,7 @@
 mat-card {
   height: 100%;
 }
+
+gantt-chart {
+  max-height: 100%;
+}

observability_ui/apps/shell/src/app/projects/runs/runs-table/run-time/run-time.component.ts

@@ -23,7 +23,7 @@ export class RunTimeComponent {
     return this._actual();
   }
 
-  currentTime = new Date()
+  currentTime = new Date();
 
   time = computed(() => {
     return this._actual() || this.expected;

observability_ui/libs/ui/src/lib/details-header/details-header.component.scss

@@ -39,4 +39,5 @@ $right-side-width: 400px;
 
 .right-side {
   width: $right-side-width;
+  z-index: 10;
 }

observability_ui/libs/ui/src/lib/gantt-chart/gantt-chart.component.scss

@@ -54,7 +54,7 @@ $task-name-width: 250px;
 
   flex: 1;
   top: 32px;
-  height: 100%;
+  height: calc(100% - 32px);
   position: absolute;
   pointer-events: none;
   margin-left: $task-name-width;
@@ -63,6 +63,7 @@ $task-name-width: 250px;
     @include mixins.flex-column($justify: flex-start, $align: normal);
     @include mixins.font-style($style: caption);
 
+    flex: auto;
     height: 100%;
     margin-top: -20px;
     margin-left: -26px;

observability_ui/libs/ui/src/lib/pipes/parseDate/parseDate.pipe.spec.ts

@@ -2,6 +2,13 @@ import { TestBed } from '@angular/core/testing';
 import { ParseDatePipe } from './parseDate.pipe';
 import * as parseDate from '@observability-ui/core';
 
+jest.mock('@observability-ui/core', () => {
+  return {
+    __esModule: true,
+    ...jest.requireActual('@observability-ui/core')
+  };
+});
+
 describe('parseDate.pipe', () => {
   let pipe: ParseDatePipe;