diff --git a/.gitignore b/.gitignore index 6f0b170..aa8b3f6 100644 --- a/.gitignore +++ b/.gitignore @@ -169,3 +169,6 @@ observability_ui/node_modules/ # Terraform .terraform* + +# Helm +**/*.tgz diff --git a/common/apscheduler_extensions.py b/common/apscheduler_extensions.py index 1c935dc..18fa404 100644 --- a/common/apscheduler_extensions.py +++ b/common/apscheduler_extensions.py @@ -58,7 +58,7 @@ def validate_cron_expression(expression: str) -> list[str]: if len(values) != 5: errors.append(f"Got {len(values)} fields. Expected 5.") - for (field_name, field_class), value in zip(CRON_EXPRESSION_FIELDS, values, strict=True): + for (field_name, field_class), value in zip(CRON_EXPRESSION_FIELDS, values, strict=False): try: field_class(field_name, value) except ValueError: diff --git a/common/entity_services/project_service.py b/common/entity_services/project_service.py index 005732d..6086e5a 100644 --- a/common/entity_services/project_service.py +++ b/common/entity_services/project_service.py @@ -58,7 +58,7 @@ def get_test_outcomes_with_rules( query = TestOutcome.select(TestOutcome).where(TestOutcome.component.in_(project.components)) if rules.search is not None: query = query.where( - ((TestOutcome.name ** f"%{rules.search}%") | (TestOutcome.description ** f"%{rules.search}%")) + (TestOutcome.name ** f"%{rules.search}%") | (TestOutcome.description ** f"%{rules.search}%") ) if filters: if statuses := filters.statuses: diff --git a/deploy/charts/observability-app/Chart.yaml b/deploy/charts/observability-app/Chart.yaml index c61476d..0ba607f 100644 --- a/deploy/charts/observability-app/Chart.yaml +++ b/deploy/charts/observability-app/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: dataops-observability-app type: application appVersion: "2.x.x" -version: "2.1.0" +version: "2.2.0" description: DataOps Observability home: https://datakitchen.io diff --git a/deploy/charts/observability-app/templates/agent-api.yaml b/deploy/charts/observability-app/templates/agent-api.yaml index 3b56d12..2233f88 100644 --- a/deploy/charts/observability-app/templates/agent-api.yaml +++ b/deploy/charts/observability-app/templates/agent-api.yaml @@ -61,6 +61,10 @@ spec: name: {{ .Values.observability.keys_secrets_name | quote }} key: AGENT_API_KEY_FLASK_SECRET {{- end }} + {{- with .Values.extraVolumeMounts }} + volumeMounts: + {{ toYaml . | nindent 12 }} + {{- end }} command: [ "/dk/bin/gunicorn" ] args: [ "-c", "/dk/gunicorn.conf.py", "agent_api.app:app" ] readinessProbe: @@ -74,6 +78,10 @@ spec: - name: Host value: {{ tpl .Values.agent_api.hostname . | quote }} {{- end }} + {{- with .Values.extraVolumes }} + volumes: + {{ toYaml . | nindent 8 }} + {{- end }} {{- with .Values.agent_api.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/deploy/charts/observability-app/templates/cli-tool-hook.yaml b/deploy/charts/observability-app/templates/cli-tool-hook.yaml index 9ace34c..311e8e6 100644 --- a/deploy/charts/observability-app/templates/cli-tool-hook.yaml +++ b/deploy/charts/observability-app/templates/cli-tool-hook.yaml @@ -1,8 +1,8 @@ -{{- range .Values.cli_hook.enable }} +{{- range $index, $job := .Values.cli_hook.enable }} apiVersion: batch/v1 kind: Job metadata: - name: "cli-job" + name: cli-job-{{ default $index .name }} annotations: "helm.sh/hook": {{ join ", " .phases }} "helm.sh/hook-weight": {{ default 0 .weight | quote }} @@ -10,13 +10,14 @@ metadata: spec: template: metadata: - name: "cli-job" + name: cli-job-{{ default $index .name }} spec: restartPolicy: Never {{- with $.Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + serviceAccountName: {{ include "observability.serviceAccountName" $ }} containers: - name: cli-hook image: {{ include "observability.cli_hook.image" $ | quote }} @@ -25,9 +26,21 @@ spec: {{- include "observability.environment.base" $ | nindent 12 }} {{- include "observability.environment.database" $ | nindent 12 }} {{- include "observability.environment.pythonpath" $ | nindent 12 }} + {{- with $.Values.extraVolumeMounts }} + volumeMounts: + {{ toYaml . | nindent 12 }} + {{- end }} command: {{- range .command }} - {{ . | quote -}} {{- end }} + args: + {{- range .args }} + - {{ . | quote -}} + {{- end }} + {{- with $.Values.extraVolumes }} + volumes: + {{ toYaml . | nindent 8 }} + {{- end }} --- {{- end }} diff --git a/deploy/charts/observability-app/templates/event-api.yaml b/deploy/charts/observability-app/templates/event-api.yaml index 41545a2..baeb6d1 100644 --- a/deploy/charts/observability-app/templates/event-api.yaml +++ b/deploy/charts/observability-app/templates/event-api.yaml @@ -62,6 +62,10 @@ spec: name: {{ .Values.observability.keys_secrets_name | quote }} key: EVENTS_KEY_FLASK_SECRET {{- end }} + {{- with .Values.extraVolumeMounts }} + volumeMounts: + {{ toYaml . | nindent 12 }} + {{- end }} command: [ "/dk/bin/gunicorn" ] args: [ "-c", "/dk/gunicorn.conf.py", "event_api.app:app" ] readinessProbe: @@ -75,7 +79,10 @@ spec: - name: Host value: {{ tpl .Values.event_api.hostname . | quote }} {{- end }} - + {{- with .Values.extraVolumes }} + volumes: + {{ toYaml . | nindent 8 }} + {{- end }} {{- with .Values.event_api.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/deploy/charts/observability-app/templates/observability-api.yaml b/deploy/charts/observability-app/templates/observability-api.yaml index ee5a2ec..b4ac613 100644 --- a/deploy/charts/observability-app/templates/observability-api.yaml +++ b/deploy/charts/observability-app/templates/observability-api.yaml @@ -61,6 +61,10 @@ spec: name: {{ .Values.observability.keys_secrets_name | quote }} key: OBSERVABILITY_KEY_FLASK_SECRET {{- end }} + {{- with .Values.extraVolumeMounts }} + volumeMounts: + {{ toYaml . | nindent 12 }} + {{- end }} command: [ "/dk/bin/gunicorn" ] args: [ "-c", "/dk/gunicorn.conf.py", "observability_api.app:app" ] readinessProbe: @@ -74,6 +78,10 @@ spec: - name: Host value: {{ tpl .Values.observability_api.hostname . | quote }} {{- end }} + {{- with .Values.extraVolumes }} + volumes: + {{ toYaml . | nindent 8 }} + {{- end }} {{- with .Values.observability_api.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/deploy/charts/observability-app/templates/rules-engine.yaml b/deploy/charts/observability-app/templates/rules-engine.yaml index a1f9b8a..afaa837 100644 --- a/deploy/charts/observability-app/templates/rules-engine.yaml +++ b/deploy/charts/observability-app/templates/rules-engine.yaml @@ -45,8 +45,16 @@ spec: {{- include "observability.environment.database" . | nindent 12 }} {{- include "observability.environment.kafka" . | nindent 12 }} {{- include "observability.environment.smtp" . | nindent 12 }} + {{- with .Values.extraVolumeMounts }} + volumeMounts: + {{ toYaml . | nindent 12 }} + {{- end }} command: [ "/dk/bin/rules-engine" ] {{- include "observability.probes.readiness_cmd" "rules-engine" | nindent 10 -}} + {{- with .Values.extraVolumes }} + volumes: + {{ toYaml . | nindent 8 }} + {{- end }} {{- with .Values.rules_engine.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/deploy/charts/observability-app/templates/run-manager.yaml b/deploy/charts/observability-app/templates/run-manager.yaml index 0b24e12..78fe570 100644 --- a/deploy/charts/observability-app/templates/run-manager.yaml +++ b/deploy/charts/observability-app/templates/run-manager.yaml @@ -44,8 +44,16 @@ spec: {{- include "observability.environment.base" . | nindent 12 }} {{- include "observability.environment.database" . | nindent 12 }} {{- include "observability.environment.kafka" . | nindent 12 }} + {{- with .Values.extraVolumeMounts }} + volumeMounts: + {{ toYaml . | nindent 12 }} + {{- end }} command: ["/dk/bin/run-manager"] {{- include "observability.probes.readiness_cmd" "run-manager" | nindent 10 -}} + {{- with .Values.extraVolumes }} + volumes: + {{ toYaml . | nindent 8 }} + {{- end }} {{- with .Values.run_manager.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/deploy/charts/observability-app/templates/scheduler.yaml b/deploy/charts/observability-app/templates/scheduler.yaml index e790685..0110174 100644 --- a/deploy/charts/observability-app/templates/scheduler.yaml +++ b/deploy/charts/observability-app/templates/scheduler.yaml @@ -44,8 +44,16 @@ spec: {{- include "observability.environment.base" . | nindent 12 }} {{- include "observability.environment.database" . | nindent 12 }} {{- include "observability.environment.kafka" . | nindent 12 }} + {{- with .Values.extraVolumeMounts }} + volumeMounts: + {{ toYaml . | nindent 12 }} + {{- end }} command: [ "/dk/bin/scheduler" ] {{- include "observability.probes.readiness_cmd" "scheduler" | nindent 10 -}} + {{- with .Values.extraVolumes }} + volumes: + {{ toYaml . | nindent 8 }} + {{- end }} {{- with .Values.scheduler.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/observability_ui/apps/shell/src/app/projects/instances/instance-tests/instance-tests.component.html b/observability_ui/apps/shell/src/app/projects/instances/instance-tests/instance-tests.component.html index 48204a4..f00e692 100644 --- a/observability_ui/apps/shell/src/app/projects/instances/instance-tests/instance-tests.component.html +++ b/observability_ui/apps/shell/src/app/projects/instances/instance-tests/instance-tests.component.html @@ -183,7 +183,7 @@ Start Time -
{{test.start_time | date: 'hh:mm:ss a'}}
+
{{test.start_time | date: 'h:mm:ss a'}}
 End Time -
{{test.end_time | date: 'hh:mm:ss a'}}
+
{{test.end_time | date: 'h:mm:ss a'}}
 + [timeformat]="timeformat$ | async"> @@ -49,8 +48,10 @@
{{ run.pipeline.display_name }} + Key: {{ run.pipeline.key ?? '-' }} {{ 'runStatus.' + run.status | translate }} - Key: {{ run.key ?? '-' }} + Run Key: {{ run.key ?? '-' }} Start Time: {{ run.start_time | date: (timeformat$ | async) }} { const timespan = end.getTime() - start.getTime(); if (timespan <= this.minTimespanInMilliseconds) { - return 'hh:mm:ss a'; + return 'MMM d, h:mm:ss a'; } - return 'hh:mm a'; - }), - startWith('hh:mm a'), - ); - spansMultipleDays$ = combineLatest([ this.ganttStartDate$, this.ganttEndDate$ ]).pipe( - map(([ start, end ]) => { - return (end.getTime() - start.getTime()) > 24 * 60 * 60 * 1000; + return 'MMM d, h:mm a'; }), + startWith('MMM d, h:mm a'), ); private minTimespanInMilliseconds: number = 1 * 60 * 1000; diff --git a/observability_ui/apps/shell/src/app/projects/overview/overview.component.html b/observability_ui/apps/shell/src/app/projects/overview/overview.component.html index 0ce9b2d..76a955f 100644 --- a/observability_ui/apps/shell/src/app/projects/overview/overview.component.html +++ b/observability_ui/apps/shell/src/app/projects/overview/overview.component.html @@ -145,9 +145,9 @@

{{ 'overview' | translate }}

{{ instance.errorAlertsCount }} errors, {{ instance.warningAlertsCount }} warnings Total Runs: {{ instance.runsCount }} - Start Time: {{ instance.start_time | date:'MMM d, hh:mm a' }} + Start Time: {{ instance.start_time | date:'MMM d, h:mm a' }} End Time: {{ instance.end_time | date:'MMM d, hh:mm a' }} + class="gantt-bar-tooltip--line gantt-bar-tooltip--end">End Time: {{ instance.end_time | date:'MMM d, h:mm a' }} Duration: {{ instance.start_time | duration:instance.end_time }} @@ -155,12 +155,12 @@

{{ 'overview' | translate }}

- Expected Start Time: {{ instance.start_time | date:'MMM d, hh:mm a' }} + Expected Start Time: {{ instance.start_time | date:'MMM d, h:mm a' }} - Expected End Time: {{ instance.end_time | date:'MMM d, hh:mm a' }} + Expected End Time: {{ instance.end_time | date:'MMM d, h:mm a' }} @@ -246,9 +246,9 @@

{{ 'overview' | translate }}

{{ directive.value.active ? 'Active' : 'Ended' }} {{ directive.value.errorAlertsCount }} errors, {{ directive.value.warningAlertsCount }} warnings - Start Time: {{ directive.value.start_time | date:'MMM d, hh:mm a' }} + Start Time: {{ directive.value.start_time | date:'MMM d, h:mm a' }} End Time: {{ directive.value.end_time | date:'MMM d, hh:mm a' }} + class="gantt-bar-tooltip--line gantt-bar-tooltip--end">End Time: {{ directive.value.end_time | date:'MMM d, h:mm a' }} Duration: {{ directive.value.start_time | duration:directive.value.end_time }} Total Runs: {{ directive.value.runsCount }} @@ -256,12 +256,12 @@

{{ 'overview' | translate }}

- Expected Start Time: {{ directive.value.start_time | date:'MMM d, hh:mm a' }} + Expected Start Time: {{ directive.value.start_time | date:'MMM d, h:mm a' }} - Expected End Time: {{ directive.value.end_time | date:'MMM d, hh:mm a' }} + Expected End Time: {{ directive.value.end_time | date:'MMM d, h:mm a' }} @@ -290,9 +290,9 @@

{{ 'overview' | translate }}

{{ 'runStatus.' + run.status | translate }} Key: {{ run.key ?? '-' }} Start Time: {{ run.start_time | date: 'MMM d, hh:mm a' }} + class="gantt-bar-tooltip--line gantt-bar-tooltip--start">Start Time: {{ run.start_time | date: 'MMM d, h:mm a' }} End Time: {{ run.end_time | date: 'MMM d, hh:mm a' }} + class="gantt-bar-tooltip--line gantt-bar-tooltip--end">End Time: {{ run.end_time | date: 'MMM d, h:mm a' }} Duration: {{ run.start_time | duration: (run.end_time ?? (now$ | async)) }}
diff --git a/observability_ui/apps/shell/src/app/projects/runs/run-tests/run-tests.component.html b/observability_ui/apps/shell/src/app/projects/runs/run-tests/run-tests.component.html index da72a05..34bf607 100644 --- a/observability_ui/apps/shell/src/app/projects/runs/run-tests/run-tests.component.html +++ b/observability_ui/apps/shell/src/app/projects/runs/run-tests/run-tests.component.html @@ -114,6 +114,50 @@
{{test.description || '-'}}
 + + +
+ {{ 'table' | translate }} + {{'columns' | translate}} +
+ + + +
+ {{ testgen?.table ?? '' }} + {{ testgen?.columns ?? '-' }} +
+ + + + + +
+ Dimension + Test Type +
+ + +
+
+ + + {{ 'testgen.testDimension.' + dimension | translate: {}:dimension }}{{ !last ? ', ' : '' }} + + +
+ {{ test.type ? ('testgen.testType.' + test.type | translate: {}:test.type) : '-' }} +
+ + @@ -121,7 +165,7 @@ Start Time -
{{test.start_time | date: 'hh:mm:ss a'}}
+
{{test.start_time | date: 'h:mm:ss a'}}
 End Time -
{{test.end_time | date: 'hh:mm:ss a'}}
+
{{test.end_time | date: 'h:mm:ss a'}}
 + + + - + diff --git a/observability_ui/apps/shell/src/app/projects/runs/run-tests/run-tests.component.scss b/observability_ui/apps/shell/src/app/projects/runs/run-tests/run-tests.component.scss index c5c3200..f7246c8 100644 --- a/observability_ui/apps/shell/src/app/projects/runs/run-tests/run-tests.component.scss +++ b/observability_ui/apps/shell/src/app/projects/runs/run-tests/run-tests.component.scss @@ -191,6 +191,17 @@ table-wrapper::ng-deep { flex: 0 0 106px; justify-content: flex-end; } + + .mat-header-cell, + .mat-cell { + .multiline-cell { + @include mixins.flex-column($align: flex-start); + + span.caption { + @include mixins.font-style($style: caption, $color: disabled); + } + } + } } .description { diff --git a/observability_ui/apps/shell/src/app/projects/runs/run-timeline/run-timeline.component.html b/observability_ui/apps/shell/src/app/projects/runs/run-timeline/run-timeline.component.html index 96826cf..1a828bb 100644 --- a/observability_ui/apps/shell/src/app/projects/runs/run-timeline/run-timeline.component.html +++ b/observability_ui/apps/shell/src/app/projects/runs/run-timeline/run-timeline.component.html @@ -12,8 +12,7 @@ [start]="ganttStartDate$ | async" [end]="ganttEndDate$ | async" [nowBar]="(run$ | async)?.active" - [timeformat]="timeformat$ | async" - [showDays]="spansMultipleDays$ | async"> + [timeformat]="timeformat$ | async"> diff --git a/observability_ui/apps/shell/src/app/projects/runs/run-timeline/run-timeline.component.ts b/observability_ui/apps/shell/src/app/projects/runs/run-timeline/run-timeline.component.ts index 88eca2d..616708f 100644 --- a/observability_ui/apps/shell/src/app/projects/runs/run-timeline/run-timeline.component.ts +++ b/observability_ui/apps/shell/src/app/projects/runs/run-timeline/run-timeline.component.ts @@ -60,16 +60,11 @@ export class RunTimelineComponent implements OnInit, OnDestroy { map(([ start, end ]) => { const timespan = end.getTime() - start.getTime(); if (timespan <= this.minTimespanInMilliseconds) { - return 'hh:mm:ss a'; + return 'MMM d, h:mm:ss a'; } - return 'hh:mm a'; - }), - startWith('hh:mm a'), - ); - spansMultipleDays$ = combineLatest([ this.ganttStartDate$, this.ganttEndDate$ ]).pipe( - map(([ start, end ]) => { - return (end.getTime() - start.getTime()) > 24 * 60 * 60 * 1000; + return 'MMM d, h:mm a'; }), + startWith('MMM d, h:mm a'), ); private subscriptions: Subscription[] = []; diff --git a/observability_ui/libs/ui/src/lib/fields/schedule-field/schedule-field.component.html b/observability_ui/libs/ui/src/lib/fields/schedule-field/schedule-field.component.html index ab71c8e..b9be9ef 100644 --- a/observability_ui/libs/ui/src/lib/fields/schedule-field/schedule-field.component.html +++ b/observability_ui/libs/ui/src/lib/fields/schedule-field/schedule-field.component.html @@ -64,7 +64,7 @@
Cron Expression: {{ cronExpression$ | async }} - Next Time: {{ nextTime$ | async | date: 'MMM d, YYYY hh:mm a' }} + Next Time: {{ nextTime$ | async | date: 'MMM d, YYYY h:mm a' }}
cron expressions diff --git a/observability_ui/libs/ui/src/lib/gantt-chart/gantt-chart.component.html b/observability_ui/libs/ui/src/lib/gantt-chart/gantt-chart.component.html index 4446630..507aa36 100644 --- a/observability_ui/libs/ui/src/lib/gantt-chart/gantt-chart.component.html +++ b/observability_ui/libs/ui/src/lib/gantt-chart/gantt-chart.component.html @@ -7,7 +7,7 @@ class="now-layer" [style.left]="now()?.offset + 'px'"> - {{ now()?.time | date:'hh:mm a' }} + {{ now()?.time | date:'h:mm a' }} diff --git a/observability_ui/libs/ui/src/lib/gantt-chart/gantt-chart.component.spec.ts b/observability_ui/libs/ui/src/lib/gantt-chart/gantt-chart.component.spec.ts index e06cc38..dbbd045 100644 --- a/observability_ui/libs/ui/src/lib/gantt-chart/gantt-chart.component.spec.ts +++ b/observability_ui/libs/ui/src/lib/gantt-chart/gantt-chart.component.spec.ts @@ -116,7 +116,7 @@ describe('GanttChartComponent', () => { }); it('should render the ticks', () => { - const format = 'hh:mm a'; + const format = 'h:mm a'; const start = testComponent.start; const end = testComponent.end; diff --git a/observability_ui/libs/ui/src/lib/gantt-chart/gantt-chart.component.ts b/observability_ui/libs/ui/src/lib/gantt-chart/gantt-chart.component.ts index 2336d59..2f449b0 100644 --- a/observability_ui/libs/ui/src/lib/gantt-chart/gantt-chart.component.ts +++ b/observability_ui/libs/ui/src/lib/gantt-chart/gantt-chart.component.ts @@ -31,12 +31,8 @@ export class GanttChartComponent implements AfterContentInit { @Input() nowBar: boolean = false; - @Input() set showDays(value: boolean) { - this._showDays.set(value); - } - @Input() set timeformat(value: string) { - this._timeformat.set(value ?? 'hh:mm a'); + this._timeformat.set(value ?? 'h:mm a'); } ticks: Signal> = computed(() => { @@ -46,18 +42,12 @@ export class GanttChartComponent implements AfterContentInit { } const format = this._timeformat(); - const showDays = this._showDays(); const span = end.getTime() - start.getTime(); const tickSpan = Math.floor(span / this.ticksCount); const extraTicks = Array(this.ticksCount).fill(0).map((_, idx) => new Date(start.getTime() + (idx * tickSpan))); const ticks = [ start, ...extraTicks.slice(1), end ].map(time => ({ format, time })); - if (showDays) { - ticks[0].format = `MMM d ${format}`; - ticks[ticks.length - 1].format = `MMM d ${format}`; - } - return ticks; }); @@ -139,8 +129,7 @@ export class GanttChartComponent implements AfterContentInit { private _regroup: WritableSignal = signal(0); private _resized: WritableSignal = signal(0); - private _showDays: WritableSignal = signal(false); - private _timeformat: WritableSignal = signal('hh:mm a'); + private _timeformat: WritableSignal = signal('h:mm a'); private _dateRange: WritableSignal<{ start?: Date, end?: Date }> = signal({ start: undefined, end: undefined diff --git a/observability_ui/libs/ui/styles/_gantt-chart.scss b/observability_ui/libs/ui/styles/_gantt-chart.scss index b7cf450..f69d877 100644 --- a/observability_ui/libs/ui/styles/_gantt-chart.scss +++ b/observability_ui/libs/ui/styles/_gantt-chart.scss @@ -52,6 +52,10 @@ gantt-chart { font-weight: 700; } + &--status { + margin-bottom: variables.$space-xs; + } + &--key, &--start, &--end,