diff --git a/web/client/openapi.json b/web/client/openapi.json index 6ce58ee047..2cc0e84705 100644 --- a/web/client/openapi.json +++ b/web/client/openapi.json @@ -883,6 +883,13 @@ "allOf": [{ "$ref": "#/components/schemas/NodeType" }], "default": "model" }, + "parents": { + "items": { "type": "string" }, + "type": "array", + "uniqueItems": true, + "title": "Parents", + "default": [] + }, "interval": { "items": { "type": "string" }, "type": "array", @@ -902,6 +909,13 @@ "allOf": [{ "$ref": "#/components/schemas/NodeType" }], "default": "model" }, + "parents": { + "items": { "type": "string" }, + "type": "array", + "uniqueItems": true, + "title": "Parents", + "default": [] + }, "completed": { "type": "integer", "title": "Completed" }, "total": { "type": "integer", "title": "Total" }, "start": { "type": "integer", "title": "Start" }, @@ -972,6 +986,18 @@ { "$ref": "#/components/schemas/PlanOptions" }, { "type": "null" } ] + }, + "categories": { + "anyOf": [ + { + "additionalProperties": { + "$ref": "#/components/schemas/SnapshotChangeCategory" + }, + "type": "object" + }, + { "type": "null" } + ], + "title": "Categories" } }, "type": "object", @@ -1006,6 +1032,13 @@ "allOf": [{ "$ref": "#/components/schemas/NodeType" }], "default": "model" }, + "parents": { + "items": { "type": "string" }, + "type": "array", + "uniqueItems": true, + "title": "Parents", + "default": [] + }, "diff": { "type": "string", "title": "Diff" }, "indirect": { "items": { "$ref": "#/components/schemas/ChangeDisplay" }, @@ -1013,6 +1046,12 @@ "title": "Indirect", "default": [] }, + "direct": { + "items": { "$ref": "#/components/schemas/ChangeDisplay" }, + "type": "array", + "title": "Direct", + "default": [] + }, "change_category": { "anyOf": [ { "$ref": "#/components/schemas/SnapshotChangeCategory" }, @@ -1031,6 +1070,13 @@ "node_type": { "allOf": [{ "$ref": "#/components/schemas/NodeType" }], "default": "model" + }, + "parents": { + "items": { "type": "string" }, + "type": "array", + "uniqueItems": true, + "title": "Parents", + "default": [] } }, "type": "object", @@ -1045,10 +1091,11 @@ "allOf": [{ "$ref": "#/components/schemas/NodeType" }], "default": "model" }, - "direct": { - "items": { "$ref": "#/components/schemas/ChangeDisplay" }, + "parents": { + "items": { "type": "string" }, "type": "array", - "title": "Direct", + "uniqueItems": true, + "title": "Parents", "default": [] } }, diff --git a/web/client/src/api/index.ts b/web/client/src/api/index.ts index b7a2fd9ea3..0b83dda013 100644 --- a/web/client/src/api/index.ts +++ b/web/client/src/api/index.ts @@ -274,6 +274,7 @@ export function useApiPlanRun( inputs?: { planDates?: PlanDates planOptions?: PlanOptions + categories?: BodyInitiateApplyApiCommandsApplyPostCategories }, options?: ApiOptions, ): UseQueryWithTimeoutOptions { @@ -286,6 +287,7 @@ export function useApiPlanRun( environment, plan_dates: inputs?.planDates, plan_options: inputs?.planOptions, + categories: inputs?.categories, }, { signal }, ) diff --git a/web/client/src/library/components/environmentDetails/SelectEnvironment.tsx b/web/client/src/library/components/environmentDetails/SelectEnvironment.tsx index 84c0e792c5..c77502172e 100644 --- a/web/client/src/library/components/environmentDetails/SelectEnvironment.tsx +++ b/web/client/src/library/components/environmentDetails/SelectEnvironment.tsx @@ -121,7 +121,7 @@ export function SelectEnvironment({ leaveFrom="opacity-100" leaveTo="opacity-0" > -
+
{Array.from(environments).map(env => ( {truncate(decodeURI(label), 50, 20)} - - {count} - + {isNotNil(count) && ( + + {count} + + )}
{hasRight && ( diff --git a/web/client/src/library/components/graph/ModelNode.tsx b/web/client/src/library/components/graph/ModelNode.tsx index 93c849d1cf..3bd205284f 100644 --- a/web/client/src/library/components/graph/ModelNode.tsx +++ b/web/client/src/library/components/graph/ModelNode.tsx @@ -100,12 +100,12 @@ export default function ModelNode({ [setSelectedNodes, highlightedNodeModels], ) - const splat = highlightedNodes?.['*'] + const splat = highlightedNodes['*'] const hasSelectedColumns = columns.some(({ name }) => connections.get(toID(id, name)), ) const hasHighlightedNodes = Object.keys(highlightedNodes).length > 0 - const highlighted = Object.keys(highlightedNodes ?? {}).find(key => + const highlighted = Object.keys(highlightedNodes).find(key => highlightedNodes[key]!.includes(id), ) const isMainNode = mainNode === id @@ -197,7 +197,7 @@ export default function ModelNode({ ? undefined : handleSelect } - count={columns.length} + count={hasHighlightedNodes ? undefined : columns.length} /> {showColumns && ( - + {children} @@ -80,7 +80,7 @@ function ModalConfirmationDetails({ details: string[] }): JSX.Element { return ( -
    +
      {details.map(detail => (
    • s.setTests) - const isInitialPlanRun = - isNil(environment?.isDefault) || isTrue(environment?.isDefault) - - const planPayload = usePlanPayload({ environment, isInitialPlanRun }) - const applyPayload = useApplyPayload({ isInitialPlanRun }) + const planPayload = usePlanPayload() + const applyPayload = useApplyPayload() const { refetch: planRun, cancel: cancelRequestPlanRun } = useApiPlanRun( environment.name, @@ -47,6 +43,7 @@ function Plan(): JSX.Element { { type: EnumPlanActions.ResetPlanDates }, { type: EnumPlanActions.ResetPlanOptions }, { type: EnumPlanActions.ResetTestsReport }, + { type: EnumPlanActions.ResetCategories }, ]) } diff --git a/web/client/src/library/components/plan/PlanActions.tsx b/web/client/src/library/components/plan/PlanActions.tsx index ca31216b43..aa38b7d6ee 100644 --- a/web/client/src/library/components/plan/PlanActions.tsx +++ b/web/client/src/library/components/plan/PlanActions.tsx @@ -1,7 +1,7 @@ import { type MouseEvent } from 'react' import useActiveFocus from '~/hooks/useActiveFocus' import { EnumSize, EnumVariant } from '~/types/enum' -import { isFalse } from '~/utils' +import { isFalse, isNil } from '~/utils' import { Button } from '../button/Button' import { EnumPlanAction, ModelPlanAction } from '@models/plan-action' import { useStorePlan } from '@context/plan' @@ -10,6 +10,7 @@ import { Transition } from '@headlessui/react' import { useNavigate } from 'react-router-dom' import { SelectEnvironment } from '@components/environmentDetails/SelectEnvironment' import { AddEnvironment } from '@components/environmentDetails/AddEnvironment' +import { usePlan } from './context' export default function PlanActions({ run, @@ -23,6 +24,7 @@ export default function PlanActions({ reset: () => void }): JSX.Element { const navigate = useNavigate() + const { change_categorization } = usePlan() const modules = useStoreContext(s => s.modules) const environment = useStoreContext(s => s.environment) @@ -58,15 +60,26 @@ export default function PlanActions({ function handleApply(e: MouseEvent): void { e.stopPropagation() - if (environment.isProd && isFalse(environment.isInitial)) { + const isProd = environment.isProd && isFalse(environment.isInitial) + const hasUncategorized = Array.from(change_categorization.values()).some( + c => isNil(c.category), + ) + + if (isProd) { addConfirmation({ headline: 'Applying Plan Directly On Prod Environment!', - description: `Are you sure you want to apply your changes directly on prod? Safer choice will be to select or add new environment first.`, + tagline: 'Safer choice will be to select or add new environment first.', + description: + 'Are you sure you want to apply your changes directly on prod?', yesText: `Yes, Run ${environment.name}`, noText: 'No, Cancel', - action() { - apply() - }, + action: apply, + details: hasUncategorized + ? [ + 'ATTENTION!', + '[Breaking Change] category will be applied to all uncategorized changes', + ] + : undefined, children: (

      {`${ @@ -95,7 +108,20 @@ export default function PlanActions({ ), }) } else { - apply() + if (hasUncategorized) { + addConfirmation({ + headline: 'Some changes are missing categorization!', + description: 'Are you sure you want to proceed?', + details: [ + '[Breaking Change] category will be applied to all uncategorized changes', + ], + yesText: 'Yes, Apply', + noText: 'No, Cancel', + action: apply, + }) + } else { + apply() + } } } diff --git a/web/client/src/library/components/plan/PlanApplyStageTracker.tsx b/web/client/src/library/components/plan/PlanApplyStageTracker.tsx index 13c04edfa4..8fdf2ea27c 100644 --- a/web/client/src/library/components/plan/PlanApplyStageTracker.tsx +++ b/web/client/src/library/components/plan/PlanApplyStageTracker.tsx @@ -16,7 +16,6 @@ import { isNil, toDateFormat, isFalse, - isArrayEmpty, } from '../../../utils' import { EnumPlanChangeType, usePlan } from './context' import { getPlanOverviewDetails } from './help' @@ -37,7 +36,6 @@ import { } from '@api/client' import { type PlanTrackerMeta } from '@models/tracker-plan' import { type Tests, useStoreProject } from '@context/project' -import { type ModelSQLMeshChangeDisplay } from '@models/sqlmesh-change-display' export default function PlanApplyStageTracker(): JSX.Element { const tests = useStoreProject(s => s.tests) @@ -160,6 +158,8 @@ function StageChanges({ isOpen = false }: { isOpen?: boolean }): JSX.Element { } function StageBackfills({ isOpen = false }: { isOpen?: boolean }): JSX.Element { + const { change_categorization } = usePlan() + const planApply = useStorePlan(s => s.planApply) const planOverview = useStorePlan(s => s.planOverview) const planCancel = useStorePlan(s => s.planCancel) @@ -167,7 +167,40 @@ function StageBackfills({ isOpen = false }: { isOpen?: boolean }): JSX.Element { const { meta, stageBackfills, backfills, hasBackfills } = getPlanOverviewDetails(planApply, planOverview, planCancel) const tempMeta = stageBackfills?.meta ?? meta - const showBackfills = tempMeta?.status === Status.init || isTrue(hasBackfills) + + const categories = useMemo( + () => + Array.from(change_categorization.values()).reduce( + (acc, { category, change }) => { + if (category?.value !== SnapshotChangeCategory.NUMBER_3) { + acc.add(change.name) + } + + if ( + isNil(category) || + category.value === SnapshotChangeCategory.NUMBER_1 + ) { + change.indirect?.forEach(c => acc.add(c.name)) + } + + return acc + }, + new Set(), + ), + [change_categorization], + ) + + const changes = useMemo( + () => + change_categorization.size > 0 + ? backfills.filter(backfill => categories.has(backfill.name)) + : backfills, + [backfills, categories], + ) + + const showBackfills = + (tempMeta?.status === Status.init || isTrue(hasBackfills)) && + (change_categorization.size > 0 ? changes.length > 0 : true) return showBackfills ? ( } @@ -356,72 +389,33 @@ function StageRestate(): JSX.Element { function StageBackfill(): JSX.Element { const planApply = useStorePlan(s => s.planApply) const planAction = useStorePlan(s => s.planAction) + const planOverview = useStorePlan(s => s.planOverview) + const planCancel = useStorePlan(s => s.planCancel) const environment = planApply.environment const stageBackfill = planApply.stageBackfill - const { change_categorization } = usePlan() + const { backfills } = getPlanOverviewDetails( + planApply, + planOverview, + planCancel, + ) - const categories = useMemo( + const tasks = useMemo( () => - Array.from(change_categorization.values()).reduce< - Record - >((acc, { category, change }) => { - change.indirect?.forEach(c => { - if (isNil(acc[c.name])) { - acc[c.name] = [] - } + Object.values(planApply.tasks).reduce( + (acc, task) => { + const backfill = backfills.find(b => b.name === task.name) - acc[c.name]?.push(category?.value !== SnapshotChangeCategory.NUMBER_1) - }) + task.interval = backfill?.interval ?? [] - if (category?.value === SnapshotChangeCategory.NUMBER_3) { - acc[change.name] = [true] - } + acc[task.name] = task - return acc - }, {}), - [change_categorization], - ) - - const tasks = useMemo( - () => - isArrayEmpty(planApply.backfills) - ? Object.entries(planApply.tasks).reduce( - (acc: Record, [name, task]) => { - const choices = categories[name] - const shouldExclude = isNil(choices) - ? false - : choices.every(Boolean) - - if (shouldExclude) return acc - - acc[name] = task - - return acc - }, - {}, - ) - : planApply.backfills.reduce( - (acc: Record, model) => { - const taskBackfill = planApply.tasks[model.name] ?? model - - taskBackfill.interval = model.interval ?? [] - - const choices = categories[model.name] - const shouldExclude = isNil(choices) - ? false - : choices.every(Boolean) - - if (shouldExclude) return acc - - acc[model.name] = taskBackfill - - return acc - }, - {}, - ), - [planApply.backfills, planApply.tasks, categories], + return acc + }, + {}, + ), + [backfills, planApply.tasks], ) if (isNil(stageBackfill) || isNil(environment)) return <> diff --git a/web/client/src/library/components/plan/PlanChangePreview.tsx b/web/client/src/library/components/plan/PlanChangePreview.tsx index ee14214533..17fd887ccf 100644 --- a/web/client/src/library/components/plan/PlanChangePreview.tsx +++ b/web/client/src/library/components/plan/PlanChangePreview.tsx @@ -5,7 +5,7 @@ import { PlusIcon, MinusIcon, ArrowPathRoundedSquareIcon, -} from '@heroicons/react/24/solid' +} from '@heroicons/react/20/solid' import clsx from 'clsx' import { Divider } from '../divider/Divider' import { @@ -15,13 +15,19 @@ import { usePlanDispatch, type PlanChangeType, } from './context' -import { isArrayNotEmpty, isNil, isNotNil, truncate } from '@utils/index' +import { + isArrayNotEmpty, + isNil, + isNotNil, + isStringNotEmpty, + truncate, +} from '@utils/index' import LineageFlowProvider from '@components/graph/context' import { useStoreContext } from '@context/context' import ModelLineage from '@components/graph/ModelLineage' import { type ModelSQLMeshChangeDisplay } from '@models/sqlmesh-change-display' -import { useEffect } from 'react' import { type SnapshotChangeCategory } from '@api/client' +import { useEffect } from 'react' function PlanChangePreview({ children, @@ -37,7 +43,7 @@ function PlanChangePreview({ return (
      )} - {truncate(change.displayViewName, 50, 25)} - +

      • ))}
    @@ -123,7 +129,7 @@ function PlanChangePreviewDirect({ disabled?: boolean }): JSX.Element { const dispatch = usePlanDispatch() - const { categories } = usePlan() + const { categories, change_categorization } = usePlan() const models = useStoreContext(s => s.models) @@ -131,9 +137,9 @@ function PlanChangePreviewDirect({ dispatch( changes.map(change => ({ type: EnumPlanActions.Category, - category: categories.find( - ({ value }) => value === change.change_category, - ), + category: + change_categorization.get(change.name)?.category ?? + categories.find(({ value }) => value === change.change_category), change, })), ) @@ -162,22 +168,39 @@ function PlanChangePreviewDirect({ ) })()} - + + {isArrayNotEmpty(change.direct) && ( + + )} {isArrayNotEmpty(change.indirect) && ( )} - + { + dispatch({ + type: EnumPlanActions.Category, + category: categories.find( + ({ value }) => value === category, + ), + change, + }) + }} /> - +
    - {isNotNil(change?.diff) && ( - + {isNotNil(change) && isStringNotEmpty(change.diff) && ( + )} {(() => { const model = models.get(change.name) @@ -185,19 +208,21 @@ function PlanChangePreviewDirect({ if (isNil(model)) return <> return ( -
    +
    c.name) ?? [], - 'border-4 border-secondary-500 dark:border-primary-500 bg-secondary-500 dark:bg-primary-500 text-bg-secondary-500 dark:text-light': + 'border-4 border-secondary-500 dark:border-primary-500 bg-secondary-500 dark:bg-primary-500 text-bg-secondary-500 dark:bg-primary-500 ring-8 ring-brand-50': [change.name], + 'border-4 border-secondary-500 dark:border-primary-500 bg-secondary-500 dark:bg-primary-500 text-bg-secondary-500 dark:bg-primary-500': + change.direct?.map(c => c.name) ?? [], '*': [ 'border-4 border-neutral-500 bg-neutral-500 text-neutral-600 dark:text-light', ], @@ -221,12 +246,12 @@ function PlanChangePreviewDirect({ function ChangeCategories({ change, disabled = false, + onChange, }: { disabled?: boolean change: ModelSQLMeshChangeDisplay + onChange?: (category: SnapshotChangeCategory) => void }): JSX.Element { - const dispatch = usePlanDispatch() - const { change_categorization, categories } = usePlan() return ( @@ -236,17 +261,11 @@ function ChangeCategories({ disabled && 'pointer-events-none opacity-50 cursor-not-allowed', )} disabled={disabled} - value={ + defaultValue={ change_categorization.get(change.name)?.category?.value ?? change.change_category } - onChange={(category: SnapshotChangeCategory) => { - dispatch({ - type: EnumPlanActions.Category, - category: categories.find(({ value }) => value === category), - change, - }) - }} + onChange={onChange} > {categories.map(category => ( - {isArrayNotEmpty(change.direct) ? ( - - {({ open }) => ( - <> - - - {(() => { - const Tag = open ? MinusCircleIcon : PlusCircleIcon - - return ( - - ) - })()} - - - - - - )} - - ) : ( - - )} + ))}
@@ -341,12 +335,18 @@ function PlanChangePreviewTitle({ return (
- - + + {change.displayViewName} - - {isNotNil(category) && ( - + + {isNil(category) ? ( + isNotNil(change.indirect) && ( + + Categorize Manually + + ) + ) : ( + {category.name} )} @@ -357,16 +357,18 @@ function PlanChangePreviewTitle({ function PlanChangePreviewRelations({ type, models, + className, }: { type: 'direct' | 'indirect' models: ModelSQLMeshChangeDisplay[] + className?: string }): JSX.Element { return (
    {models.map(model => ( @@ -384,7 +386,7 @@ function PlanChangePreviewRelations({ function PlanChangePreviewDiff({ diff }: { diff: string }): JSX.Element { return ( -
    + 
             {diff.split('\n').map((line: string, idx: number) => (
           
    
       
    
         
-          
+          
             {({ open }) => (
               <>
                 ,
+        PlanDetails,
+        { change_categorization: Map }
+      >({}, plan, {
+        change_categorization: new Map(),
+      })
+    }
     case EnumPlanActions.ResetPlanDates: {
       return Object.assign<
         Record,
diff --git a/web/client/src/library/components/plan/hooks.ts b/web/client/src/library/components/plan/hooks.ts
index e97931fdcc..a7e26aca98 100644
--- a/web/client/src/library/components/plan/hooks.ts
+++ b/web/client/src/library/components/plan/hooks.ts
@@ -6,17 +6,15 @@ import {
   type PlanOptions,
   SnapshotChangeCategory,
 } from '~/api/client'
-import { type ModelEnvironment } from '~/models/environment'
-import { isStringEmptyOrNil } from '~/utils'
+import { isNil, isStringEmptyOrNil, isTrue } from '~/utils'
 import { usePlan } from './context'
+import { useStoreContext } from '@context/context'
 
-export function usePlanPayload({
-  environment,
-  isInitialPlanRun,
-}: {
-  environment: ModelEnvironment
-  isInitialPlanRun: boolean
-}): { planDates?: PlanDates; planOptions: PlanOptions } {
+export function usePlanPayload(options?: PlanOptions): {
+  planDates?: PlanDates
+  planOptions: PlanOptions
+  categories: BodyInitiateApplyApiCommandsApplyPostCategories
+} {
   const {
     start,
     end,
@@ -24,13 +22,19 @@ export function usePlanPayload({
     no_gaps,
     skip_backfill,
     forward_only,
+    include_unmodified,
     no_auto_categorization,
     restate_models,
-    create_from,
-    include_unmodified,
     auto_apply,
+    create_from,
+    change_categorization,
   } = usePlan()
 
+  const environment = useStoreContext(s => s.environment)
+
+  const isInitialPlanRun =
+    isNil(environment?.isDefault) || isTrue(environment?.isDefault)
+
   const planDates = useMemo(() => {
     if (environment.isProd) return
 
@@ -75,17 +79,31 @@ export function usePlanPayload({
     auto_apply,
   ])
 
+  const categories = useMemo(() => {
+    return Array.from(
+      change_categorization.values(),
+    ).reduce(
+      (acc, { category, change }) => {
+        acc[change.displayName] =
+          category?.value ?? SnapshotChangeCategory.NUMBER_1
+
+        return acc
+      },
+      {},
+    )
+  }, [change_categorization])
+
   return {
-    planOptions,
+    planOptions: {
+      ...planOptions,
+      ...options,
+    },
     planDates,
+    categories,
   }
 }
 
-export function useApplyPayload({
-  isInitialPlanRun,
-}: {
-  isInitialPlanRun: boolean
-}): {
+export function useApplyPayload(): {
   planDates?: PlanDates
   planOptions: PlanOptions
   categories: BodyInitiateApplyApiCommandsApplyPostCategories
@@ -104,6 +122,11 @@ export function useApplyPayload({
     change_categorization,
   } = usePlan()
 
+  const environment = useStoreContext(s => s.environment)
+
+  const isInitialPlanRun =
+    isNil(environment?.isDefault) || isTrue(environment?.isDefault)
+
   const planDates = useMemo(() => {
     if (isInitialPlanRun) return
 
@@ -118,7 +141,8 @@ export function useApplyPayload({
       change_categorization.values(),
     ).reduce(
       (acc, { category, change }) => {
-        acc[change.name] = category?.value ?? SnapshotChangeCategory.NUMBER_1
+        acc[change.displayName] =
+          category?.value ?? SnapshotChangeCategory.NUMBER_1
 
         return acc
       },
diff --git a/web/client/src/library/components/report/ReportErrors.tsx b/web/client/src/library/components/report/ReportErrors.tsx
index a6c44209ee..b1f1d140cf 100644
--- a/web/client/src/library/components/report/ReportErrors.tsx
+++ b/web/client/src/library/components/report/ReportErrors.tsx
@@ -66,7 +66,7 @@ export default function ReportErrors(): JSX.Element {
             leaveFrom="opacity-100 translate-y-0"
             leaveTo="opacity-0 translate-y-1"
           >
-            
+