Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions extension/src/experiments/checkpoints/collect.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,34 @@ describe('collectHasCheckpoints', () => {
expect(hasCheckpoints).toBe(false)
})

it('should not fail if a train stage is not provided', () => {
const hasCheckpoints = collectHasCheckpoints({
stages: {
extract: {
cmd: 'tar -xzf data/images.tar.gz --directory data',
deps: ['data/images.tar.gz'],
outs: [{ 'data/images/': { cache: false } }]
}
}
} as PartialDvcYaml)

expect(hasCheckpoints).toBe(false)
})

it('should return true if any stage has checkpoints', () => {
const hasCheckpoints = collectHasCheckpoints({
stages: {
extract: {
cmd: 'tar -xzf data/images.tar.gz --directory data',
deps: ['data/images.tar.gz'],
outs: [{ 'data/images/': { cache: false, checkpoint: true } }]
}
}
} as PartialDvcYaml)

expect(hasCheckpoints).toBe(true)
})

it('should correctly classify a more complex dvc.yaml without checkpoint', () => {
const hasCheckpoints = collectHasCheckpoints({
stages: {
Expand Down
21 changes: 15 additions & 6 deletions extension/src/experiments/checkpoints/collect.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,22 @@
import { PartialDvcYaml } from '../../fileSystem'
import { Out, PartialDvcYaml } from '../../fileSystem'

export const collectHasCheckpoints = (yaml: PartialDvcYaml): boolean => {
return !!yaml.stages.train.outs.some(out => {
const stageHasCheckpoints = (outs: Out[] = []): boolean => {
for (const out of outs) {
if (typeof out === 'string') {
return false
continue
}

if (Object.values(out).some(file => file?.checkpoint)) {
return true
}
})
}
return false
}

export const collectHasCheckpoints = (yaml: PartialDvcYaml): boolean => {
for (const stage of Object.values(yaml?.stages || {})) {
if (stageHasCheckpoints(stage?.outs)) {
return true
}
}
return false
}
8 changes: 7 additions & 1 deletion extension/src/fileSystem/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,15 @@ export const isSameOrChild = (root: string, path: string) => {
return !rel.startsWith('..')
}

export type Out =
| string
| Record<string, { checkpoint?: boolean; cache?: boolean }>

export type PartialDvcYaml = {
stages: {
train: { outs: (string | Record<string, { checkpoint?: boolean }>)[] }
[stage: string]: {
outs?: Out[]
}
}
}

Expand Down