This repository has been archived by the owner on Oct 9, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 53
/
paths.go
61 lines (52 loc) · 2.7 KB
/
paths.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
package ioutils
import (
"context"
"github.com/flyteorg/flytestdlib/logger"
"github.com/flyteorg/flytestdlib/storage"
)
const (
// InputsSuffix specifies the name of the file that contains the task inputs in the form core.LiteralMap
InputsSuffix = "inputs.pb"
// TaskTemplateSuffix In case a task requests for a task template, it is passed into the task using this filename.
// The format is of type core.TaskTemplate
TaskTemplateSuffix = "task.pb"
// FuturesSuffix specifies that for dynamic workflows, the futures files is written with this filename/suffix.
// The format is core.DynamicJobSpec
FuturesSuffix = "futures.pb"
// OutputsSuffix specifies that outputs are assumed to be written to this "file"/"suffix" under the given prefix
// The outputs file has a format of core.LiteralMap
OutputsSuffix = "outputs.pb"
// deckSuffix specifies that deck file are assumed to be written to this "file"/"suffix" under the given prefix
// The deck file has a format of HTML
deckSuffix = "deck.html"
// ErrorsSuffix specifies that the errors are written to this prefix/file under the given prefix. The Error File
// has a format of core.ErrorDocument
ErrorsSuffix = "error.pb"
IndexLookupSuffix = "indexlookup.pb"
// CheckpointPrefix specifies the common prefix that can be used as a directory where all the checkpoint information
// will be stored. This directory is under the raw output-prefix path
CheckpointPrefix = "_flytecheckpoints"
)
// ConstructCheckpointPath returns a checkpoint path under the given `base` / rawOutputPrefix, following the conventions of
// the store
func ConstructCheckpointPath(store storage.ReferenceConstructor, rawOutputPrefix storage.DataReference) storage.DataReference {
if len(rawOutputPrefix) == 0 {
return ""
}
return constructPath(store, rawOutputPrefix, CheckpointPrefix)
}
func constructPath(store storage.ReferenceConstructor, base storage.DataReference, suffix string) storage.DataReference {
res, err := store.ConstructReference(context.Background(), base, suffix)
if err != nil {
logger.Error(context.Background(), "Failed to construct path. Base[%v] Error: %v", base, err)
}
return res
}
// GetTaskTemplatePath returns a protobuf file path where TaskTemplate is stored
func GetTaskTemplatePath(ctx context.Context, store storage.ReferenceConstructor, base storage.DataReference) (storage.DataReference, error) {
return store.ConstructReference(ctx, base, TaskTemplateSuffix)
}
// GetIndexLookupPath returns the indexpath suffixed to IndexLookupSuffix
func GetIndexLookupPath(ctx context.Context, store storage.ReferenceConstructor, prefix storage.DataReference) (res storage.DataReference, err error) {
return store.ConstructReference(ctx, prefix, IndexLookupSuffix)
}