Skip to content

Commit 752e76c

Browse files
1vn1vn
authored
Refactor away metadata_key (#105)
1 parent 48d9700 commit 752e76c

File tree

18 files changed

+29
-66
lines changed

18 files changed

+29
-66
lines changed

‎pkg/consts/consts.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ var (
5454
LogPrefixesDir = "log_prefixes"
5555
RawColumnsDir = "raw_columns"
5656
TransformedColumnsDir = "transformed_columns"
57+
MetadataDir = "metadata"
5758

5859
TelemetryURL = "https://telemetry.cortexlabs.dev"
5960
)

‎pkg/operator/api/context/context.go‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ type Context struct {
3030
CortexConfig *config.CortexConfig `json:"cortex_config"`
3131
DatasetVersion string `json:"dataset_version"`
3232
Root string `json:"root"`
33+
MetadataRoot string `json:"metadata_root"`
3334
RawDataset RawDataset `json:"raw_dataset"`
3435
StatusPrefix string `json:"status_prefix"`
3536
App *App `json:"app"`
@@ -46,16 +47,14 @@ type Context struct {
4647
}
4748

4849
type RawDataset struct {
49-
Key string `json:"key"`
50-
MetadataKey string `json:"metadata_key"`
50+
Key string `json:"key"`
5151
}
5252

5353
type Resource interface {
5454
userconfig.Resource
5555
GetID() string
5656
GetIDWithTags() string
5757
GetResourceFields() *ResourceFields
58-
GetMetadataKey() string
5958
}
6059

6160
type ComputedResource interface {
@@ -73,7 +72,6 @@ type ResourceFields struct {
7372
ID string `json:"id"`
7473
IDWithTags string `json:"id_with_tags"`
7574
ResourceType resource.Type `json:"resource_type"`
76-
MetadataKey string `json:"metadata_key"`
7775
}
7876

7977
type ComputedResourceFields struct {
@@ -93,10 +91,6 @@ func (r *ResourceFields) GetResourceFields() *ResourceFields {
9391
return r
9492
}
9593

96-
func (r *ResourceFields) GetMetadataKey() string {
97-
return r.MetadataKey
98-
}
99-
10094
func (r *ComputedResourceFields) GetWorkloadID() string {
10195
return r.WorkloadID
10296
}

‎pkg/operator/context/aggregates.go‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -80,21 +80,18 @@ func getAggregates(
8080
buf.WriteString(aggregateConfig.Tags.ID())
8181
idWithTags := hash.Bytes(buf.Bytes())
8282

83-
aggregateRootKey := filepath.Join(
83+
aggregateKey := filepath.Join(
8484
root,
8585
consts.AggregatesDir,
8686
id,
87-
)
88-
aggregateKey := aggregateRootKey + ".msgpack"
89-
aggregateMetadataKey := aggregateRootKey + "_metadata.json"
87+
) + ".msgpack"
9088

9189
aggregates[aggregateConfig.Name] = &context.Aggregate{
9290
ComputedResourceFields: &context.ComputedResourceFields{
9391
ResourceFields: &context.ResourceFields{
9492
ID: id,
9593
IDWithTags: idWithTags,
9694
ResourceType: resource.AggregateType,
97-
MetadataKey: aggregateMetadataKey,
9895
},
9996
},
10097
Aggregate: aggregateConfig,

‎pkg/operator/context/aggregators.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,6 @@ func newAggregator(
106106
ID: id,
107107
IDWithTags: id,
108108
ResourceType: resource.AggregatorType,
109-
MetadataKey: filepath.Join(consts.AggregatorsDir, id+"_metadata.json"),
110109
},
111110
Aggregator: &aggregatorConfig,
112111
Namespace: namespace,

‎pkg/operator/context/apis.go‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,7 @@ package context
1818

1919
import (
2020
"bytes"
21-
"path/filepath"
2221

23-
"github.com/cortexlabs/cortex/pkg/consts"
2422
"github.com/cortexlabs/cortex/pkg/lib/hash"
2523
"github.com/cortexlabs/cortex/pkg/operator/api/context"
2624
"github.com/cortexlabs/cortex/pkg/operator/api/resource"
@@ -50,7 +48,6 @@ func getAPIs(config *userconfig.Config,
5048
ID: id,
5149
IDWithTags: idWithTags,
5250
ResourceType: resource.APIType,
53-
MetadataKey: filepath.Join(consts.APIsDir, id+"_metadata.json"),
5451
},
5552
},
5653
API: apiConfig,

‎pkg/operator/context/constants.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@ func newConstant(constantConfig userconfig.Constant) (*context.Constant, error)
5959
ID: id,
6060
IDWithTags: idWithTags,
6161
ResourceType: resource.ConstantType,
62-
MetadataKey: filepath.Join(consts.ConstantsDir, id+"_metadata.json"),
6362
},
6463
Constant: &constantConfig,
6564
Key: filepath.Join(consts.ConstantsDir, id+".msgpack"),

‎pkg/operator/context/context.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,14 @@ func New(
117117
ctx.DatasetVersion,
118118
ctx.Environment.ID,
119119
)
120+
121+
ctx.MetadataRoot = filepath.Join(
122+
ctx.Root,
123+
consts.MetadataDir,
124+
)
125+
120126
ctx.RawDataset = context.RawDataset{
121-
Key: filepath.Join(ctx.Root, consts.RawDataDir, "raw.parquet"),
122-
MetadataKey: filepath.Join(ctx.Root, consts.RawDataDir, "metadata.json"),
127+
Key: filepath.Join(ctx.Root, consts.RawDataDir, "raw.parquet"),
123128
}
124129

125130
ctx.StatusPrefix = StatusPrefix(ctx.App.Name)

‎pkg/operator/context/models.go‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,6 @@ func getModels(
9696
ID: modelID,
9797
IDWithTags: modelID,
9898
ResourceType: resource.ModelType,
99-
MetadataKey: filepath.Join(root, consts.ModelsDir, modelID+"_metadata.json"),
10099
},
101100
},
102101
Model: modelConfig,
@@ -114,7 +113,6 @@ func getModels(
114113
ID: datasetID,
115114
IDWithTags: datasetIDWithTags,
116115
ResourceType: resource.TrainingDatasetType,
117-
MetadataKey: filepath.Join(datasetRoot, "metadata.json"),
118116
},
119117
},
120118
ModelName: modelConfig.Name,

‎pkg/operator/context/python_packages.go‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@ func loadPythonPackages(files map[string][]byte, datasetVersion string) (context
6464
ResourceFields: &context.ResourceFields{
6565
ID: id,
6666
ResourceType: resource.PythonPackageType,
67-
MetadataKey: filepath.Join(consts.PythonPackagesDir, id, "metadata.json"),
6867
},
6968
},
7069
SrcKey: filepath.Join(consts.PythonPackagesDir, id, "src.txt"),
@@ -103,7 +102,6 @@ func loadPythonPackages(files map[string][]byte, datasetVersion string) (context
103102
ResourceFields: &context.ResourceFields{
104103
ID: id,
105104
ResourceType: resource.PythonPackageType,
106-
MetadataKey: filepath.Join(consts.PythonPackagesDir, id, "metadata.json"),
107105
},
108106
},
109107
SrcKey: filepath.Join(consts.PythonPackagesDir, id, "src.zip"),

‎pkg/operator/context/raw_columns.go‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,7 @@ package context
1818

1919
import (
2020
"bytes"
21-
"path/filepath"
2221

23-
"github.com/cortexlabs/cortex/pkg/consts"
2422
"github.com/cortexlabs/cortex/pkg/lib/configreader"
2523
"github.com/cortexlabs/cortex/pkg/lib/errors"
2624
"github.com/cortexlabs/cortex/pkg/lib/hash"
@@ -59,7 +57,6 @@ func getRawColumns(
5957
ID: id,
6058
IDWithTags: idWithTags,
6159
ResourceType: resource.RawColumnType,
62-
MetadataKey: filepath.Join(consts.RawColumnsDir, id+"_metadata.json"),
6360
},
6461
},
6562
RawIntColumn: typedColumnConfig,
@@ -77,7 +74,6 @@ func getRawColumns(
7774
ID: id,
7875
IDWithTags: idWithTags,
7976
ResourceType: resource.RawColumnType,
80-
MetadataKey: filepath.Join(consts.RawColumnsDir, id+"_metadata.json"),
8177
},
8278
},
8379
RawFloatColumn: typedColumnConfig,
@@ -93,7 +89,6 @@ func getRawColumns(
9389
ID: id,
9490
IDWithTags: idWithTags,
9591
ResourceType: resource.RawColumnType,
96-
MetadataKey: filepath.Join(consts.RawColumnsDir, id+"_metadata.json"),
9792
},
9893
},
9994
RawStringColumn: typedColumnConfig,

0 commit comments

Comments
 (0)