/
config.go
111 lines (102 loc) · 4.95 KB
/
config.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
package serviceenv
// Configuration is the generic configuration structure used to access configuration fields.
type Configuration struct {
*GlobalConfiguration
*PachdSpecificConfiguration
*WorkerSpecificConfiguration
}
// GlobalConfiguration contains the global configuration.
type GlobalConfiguration struct {
FeatureFlags
EtcdHost string `env:"ETCD_SERVICE_HOST,required"`
EtcdPort string `env:"ETCD_SERVICE_PORT,required"`
PPSWorkerPort uint16 `env:"PPS_WORKER_GRPC_PORT,default=80"`
Port uint16 `env:"PORT,default=650"`
HTTPPort uint16 `env:"HTTP_PORT,default=652"`
PeerPort uint16 `env:"PEER_PORT,default=653"`
PPSEtcdPrefix string `env:"PPS_ETCD_PREFIX,default=pachyderm_pps"`
Namespace string `env:"NAMESPACE,default=default"`
StorageRoot string `env:"PACH_ROOT,default=/pach"`
}
// PachdFullConfiguration contains the full pachd configuration.
type PachdFullConfiguration struct {
GlobalConfiguration
PachdSpecificConfiguration
}
// PachdSpecificConfiguration contains the pachd specific configuration.
type PachdSpecificConfiguration struct {
StorageConfiguration
NumShards uint64 `env:"NUM_SHARDS,default=32"`
StorageBackend string `env:"STORAGE_BACKEND,default="`
StorageHostPath string `env:"STORAGE_HOST_PATH,default="`
EtcdPrefix string `env:"ETCD_PREFIX,default="`
PFSEtcdPrefix string `env:"PFS_ETCD_PREFIX,default=pachyderm_pfs"`
AuthEtcdPrefix string `env:"PACHYDERM_AUTH_ETCD_PREFIX,default=pachyderm_auth"`
EnterpriseEtcdPrefix string `env:"PACHYDERM_ENTERPRISE_ETCD_PREFIX,default=pachyderm_enterprise"`
KubeAddress string `env:"KUBERNETES_PORT_443_TCP_ADDR,required"`
Metrics bool `env:"METRICS,default=true"`
Init bool `env:"INIT,default=false"`
BlockCacheBytes string `env:"BLOCK_CACHE_BYTES,default=1G"`
PFSCacheSize string `env:"PFS_CACHE_SIZE,default=0"`
WorkerImage string `env:"WORKER_IMAGE,default="`
WorkerSidecarImage string `env:"WORKER_SIDECAR_IMAGE,default="`
WorkerImagePullPolicy string `env:"WORKER_IMAGE_PULL_POLICY,default="`
LogLevel string `env:"LOG_LEVEL,default=info"`
IAMRole string `env:"IAM_ROLE,default="`
ImagePullSecret string `env:"IMAGE_PULL_SECRET,default="`
NoExposeDockerSocket bool `env:"NO_EXPOSE_DOCKER_SOCKET,default=false"`
ExposeObjectAPI bool `env:"EXPOSE_OBJECT_API,default=false"`
MemoryRequest string `env:"PACHD_MEMORY_REQUEST,default=1T"`
WorkerUsesRoot bool `env:"WORKER_USES_ROOT,default=true"`
S3GatewayPort uint16 `env:"S3GATEWAY_PORT,default=600"`
DeploymentID string `env:"CLUSTER_DEPLOYMENT_ID,default="`
RequireCriticalServersOnly bool `env:"REQUIRE_CRITICAL_SERVERS_ONLY",default=false"`
}
// StorageConfiguration contains the storage configuration.
type StorageConfiguration struct {
StorageMemoryThreshold int64 `env:"STORAGE_MEMORY_THRESHOLD"`
StorageShardThreshold int64 `env:"STORAGE_SHARD_THRESHOLD"`
StorageLevelZeroSize int64 `env:"STORAGE_LEVEL_ZERO_SIZE"`
StorageLevelSizeBase int `env:"STORAGE_LEVEL_SIZE_BASE"`
StorageUploadConcurrencyLimit int `env:"STORAGE_UPLOAD_CONCURRENCY_LIMIT,default=100"`
}
// WorkerFullConfiguration contains the full worker configuration.
type WorkerFullConfiguration struct {
GlobalConfiguration
WorkerSpecificConfiguration
}
// WorkerSpecificConfiguration contains the worker specific configuration.
type WorkerSpecificConfiguration struct {
// Worker gets its own IP here, via the k8s downward API. It then writes that
// IP back to etcd so that pachd can discover it
PPSWorkerIP string `env:"PPS_WORKER_IP,required"`
// The name of the pipeline that this worker belongs to
PPSPipelineName string `env:"PPS_PIPELINE_NAME,required"`
// The ID of the commit that contains the pipeline spec.
PPSSpecCommitID string `env:"PPS_SPEC_COMMIT,required"`
// The name of this pod
PodName string `env:"PPS_POD_NAME,required"`
}
// FeatureFlags contains the configuration for feature flags.
type FeatureFlags struct {
NewStorageLayer bool `env:"NEW_STORAGE_LAYER,default=false"`
}
// NewConfiguration creates a generic configuration from a specific type of configuration.
func NewConfiguration(config interface{}) *Configuration {
configuration := &Configuration{}
switch v := config.(type) {
case *GlobalConfiguration:
configuration.GlobalConfiguration = v
return configuration
case *PachdFullConfiguration:
configuration.GlobalConfiguration = &v.GlobalConfiguration
configuration.PachdSpecificConfiguration = &v.PachdSpecificConfiguration
return configuration
case *WorkerFullConfiguration:
configuration.GlobalConfiguration = &v.GlobalConfiguration
configuration.WorkerSpecificConfiguration = &v.WorkerSpecificConfiguration
return configuration
default:
return nil
}
}