/
s3conf.cpp
129 lines (95 loc) · 4.14 KB
/
s3conf.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
#include "s3conf.h"
#include "s3macros.h"
#include "s3params.h"
#include <arpa/inet.h>
#ifndef S3_STANDALONE
extern "C" {
void write_log(const char* fmt, ...) __attribute__((format(printf, 1, 2)));
}
#endif
// configurable parameters
int32_t s3ext_segid = -1;
int32_t s3ext_segnum = -1;
string s3ext_logserverhost;
int32_t s3ext_loglevel = EXT_WARNING;
int32_t s3ext_logtype = INTERNAL_LOG;
int32_t s3ext_logserverport = -1;
int32_t s3ext_logsock_udp = -1;
struct sockaddr_in s3ext_logserveraddr;
S3Params InitConfig(const string& urlWithOptions) {
#ifdef S3_STANDALONE
s3ext_segid = 0;
s3ext_segnum = 1;
#else
s3ext_segid = GpIdentity.segindex;
s3ext_segnum = GpIdentity.numsegments;
#endif
if (s3ext_segid == -1 && s3ext_segnum > 0) {
s3ext_segid = 0;
s3ext_segnum = 1;
}
string sourceUrl = TruncateOptions(urlWithOptions);
S3_CHECK_OR_DIE(!sourceUrl.empty(), S3RuntimeError, "URL not found from location string");
string configPath = GetOptS3(urlWithOptions, "config");
if (configPath.empty()) {
S3WARN("The 'config' parameter is not provided, use default value 's3/s3.conf'.");
configPath = "s3/s3.conf";
}
string configSection = GetOptS3(urlWithOptions, "section");
if (configSection.empty()) {
configSection = "default";
}
// region could be empty
string urlRegion = GetOptS3(urlWithOptions, "region");
// read configurations from file
Config s3Cfg(configPath);
S3_CHECK_OR_DIE(s3Cfg.Handle() != NULL, S3RuntimeError,
"Failed to parse config file '" + configPath + "', or it doesn't exist");
S3_CHECK_OR_DIE(s3Cfg.SectionExist(configSection), S3ConfigError,
"Selected section '" + configSection +
"' does not exist, please check your configuration file",
configSection);
bool useHttps = s3Cfg.GetBool(configSection, "encryption", "true");
bool verifyCert = s3Cfg.GetBool(configSection, "verifycert", "true");
string version = s3Cfg.Get(configSection, "version", "");
S3Params params(sourceUrl, useHttps, version, urlRegion);
string sse_type = s3Cfg.Get(configSection, "server_side_encryption", "none");
if (sse_type == "sse-s3") {
params.setSSEType(SSE_S3);
} else {
params.setSSEType(SSE_NONE);
}
string content = s3Cfg.Get(configSection, "loglevel", "WARNING");
s3ext_loglevel = getLogLevel(content.c_str());
content = s3Cfg.Get(configSection, "logtype", "INTERNAL");
s3ext_logtype = getLogType(content.c_str());
params.setDebugCurl(s3Cfg.GetBool(configSection, "debug_curl", "false"));
params.setCred(s3Cfg.Get(configSection, "accessid", ""), s3Cfg.Get(configSection, "secret", ""),
s3Cfg.Get(configSection, "token", ""));
s3ext_logserverhost = s3Cfg.Get(configSection, "logserverhost", "127.0.0.1");
s3ext_logserverport = s3Cfg.SafeScan("logserverport", configSection, 1111, 1, 65535);
int64_t numOfChunks = s3Cfg.SafeScan("threadnum", configSection, 4, 1, 8);
params.setNumOfChunks(numOfChunks);
int64_t chunkSize = s3Cfg.SafeScan("chunksize", configSection, 64 * 1024 * 1024,
8 * 1024 * 1024, 128 * 1024 * 1024);
params.setChunkSize(chunkSize);
int64_t lowSpeedLimit = s3Cfg.SafeScan("low_speed_limit", configSection, 10240, 0, INT_MAX);
params.setLowSpeedLimit(lowSpeedLimit);
int64_t lowSpeedTime = s3Cfg.SafeScan("low_speed_time", configSection, 60, 0, INT_MAX);
params.setLowSpeedTime(lowSpeedTime);
params.setProxy(s3Cfg.Get(configSection, "proxy", ""));
params.setVerifyCert(verifyCert);
CheckEssentialConfig(params);
return params;
}
void CheckEssentialConfig(const S3Params& params) {
if (params.getCred().accessID.empty()) {
S3_CHECK_OR_DIE(false, S3ConfigError, "\"FATAL: access id not set\"", "accessid");
}
if (params.getCred().secret.empty()) {
S3_CHECK_OR_DIE(false, S3ConfigError, "\"FATAL: secret id not set\"", "secret");
}
if (s3ext_segnum <= 0) {
S3_CHECK_OR_DIE(false, S3ConfigError, "\"FATAL: segment info is invalid\"", "segment");
}
}