gpcloud: support proxy

Add an option of proxy in the configuration. Set proxy for cURL, http, https, socks4, socks4a, socks5 and socks5h are supported. Signed-off-by: Xiaoran Wang <xiwang@pivotal.io>
greenplum-db · Jul 19, 2017 · 67f7dd1 · 67f7dd1
1 parent 57a11a9
commit 67f7dd1
Show file tree

Hide file tree

Showing 15 changed files with 124 additions and 9 deletions.
diff --git a/gpAux/extensions/gpcloud/bin/gpcheckcloud/gpcheckcloud.cpp b/gpAux/extensions/gpcloud/bin/gpcheckcloud/gpcheckcloud.cpp
@@ -135,7 +135,8 @@ void printTemplate() {
         "low_speed_limit = 10240\n"
         "low_speed_time = 60\n"
         "encryption = true\n"
-        "autocompress = true\n");
+        "autocompress = true\n"
+        "proxy = \"\"\n");
 }
 
 void printBucketContents(const ListBucketResult &result) {

diff --git a/gpAux/extensions/gpcloud/include/s3params.h b/gpAux/extensions/gpcloud/include/s3params.h
@@ -129,6 +129,14 @@ class S3Params {
         this->sseType = sseType;
     }
 
+    const string& getProxy() const {
+        return proxy;
+    }
+
+    void setProxy(string proxy) {
+        this->proxy = proxy;
+    }
+
    private:
     S3Url s3Url;  // original url to read/write.
 
@@ -142,6 +150,8 @@ class S3Params {
     uint64_t lowSpeedLimit;  // low speed limit
     uint64_t lowSpeedTime;   // low speed timeout
 
+    string proxy;  // proxy
+
     bool debugCurl;     // debug curl or not
     bool autoCompress;  // whether to compress data before uploading
     bool verifyCert;  // This option determines whether curl verifies the authenticity of the peer's

diff --git a/gpAux/extensions/gpcloud/include/s3restful_service.h b/gpAux/extensions/gpcloud/include/s3restful_service.h
@@ -13,6 +13,7 @@
 class S3RESTfulService : public RESTfulService {
    public:
     S3RESTfulService();
+    S3RESTfulService(const string& proxy);
     S3RESTfulService(const S3Params& params);
     virtual ~S3RESTfulService();
 
@@ -30,6 +31,8 @@ class S3RESTfulService : public RESTfulService {
     uint64_t lowSpeedLimit;
     uint64_t lowSpeedTime;
 
+    string proxy;
+
     bool debugCurl;
     bool verifyCert;
 

diff --git a/gpAux/extensions/gpcloud/regress/Makefile b/gpAux/extensions/gpcloud/regress/Makefile
@@ -13,11 +13,13 @@ installcheck:
 	@mkdir -p source_replaced
 	@cp -rf input source_replaced/input
 	@cp -rf output source_replaced/output
+	@ssh -CfNqT -D 1080 localhost
 	@./generate_config_file.sh $(config_file)
 	@perl -p -i -e 's/\@config_file\@/$(config_file)/;s/\@read_prefix\@/$(read_prefix)/;s/\@write_prefix\@/$(write_prefix)/;s/\@write_encrypt_prefix\@/$(write_encrypt_prefix)/' source_replaced/input/*.source source_replaced/output/*.source
 	@cp $(top_builddir)/src/test/regress/init_file source_replaced/init_file
 	@perl -p -i -e 's/-- end_matchsubs/\nm\/\\(\.\*\\.cpp:\\d\+\\)\/\ns\/\\(\.\*\\.cpp:\\d\+\\)\/\/\n-- end_matchsubs/' source_replaced/init_file
 	-$(pgxs_dir)/src/test/regress/pg_regress --psqldir=$$GPHOME/bin/ --schedule=regress_schedule --init-file=source_replaced/init_file --srcdir=source_replaced --inputdir=source_replaced --outputdir=.
+	@pkill -xf "ssh -CfNqT -D 1080 localhost"
 	@echo "Regression ends:" `date` ]
 
 clean:

diff --git a/gpAux/extensions/gpcloud/regress/generate_config_file.sh b/gpAux/extensions/gpcloud/regress/generate_config_file.sh
@@ -37,6 +37,22 @@ if [ -n "$gpcloud_access_key_id" ] && [ -n "$gpcloud_secret_access_key" ]; then
 	threadnum = 3
 	chunksize = 16777217
 	server_side_encryption = sse-s3
+
+	[proxy]
+	accessid = "$gpcloud_access_key_id"
+	secret = "$gpcloud_secret_access_key"
+
+	threadnum = 3
+	chunksize = 16777217
+	proxy = socks5://127.0.0.1:1080
+
+	[wrong_proxy]
+	accessid = "$gpcloud_access_key_id"
+	secret = "$gpcloud_secret_access_key"
+
+	threadnum = 3
+	chunksize = 16777217
+	proxy = socks5://127.0.0.1:1090
 	EOF
 else
 	echo "Error: environment varibles \$gpcloud_access_key_id and \$gpcloud_secret_access_key are not set."

diff --git a/gpAux/extensions/gpcloud/regress/input/1_19_proxy.source b/gpAux/extensions/gpcloud/regress/input/1_19_proxy.source
@@ -0,0 +1,8 @@
+CREATE READABLE EXTERNAL TABLE s3regress_proxy (date text, time text, open float, high float,
+        low float, volume int) LOCATION('s3://s3-us-east-1.amazonaws.com/us-east-1.@read_prefix@/small17/data0000 config=@config_file@ section=proxy') FORMAT 'csv';
+
+\d s3regress_proxy
+
+SELECT round(sum(open)) FROM s3regress_proxy;
+
+DROP EXTERNAL TABLE s3regress_proxy;
diff --git a/gpAux/extensions/gpcloud/regress/input/2_07_wrong_proxy.source b/gpAux/extensions/gpcloud/regress/input/2_07_wrong_proxy.source
@@ -0,0 +1,6 @@
+CREATE READABLE EXTERNAL TABLE s3regress_wrong_proxy (date text, time text, open float, high float,
+        low float, volume int) LOCATION('s3://s3-us-east-1.amazonaws.com/us-east-1.@read_prefix@/small17/data0000 config=@config_file@ section=wrong_proxy') FORMAT 'csv';
+
+SELECT round(sum(open)) FROM s3regress_wrong_proxy;
+
+DROP EXTERNAL TABLE s3regress_wrong_proxy;
diff --git a/gpAux/extensions/gpcloud/regress/output/1_19_proxy.source b/gpAux/extensions/gpcloud/regress/output/1_19_proxy.source
@@ -0,0 +1,27 @@
+CREATE READABLE EXTERNAL TABLE s3regress_proxy (date text, time text, open float, high float,
+        low float, volume int) LOCATION('s3://s3-us-east-1.amazonaws.com/us-east-1.@read_prefix@/small17/data0000 config=@config_file@ section=proxy') FORMAT 'csv';
+\d s3regress_proxy
+External table "public.s3regress_proxy"
+ Column |       Type       | Modifiers 
+--------+------------------+-----------
+ date   | text             | 
+ time   | text             | 
+ open   | double precision | 
+ high   | double precision | 
+ low    | double precision | 
+ volume | integer          | 
+Type: readable
+Encoding: UTF8
+Format type: csv
+Format options: delimiter ',' null '' escape '"' quote '"'
+External options: {}
+External location: "s3://s3-us-east-1.amazonaws.com/us-east-1.@read_prefix@/small17/data0000 config=@config_file@ section=proxy"
+Execute on: all segments
+
+SELECT round(sum(open)) FROM s3regress_proxy;
+  round  
+---------
+ 4239338
+(1 row)
+
+DROP EXTERNAL TABLE s3regress_proxy;
diff --git a/gpAux/extensions/gpcloud/regress/output/2_07_wrong_proxy.source b/gpAux/extensions/gpcloud/regress/output/2_07_wrong_proxy.source
@@ -0,0 +1,6 @@
+CREATE READABLE EXTERNAL TABLE s3regress_wrong_proxy (date text, time text, open float, high float,
+        low float, volume int) LOCATION('s3://s3-us-east-1.amazonaws.com/us-east-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf section=wrong_proxy') FORMAT 'csv';
+SELECT round(sum(open)) FROM s3regress_wrong_proxy;
+ERROR:  Failed to init gpcloud extension
+DETAIL:  External table s3regress_wrong_proxy, file s3://s3-us-east-1.amazonaws.com/us-east-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf section=wrong_proxy
+DROP EXTERNAL TABLE s3regress_wrong_proxy;
diff --git a/gpAux/extensions/gpcloud/regress/regress_schedule b/gpAux/extensions/gpcloud/regress/regress_schedule
@@ -1,10 +1,10 @@
 test: 0_00_prepare_protocols
 
 # ~ 1s
-test: 1_03_bad_data 1_04_empty_prefix 1_05_one_line 1_06_1correct_1wrong 2_02_invalid_region 2_03_invalid_config 2_04_invalid_header 2_05_limit_zero 3_01_create_wet 3_02_quick_shoot_wet 3_11_write_with_encryption 4_01_create_invalid_wet 2_06_invalid_sub_query 1_17_no_eol_at_eof
+test: 1_03_bad_data 1_04_empty_prefix 1_05_one_line 1_06_1correct_1wrong 2_02_invalid_region 2_03_invalid_config 2_04_invalid_header 2_05_limit_zero 3_01_create_wet 3_02_quick_shoot_wet 3_11_write_with_encryption 4_01_create_invalid_wet 2_06_invalid_sub_query 1_17_no_eol_at_eof 2_07_wrong_proxy
 
 # tens of seconds
-test: 1_01_normal 1_02_log_error 1_10_all_regions 1_11_gzipped_data 1_12_no_prefix 1_13_parallel1 1_13_parallel2 1_09_partition 3_09_write_big_row 3_10_write_mixed_length_rows 1_15_normal_sub_query 1_16_multiple_files_with_header_line 1_18_all_regions_version2
+test: 1_01_normal 1_02_log_error 1_10_all_regions 1_11_gzipped_data 1_12_no_prefix 1_13_parallel1 1_13_parallel2 1_09_partition 3_09_write_big_row 3_10_write_mixed_length_rows 1_15_normal_sub_query 1_16_multiple_files_with_header_line 1_18_all_regions_version2 1_19_proxy
 
 # heavy loads, > 100s
 test: 1_07_huge_bad_data 1_08_huge_correct_data 1_14_thousands_of_files 3_03_insert_lots_of_rows 3_07_write_lots_of_files 3_04_insert_mixed_workload 3_05_insert_to_wet_from_ret 3_06_special_characters 3_08_join_query_wet_local_tbl 4_02_wet_with_mixed_format

diff --git a/gpAux/extensions/gpcloud/src/s3conf.cpp b/gpAux/extensions/gpcloud/src/s3conf.cpp
@@ -105,6 +105,8 @@ S3Params InitConfig(const string& urlWithOptions) {
     int64_t lowSpeedTime = s3Cfg.SafeScan("low_speed_time", configSection, 60, 0, INT_MAX);
     params.setLowSpeedTime(lowSpeedTime);
 
+    params.setProxy(s3Cfg.Get(configSection, "proxy", ""));
+
     params.setVerifyCert(verifyCert);
 
     CheckEssentialConfig(params);

diff --git a/gpAux/extensions/gpcloud/src/s3restful_service.cpp b/gpAux/extensions/gpcloud/src/s3restful_service.cpp
@@ -3,6 +3,16 @@
 S3RESTfulService::S3RESTfulService()
     : lowSpeedLimit(0),
       lowSpeedTime(0),
+      proxy(""),
+      debugCurl(false),
+      verifyCert(true),
+      chunkBufferSize(64 * 1024) {
+}
+
+S3RESTfulService::S3RESTfulService(const string &proxy)
+    : lowSpeedLimit(0),
+      lowSpeedTime(0),
+      proxy(proxy),
       debugCurl(false),
       verifyCert(true),
       chunkBufferSize(64 * 1024) {
@@ -19,6 +29,7 @@ S3RESTfulService::S3RESTfulService(const S3Params &params)
     this->debugCurl = params.isDebugCurl();
     this->chunkBufferSize = params.getChunkSize();
     this->verifyCert = params.isVerifyCert();
+    this->proxy = params.getProxy();
 }
 
 S3RESTfulService::~S3RESTfulService() {
@@ -84,7 +95,7 @@ size_t RESTfulServiceReadFuncCallback(char *ptr, size_t size, size_t nmemb, void
 
 struct CURLWrapper {
     CURLWrapper(const string &url, curl_slist *headers, uint64_t lowSpeedLimit,
-                uint64_t lowSpeedTime, bool debugCurl) {
+                uint64_t lowSpeedTime, bool debugCurl, string proxy) {
         curl = curl_easy_init();
         curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
         curl_easy_setopt(curl, CURLOPT_FORBID_REUSE, 1L);
@@ -96,6 +107,10 @@ struct CURLWrapper {
         if (debugCurl) {
             curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
         }
+
+        if (!proxy.empty()) {
+            curl_easy_setopt(curl, CURLOPT_PROXY, proxy.c_str());
+        }
     }
     ~CURLWrapper() {
         curl_easy_cleanup(curl);
@@ -135,7 +150,7 @@ Response S3RESTfulService::get(const string &url, HTTPHeaders &headers) {
 
     headers.CreateList();
     CURLWrapper wrapper(url, headers.GetList(), this->lowSpeedLimit, this->lowSpeedTime,
-                        this->debugCurl);
+                        this->debugCurl, this->proxy);
     CURL *curl = wrapper.curl;
 
     curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&response);
@@ -152,7 +167,7 @@ Response S3RESTfulService::put(const string &url, HTTPHeaders &headers, const S3
 
     headers.CreateList();
     CURLWrapper wrapper(url, headers.GetList(), this->lowSpeedLimit, this->lowSpeedTime,
-                        this->debugCurl);
+                        this->debugCurl, this->proxy);
     CURL *curl = wrapper.curl;
 
     curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&response);
@@ -179,7 +194,7 @@ Response S3RESTfulService::post(const string &url, HTTPHeaders &headers,
 
     headers.CreateList();
     CURLWrapper wrapper(url, headers.GetList(), this->lowSpeedLimit, this->lowSpeedTime,
-                        this->debugCurl);
+                        this->debugCurl, this->proxy);
     CURL *curl = wrapper.curl;
 
     curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&response);
@@ -208,7 +223,7 @@ ResponseCode S3RESTfulService::head(const string &url, HTTPHeaders &headers) {
 
     headers.CreateList();
     CURLWrapper wrapper(url, headers.GetList(), this->lowSpeedLimit, this->lowSpeedTime,
-                        this->debugCurl);
+                        this->debugCurl, this->proxy);
     CURL *curl = wrapper.curl;
 
     curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "HEAD");
@@ -225,7 +240,7 @@ Response S3RESTfulService::deleteRequest(const string &url, HTTPHeaders &headers
 
     headers.CreateList();
     CURLWrapper wrapper(url, headers.GetList(), this->lowSpeedLimit, this->lowSpeedTime,
-                        this->debugCurl);
+                        this->debugCurl, this->proxy);
     CURL *curl = wrapper.curl;
 
     curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&response);

diff --git a/gpAux/extensions/gpcloud/test/data/s3test.conf b/gpAux/extensions/gpcloud/test/data/s3test.conf
@@ -71,3 +71,8 @@ low_speed_time = 600
 verifycert = false
 secret = "secret_test"
 accessid = "accessid_test"
+
+[proxy]
+secret = "secret_test"
+accessid = "accessid_test"
+proxy = "https://127.0.0.1:8080"
diff --git a/gpAux/extensions/gpcloud/test/s3conf_test.cpp b/gpAux/extensions/gpcloud/test/s3conf_test.cpp
@@ -96,3 +96,8 @@ TEST(Config, SkipVerify) {
     S3Params params = InitConfig("s3://abc/a config=data/s3test.conf section=skip_verify");
     EXPECT_FALSE(params.isVerifyCert());
 }
+
+TEST(Config, Proxy) {
+    S3Params params = InitConfig("s3://abc/a config=data/s3test.conf section=proxy");
+    EXPECT_EQ("https://127.0.0.1:8080", params.getProxy());
+}
diff --git a/gpAux/extensions/gpcloud/test/s3restful_service_test.cpp b/gpAux/extensions/gpcloud/test/s3restful_service_test.cpp
@@ -320,3 +320,12 @@ TEST(S3RESTfulService, DISABLED_DeleteToDummyServerWithData) {
     Response resp = service.deleteRequest(url, headers);
     EXPECT_EQ(RESPONSE_OK, resp.getStatus());
 }
+
+TEST(S3RESTfulService, GetWithWrongProxy) {
+    HTTPHeaders headers;
+    S3RESTfulService service("https://127.0.0.1:8080");
+
+    string url = "https://www.bing.com/";
+
+    EXPECT_THROW(service.get(url, headers), S3ConnectionError);
+}