merge pingcap/release-3.1 until (tikv#7156)

support tls for status server (tikv#7142)

Signed-off-by: Ryan Leung <rleungx@gmail.com>

backup: unblock worker by releasing lock as soon as possible (tikv#7202)

Signed-off-by: Neil Shen <overvenus@gmail.com>

import: handle ingest sst error properly (tikv#7135) (tikv#7152)

Signed-off-by: Neil Shen <overvenus@gmail.com>

raftstore: set sync hint on ingest sst commands (tikv#7151) (tikv#7156)

Signed-off-by: Neil Shen <overvenus@gmail.com>

No panic if exec admin cmd fail

Cargo.toml

@@ -62,6 +62,10 @@ tikv_alloc = { path = "components/tikv_alloc", default-features = false }
 tokio-core = "0.1"
 tokio-timer = "0.2"
 tokio-executor = "0.1"
+tokio-tcp = "0.1"
+tokio-openssl = "0.2"
+tokio-io = "0.1"
+openssl = "0.10"
 serde = "1.0"
 serde_json = "1.0"
 serde_derive = "1.0"
@@ -130,6 +134,7 @@ panic_hook = { path = "components/panic_hook" }
 # of `real_blackbox` feature.
 criterion = { version = "0.2.11", features=['real_blackbox'] }
 arrow = "0.10.0"
+hyper-openssl = "0.7"
 
 [target.'cfg(target_os = "linux")'.dependencies]
 procinfo = { git = "https://github.com/tikv/procinfo-rs" }

components/backup/src/endpoint.rs

@@ -517,9 +517,17 @@ impl<E: Engine, R: RegionInfoProvider> Endpoint<E, R> {
         let store_id = self.store_id;
         // TODO: make it async.
         self.pool.borrow_mut().spawn(lazy(move || loop {
-            let mut progress = prs.lock().unwrap();
-            let branges = progress.forward(WORKER_TAKE_RANGE);
-            let is_raw_kv = progress.is_raw_kv;
+            let (branges, is_raw_kv, cf) = {
+                // Release lock as soon as possible.
+                // It is critical to speed up backup, otherwise workers are
+                // blocked by each other.
+                let mut progress = prs.lock().unwrap();
+                (
+                    progress.forward(WORKER_TAKE_RANGE),
+                    progress.is_raw_kv,
+                    progress.cf,
+                )
+            };
             if branges.is_empty() {
                 return Ok(());
             }
@@ -541,7 +549,7 @@ impl<E: Engine, R: RegionInfoProvider> Endpoint<E, R> {
                 let name = backup_file_name(store_id, &brange.region, key);
 
                 let res = if is_raw_kv {
-                    brange.backup_raw_kv_to_file(&engine, db.clone(), &storage, name, progress.cf)
+                    brange.backup_raw_kv_to_file(&engine, db.clone(), &storage, name, cf)
                 } else {
                     brange.backup_to_file(&engine, db.clone(), &storage, name, backup_ts, start_ts)
                 };

components/test_util/data/ca.pem

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDojCCAoqgAwIBAgIUdZFW8VQoZZzek8cA+5GGu6ZInjowDQYJKoZIhvcNAQEL
+BQAwVzELMAkGA1UEBhMCQ04xEDAOBgNVBAgTB0JlaWppbmcxEDAOBgNVBAcTB0Jl
+aWppbmcxEDAOBgNVBAoTB1BpbmdDQVAxEjAQBgNVBAMTCU15IG93biBDQTAeFw0x
+OTA5MDIwNjEyMDBaFw0yNDA4MzEwNjEyMDBaMFcxCzAJBgNVBAYTAkNOMRAwDgYD
+VQQIEwdCZWlqaW5nMRAwDgYDVQQHEwdCZWlqaW5nMRAwDgYDVQQKEwdQaW5nQ0FQ
+MRIwEAYDVQQDEwlNeSBvd24gQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQDcDtQ7UX+xlVY0vpklp1uUmPoFsN0U6fqRzHU+LvYS5AM5RPJMVLiKBiSi
+zGsB+XPmXZ8H7rZZ+osZsEmDIF3HdyiSNpPNzRJKxsz4KVRzfoKZXL9D41TpuE27
++7tN6qGytYrnAy8cHMA0S1TnQ0biOFTcXZrwh5lvlIcx7ceUamGuEl94tblxSSJl
+2SkpHkKIDv0kcgoGmmh4y8SzAtmnwcCjkCSoITvvwKklp5830pFKOnpN9uZJzkXa
+tuUSpSji/JG79nQfH91LtL7xMprORVtg9YAa3aJm0Uf33WFvaCTSrt//7CVK8nqK
+xayS3u7dNH3GV9b81OGtlR76leFlAgMBAAGjZjBkMA4GA1UdDwEB/wQEAwIBBjAS
+BgNVHRMBAf8ECDAGAQH/AgECMB0GA1UdDgQWBBS3hxTaN9B7eF8xr0DKLZ3b5vFn
+rDAfBgNVHSMEGDAWgBS3hxTaN9B7eF8xr0DKLZ3b5vFnrDANBgkqhkiG9w0BAQsF
+AAOCAQEAi9WiEvTQQjmb7ekXHf1tKwdLNu5akQXIwTKeZSWRSeMgqVQcoyTZMPBX
+ythl6K3175RUIMtCwO4uZTOpRU1mTl0pIjoEcJGHYX91zyA5BjWahXZttvt7/hyX
+UwJN9clBXLfZTCp1ysLCtarLcip4WxWNsxEwXFUisE2gbu3F9ELHAbRSVUe/CwC6
+8BkY+G+fovazjGoTV4NadJVFRzTR/zsWkBNllBOBTrop8FH23ePVh3hXafzJlcip
+bDbRxNqSzNtLr88mwswklgiIHXF6PY2TkyscsXVkHPAswZnrv4lLov7M3VjL8ITA
+uYm4Me5Tmj+6pb+Foky15+ehmicQbA==
+-----END CERTIFICATE-----

components/test_util/data/key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAsRpq/E/VC82YxsC5LlKFvI9HJuchMtKskn53anW4rNE3sfN0
+WDS6qCyxNumUVBqO98J18xxbz/XkV7aP6TcXZrNgEqw07PZWTDoyZVi+n9HXyWwl
+BeiE2WWrCESqsar+cXV5UE3oE7Y4CT56tMN+awKqnf1zLyRl9DlqSg1/GabheVzz
+fGhdqddqdpAZcaOHH8UMEWdnZ4qTFaaGNRlrRy3W0VjzgIocQorpvvtZkQM5iCxx
+z9wuF9/6gGdopTA0J2SvZKa+oI/867NLpN5Hx+cn/ThHhCTh1N34Ulloa0aiou72
+mGgyMIdQxYAsRnG62EHn+9aPtegIjQd13Be9/wIDAQABAoIBAHJ8v3iIKxNMP10M
+rSlS032HqdluRLnUExdIhe3eWBnvze9NkIKM47Vf3te+u9J6sL1dil40kO2o6YoC
+TJnYsVoEzzCC/lvJCxSP8pAthF1QjAx7yps9KtRWsu/PZAEipwW1iUzub/5+J09i
+gnRkhE6tFJq5g0KQZxAwJZPlkaqEcZIOObfh9zD9hutvCPmXBtB600EbQU4XzyjP
+KaU08LtNZVm4mhKMuhXuFt8LBkjjfuw6zNcjsvgMkyflFTLc/SgWWIpq1ALHQCsq
+OiFfTPyuLy+8tGTbawvRIqiHHRd23XttPcfkdfWbNVTSBfodTOhXGFaVYbJ6EVA4
+OzVzftECgYEAz/D99wpWbjU8vye5cjKjZCY/+QnY0t76YsUmfD9+fQNBDSQnKCKj
+6nO6oYFQ9RI/vPMfrNX0sqo5hKfufNBCr/MILDXR6vtcEuaqd84DgaPVPRjHef6v
+paYUi0Enb3gF3LXYggTN1mz9leEW8BablTN/DLP5AAvMfM/XSkVzlIsCgYEA2gjc
+mcUDL1smAvriFVmpD4IrPzaZ9kINOfFNqkp/+y7S0BZGeS5ESSodrs0CIojttp3o
+9GL7QLhZ9DehJWfh2qfA5mvzKGzUeM2oapR2Ts/m3voS4ErPTm+cTBOjRe3gGSSN
+4sAJ5LA071RfNjEZBSktow//WX/oWrhIyovnxt0CgYBxyge/4xlO77URSdSySEGf
+MUs6pYfQRRKxb/9SaJB4KoqzfUAsN2CJkNDlRlWd9mGIrWZ89wwTpREapabdCD4l
++JFVWBJKS0ikUzOfoc3LaHLtHx0xhgxqUkrVtU62MfDLSXt0Etrs5vGRzf32Xfi/
+mdGBiw7MVqiM+FNwojbQZwKBgDly5E1P78rmhVl7qV5exYDkl2iMhnywYrPFtOUN
+xDL2320csWz0l+F/S1rngYx/78KSUPMzsWgYKvuCPN+SQ5xNXzJXdzZLlqBN7/ZF
+L/cMKJTP53FZxM2x8sjI09h1GPsG+quoVfL/yrLU1FF/FkyZ0QCKEooOfbaJoARe
+YK+xAoGAfT0P200WsLKRl73XYJZNYQl5+h5s7Sk9J8QuPwFWqm/mGwYKTLI042jg
+lsAym4krAR0c1CHTW3aHRimYpYbi7/kztZU1zUQgcGL+79afer3ZuFF7mGzR+I/r
+yOQ2dEfmVASfl/fMh1qyExpcCaMuejaODWyILlxOwvnywHWMSCU=
+-----END RSA PRIVATE KEY-----

components/test_util/data/server.pem

@@ -1,27 +1,22 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEpAIBAAKCAQEA2i5r+HQVIEbKr7wHemWCmbkkmbS0NSogGf9VgZuw033Qo3gc
-gqL6NvYx7nIk+615UnFHUftQhnuWFjLIXHTtg0VgIg4uGU6gAw7DUygspQ+clGuD
-66796ThF2UuDKDzWmgRzi8xzWiF3L9RhCv5dT+07b6+0QdLFUsjDKFqtfZ+YuE41
-wVTZpmgxemayd1D0zrisAn0CMpUKZOY2btpW38K6oI5nq2PzZRU8GtEfCKLgKoFN
-zR/RtGYz2mnnBvs4WufPSPILblNqB1k1zXJYzqLxy3rxhbAWxaXG2hYZxkwhSRIg
-8yMH6Hkvehzm3fu2rgWOra+sR/D1ujMZNZWh4QIDAQABAoIBAEfb/EGzcfXUexNQ
-OaJNZqtcuDpLswLDohkN6MqsTZwKlzoP6Ev6g7Cwe5eOTrH527iUiKnuvQHeGHut
-NCKHfGa85cGxq+s34ym+pgRweevPbYHQu31XgFdc6lx8K3GIQCIwDyJfLyrjVM8T
-AvdM+czGVMofM55uXgE7EFPtMbDq0JKHj/A0uAd8+3WKTWebshVxi2h7ejGQK5fO
-xi3WzeeAYu65lX7rrqDT9bFZFXuyxd5mhZFJYxG9KxYgI3rRhYLuLDMZHLT0N0wn
-M2mcf4lRgoXQr+stzyT5P43fu1QM+FaNZ8uhBUGJm6nRi3cw8QIqWfTzvb22PLvk
-NPhznLECgYEA/sbfqqAfm7GH8WGWfAKJLTd4tdoAHn17udTAZWn5Kegy0q+81YMj
-BIiETrnpGQRxIZnCJw+aHU/lic5dbkD0ZwvDR5+CNzQjxs4y/hTpjNEsaCKyArSm
-jWvuy6mYWcwM1Lp5yuUpL7VQk9x7maXD+7a97xYL2/9t3/CyYS/+zqUCgYEA2zqS
-Rbor7dbVkKgDzgRtt8GGC2k+yz1U1rBnHke/OSfUSUBbrdUr7ybsAa3qon02LGA7
-+qidDY4j+s+PQfTr4SMTCOXlV1DpMpJd5+qh5yPXe+XThu+FWh1CXdcWI0XT+/9B
-0AwLgWzAuRU/m1JIBhMZakEBasRzQiqB1deWvY0CgYBmM/0xgz5qxJLWH+GwKYxB
-2UjRGnyFvqzNZS0xAYv0ZbNNlTXZKNv5S5JXynhZktCXPAkIhle6fnyEBYaxXdkt
-JSjXKIOiBYZ8j+cgyd7OoHKB67khILrXbH7EsGnvS82x4IRPAhK9kqyaRA5JGpg9
-95bFvEBRpmu7M+E633gGCQKBgQCSjYYpDLq/JUXhjR/2AinilIFqcXHj5d0oJAbb
-TDU+HS0hxt9CxuW22vscaEoZU8D6S17tQviyjhnpWgW3nuZsu8jGwwDcrR8niocy
-OT0ASoqLrekJJGeuBS9PkjCfZde/dzVkwhiS7cOsNtMtnwS84tmzmT88Q5WVXtsq
-vBNuJQKBgQDgf4GyhMwooWWJy7dkgrLdd5/WIrBIgXi3l731GytEDaQZyWGgNpXN
-jiW67honEyX/TX59Fw5nU3eFodvjsPZFDHvhFb6QPUikgnY4TEuR03wxiiTzQO5a
-zcG3Oyw62OSzUuI7Yu7nDkV5Pr1BYvHoL3mM7NqqrQOpD9zMuLv1Gg==
------END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIIDlTCCAn2gAwIBAgIUGKdjy/Uqp64ZiwqMwpTMGP5tKT0wDQYJKoZIhvcNAQEL
+BQAwVzELMAkGA1UEBhMCQ04xEDAOBgNVBAgTB0JlaWppbmcxEDAOBgNVBAcTB0Jl
+aWppbmcxEDAOBgNVBAoTB1BpbmdDQVAxEjAQBgNVBAMTCU15IG93biBDQTAgFw0x
+OTA5MDIwNjEzMDBaGA8yMTE5MDgwOTA2MTMwMFowFjEUMBIGA1UEAxMLdGlrdi1z
+ZXJ2ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCxGmr8T9ULzZjG
+wLkuUoW8j0cm5yEy0qySfndqdbis0Tex83RYNLqoLLE26ZRUGo73wnXzHFvP9eRX
+to/pNxdms2ASrDTs9lZMOjJlWL6f0dfJbCUF6ITZZasIRKqxqv5xdXlQTegTtjgJ
+Pnq0w35rAqqd/XMvJGX0OWpKDX8ZpuF5XPN8aF2p12p2kBlxo4cfxQwRZ2dnipMV
+poY1GWtHLdbRWPOAihxCium++1mRAzmILHHP3C4X3/qAZ2ilMDQnZK9kpr6gj/zr
+s0uk3kfH5yf9OEeEJOHU3fhSWWhrRqKi7vaYaDIwh1DFgCxGcbrYQef71o+16AiN
+B3XcF73/AgMBAAGjgZcwgZQwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsG
+AQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBTw7yUYqbAv
+BJw3zZctLUfUi0vyqzAfBgNVHSMEGDAWgBS3hxTaN9B7eF8xr0DKLZ3b5vFnrDAV
+BgNVHREEDjAMhwSsEAUohwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQCBljfge2fC
+5X+tt1v7AkWoH5xpymEVvuIWWJmT/6FNTn1rdnIaxWCQzJbBCXjZS/75lKnwfrTB
+ZK7iMv1GQaBevT/qm+7GcApsr5nFrI/MvzrvY+XRqvU8gsRhUjHYI+JPLGWxhzZD
+pQdJTAGvsDLHu1VVdHR2KsE4M8ceGq58f7zPSq/suf+8SYEOFP8zfuXX1HfUrFVe
+69ZQw8PZh4EYL0PYtE5BYfe9iJyFNNtZiejiribMQz/NtNkKM3M+Hm40ULGuwHXq
+bKDjDq1PvmpVb/kKO/xADTIAbqproXETZ4W2keI3hwm6NxysvEbYV9+puQBXQqwT
+KOt9Lo4ofSAF
+-----END CERTIFICATE-----

components/test_util/src/security.rs

@@ -7,10 +7,10 @@ use tikv_util::security::SecurityConfig;
 pub fn new_security_cfg() -> SecurityConfig {
     let p = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
     SecurityConfig {
-        ca_path: format!("{}", p.join("data/ca.crt").display()),
-        cert_path: format!("{}", p.join("data/server.crt").display()),
-        key_path: format!("{}", p.join("data/server.pem").display()),
-        override_ssl_target: "example.com".to_owned(),
+        ca_path: format!("{}", p.join("data/ca.pem").display()),
+        cert_path: format!("{}", p.join("data/server.pem").display()),
+        key_path: format!("{}", p.join("data/key.pem").display()),
+        override_ssl_target: "".to_owned(),
         cipher_file: "".to_owned(),
     }
 }

src/import/service.rs

@@ -1,24 +1,13 @@
 // Copyright 2018 TiKV Project Authors. Licensed under Apache-2.0.
 
-use futures::Future;
-use grpcio::{RpcContext, RpcStatus, RpcStatusCode, UnarySink};
+use grpcio::{RpcStatus, RpcStatusCode};
 
 use super::Error;
 
 pub fn make_rpc_error(err: Error) -> RpcStatus {
     RpcStatus::new(RpcStatusCode::Unknown, Some(format!("{:?}", err)))
 }
 
-pub fn send_rpc_error<M, E>(ctx: RpcContext<'_>, sink: UnarySink<M>, error: E)
-where
-    Error: From<E>,
-{
-    let err = make_rpc_error(Error::from(error));
-    ctx.spawn(sink.fail(err).map_err(|e| {
-        warn!("send rpc failed"; "err" => %e);
-    }));
-}
-
 macro_rules! send_rpc_response {
     ($res:ident, $sink:ident, $label:ident, $timer:ident) => {{
         let res = match $res {

src/import/sst_service.rs

@@ -207,7 +207,12 @@ impl<Router: RaftStoreRouter> ImportSst for ImportSSTService<Router> {
 
         let (cb, future) = paired_future_callback();
         if let Err(e) = self.router.send_command(cmd, Callback::Write(cb)) {
-            return send_rpc_error(ctx, sink, e);
+            let mut resp = IngestResponse::default();
+            resp.set_error(e.into());
+            ctx.spawn(sink.success(resp).map_err(|e| {
+                warn!("send rpc failed"; "err" => %e);
+            }));
+            return;
         }
 
         ctx.spawn(