MDEV-37143 mariabackup fails on Windows with "SSL certificate is self-signed"

This is the same as MDEV-35368, which was  previously incompletely fixed
(on *nix-only, for unix socket connections)

This time, we fix it compatibly to Connector/C, by not verifying
server certificate for local connections, which, in addition to socket
and named pipe, are also "127.0.0.1" and "::1", and on Windows "localhost"
as well.

The corresponding code in Connector/C is was added by
1287c901dc8515823d28edcebfe4be65e6c5a6b3.

It remain a good question whether mariabackup should use SSL at all
since all it does are local connections, for "BACKUP STAGE" stuff.

Author: vaintroub

mysql-test/suite/mariabackup/backup_ssl.result

@@ -15,3 +15,8 @@ DROP USER backup_user;
 # MDEV-32473 --disable-ssl doesn't disable it
 #
 # tcp skip-ssl
+#
+# MDEV-37143 Mariadb-backup fails on Windows with SSL certificate is self-signed error
+#
+# do not fail with passwordless with default protocol
+# do not fail with passwordless with 127.0.0.1 TCP

mysql-test/suite/mariabackup/backup_ssl.test

@@ -19,8 +19,14 @@ echo # MDEV-31855 validate ssl certificates using client password in the interna
 echo #;
 # fails to connect, passwordless root
 echo # tcp ssl ssl-verify-server-cert;
+let $host=;
+if ($MARIADB_UPGRADE_EXE) {
+  # On Windows, we need host different from "127.0.0.1","::1" or "localhost"
+  # to trigger self-signed error
+  let $host=--host=127.0.0.2;
+}
 error 1;
-exec $XTRABACKUP --no-defaults --protocol=tcp --user=root --port=$MASTER_MYPORT --backup --target-dir=$targetdir;
+exec $XTRABACKUP --no-defaults $host --protocol=tcp --user=root --port=$MASTER_MYPORT --backup --target-dir=$targetdir;
 
 --echo #
 --echo # MDEV-32473 --disable-ssl doesn't disable it
@@ -29,3 +35,17 @@ exec $XTRABACKUP --no-defaults --protocol=tcp --user=root --port=$MASTER_MYPORT
 echo # tcp skip-ssl;
 exec $XTRABACKUP --no-defaults --protocol=tcp --user=root --skip-ssl --port=$MASTER_MYPORT --backup --target-dir=$targetdir;
 rmdir $targetdir;
+
+--echo #
+--echo # MDEV-37143 Mariadb-backup fails on Windows with SSL certificate is self-signed error
+--echo #
+--echo # do not fail with passwordless with default protocol
+let $port_or_socket=--socket=$MASTER_MYSOCK;
+if ($MARIADB_UPGRADE_EXE) { # windows
+  let  $port_or_socket=--port=$MASTER_MYPORT;
+}
+exec $XTRABACKUP --no-defaults --user=root --backup $port_or_socket --target-dir=$targetdir;
+rmdir $targetdir;
+--echo # do not fail with passwordless with 127.0.0.1 TCP
+exec $XTRABACKUP --no-defaults --host=127.0.0.1 --protocol=tcp --port=$MASTER_MYPORT --user=root --backup --target-dir=$targetdir;
+rmdir $targetdir;

sql-common/client.c

@@ -1994,6 +1994,29 @@ static int send_change_user_packet(MCPVIO_EXT *mpvio,
   return res;
 }
 
+/**
+  Checks if self-signed certificate error should be ignored.
+*/
+static my_bool is_local_connection(const char *hostname, enum enum_vio_type viotype)
+{
+  const char *local_host_names[]= {
+#ifdef _WIN32
+  "localhost",
+#endif
+  "127.0.0.1", "::1"};
+  size_t i;
+
+  if (viotype != VIO_TYPE_TCPIP || !hostname)
+    return TRUE;
+
+  for (i= 0; i < array_elements(local_host_names); i++)
+  {
+    if (strcmp(hostname, local_host_names[i]) == 0)
+      return TRUE;
+  }
+  return FALSE;
+}
+
 #define MAX_CONNECTION_ATTR_STORAGE_LENGTH 65536
 
 /**
@@ -2122,6 +2145,7 @@ static int send_client_reply_packet(MCPVIO_EXT *mpvio,
     enum enum_ssl_init_error ssl_init_error;
     const char *cert_error;
     unsigned long ssl_error;
+    my_bool is_local;
 
     /*
       Send mysql->client_flag, max_packet_size - unencrypted otherwise
@@ -2169,10 +2193,11 @@ static int send_client_reply_packet(MCPVIO_EXT *mpvio,
     }
     DBUG_PRINT("info", ("IO layer change done!"));
 
+    is_local= is_local_connection(mysql->host, vio_type);
     /* Verify server cert */
     if ((!mysql->options.extension ||
          !mysql->options.extension->tls_allow_invalid_server_cert) &&
-        ssl_verify_server_cert(mysql, &cert_error, vio_type == VIO_TYPE_SOCKET))
+        ssl_verify_server_cert(mysql, &cert_error, is_local))
     {
       set_mysql_extended_error(mysql, CR_SSL_CONNECTION_ERROR, unknown_sqlstate,
                                ER(CR_SSL_CONNECTION_ERROR), cert_error);
@@ -2181,14 +2206,13 @@ static int send_client_reply_packet(MCPVIO_EXT *mpvio,
     if (mysql->tls_self_signed_error)
     {
       /*
-        If the transport is secure (see opt_require_secure_transport) we
-        allow a self-signed cert as we know it came from the server.
+        If connection is local, we allow self-signed cert.
 
         If no password or plugin uses insecure protocol - refuse the cert.
 
         Otherwise one last cert check after auth.
       */
-      if (vio_type == VIO_TYPE_SOCKET)
+      if (is_local)
         mysql->tls_self_signed_error= 0;
       else if (!mysql->passwd || !mysql->passwd[0] ||
                !mpvio->plugin->hash_password_bin)