Fix skip_unavailable_shards in case of unavailable hosts

src/QueryPipeline/RemoteQueryExecutor.cpp

@@ -216,7 +216,15 @@ void RemoteQueryExecutor::sendQuery(ClientInfo::QueryKind query_kind)
 
     const auto & settings = context->getSettingsRef();
     if (settings.skip_unavailable_shards && 0 == connections->size())
+    {
+        /// To avoid sending the query again in the read(), we need to update the following flags:
+        std::lock_guard guard(was_cancelled_mutex);
+        was_cancelled = true;
+        finished = true;
+        sent_query = true;
+
         return;
+    }
 
     /// Query cannot be canceled in the middle of the send query,
     /// since there are multiple packets:

tests/queries/0_stateless/01956_skip_unavailable_shards_excessive_attempts.reference

@@ -1 +1,10 @@
-Connection failed at try №1,
+255.255.255.255
+HedgedConnectionsFactory: Connection failed at try №1
+executeQuery: Code: 519.: All attempts to get table structure failed.
+127.2,255.255.255.255
+0
+HedgedConnectionsFactory: Connection failed at try №1
+255.255.255.255,127.2
+0
+HedgedConnectionsFactory: Connection failed at try №1
+HedgedConnectionsFactory: Connection failed at try №1

tests/queries/0_stateless/01956_skip_unavailable_shards_excessive_attempts.sh

@@ -1,14 +1,36 @@
 #!/usr/bin/env bash
-# Tags: shard
-
-CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=trace
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-opts=(
-    "--connections_with_failover_max_tries=1"
-    "--skip_unavailable_shards=1"
-)
-$CLICKHOUSE_CLIENT --query "select * from remote('255.255.255.255', system.one)" "${opts[@]}" 2>&1 | grep -o 'Connection failed at try.*,'
+stderr="$(mktemp "$CURDIR/clickhouse.stderr.XXXXXX.log")"
+trap 'rm -f "$stderr"' EXIT
+
+function process_log_safe()
+{
+    grep "^\\[" "$@" | sed -e 's/.*> //' -e 's/, reason.*//' -e 's/ DB::NetException//' -e 's/ Log: //'
+}
+function execute_query()
+{
+    local hosts=$1 && shift
+    local opts=(
+        "--connections_with_failover_max_tries=1"
+        "--skip_unavailable_shards=1"
+    )
+
+    echo "$hosts"
+    # NOTE: we cannot use process substition here for simplicity because they are async, i.e.:
+    #
+    #   clickhouse-client 2> >(wc -l)
+    #
+    # May dump output of "wc -l" after some other programs.
+    $CLICKHOUSE_CLIENT "${opts[@]}" --query "select * from remote('$hosts', system.one)" 2>"$stderr"
+    process_log_safe "$stderr"
+}
+execute_query 255.255.255.255
+execute_query 127.2,255.255.255.255
+# This will print two errors because there will be two attempts for 255.255.255.255:
+# - first for obtaining structure of the table
+# - second for the query
+execute_query 255.255.255.255,127.2