From 9b499ed85671b8a58c5ea5c9ac62652a39edcb9e Mon Sep 17 00:00:00 2001
From: Gimi Liang <gimi.liang@timeplus.io>
Date: Thu, 1 Feb 2024 22:47:01 -0800
Subject: [PATCH] enhancement: input_format_skip_unknown_fields default to true
 for external streams

---
 src/Storages/ExternalStream/StorageExternalStreamImpl.h | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/Storages/ExternalStream/StorageExternalStreamImpl.h b/src/Storages/ExternalStream/StorageExternalStreamImpl.h
index eb7e0362947..35f35187f15 100644
--- a/src/Storages/ExternalStream/StorageExternalStreamImpl.h
+++ b/src/Storages/ExternalStream/StorageExternalStreamImpl.h
@@ -12,7 +12,12 @@ namespace DB
 class StorageExternalStreamImpl : public std::enable_shared_from_this<StorageExternalStreamImpl>
 {
 public:
-    explicit StorageExternalStreamImpl(std::unique_ptr<ExternalStreamSettings> settings_): settings(std::move(settings_)) {}
+    explicit StorageExternalStreamImpl(std::unique_ptr<ExternalStreamSettings> settings_): settings(std::move(settings_)) {
+        /// Make it easier for people to ingest data from external streams. A lot of times people didn't see data coming
+        /// only because the external stream does not have all the fields.
+        if (!settings->input_format_skip_unknown_fields.changed)
+            settings->input_format_skip_unknown_fields = true;
+    }
     virtual ~StorageExternalStreamImpl() = default;
 
     virtual void startup() = 0;