[Feature](Compaction) Add compaction profile manager with HTTP API for historical tracking

be/src/cloud/cloud_base_compaction.h

@@ -40,6 +40,8 @@ class CloudBaseCompaction : public CloudCompactionMixin {
     Status pick_rowsets_to_compact();
 
     std::string_view compaction_name() const override { return "CloudBaseCompaction"; }
+    CompactionProfileType profile_type() const override { return CompactionProfileType::BASE; }
+    int64_t input_segments_num() const override { return _input_segments; }
 
     Status modify_rowsets() override;
 

be/src/cloud/cloud_cumulative_compaction.h

@@ -45,6 +45,10 @@ class CloudCumulativeCompaction : public CloudCompactionMixin {
     Status pick_rowsets_to_compact();
 
     std::string_view compaction_name() const override { return "CloudCumulativeCompaction"; }
+    CompactionProfileType profile_type() const override {
+        return CompactionProfileType::CUMULATIVE;
+    }
+    int64_t input_segments_num() const override { return _input_segments; }
 
     Status modify_rowsets() override;
 

be/src/cloud/cloud_full_compaction.h

@@ -41,6 +41,8 @@ class CloudFullCompaction : public CloudCompactionMixin {
     Status pick_rowsets_to_compact();
 
     std::string_view compaction_name() const override { return "CloudFullCompaction"; }
+    CompactionProfileType profile_type() const override { return CompactionProfileType::FULL; }
+    int64_t input_segments_num() const override { return _input_segments; }
 
     Status modify_rowsets() override;
     Status garbage_collection() override;

be/src/cloud/cloud_index_change_compaction.h

@@ -55,6 +55,10 @@ class CloudIndexChangeCompaction : public CloudCompactionMixin {
 
 protected:
     std::string_view compaction_name() const override { return "CloudIndexChangeCompaction"; }
+    CompactionProfileType profile_type() const override {
+        return CompactionProfileType::INDEX_CHANGE;
+    }
+    int64_t input_segments_num() const override { return _input_segments; }
 
     // if cumu rowset is modified, cumu compaction should sync rowset before execute.
     // if base rowset is modified, base compaction should sync rowset before execute.

be/src/common/config.cpp

@@ -436,6 +436,8 @@ DEFINE_Bool(enable_low_cardinality_cache_code, "true");
 DEFINE_mBool(enable_compaction_checksum, "false");
 // whether disable automatic compaction task
 DEFINE_mBool(disable_auto_compaction, "false");
+// max number of compaction profile records to keep in memory, 0 to disable
+DEFINE_mInt32(compaction_profile_max_records, "500");
 // whether enable vertical compaction
 DEFINE_mBool(enable_vertical_compaction, "true");
 // whether enable ordered data compaction

be/src/common/config.h

@@ -501,6 +501,8 @@ DECLARE_Bool(enable_low_cardinality_cache_code);
 DECLARE_mBool(enable_compaction_checksum);
 // whether disable automatic compaction task
 DECLARE_mBool(disable_auto_compaction);
+// max number of compaction profile records to keep in memory, 0 to disable
+DECLARE_mInt32(compaction_profile_max_records);
 // whether enable vertical compaction
 DECLARE_mBool(enable_vertical_compaction);
 // whether enable ordered data compaction

be/src/service/http/action/compaction_profile_action.cpp

@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "service/http/action/compaction_profile_action.h"
+
+#include <rapidjson/document.h>
+#include <rapidjson/prettywriter.h>
+#include <rapidjson/stringbuffer.h>
+
+#include <exception>
+#include <string>
+
+#include "service/http/http_channel.h"
+#include "service/http/http_headers.h"
+#include "service/http/http_request.h"
+#include "service/http/http_status.h"
+#include "storage/compaction/compaction_profile_mgr.h"
+
+namespace doris {
+
+CompactionProfileAction::CompactionProfileAction(ExecEnv* exec_env, TPrivilegeHier::type hier,
+                                                 TPrivilegeType::type ptype)
+        : HttpHandlerWithAuth(exec_env, hier, ptype) {}
+
+void CompactionProfileAction::handle(HttpRequest* req) {
+    req->add_output_header(HttpHeaders::CONTENT_TYPE, "application/json");
+
+    int64_t tablet_id = 0;
+    int64_t top_n = 0;
+
+    const auto& tablet_id_str = req->param("tablet_id");
+    if (!tablet_id_str.empty()) {
+        try {
+            tablet_id = std::stoll(tablet_id_str);
+        } catch (const std::exception&) {
+            HttpChannel::send_reply(
+                    req, HttpStatus::BAD_REQUEST,
+                    R"({"status": "Failed", "msg": "invalid tablet_id parameter"})");
+            return;
+        }
+    }
+
+    const auto& top_n_str = req->param("top_n");
+    if (!top_n_str.empty()) {
+        try {
+            top_n = std::stoll(top_n_str);
+        } catch (const std::exception&) {
+            HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST,
+                                    R"({"status": "Failed", "msg": "invalid top_n parameter"})");
+            return;
+        }
+        if (top_n < 0) {
+            HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST,
+                                    R"({"status": "Failed", "msg": "top_n must be non-negative"})");
+            return;
+        }
+    }
+
+    auto records = CompactionProfileManager::instance()->get_records(tablet_id, top_n);
+
+    rapidjson::Document root;
+    root.SetObject();
+    auto& allocator = root.GetAllocator();
+
+    root.AddMember("status", "Success", allocator);
+
+    rapidjson::Value profiles(rapidjson::kArrayType);
+    for (const auto& record : records) {
+        rapidjson::Value obj;
+        record.to_json(obj, allocator);
+        profiles.PushBack(obj, allocator);
+    }
+    root.AddMember("compaction_profiles", profiles, allocator);
+
+    rapidjson::StringBuffer str_buf;
+    rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(str_buf);
+    root.Accept(writer);
+
+    HttpChannel::send_reply(req, HttpStatus::OK, str_buf.GetString());
+}
+
+} // namespace doris

be/src/service/http/action/compaction_profile_action.h

@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "service/http/http_handler_with_auth.h"
+
+namespace doris {
+class HttpRequest;
+class ExecEnv;
+
+class CompactionProfileAction : public HttpHandlerWithAuth {
+public:
+    CompactionProfileAction(ExecEnv* exec_env, TPrivilegeHier::type hier,
+                            TPrivilegeType::type ptype);
+    ~CompactionProfileAction() override = default;
+
+    void handle(HttpRequest* req) override;
+};
+
+} // namespace doris

be/src/service/http_service.cpp

@@ -43,6 +43,7 @@
 #include "service/http/action/checksum_action.h"
 #include "service/http/action/clear_cache_action.h"
 #include "service/http/action/compaction_action.h"
+#include "service/http/action/compaction_profile_action.h"
 #include "service/http/action/compaction_score_action.h"
 #include "service/http/action/config_action.h"
 #include "service/http/action/debug_point_action.h"
@@ -280,6 +281,11 @@ Status HttpService::start() {
     ShrinkMemAction* shrink_mem_action = _pool.add(new ShrinkMemAction(_env));
     _ev_http_server->register_handler(HttpMethod::GET, "/api/shrink_mem", shrink_mem_action);
 
+    CompactionProfileAction* compaction_profile_action = _pool.add(
+            new CompactionProfileAction(_env, TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN));
+    _ev_http_server->register_handler(HttpMethod::GET, "/api/compaction/profile",
+                                      compaction_profile_action);
+
 #ifndef BE_TEST
     auto& engine = _env->storage_engine();
     if (config::is_cloud_mode()) {

be/src/storage/compaction/base_compaction.h

@@ -43,6 +43,7 @@ class BaseCompaction final : public CompactionMixin {
 private:
     Status pick_rowsets_to_compact();
     std::string_view compaction_name() const override { return "base compaction"; }
+    CompactionProfileType profile_type() const override { return CompactionProfileType::BASE; }
 
     ReaderType compaction_type() const override { return ReaderType::READER_BASE_COMPACTION; }
 

be/src/storage/compaction/cold_data_compaction.h

@@ -35,6 +35,7 @@ class ColdDataCompaction final : public CompactionMixin {
 
 private:
     std::string_view compaction_name() const override { return "cold data compaction"; }
+    CompactionProfileType profile_type() const override { return CompactionProfileType::COLD_DATA; }
     ReaderType compaction_type() const override { return ReaderType::READER_COLD_DATA_COMPACTION; }
 
     Status construct_output_rowset_writer(RowsetWriterContext& ctx) override;