diff --git a/R-package/src/xgboost_R.cc b/R-package/src/xgboost_R.cc
index 0d5311705d11..cb86ef46905c 100644
--- a/R-package/src/xgboost_R.cc
+++ b/R-package/src/xgboost_R.cc
@@ -1,17 +1,14 @@
 // Copyright (c) 2014 by Contributors
-#include <cstring>
-#include <cstdio>
+#include <dmlc/logging.h>
+#include <dmlc/omp.h>
+#include <xgboost/c_api.h>
 #include <vector>
 #include <string>
 #include <utility>
+#include <cstring>
+#include <cstdio>
 #include <sstream>
-
-#include <dmlc/logging.h>
-#include <dmlc/omp.h>
-#include <xgboost/c_api.h>
-
-#include "xgboost_R.h"
-
+#include "./xgboost_R.h"
 
 /*!
  * \brief macro to annotate begin of api
diff --git a/include/xgboost/base.h b/include/xgboost/base.h
index 81b618a928e6..4802426112e5 100644
--- a/include/xgboost/base.h
+++ b/include/xgboost/base.h
@@ -6,15 +6,14 @@
 #ifndef XGBOOST_BASE_H_
 #define XGBOOST_BASE_H_
 
+#include <dmlc/base.h>
+#include <dmlc/omp.h>
 #include <cmath>
 #include <iostream>
 #include <vector>
 #include <string>
 #include <utility>
 
-#include <dmlc/base.h>
-#include <dmlc/omp.h>
-
 /*!
  * \brief string flag for R library, to leave hooks when needed.
  */
diff --git a/include/xgboost/data.h b/include/xgboost/data.h
index 908b8a15b5b6..d7ab895a8dba 100644
--- a/include/xgboost/data.h
+++ b/include/xgboost/data.h
@@ -7,13 +7,6 @@
 #ifndef XGBOOST_DATA_H_
 #define XGBOOST_DATA_H_
 
-#include <memory>
-#include <numeric>
-#include <algorithm>
-#include <string>
-#include <utility>
-#include <vector>
-
 #include <dmlc/base.h>
 #include <dmlc/data.h>
 #include <dmlc/serializer.h>
@@ -22,6 +15,13 @@
 #include <xgboost/span.h>
 #include <xgboost/host_device_vector.h>
 
+#include <memory>
+#include <numeric>
+#include <algorithm>
+#include <string>
+#include <utility>
+#include <vector>
+
 namespace xgboost {
 // forward declare dmatrix.
 class DMatrix;
diff --git a/include/xgboost/feature_map.h b/include/xgboost/feature_map.h
index 47c20f8e27b5..a48e28ba1bfa 100644
--- a/include/xgboost/feature_map.h
+++ b/include/xgboost/feature_map.h
@@ -7,13 +7,13 @@
 #ifndef XGBOOST_FEATURE_MAP_H_
 #define XGBOOST_FEATURE_MAP_H_
 
+#include <xgboost/logging.h>
+
 #include <vector>
 #include <string>
 #include <cstring>
 #include <iostream>
 
-#include <xgboost/logging.h>
-
 namespace xgboost {
 /*!
  * \brief Feature map data structure to help text model dump.
diff --git a/include/xgboost/gbm.h b/include/xgboost/gbm.h
index 95c54b959649..d359f5c2965b 100644
--- a/include/xgboost/gbm.h
+++ b/include/xgboost/gbm.h
@@ -8,13 +8,6 @@
 #ifndef XGBOOST_GBM_H_
 #define XGBOOST_GBM_H_
 
-#include <vector>
-#include <utility>
-#include <string>
-#include <functional>
-#include <unordered_map>
-#include <memory>
-
 #include <dmlc/registry.h>
 #include <dmlc/any.h>
 #include <xgboost/base.h>
@@ -22,6 +15,13 @@
 #include <xgboost/host_device_vector.h>
 #include <xgboost/model.h>
 
+#include <vector>
+#include <utility>
+#include <string>
+#include <functional>
+#include <unordered_map>
+#include <memory>
+
 namespace xgboost {
 
 class Json;
diff --git a/include/xgboost/generic_parameters.h b/include/xgboost/generic_parameters.h
index 19e77b3042ec..752342b9a90c 100644
--- a/include/xgboost/generic_parameters.h
+++ b/include/xgboost/generic_parameters.h
@@ -5,11 +5,11 @@
 #ifndef XGBOOST_GENERIC_PARAMETERS_H_
 #define XGBOOST_GENERIC_PARAMETERS_H_
 
-#include <string>
-
 #include <xgboost/logging.h>
 #include <xgboost/parameter.h>
 
+#include <string>
+
 namespace xgboost {
 struct GenericParameter : public XGBoostParameter<GenericParameter> {
   // Constant representing the device ID of CPU.
diff --git a/include/xgboost/json.h b/include/xgboost/json.h
index 59861aa44e0c..c190505bf892 100644
--- a/include/xgboost/json.h
+++ b/include/xgboost/json.h
@@ -4,15 +4,15 @@
 #ifndef XGBOOST_JSON_H_
 #define XGBOOST_JSON_H_
 
+#include <xgboost/logging.h>
+#include <xgboost/parameter.h>
+#include <string>
+
 #include <map>
 #include <memory>
 #include <vector>
 #include <functional>
 #include <utility>
-#include <string>
-
-#include <xgboost/logging.h>
-#include <xgboost/parameter.h>
 
 namespace xgboost {
 
diff --git a/include/xgboost/json_io.h b/include/xgboost/json_io.h
index 261d71097766..a58afe172b0d 100644
--- a/include/xgboost/json_io.h
+++ b/include/xgboost/json_io.h
@@ -3,6 +3,7 @@
  */
 #ifndef XGBOOST_JSON_IO_H_
 #define XGBOOST_JSON_IO_H_
+#include <xgboost/json.h>
 
 #include <memory>
 #include <string>
@@ -13,8 +14,6 @@
 #include <sstream>
 #include <locale>
 
-#include <xgboost/json.h>
-
 namespace xgboost {
 
 template <typename Allocator>
diff --git a/include/xgboost/learner.h b/include/xgboost/learner.h
index cbe064aca8d0..a608bc1b8206 100644
--- a/include/xgboost/learner.h
+++ b/include/xgboost/learner.h
@@ -8,12 +8,6 @@
 #ifndef XGBOOST_LEARNER_H_
 #define XGBOOST_LEARNER_H_
 
-#include <utility>
-#include <map>
-#include <memory>
-#include <string>
-#include <vector>
-
 #include <dmlc/any.h>
 #include <rabit/rabit.h>
 #include <xgboost/base.h>
@@ -23,6 +17,12 @@
 #include <xgboost/host_device_vector.h>
 #include <xgboost/model.h>
 
+#include <utility>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
 namespace xgboost {
 
 class Metric;
diff --git a/include/xgboost/linear_updater.h b/include/xgboost/linear_updater.h
index d46abdcb2133..39a0c324a958 100644
--- a/include/xgboost/linear_updater.h
+++ b/include/xgboost/linear_updater.h
@@ -3,11 +3,6 @@
  */
 #pragma once
 
-#include <functional>
-#include <string>
-#include <utility>
-#include <vector>
-
 #include <dmlc/registry.h>
 #include <xgboost/base.h>
 #include <xgboost/data.h>
@@ -15,6 +10,12 @@
 #include <xgboost/host_device_vector.h>
 #include <xgboost/model.h>
 
+#include <functional>
+#include <string>
+#include <utility>
+#include <vector>
+
+
 namespace xgboost {
 
 class Json;
diff --git a/include/xgboost/logging.h b/include/xgboost/logging.h
index ff8570db4b2f..07a2e67cf8c9 100644
--- a/include/xgboost/logging.h
+++ b/include/xgboost/logging.h
@@ -8,18 +8,18 @@
 #ifndef XGBOOST_LOGGING_H_
 #define XGBOOST_LOGGING_H_
 
-#include <sstream>
-#include <map>
-#include <string>
-#include <utility>
-#include <vector>
-
 #include <dmlc/logging.h>
 #include <dmlc/thread_local.h>
 
 #include <xgboost/base.h>
 #include <xgboost/parameter.h>
 
+#include <sstream>
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
 namespace xgboost {
 
 class BaseLogger {
diff --git a/include/xgboost/metric.h b/include/xgboost/metric.h
index b7cc77a8c02e..8e9d2afe6ba5 100644
--- a/include/xgboost/metric.h
+++ b/include/xgboost/metric.h
@@ -7,11 +7,6 @@
 #ifndef XGBOOST_METRIC_H_
 #define XGBOOST_METRIC_H_
 
-#include <vector>
-#include <string>
-#include <functional>
-#include <utility>
-
 #include <dmlc/registry.h>
 #include <xgboost/model.h>
 #include <xgboost/generic_parameters.h>
@@ -19,6 +14,11 @@
 #include <xgboost/base.h>
 #include <xgboost/host_device_vector.h>
 
+#include <vector>
+#include <string>
+#include <functional>
+#include <utility>
+
 namespace xgboost {
 /*!
  * \brief interface of evaluation metric used to evaluate model performance.
diff --git a/include/xgboost/objective.h b/include/xgboost/objective.h
index 957ff9da447d..1c0942ed0519 100644
--- a/include/xgboost/objective.h
+++ b/include/xgboost/objective.h
@@ -7,11 +7,6 @@
 #ifndef XGBOOST_OBJECTIVE_H_
 #define XGBOOST_OBJECTIVE_H_
 
-#include <vector>
-#include <utility>
-#include <string>
-#include <functional>
-
 #include <dmlc/registry.h>
 #include <xgboost/base.h>
 #include <xgboost/data.h>
@@ -19,6 +14,11 @@
 #include <xgboost/generic_parameters.h>
 #include <xgboost/host_device_vector.h>
 
+#include <vector>
+#include <utility>
+#include <string>
+#include <functional>
+
 namespace xgboost {
 
 /*! \brief interface of objective function */
diff --git a/include/xgboost/parameter.h b/include/xgboost/parameter.h
index 465c101e4481..3186651eda16 100644
--- a/include/xgboost/parameter.h
+++ b/include/xgboost/parameter.h
@@ -8,11 +8,10 @@
 #ifndef XGBOOST_PARAMETER_H_
 #define XGBOOST_PARAMETER_H_
 
-#include <string>
-#include <type_traits>
-
 #include <dmlc/parameter.h>
 #include <xgboost/base.h>
+#include <string>
+#include <type_traits>
 
 /*!
  * \brief Specialization of FieldEntry for enum class (backed by int)
diff --git a/include/xgboost/predictor.h b/include/xgboost/predictor.h
index b1790e7771ea..0e3f7568ca08 100644
--- a/include/xgboost/predictor.h
+++ b/include/xgboost/predictor.h
@@ -5,6 +5,10 @@
  *  performs predictions for a gradient booster.
  */
 #pragma once
+#include <xgboost/base.h>
+#include <xgboost/data.h>
+#include <xgboost/generic_parameters.h>
+#include <xgboost/host_device_vector.h>
 
 #include <functional>
 #include <memory>
@@ -14,11 +18,6 @@
 #include <vector>
 #include <mutex>
 
-#include <xgboost/base.h>
-#include <xgboost/data.h>
-#include <xgboost/generic_parameters.h>
-#include <xgboost/host_device_vector.h>
-
 // Forward declarations
 namespace xgboost {
 class TreeUpdater;
diff --git a/include/xgboost/span.h b/include/xgboost/span.h
index 04b737666389..29b0b9a7930d 100644
--- a/include/xgboost/span.h
+++ b/include/xgboost/span.h
@@ -29,15 +29,15 @@
 #ifndef XGBOOST_SPAN_H_
 #define XGBOOST_SPAN_H_
 
+#include <xgboost/base.h>
+#include <xgboost/logging.h>
+
 #include <cinttypes>          // size_t
 #include <limits>             // numeric_limits
 #include <iterator>
 #include <type_traits>
 #include <cstdio>
 
-#include <xgboost/base.h>
-#include <xgboost/logging.h>
-
 /*!
  * The version number 1910 is picked up from GSL.
  *
diff --git a/include/xgboost/tree_model.h b/include/xgboost/tree_model.h
index aa728a918fbb..e7f6dc8ec089 100644
--- a/include/xgboost/tree_model.h
+++ b/include/xgboost/tree_model.h
@@ -7,14 +7,6 @@
 #ifndef XGBOOST_TREE_MODEL_H_
 #define XGBOOST_TREE_MODEL_H_
 
-#include <limits>
-#include <vector>
-#include <string>
-#include <cstring>
-#include <algorithm>
-#include <tuple>
-#include <stack>
-
 #include <dmlc/io.h>
 #include <dmlc/parameter.h>
 
@@ -24,6 +16,14 @@
 #include <xgboost/feature_map.h>
 #include <xgboost/model.h>
 
+#include <limits>
+#include <vector>
+#include <string>
+#include <cstring>
+#include <algorithm>
+#include <tuple>
+#include <stack>
+
 namespace xgboost {
 
 struct PathElement;  // forward declaration
diff --git a/include/xgboost/tree_updater.h b/include/xgboost/tree_updater.h
index 805d6161c8a1..a091c81b045b 100644
--- a/include/xgboost/tree_updater.h
+++ b/include/xgboost/tree_updater.h
@@ -8,11 +8,6 @@
 #ifndef XGBOOST_TREE_UPDATER_H_
 #define XGBOOST_TREE_UPDATER_H_
 
-#include <functional>
-#include <vector>
-#include <utility>
-#include <string>
-
 #include <dmlc/registry.h>
 #include <xgboost/base.h>
 #include <xgboost/data.h>
@@ -21,6 +16,11 @@
 #include <xgboost/host_device_vector.h>
 #include <xgboost/model.h>
 
+#include <functional>
+#include <vector>
+#include <utility>
+#include <string>
+
 namespace xgboost {
 
 class Json;
diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc
index cc50c956fd3c..360baf8af7f1 100644
--- a/src/c_api/c_api.cc
+++ b/src/c_api/c_api.cc
@@ -1,4 +1,7 @@
 // Copyright (c) 2014-2020 by Contributors
+#include <rabit/rabit.h>
+#include <rabit/c_api.h>
+
 #include <cstdio>
 #include <cstring>
 #include <fstream>
@@ -7,9 +10,6 @@
 #include <string>
 #include <memory>
 
-#include <rabit/rabit.h>
-#include <rabit/c_api.h>
-
 #include "xgboost/base.h"
 #include "xgboost/data.h"
 #include "xgboost/host_device_vector.h"
diff --git a/src/cli_main.cc b/src/cli_main.cc
index dc4592099903..5c602f374b2a 100644
--- a/src/cli_main.cc
+++ b/src/cli_main.cc
@@ -7,14 +7,6 @@
 #define _CRT_SECURE_NO_WARNINGS
 #define _CRT_SECURE_NO_DEPRECATE
 #define NOMINMAX
-
-#include <iomanip>
-#include <ctime>
-#include <string>
-#include <cstdio>
-#include <cstring>
-#include <vector>
-
 #include <dmlc/timer.h>
 
 #include <xgboost/learner.h>
@@ -23,6 +15,12 @@
 #include <xgboost/logging.h>
 #include <xgboost/parameter.h>
 
+#include <iomanip>
+#include <ctime>
+#include <string>
+#include <cstdio>
+#include <cstring>
+#include <vector>
 #include "common/common.h"
 #include "common/config.h"
 #include "common/io.h"
diff --git a/src/common/base64.h b/src/common/base64.h
index ed8f7d759534..4c876b5f8ddd 100644
--- a/src/common/base64.h
+++ b/src/common/base64.h
@@ -8,11 +8,10 @@
 #ifndef XGBOOST_COMMON_BASE64_H_
 #define XGBOOST_COMMON_BASE64_H_
 
+#include <xgboost/logging.h>
 #include <cctype>
 #include <cstdio>
 #include <string>
-
-#include "xgboost/logging.h"
 #include "./io.h"
 
 namespace xgboost {
diff --git a/src/common/common.h b/src/common/common.h
index f4c82e13e7a7..f9300f3d583c 100644
--- a/src/common/common.h
+++ b/src/common/common.h
@@ -6,6 +6,9 @@
 #ifndef XGBOOST_COMMON_COMMON_H_
 #define XGBOOST_COMMON_COMMON_H_
 
+#include <xgboost/base.h>
+#include <xgboost/logging.h>
+
 #include <exception>
 #include <limits>
 #include <type_traits>
@@ -13,9 +16,6 @@
 #include <string>
 #include <sstream>
 
-#include <xgboost/base.h>
-#include <xgboost/logging.h>
-
 #if defined(__CUDACC__)
 #include <thrust/system/cuda/error.h>
 #include <thrust/system_error.h>
diff --git a/src/common/compressed_iterator.h b/src/common/compressed_iterator.h
index a042d1ad46f6..9f60722fb982 100644
--- a/src/common/compressed_iterator.h
+++ b/src/common/compressed_iterator.h
@@ -3,11 +3,11 @@
  * \file compressed_iterator.h
  */
 #pragma once
+#include <xgboost/base.h>
 #include <cmath>
 #include <cstddef>
 #include <algorithm>
 
-#include "xgboost/base.h"
 #include "common.h"
 
 #ifdef __CUDACC__
diff --git a/src/common/hist_util.cc b/src/common/hist_util.cc
index a5b4db1556a2..e3ca953d2e4b 100644
--- a/src/common/hist_util.cc
+++ b/src/common/hist_util.cc
@@ -2,21 +2,19 @@
  * Copyright 2017-2020 by Contributors
  * \file hist_util.cc
  */
-#include <numeric>
-#include <vector>
-
 #include <dmlc/timer.h>
 #include <dmlc/omp.h>
 
 #include <rabit/rabit.h>
+#include <numeric>
+#include <vector>
 
 #include "xgboost/base.h"
+#include "../common/common.h"
 #include "hist_util.h"
 #include "random.h"
 #include "column_matrix.h"
 #include "quantile.h"
-
-#include "../common/common.h"
 #include "./../tree/updater_quantile_hist.h"
 
 #if defined(XGBOOST_MM_PREFETCH_PRESENT)
diff --git a/src/common/hist_util.cu b/src/common/hist_util.cu
index cde46a2b41d9..716d49f9ac37 100644
--- a/src/common/hist_util.cu
+++ b/src/common/hist_util.cu
@@ -1,10 +1,6 @@
 /*!
  * Copyright 2018 XGBoost contributors
  */
-#include <memory>
-#include <mutex>
-#include <utility>
-#include <vector>
 
 #include <xgboost/logging.h>
 
@@ -17,6 +13,11 @@
 #include <thrust/binary_search.h>
 #include <thrust/execution_policy.h>
 
+#include <memory>
+#include <mutex>
+#include <utility>
+#include <vector>
+
 #include "device_helpers.cuh"
 #include "hist_util.h"
 #include "hist_util.cuh"
diff --git a/src/common/hist_util.h b/src/common/hist_util.h
index aef3f792147a..d6096030182f 100644
--- a/src/common/hist_util.h
+++ b/src/common/hist_util.h
@@ -7,6 +7,8 @@
 #ifndef XGBOOST_COMMON_HIST_UTIL_H_
 #define XGBOOST_COMMON_HIST_UTIL_H_
 
+#include <xgboost/data.h>
+#include <xgboost/generic_parameters.h>
 #include <limits>
 #include <vector>
 #include <algorithm>
@@ -14,9 +16,6 @@
 #include <utility>
 #include <map>
 
-#include "xgboost/data.h"
-#include "xgboost/generic_parameters.h"
-
 #include "row_set.h"
 #include "threading_utils.h"
 #include "../tree/param.h"
diff --git a/src/common/host_device_vector.cc b/src/common/host_device_vector.cc
index e5dc746c0035..1dd9997faa30 100644
--- a/src/common/host_device_vector.cc
+++ b/src/common/host_device_vector.cc
@@ -4,12 +4,12 @@
 #ifndef XGBOOST_USE_CUDA
 
 // dummy implementation of HostDeviceVector in case CUDA is not used
+
+#include <xgboost/base.h>
+#include <xgboost/data.h>
 #include <cstdint>
 #include <memory>
 #include <utility>
-
-#include "xgboost/base.h"
-#include "xgboost/data.h"
 #include "xgboost/host_device_vector.h"
 
 namespace xgboost {
diff --git a/src/common/host_device_vector.cu b/src/common/host_device_vector.cu
index 87e58cc5bf6d..3ffcf757e846 100644
--- a/src/common/host_device_vector.cu
+++ b/src/common/host_device_vector.cu
@@ -1,12 +1,14 @@
 /*!
  * Copyright 2017 XGBoost contributors
  */
-#include <algorithm>
-#include <cstdint>
 
 #include <thrust/fill.h>
 #include <thrust/device_ptr.h>
 
+#include <algorithm>
+#include <cstdint>
+#include <mutex>
+
 #include "xgboost/data.h"
 #include "xgboost/host_device_vector.h"
 #include "device_helpers.cuh"
diff --git a/src/common/io.h b/src/common/io.h
index 6b9cac43ffc2..528296dc76cc 100644
--- a/src/common/io.h
+++ b/src/common/io.h
@@ -7,11 +7,11 @@
 
 #ifndef XGBOOST_COMMON_IO_H_
 #define XGBOOST_COMMON_IO_H_
-#include <string>
-#include <cstring>
 
 #include <dmlc/io.h>
 #include <rabit/rabit.h>
+#include <string>
+#include <cstring>
 
 #include "common.h"
 
diff --git a/src/common/math.h b/src/common/math.h
index 4e5add27ed9f..41905310e70d 100644
--- a/src/common/math.h
+++ b/src/common/math.h
@@ -7,14 +7,14 @@
 #ifndef XGBOOST_COMMON_MATH_H_
 #define XGBOOST_COMMON_MATH_H_
 
+#include <xgboost/base.h>
+
 #include <algorithm>
 #include <cmath>
 #include <limits>
 #include <utility>
 #include <vector>
 
-#include <xgboost/base.h>
-
 namespace xgboost {
 namespace common {
 /*!
diff --git a/src/common/probability_distribution.cc b/src/common/probability_distribution.cc
index 593e87fd97cf..51bcc495b551 100644
--- a/src/common/probability_distribution.cc
+++ b/src/common/probability_distribution.cc
@@ -4,9 +4,9 @@
  * \brief Implementation of a few useful probability distributions
  * \author Avinash Barnwal and Hyunsu Cho
  */
-#include <cmath>
 
-#include "xgboost/logging.h"
+#include <xgboost/logging.h>
+#include <cmath>
 #include "probability_distribution.h"
 
 namespace xgboost {
diff --git a/src/common/quantile.h b/src/common/quantile.h
index 569f06aaf79c..067b041bc4e9 100644
--- a/src/common/quantile.h
+++ b/src/common/quantile.h
@@ -7,15 +7,14 @@
 #ifndef XGBOOST_COMMON_QUANTILE_H_
 #define XGBOOST_COMMON_QUANTILE_H_
 
+#include <dmlc/base.h>
+#include <xgboost/logging.h>
 #include <cmath>
 #include <vector>
 #include <cstring>
 #include <algorithm>
 #include <iostream>
 
-#include <dmlc/base.h>
-#include <xgboost/logging.h>
-
 namespace xgboost {
 namespace common {
 /*!
diff --git a/src/common/random.h b/src/common/random.h
index 4ee61387dd40..45af80ce030b 100644
--- a/src/common/random.h
+++ b/src/common/random.h
@@ -7,6 +7,8 @@
 #ifndef XGBOOST_COMMON_RANDOM_H_
 #define XGBOOST_COMMON_RANDOM_H_
 
+#include <rabit/rabit.h>
+#include <xgboost/logging.h>
 #include <algorithm>
 #include <vector>
 #include <limits>
@@ -15,8 +17,6 @@
 #include <numeric>
 #include <random>
 
-#include "rabit/rabit.h"
-#include "xgboost/logging.h"
 #include "xgboost/host_device_vector.h"
 
 namespace xgboost {
diff --git a/src/common/row_set.h b/src/common/row_set.h
index 2ac1cf467ed3..25f7c739d8d7 100644
--- a/src/common/row_set.h
+++ b/src/common/row_set.h
@@ -7,12 +7,11 @@
 #ifndef XGBOOST_COMMON_ROW_SET_H_
 #define XGBOOST_COMMON_ROW_SET_H_
 
+#include <xgboost/data.h>
 #include <algorithm>
 #include <vector>
 #include <utility>
 
-#include "xgboost/data.h"
-
 namespace xgboost {
 namespace common {
 
diff --git a/src/common/survival_util.cc b/src/common/survival_util.cc
index 4d78a49ca8bd..5630da35593b 100644
--- a/src/common/survival_util.cc
+++ b/src/common/survival_util.cc
@@ -6,10 +6,9 @@
  * \author Avinash Barnwal, Hyunsu Cho and Toby Hocking
  */
 
+#include <dmlc/registry.h>
 #include <algorithm>
 #include <cmath>
-
-#include <dmlc/registry.h>
 #include "survival_util.h"
 
 /*
diff --git a/src/common/survival_util.h b/src/common/survival_util.h
index fac8b4728acd..50c6aab5109b 100644
--- a/src/common/survival_util.h
+++ b/src/common/survival_util.h
@@ -8,9 +8,8 @@
 #ifndef XGBOOST_COMMON_SURVIVAL_UTIL_H_
 #define XGBOOST_COMMON_SURVIVAL_UTIL_H_
 
+#include <xgboost/parameter.h>
 #include <memory>
-
-#include "xgboost/parameter.h"
 #include "probability_distribution.h"
 
 DECLARE_FIELD_ENUM_CLASS(xgboost::common::ProbabilityDistributionType);
diff --git a/src/common/timer.cc b/src/common/timer.cc
index 1cbf869bcb5d..49d08a35cae6 100644
--- a/src/common/timer.cc
+++ b/src/common/timer.cc
@@ -1,15 +1,14 @@
 /*!
  * Copyright by Contributors 2019
  */
+#include <rabit/rabit.h>
 #include <algorithm>
 #include <type_traits>
 #include <utility>
 #include <vector>
 #include <sstream>
-
-#include <rabit/rabit.h>
-#include "xgboost/json.h"
 #include "timer.h"
+#include "xgboost/json.h"
 
 #if defined(XGBOOST_USE_NVTX)
 #include <nvToolsExt.h>
diff --git a/src/common/timer.h b/src/common/timer.h
index 3fd54e18659e..b2dc2a428d13 100644
--- a/src/common/timer.h
+++ b/src/common/timer.h
@@ -2,6 +2,7 @@
  * Copyright by Contributors 2017-2019
  */
 #pragma once
+#include <xgboost/logging.h>
 #include <chrono>
 #include <iostream>
 #include <map>
@@ -9,8 +10,6 @@
 #include <utility>
 #include <vector>
 
-#include <xgboost/logging.h>
-
 namespace xgboost {
 namespace common {
 
diff --git a/src/common/transform.h b/src/common/transform.h
index 3d5970dec0a9..fa2d0d3794a3 100644
--- a/src/common/transform.h
+++ b/src/common/transform.h
@@ -3,14 +3,15 @@
  */
 #ifndef XGBOOST_COMMON_TRANSFORM_H_
 #define XGBOOST_COMMON_TRANSFORM_H_
-#include <utility>
-#include <vector>
-#include <type_traits>  // enable_if
 
 #include <dmlc/omp.h>
 #include <dmlc/common.h>
 
-#include "xgboost/data.h"
+#include <xgboost/data.h>
+#include <utility>
+#include <vector>
+#include <type_traits>  // enable_if
+
 #include "xgboost/host_device_vector.h"
 #include "xgboost/span.h"
 
diff --git a/src/common/version.cc b/src/common/version.cc
index c0cdffb99607..3fb2e5c24392 100644
--- a/src/common/version.cc
+++ b/src/common/version.cc
@@ -1,12 +1,12 @@
 /*!
  * Copyright 2019 XGBoost contributors
  */
+#include <dmlc/io.h>
+
 #include <string>
 #include <tuple>
 #include <vector>
 
-#include <dmlc/io.h>
-
 #include "xgboost/logging.h"
 #include "xgboost/json.h"
 #include "xgboost/version_config.h"
diff --git a/src/common/version.h b/src/common/version.h
index 3c3b97f16ed7..96885f6a8a82 100644
--- a/src/common/version.h
+++ b/src/common/version.h
@@ -4,10 +4,10 @@
 #ifndef XGBOOST_COMMON_VERSION_H_
 #define XGBOOST_COMMON_VERSION_H_
 
+#include <dmlc/io.h>
 #include <string>
 #include <tuple>
 
-#include <dmlc/io.h>
 #include "xgboost/base.h"
 
 namespace xgboost {
diff --git a/src/data/adapter.h b/src/data/adapter.h
index cd26d0bf44f9..e252e1da7a1f 100644
--- a/src/data/adapter.h
+++ b/src/data/adapter.h
@@ -4,6 +4,8 @@
  */
 #ifndef XGBOOST_DATA_ADAPTER_H_
 #define XGBOOST_DATA_ADAPTER_H_
+#include <dmlc/data.h>
+
 #include <cstddef>
 #include <functional>
 #include <limits>
@@ -12,8 +14,6 @@
 #include <utility>
 #include <vector>
 
-#include <dmlc/data.h>
-
 #include "xgboost/logging.h"
 #include "xgboost/base.h"
 #include "xgboost/data.h"
diff --git a/src/data/data.cc b/src/data/data.cc
index ff907d03135a..2a620b7d444a 100644
--- a/src/data/data.cc
+++ b/src/data/data.cc
@@ -2,11 +2,8 @@
  * Copyright 2015-2020 by Contributors
  * \file data.cc
  */
-#include <cstring>
-#include <vector>
-#include <string>
-
 #include <dmlc/registry.h>
+#include <cstring>
 
 #include "dmlc/io.h"
 #include "xgboost/data.h"
diff --git a/src/data/device_dmatrix.cu b/src/data/device_dmatrix.cu
index 88d6a33a0a8b..d11d01b16f2a 100644
--- a/src/data/device_dmatrix.cu
+++ b/src/data/device_dmatrix.cu
@@ -3,16 +3,14 @@
  * \file device_dmatrix.cu
  * \brief Device-memory version of DMatrix.
  */
-#include <memory>
-#include <utility>
 
 #include <thrust/execution_policy.h>
 #include <thrust/iterator/discard_iterator.h>
 #include <thrust/iterator/transform_output_iterator.h>
-
 #include <xgboost/base.h>
 #include <xgboost/data.h>
-
+#include <memory>
+#include <utility>
 #include "../common/hist_util.h"
 #include "adapter.h"
 #include "device_adapter.cuh"
diff --git a/src/data/device_dmatrix.h b/src/data/device_dmatrix.h
index ad7e8aa3b904..781461baaa9e 100644
--- a/src/data/device_dmatrix.h
+++ b/src/data/device_dmatrix.h
@@ -6,11 +6,11 @@
 #ifndef XGBOOST_DATA_DEVICE_DMATRIX_H_
 #define XGBOOST_DATA_DEVICE_DMATRIX_H_
 
-#include <memory>
-
 #include <xgboost/base.h>
 #include <xgboost/data.h>
 
+#include <memory>
+
 #include "adapter.h"
 #include "simple_batch_iterator.h"
 #include "simple_dmatrix.h"
diff --git a/src/data/ellpack_page_source.h b/src/data/ellpack_page_source.h
index c5f787f7fe1a..a1ce587c2c75 100644
--- a/src/data/ellpack_page_source.h
+++ b/src/data/ellpack_page_source.h
@@ -5,11 +5,10 @@
 #ifndef XGBOOST_DATA_ELLPACK_PAGE_SOURCE_H_
 #define XGBOOST_DATA_ELLPACK_PAGE_SOURCE_H_
 
+#include <xgboost/data.h>
 #include <memory>
 #include <string>
 
-#include <xgboost/data.h>
-
 #include "../common/timer.h"
 #include "../common/hist_util.h"
 #include "sparse_page_source.h"
diff --git a/src/data/simple_dmatrix.h b/src/data/simple_dmatrix.h
index 7958785c6ae6..9d2130b4195e 100644
--- a/src/data/simple_dmatrix.h
+++ b/src/data/simple_dmatrix.h
@@ -7,11 +7,12 @@
 #ifndef XGBOOST_DATA_SIMPLE_DMATRIX_H_
 #define XGBOOST_DATA_SIMPLE_DMATRIX_H_
 
+#include <xgboost/base.h>
+#include <xgboost/data.h>
+
 #include <memory>
 #include <string>
 
-#include <xgboost/base.h>
-#include <xgboost/data.h>
 
 namespace xgboost {
 namespace data {
diff --git a/src/data/sparse_page_dmatrix.h b/src/data/sparse_page_dmatrix.h
index 79c95c5813e2..393172658695 100644
--- a/src/data/sparse_page_dmatrix.h
+++ b/src/data/sparse_page_dmatrix.h
@@ -7,14 +7,13 @@
 #ifndef XGBOOST_DATA_SPARSE_PAGE_DMATRIX_H_
 #define XGBOOST_DATA_SPARSE_PAGE_DMATRIX_H_
 
+#include <xgboost/data.h>
 #include <algorithm>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 
-#include "xgboost/data.h"
-
 #include "ellpack_page_source.h"
 #include "sparse_page_source.h"
 
diff --git a/src/data/sparse_page_source.h b/src/data/sparse_page_source.h
index 214795fad1f1..108af403b1a3 100644
--- a/src/data/sparse_page_source.h
+++ b/src/data/sparse_page_source.h
@@ -7,6 +7,9 @@
 #ifndef XGBOOST_DATA_SPARSE_PAGE_SOURCE_H_
 #define XGBOOST_DATA_SPARSE_PAGE_SOURCE_H_
 
+#include <dmlc/threadediter.h>
+#include <dmlc/timer.h>
+
 #include <algorithm>
 #include <limits>
 #include <locale>
@@ -16,9 +19,6 @@
 #include <vector>
 #include <fstream>
 
-#include <dmlc/threadediter.h>
-#include <dmlc/timer.h>
-
 #include "xgboost/base.h"
 #include "xgboost/data.h"
 
diff --git a/src/data/sparse_page_writer.h b/src/data/sparse_page_writer.h
index f303f84277f9..f63fcf0f8d7d 100644
--- a/src/data/sparse_page_writer.h
+++ b/src/data/sparse_page_writer.h
@@ -6,6 +6,8 @@
 #ifndef XGBOOST_DATA_SPARSE_PAGE_WRITER_H_
 #define XGBOOST_DATA_SPARSE_PAGE_WRITER_H_
 
+#include <xgboost/data.h>
+#include <dmlc/io.h>
 #include <vector>
 #include <algorithm>
 #include <cstring>
@@ -14,12 +16,9 @@
 #include <memory>
 #include <functional>
 
-#include <xgboost/data.h>
-#include <dmlc/io.h>
-
 #if DMLC_ENABLE_STD_THREAD
 #include <dmlc/concurrency.h>
-#include <thread>  // NOLINT
+#include <thread>
 #endif  // DMLC_ENABLE_STD_THREAD
 
 namespace xgboost {
diff --git a/src/gbm/gblinear.cc b/src/gbm/gblinear.cc
index 4742f4971076..e554f8a559e0 100644
--- a/src/gbm/gblinear.cc
+++ b/src/gbm/gblinear.cc
@@ -5,14 +5,14 @@
  *        the update rule is parallel coordinate descent (shotgun)
  * \author Tianqi Chen
  */
+#include <dmlc/omp.h>
+#include <dmlc/parameter.h>
+
 #include <vector>
 #include <string>
 #include <sstream>
 #include <algorithm>
 
-#include <dmlc/omp.h>
-#include <dmlc/parameter.h>
-
 #include "xgboost/gbm.h"
 #include "xgboost/json.h"
 #include "xgboost/predictor.h"
diff --git a/src/gbm/gblinear_model.h b/src/gbm/gblinear_model.h
index 4aff2f717c7a..f2d0d9a868d3 100644
--- a/src/gbm/gblinear_model.h
+++ b/src/gbm/gblinear_model.h
@@ -2,14 +2,14 @@
  * Copyright 2018-2019 by Contributors
  */
 #pragma once
+#include <dmlc/io.h>
+#include <dmlc/parameter.h>
+#include <xgboost/learner.h>
+
 #include <vector>
 #include <string>
 #include <cstring>
 
-#include <dmlc/io.h>
-#include <dmlc/parameter.h>
-
-#include <xgboost/learner.h>
 #include "xgboost/base.h"
 #include "xgboost/feature_map.h"
 #include "xgboost/model.h"
diff --git a/src/gbm/gbm.cc b/src/gbm/gbm.cc
index a8218e6a4f90..87a6ded29042 100644
--- a/src/gbm/gbm.cc
+++ b/src/gbm/gbm.cc
@@ -3,12 +3,11 @@
  * \file gbm.cc
  * \brief Registry of gradient boosters.
  */
+#include <dmlc/registry.h>
 #include <string>
 #include <vector>
 #include <memory>
 
-#include <dmlc/registry.h>
-
 #include "xgboost/gbm.h"
 #include "xgboost/learner.h"
 #include "xgboost/generic_parameters.h"
diff --git a/src/gbm/gbtree.cc b/src/gbm/gbtree.cc
index bf5156306695..61a3021cbefd 100644
--- a/src/gbm/gbtree.cc
+++ b/src/gbm/gbtree.cc
@@ -4,6 +4,9 @@
  * \brief gradient boosted tree implementation.
  * \author Tianqi Chen
  */
+#include <dmlc/omp.h>
+#include <dmlc/parameter.h>
+
 #include <vector>
 #include <memory>
 #include <utility>
@@ -11,9 +14,6 @@
 #include <limits>
 #include <algorithm>
 
-#include <dmlc/omp.h>
-#include <dmlc/parameter.h>
-
 #include "xgboost/data.h"
 #include "xgboost/gbm.h"
 #include "xgboost/logging.h"
diff --git a/src/gbm/gbtree.h b/src/gbm/gbtree.h
index ad9c63d141aa..534c3ad5469a 100644
--- a/src/gbm/gbtree.h
+++ b/src/gbm/gbtree.h
@@ -7,6 +7,8 @@
 #ifndef XGBOOST_GBM_GBTREE_H_
 #define XGBOOST_GBM_GBTREE_H_
 
+#include <dmlc/omp.h>
+
 #include <vector>
 #include <map>
 #include <memory>
@@ -14,7 +16,6 @@
 #include <string>
 #include <unordered_map>
 
-#include <dmlc/omp.h>
 #include "xgboost/base.h"
 #include "xgboost/data.h"
 #include "xgboost/logging.h"
diff --git a/src/gbm/gbtree_model.h b/src/gbm/gbtree_model.h
index 374f31ef9bea..7ac7d8f470a2 100644
--- a/src/gbm/gbtree_model.h
+++ b/src/gbm/gbtree_model.h
@@ -4,10 +4,6 @@
  */
 #ifndef XGBOOST_GBM_GBTREE_MODEL_H_
 #define XGBOOST_GBM_GBTREE_MODEL_H_
-#include <memory>
-#include <utility>
-#include <string>
-#include <vector>
 
 #include <dmlc/parameter.h>
 #include <dmlc/io.h>
@@ -16,6 +12,11 @@
 #include <xgboost/parameter.h>
 #include <xgboost/learner.h>
 
+#include <memory>
+#include <utility>
+#include <string>
+#include <vector>
+
 namespace xgboost {
 
 class Json;
diff --git a/src/learner.cc b/src/learner.cc
index e6c3fee802dc..34649480c5ce 100644
--- a/src/learner.cc
+++ b/src/learner.cc
@@ -4,6 +4,10 @@
  * \brief Implementation of learning algorithm.
  * \author Tianqi Chen
  */
+#include <dmlc/io.h>
+#include <dmlc/parameter.h>
+#include <dmlc/thread_local.h>
+
 #include <atomic>
 #include <mutex>
 #include <algorithm>
@@ -16,11 +20,7 @@
 #include <utility>
 #include <vector>
 
-#include <dmlc/io.h>
-#include <dmlc/parameter.h>
-#include <dmlc/thread_local.h>
-#include <dmlc/any.h>
-
+#include "dmlc/any.h"
 #include "xgboost/base.h"
 #include "xgboost/data.h"
 #include "xgboost/model.h"
diff --git a/src/logging.cc b/src/logging.cc
index c8aa1d320e92..f03e1be02d01 100644
--- a/src/logging.cc
+++ b/src/logging.cc
@@ -4,11 +4,11 @@
  * \brief Implementation of loggers.
  * \author Tianqi Chen
  */
+#include <rabit/rabit.h>
+
 #include <iostream>
 #include <map>
 
-#include <rabit/rabit.h>
-
 #include "xgboost/parameter.h"
 #include "xgboost/logging.h"
 
diff --git a/src/metric/elementwise_metric.cu b/src/metric/elementwise_metric.cu
index 82e8be3a981b..85dead9c3d98 100644
--- a/src/metric/elementwise_metric.cu
+++ b/src/metric/elementwise_metric.cu
@@ -6,11 +6,10 @@
  *
  *  The expressions like wsum == 0 ? esum : esum / wsum is used to handle empty dataset.
  */
-#include <cmath>
-
 #include <rabit/rabit.h>
 #include <xgboost/metric.h>
 #include <dmlc/registry.h>
+#include <cmath>
 
 #include "metric_common.h"
 #include "../common/math.h"
diff --git a/src/metric/multiclass_metric.cu b/src/metric/multiclass_metric.cu
index b019f2e47046..377a05010ea3 100644
--- a/src/metric/multiclass_metric.cu
+++ b/src/metric/multiclass_metric.cu
@@ -4,10 +4,9 @@
  * \brief evaluation metrics for multiclass classification.
  * \author Kailong Chen, Tianqi Chen
  */
-#include <cmath>
-
 #include <rabit/rabit.h>
-#include "xgboost/metric.h"
+#include <xgboost/metric.h>
+#include <cmath>
 
 #include "metric_common.h"
 #include "../common/math.h"
diff --git a/src/metric/rank_metric.cc b/src/metric/rank_metric.cc
index 61df433559ad..b55c764d2139 100644
--- a/src/metric/rank_metric.cc
+++ b/src/metric/rank_metric.cc
@@ -19,15 +19,15 @@
 //   this cannot be used when the translation unit is compiled using the 'nvcc' compiler (as the
 //   corresponding headers that brings in those function declaration can't be included with CUDA).
 //   This precludes the CPU and GPU logic to coexist inside a .cu file
-#include <cmath>
-#include <vector>
 
 #include <rabit/rabit.h>
+#include <xgboost/metric.h>
 #include <dmlc/registry.h>
+#include <cmath>
 
-#include "xgboost/metric.h"
-#include "xgboost/host_device_vector.h"
+#include <vector>
 
+#include "xgboost/host_device_vector.h"
 #include "../common/math.h"
 #include "metric_common.h"
 
diff --git a/src/metric/rank_metric.cu b/src/metric/rank_metric.cu
index 9bcbc68824ed..290b1fb023c5 100644
--- a/src/metric/rank_metric.cu
+++ b/src/metric/rank_metric.cu
@@ -4,9 +4,6 @@
  * \brief prediction rank based metrics.
  * \author Kailong Chen, Tianqi Chen
  */
-#include <cmath>
-#include <vector>
-
 #include <rabit/rabit.h>
 #include <dmlc/registry.h>
 
@@ -14,6 +11,9 @@
 #include <xgboost/host_device_vector.h>
 #include <thrust/iterator/discard_iterator.h>
 
+#include <cmath>
+#include <vector>
+
 #include "metric_common.h"
 
 #include "../common/math.h"
diff --git a/src/metric/survival_metric.cc b/src/metric/survival_metric.cc
index 1ea0d6f0215b..cc0614fe7fe9 100644
--- a/src/metric/survival_metric.cc
+++ b/src/metric/survival_metric.cc
@@ -4,16 +4,16 @@
  * \brief Metrics for survival analysis
  * \author Avinash Barnwal, Hyunsu Cho and Toby Hocking
  */
+
+#include <rabit/rabit.h>
+#include <xgboost/metric.h>
+#include <xgboost/host_device_vector.h>
+#include <dmlc/registry.h>
 #include <cmath>
 #include <memory>
 #include <vector>
 #include <limits>
 
-#include <dmlc/registry.h>
-#include <rabit/rabit.h>
-
-#include "xgboost/metric.h"
-#include "xgboost/host_device_vector.h"
 #include "xgboost/json.h"
 
 #include "../common/math.h"
diff --git a/src/objective/aft_obj.cc b/src/objective/aft_obj.cc
index 40882d257290..bf66d999184d 100644
--- a/src/objective/aft_obj.cc
+++ b/src/objective/aft_obj.cc
@@ -3,6 +3,10 @@
  * \file rank.cc
  * \brief Definition of aft loss.
  */
+
+#include <dmlc/omp.h>
+#include <xgboost/logging.h>
+#include <xgboost/objective.h>
 #include <vector>
 #include <limits>
 #include <algorithm>
@@ -10,10 +14,6 @@
 #include <utility>
 #include <cmath>
 
-#include <dmlc/omp.h>
-#include <xgboost/logging.h>
-#include <xgboost/objective.h>
-
 #include "xgboost/json.h"
 
 #include "../common/math.h"
diff --git a/src/objective/multiclass_obj.cu b/src/objective/multiclass_obj.cu
index c968e2685253..29af5e0d264d 100644
--- a/src/objective/multiclass_obj.cu
+++ b/src/objective/multiclass_obj.cu
@@ -4,13 +4,13 @@
  * \brief Definition of multi-class classification objectives.
  * \author Tianqi Chen
  */
+#include <dmlc/omp.h>
+
 #include <vector>
 #include <algorithm>
 #include <limits>
 #include <utility>
 
-#include <dmlc/omp.h>
-
 #include "xgboost/parameter.h"
 #include "xgboost/data.h"
 #include "xgboost/logging.h"
diff --git a/src/objective/objective.cc b/src/objective/objective.cc
index 8641f042ffe9..8f65cec0da38 100644
--- a/src/objective/objective.cc
+++ b/src/objective/objective.cc
@@ -3,11 +3,11 @@
  * \file objective.cc
  * \brief Registry of all objective functions.
  */
-#include <sstream>
-
+#include <xgboost/objective.h>
 #include <dmlc/registry.h>
 
-#include "xgboost/objective.h"
+#include <sstream>
+
 #include "xgboost/host_device_vector.h"
 
 namespace dmlc {
diff --git a/src/objective/rank_obj.cu b/src/objective/rank_obj.cu
index 44fd511939e3..e7a220dbc760 100644
--- a/src/objective/rank_obj.cu
+++ b/src/objective/rank_obj.cu
@@ -1,15 +1,14 @@
 /*!
  * Copyright 2015-2019 XGBoost contributors
  */
+#include <dmlc/omp.h>
+#include <dmlc/timer.h>
+#include <xgboost/logging.h>
+#include <xgboost/objective.h>
 #include <vector>
 #include <algorithm>
 #include <utility>
 
-#include <dmlc/omp.h>
-#include <dmlc/timer.h>
-
-#include "xgboost/logging.h"
-#include "xgboost/objective.h"
 #include "xgboost/json.h"
 #include "xgboost/parameter.h"
 
diff --git a/src/objective/regression_loss.h b/src/objective/regression_loss.h
index 2f1cf829a20b..60ede53dd645 100644
--- a/src/objective/regression_loss.h
+++ b/src/objective/regression_loss.h
@@ -3,10 +3,10 @@
  */
 #ifndef XGBOOST_OBJECTIVE_REGRESSION_LOSS_H_
 #define XGBOOST_OBJECTIVE_REGRESSION_LOSS_H_
-#include <algorithm>
 
 #include <dmlc/omp.h>
 #include <xgboost/logging.h>
+#include <algorithm>
 #include "../common/math.h"
 
 namespace xgboost {
diff --git a/src/objective/regression_obj.cu b/src/objective/regression_obj.cu
index 4bc3973d9120..810ee9a6be4a 100644
--- a/src/objective/regression_obj.cu
+++ b/src/objective/regression_obj.cu
@@ -4,13 +4,13 @@
  * \brief Definition of single-value regression and classification objectives.
  * \author Tianqi Chen, Kailong Chen
  */
-#include <cmath>
-#include <memory>
-#include <vector>
 
 #include <dmlc/omp.h>
 #include <xgboost/logging.h>
 #include <xgboost/objective.h>
+#include <cmath>
+#include <memory>
+#include <vector>
 
 #include "xgboost/host_device_vector.h"
 #include "xgboost/json.h"
diff --git a/src/predictor/cpu_predictor.cc b/src/predictor/cpu_predictor.cc
index 2a96d82ad958..1c84f4947514 100644
--- a/src/predictor/cpu_predictor.cc
+++ b/src/predictor/cpu_predictor.cc
@@ -1,13 +1,13 @@
 /*!
  * Copyright by Contributors 2017-2020
  */
+#include <dmlc/omp.h>
+#include <dmlc/any.h>
+
 #include <cstddef>
 #include <limits>
 #include <mutex>
 
-#include <dmlc/omp.h>
-#include <dmlc/any.h>
-
 #include "xgboost/base.h"
 #include "xgboost/data.h"
 #include "xgboost/predictor.h"
diff --git a/src/predictor/gpu_predictor.cu b/src/predictor/gpu_predictor.cu
index 0e57276d692f..9a498136d41a 100644
--- a/src/predictor/gpu_predictor.cu
+++ b/src/predictor/gpu_predictor.cu
@@ -1,12 +1,11 @@
 /*!
  * Copyright 2017-2020 by Contributors
  */
-#include <memory>
-
 #include <thrust/copy.h>
 #include <thrust/device_ptr.h>
 #include <thrust/device_vector.h>
 #include <thrust/fill.h>
+#include <memory>
 
 #include "xgboost/data.h"
 #include "xgboost/predictor.h"
diff --git a/src/predictor/predictor.cc b/src/predictor/predictor.cc
index 49e1e816d65f..eb68d382b421 100644
--- a/src/predictor/predictor.cc
+++ b/src/predictor/predictor.cc
@@ -1,9 +1,9 @@
 /*!
  * Copyright 2017-2020 by Contributors
  */
+#include <dmlc/registry.h>
 #include <mutex>
 
-#include <dmlc/registry.h>
 #include "xgboost/predictor.h"
 #include "xgboost/data.h"
 #include "xgboost/generic_parameters.h"
diff --git a/src/tree/constraints.cu b/src/tree/constraints.cu
index 8c23064b1e90..b6db0eda0739 100644
--- a/src/tree/constraints.cu
+++ b/src/tree/constraints.cu
@@ -1,15 +1,15 @@
 /*!
  * Copyright 2019 XGBoost contributors
  */
-#include <algorithm>
-#include <string>
-#include <set>
-
 #include <thrust/copy.h>
 #include <thrust/device_vector.h>
 #include <thrust/execution_policy.h>
 #include <thrust/iterator/counting_iterator.h>
 
+#include <algorithm>
+#include <string>
+#include <set>
+
 #include "xgboost/logging.h"
 #include "xgboost/span.h"
 #include "constraints.cuh"
diff --git a/src/tree/gpu_hist/gradient_based_sampler.cu b/src/tree/gpu_hist/gradient_based_sampler.cu
index 558e8a236fd6..eb441f39e67b 100644
--- a/src/tree/gpu_hist/gradient_based_sampler.cu
+++ b/src/tree/gpu_hist/gradient_based_sampler.cu
@@ -1,16 +1,15 @@
 /*!
  * Copyright 2019 by XGBoost Contributors
  */
-#include <algorithm>
-#include <limits>
-
 #include <thrust/functional.h>
 #include <thrust/random.h>
 #include <thrust/transform.h>
-
 #include <xgboost/host_device_vector.h>
 #include <xgboost/logging.h>
 
+#include <algorithm>
+#include <limits>
+
 #include "../../common/compressed_iterator.h"
 #include "../../common/random.h"
 #include "gradient_based_sampler.cuh"
diff --git a/src/tree/gpu_hist/histogram.cu b/src/tree/gpu_hist/histogram.cu
index bf9094556ce4..edc3046d1dcb 100644
--- a/src/tree/gpu_hist/histogram.cu
+++ b/src/tree/gpu_hist/histogram.cu
@@ -1,13 +1,12 @@
 /*!
  * Copyright 2020 by XGBoost Contributors
  */
+#include <thrust/reduce.h>
+#include <thrust/iterator/transform_iterator.h>
 #include <algorithm>
 #include <ctgmath>
 #include <limits>
 
-#include <thrust/reduce.h>
-#include <thrust/iterator/transform_iterator.h>
-
 #include "xgboost/base.h"
 #include "row_partitioner.cuh"
 
diff --git a/src/tree/gpu_hist/row_partitioner.cu b/src/tree/gpu_hist/row_partitioner.cu
index 742d8e2f76fb..2b7fbe4afda1 100644
--- a/src/tree/gpu_hist/row_partitioner.cu
+++ b/src/tree/gpu_hist/row_partitioner.cu
@@ -1,12 +1,10 @@
 /*!
  * Copyright 2017-2019 XGBoost contributors
  */
-#include <vector>
-
 #include <thrust/iterator/discard_iterator.h>
 #include <thrust/iterator/transform_output_iterator.h>
 #include <thrust/sequence.h>
-
+#include <vector>
 #include "../../common/device_helpers.cuh"
 #include "row_partitioner.cuh"
 
diff --git a/src/tree/split_evaluator.cc b/src/tree/split_evaluator.cc
index bbacca2793e8..be166156b004 100644
--- a/src/tree/split_evaluator.cc
+++ b/src/tree/split_evaluator.cc
@@ -3,6 +3,9 @@
  * \file split_evaluator.cc
  * \brief Contains implementations of different split evaluators.
  */
+#include <dmlc/json.h>
+#include <dmlc/registry.h>
+
 #include <algorithm>
 #include <unordered_set>
 #include <vector>
@@ -12,9 +15,6 @@
 #include <sstream>
 #include <utility>
 
-#include <dmlc/json.h>
-#include <dmlc/registry.h>
-
 #include "xgboost/logging.h"
 #include "xgboost/parameter.h"
 
diff --git a/src/tree/split_evaluator.h b/src/tree/split_evaluator.h
index 77b31d9162c9..2b3876491ab0 100644
--- a/src/tree/split_evaluator.h
+++ b/src/tree/split_evaluator.h
@@ -7,15 +7,15 @@
 
 #ifndef XGBOOST_TREE_SPLIT_EVALUATOR_H_
 #define XGBOOST_TREE_SPLIT_EVALUATOR_H_
+
+#include <dmlc/registry.h>
+#include <xgboost/base.h>
 #include <functional>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 
-#include <dmlc/registry.h>
-#include <xgboost/base.h>
-
 #include "param.h"
 
 #define ROOT_PARENT_ID (-1 & ((1U << 31) - 1))
diff --git a/src/tree/tree_model.cc b/src/tree/tree_model.cc
index 8818a07d94fb..8f45621ca15e 100644
--- a/src/tree/tree_model.cc
+++ b/src/tree/tree_model.cc
@@ -3,12 +3,6 @@
  * \file tree_model.cc
  * \brief model structure for tree
  */
-#include <sstream>
-#include <limits>
-#include <cmath>
-#include <iomanip>
-#include <stack>
-
 #include <dmlc/registry.h>
 #include <dmlc/json.h>
 
@@ -16,6 +10,12 @@
 #include <xgboost/logging.h>
 #include <xgboost/json.h>
 
+#include <sstream>
+#include <limits>
+#include <cmath>
+#include <iomanip>
+#include <stack>
+
 #include "param.h"
 #include "../common/common.h"
 
diff --git a/src/tree/updater_basemaker-inl.h b/src/tree/updater_basemaker-inl.h
index 42f4901dd915..66ab91982407 100644
--- a/src/tree/updater_basemaker-inl.h
+++ b/src/tree/updater_basemaker-inl.h
@@ -7,13 +7,15 @@
 #ifndef XGBOOST_TREE_UPDATER_BASEMAKER_INL_H_
 #define XGBOOST_TREE_UPDATER_BASEMAKER_INL_H_
 
+#include <rabit/rabit.h>
+
+
 #include <vector>
 #include <algorithm>
 #include <string>
 #include <limits>
 #include <utility>
 
-#include <rabit/rabit.h>
 #include "xgboost/base.h"
 #include "xgboost/json.h"
 #include "xgboost/tree_updater.h"
diff --git a/src/tree/updater_colmaker.cc b/src/tree/updater_colmaker.cc
index 563160059829..951cfdb5ec27 100644
--- a/src/tree/updater_colmaker.cc
+++ b/src/tree/updater_colmaker.cc
@@ -4,13 +4,12 @@
  * \brief use columnwise update to construct a tree
  * \author Tianqi Chen
  */
+#include <rabit/rabit.h>
 #include <memory>
 #include <vector>
 #include <cmath>
 #include <algorithm>
 
-#include <rabit/rabit.h>
-
 #include "xgboost/parameter.h"
 #include "xgboost/tree_updater.h"
 #include "xgboost/logging.h"
diff --git a/src/tree/updater_gpu_hist.cu b/src/tree/updater_gpu_hist.cu
index 8a52de9e8451..25d2645e1032 100644
--- a/src/tree/updater_gpu_hist.cu
+++ b/src/tree/updater_gpu_hist.cu
@@ -1,6 +1,9 @@
 /*!
  * Copyright 2017-2020 XGBoost contributors
  */
+#include <thrust/copy.h>
+#include <thrust/reduce.h>
+#include <xgboost/tree_updater.h>
 #include <algorithm>
 #include <cmath>
 #include <memory>
@@ -9,10 +12,6 @@
 #include <utility>
 #include <vector>
 
-#include <thrust/copy.h>
-#include <thrust/reduce.h>
-
-#include "xgboost/tree_updater.h"
 #include "xgboost/host_device_vector.h"
 #include "xgboost/parameter.h"
 #include "xgboost/span.h"
diff --git a/src/tree/updater_histmaker.cc b/src/tree/updater_histmaker.cc
index cc17b5143175..c4fdbe3c0308 100644
--- a/src/tree/updater_histmaker.cc
+++ b/src/tree/updater_histmaker.cc
@@ -4,10 +4,10 @@
  * \brief use histogram counting to construct a tree
  * \author Tianqi Chen
  */
+#include <rabit/rabit.h>
 #include <vector>
 #include <algorithm>
 
-#include <rabit/rabit.h>
 #include "xgboost/tree_updater.h"
 #include "xgboost/base.h"
 #include "xgboost/logging.h"
diff --git a/src/tree/updater_prune.cc b/src/tree/updater_prune.cc
index 5aa904ed81f9..76a8916a0598 100644
--- a/src/tree/updater_prune.cc
+++ b/src/tree/updater_prune.cc
@@ -4,18 +4,17 @@
  * \brief prune a tree given the statistics
  * \author Tianqi Chen
  */
+#include <rabit/rabit.h>
+#include <xgboost/tree_updater.h>
+
 #include <string>
 #include <memory>
 
-#include <rabit/rabit.h>
-
-#include "xgboost/tree_updater.h"
 #include "xgboost/base.h"
 #include "xgboost/json.h"
 #include "./param.h"
 #include "../common/io.h"
 #include "../common/timer.h"
-
 namespace xgboost {
 namespace tree {
 
diff --git a/src/tree/updater_quantile_hist.cc b/src/tree/updater_quantile_hist.cc
index 51b47a386e3b..30eb01a726ee 100644
--- a/src/tree/updater_quantile_hist.cc
+++ b/src/tree/updater_quantile_hist.cc
@@ -4,6 +4,9 @@
  * \brief use quantized feature values to construct a tree
  * \author Philip Cho, Tianqi Checn, Egor Smirnov
  */
+#include <dmlc/timer.h>
+#include <rabit/rabit.h>
+
 #include <cmath>
 #include <memory>
 #include <vector>
@@ -14,8 +17,6 @@
 #include <string>
 #include <utility>
 
-#include <dmlc/timer.h>
-#include <rabit/rabit.h>
 #include "xgboost/logging.h"
 #include "xgboost/tree_updater.h"
 
diff --git a/src/tree/updater_quantile_hist.h b/src/tree/updater_quantile_hist.h
index 388584df3d89..d74c16f72ceb 100644
--- a/src/tree/updater_quantile_hist.h
+++ b/src/tree/updater_quantile_hist.h
@@ -7,6 +7,10 @@
 #ifndef XGBOOST_TREE_UPDATER_QUANTILE_HIST_H_
 #define XGBOOST_TREE_UPDATER_QUANTILE_HIST_H_
 
+#include <dmlc/timer.h>
+#include <rabit/rabit.h>
+#include <xgboost/tree_updater.h>
+
 #include <memory>
 #include <vector>
 #include <string>
@@ -15,13 +19,8 @@
 #include <unordered_map>
 #include <utility>
 
-#include <dmlc/timer.h>
-#include <rabit/rabit.h>
-
-#include "xgboost/tree_updater.h"
 #include "xgboost/data.h"
 #include "xgboost/json.h"
-
 #include "constraints.h"
 #include "./param.h"
 #include "./split_evaluator.h"
diff --git a/src/tree/updater_refresh.cc b/src/tree/updater_refresh.cc
index 5748d9b3a768..d63d88c802d9 100644
--- a/src/tree/updater_refresh.cc
+++ b/src/tree/updater_refresh.cc
@@ -4,12 +4,12 @@
  * \brief refresh the statistics and leaf value on the tree on the dataset
  * \author Tianqi Chen
  */
+#include <rabit/rabit.h>
+#include <xgboost/tree_updater.h>
+
 #include <vector>
 #include <limits>
 
-#include <rabit/rabit.h>
-
-#include "xgboost/tree_updater.h"
 #include "xgboost/json.h"
 #include "./param.h"
 #include "../common/io.h"
diff --git a/src/tree/updater_skmaker.cc b/src/tree/updater_skmaker.cc
index b4468b517906..69cb4e58bbed 100644
--- a/src/tree/updater_skmaker.cc
+++ b/src/tree/updater_skmaker.cc
@@ -5,13 +5,12 @@
           a refresh is needed to make the statistics exactly correct
  * \author Tianqi Chen
  */
+#include <rabit/rabit.h>
+#include <xgboost/base.h>
+#include <xgboost/tree_updater.h>
 #include <vector>
 #include <algorithm>
 
-#include <rabit/rabit.h>
-#include "xgboost/base.h"
-#include "xgboost/tree_updater.h"
-
 #include "../common/quantile.h"
 #include "../common/group_data.h"
 #include "./updater_basemaker-inl.h"
diff --git a/src/tree/updater_sync.cc b/src/tree/updater_sync.cc
index 6ec814d4f754..578bfb83cea9 100644
--- a/src/tree/updater_sync.cc
+++ b/src/tree/updater_sync.cc
@@ -3,11 +3,11 @@
  * \file updater_sync.cc
  * \brief synchronize the tree in all distributed nodes
  */
+#include <xgboost/tree_updater.h>
 #include <vector>
 #include <string>
 #include <limits>
 
-#include "xgboost/tree_updater.h"
 #include "xgboost/json.h"
 #include "../common/io.h"