refactor: improve the perf in query

cmake/cmake.define

@@ -123,14 +123,20 @@ ELSE ()
         SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k")
     ENDIF ()
 
-    MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}")
     IF (TD_INTEL_64 OR TD_INTEL_32)
         ADD_DEFINITIONS("-msse4.2")
         IF("${FMA_SUPPORT}" MATCHES "true")
-            MESSAGE(STATUS "turn fma function support on")
+            MESSAGE(STATUS "fma function supported")
             ADD_DEFINITIONS("-mfma")
         ELSE ()
-            MESSAGE(STATUS "turn fma function support off")
+            MESSAGE(STATUS "fma function NOT supported")
+        ENDIF()
+
+        IF("${SIMD_SUPPORT}" MATCHES "true")
+            ADD_DEFINITIONS("-mavx -mavx2")
+            MESSAGE(STATUS "SIMD instructions (AVX/AVX2) is ACTIVATED")
+        ELSE()
+            MESSAGE(STATUS "SIMD instruction (AVX/AVX2)is NOT ACTIVATED")
         ENDIF()
     ENDIF ()
 

cmake/cmake.platform

@@ -1,20 +1,17 @@
 cmake_minimum_required(VERSION 3.0)
 
-MESSAGE("Current system is ${CMAKE_SYSTEM_NAME}")
-
 # init
 SET(TD_LINUX FALSE)
 SET(TD_WINDOWS FALSE)
 SET(TD_DARWIN FALSE)
 
-MESSAGE("Compiler ID: ${CMAKE_CXX_COMPILER_ID}")
 if(CMAKE_COMPILER_IS_GNUCXX MATCHES 1)
     set(CXX_COMPILER_IS_GNU TRUE)
 else()
     set(CXX_COMPILER_IS_GNU FALSE)
 endif()
 
-MESSAGE("Current system name is ${CMAKE_SYSTEM_NAME}.")
+MESSAGE("Current system: ${CMAKE_SYSTEM_NAME}")
 
 IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
 
@@ -26,6 +23,8 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
         set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup")
     ENDIF ()
 
+    MESSAGE("Current system processor: ${CMAKE_SYSTEM_PROCESSOR}")
+
     IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
 
         SET(TD_LINUX TRUE)
@@ -44,7 +43,6 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
         SET(OSTYPE "macOS")
         ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare")
 
-        MESSAGE("Current system processor is ${CMAKE_SYSTEM_PROCESSOR}.")
         IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64")
             MESSAGE("Current system arch is arm64")
             SET(TD_DARWIN_64 TRUE)
@@ -80,24 +78,22 @@ ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
 ENDIF()
 
 IF ("${CPUTYPE}" STREQUAL "")
-  MESSAGE(STATUS "The current platform " ${CMAKE_SYSTEM_PROCESSOR} " is detected")
-
   IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)")
-    MESSAGE(STATUS "The current platform is amd64")
+    MESSAGE(STATUS "Current platform is amd64")
     SET(PLATFORM_ARCH_STR "amd64")
     SET(TD_INTEL_64 TRUE)
   ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)")
-    MESSAGE(STATUS "The current platform is x86")
+    MESSAGE(STATUS "Current platform is x86")
     SET(PLATFORM_ARCH_STR "i386")
     SET(TD_INTEL_32 TRUE)
   ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l")
-    MESSAGE(STATUS "The current platform is aarch32")
+    MESSAGE(STATUS "Current platform is aarch32")
     SET(PLATFORM_ARCH_STR "arm")
     SET(TD_ARM_32 TRUE)
     ADD_DEFINITIONS("-D_TD_ARM_")
     ADD_DEFINITIONS("-D_TD_ARM_32")
   ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)")
-    MESSAGE(STATUS "The current platform is aarch64")
+    MESSAGE(STATUS "Current platform is aarch64")
     SET(PLATFORM_ARCH_STR "arm64")
     SET(TD_ARM_64 TRUE)
     ADD_DEFINITIONS("-D_TD_ARM_")
@@ -151,5 +147,5 @@ ENDIF ()
 
 MESSAGE(STATUS "platform arch:" ${PLATFORM_ARCH_STR})
 
-MESSAGE("C Compiler ID: ${CMAKE_C_COMPILER_ID}")
-MESSAGE("CXX Compiler ID: ${CMAKE_CXX_COMPILER_ID}")
+MESSAGE("C Compiler: ${CMAKE_C_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_C_COMPILER_VERSION})")
+MESSAGE("CXX Compiler: ${CMAKE_CXX_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_CXX_COMPILER_VERSION})")

cmake/cmake.version

@@ -26,7 +26,7 @@ ELSEIF (HAVE_GIT)
     SET(TD_VER_GIT "no git commit id")
   ENDIF ()
 ELSE ()
-  message(STATUS "no git cmd")
+  message(STATUS "no git found")
   SET(TD_VER_GIT "no git commit id")
 ENDIF ()
 
@@ -70,9 +70,9 @@ MESSAGE(STATUS "compatible: " ${TD_VER_COMPATIBLE})
 MESSAGE(STATUS "commit id:  " ${TD_VER_GIT})
 MESSAGE(STATUS "build date: " ${TD_VER_DATE})
 MESSAGE(STATUS "build type: " ${CMAKE_BUILD_TYPE})
-MESSAGE(STATUS "type: " ${TD_VER_VERTYPE})
-MESSAGE(STATUS "cpu:  " ${TD_VER_CPUTYPE})
-MESSAGE(STATUS "os:   " ${TD_VER_OSTYPE})
+MESSAGE(STATUS "type:       " ${TD_VER_VERTYPE})
+MESSAGE(STATUS "cpu:        " ${TD_VER_CPUTYPE})
+MESSAGE(STATUS "os:         " ${TD_VER_OSTYPE})
 MESSAGE(STATUS "============= compile version parameter information end  ============= ")
 
 STRING(REPLACE "." "_" TD_LIB_VER_NUMBER ${TD_VER_NUMBER})

include/common/tcommon.h

@@ -225,13 +225,13 @@ typedef struct SVarColAttr {
 // pBlockAgg->numOfNull == info.rows, all data are null
 // pBlockAgg->numOfNull == 0, no data are null.
 typedef struct SColumnInfoData {
-  char*         pData;    // the corresponding block data in memory
+  char*         pData;       // the corresponding block data in memory
   union {
     char*       nullbitmap;  // bitmap, one bit for each item in the list
     SVarColAttr varmeta;
   };
-  SColumnInfo info;     // column info
-  bool        hasNull;  // if current column data has null value.
+  SColumnInfo   info;        // column info
+  bool          hasNull;     // if current column data has null value.
 } SColumnInfoData;
 
 typedef struct SQueryTableDataCond {

include/common/tdatablock.h

@@ -24,26 +24,13 @@
 extern "C" {
 #endif
 
-typedef struct SCorEpSet {
-  int32_t version;
-  SEpSet  epSet;
-} SCorEpSet;
-
 typedef struct SBlockOrderInfo {
   bool             nullFirst;
   int32_t          order;
   int32_t          slotId;
   SColumnInfoData* pColData;
 } SBlockOrderInfo;
 
-int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp);
-void    addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port);
-
-bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2);
-
-void   updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet);
-SEpSet getEpSet_s(SCorEpSet* pEpSet);
-
 #define NBIT                     (3u)
 #define BitPos(_n)               ((_n) & ((1 << NBIT) - 1))
 #define BMCharPos(bm_, r_)       ((bm_)[(r_) >> NBIT])

include/common/tglobal.h

@@ -44,6 +44,8 @@ extern int32_t tsCompatibleModel;
 extern bool    tsPrintAuth;
 extern int64_t tsTickPerMin[3];
 extern int32_t tsCountAlwaysReturnValue;
+extern float   tsSelectivityRatio;
+extern int32_t tsTagFilterResCacheSize;
 
 // queue & threads
 extern int32_t tsNumOfRpcThreads;

include/common/tmisce.h

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TMISCE_H
+#define TDENGINE_TMISCE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "tmsg.h"
+
+typedef struct SCorEpSet {
+  int32_t version;
+  SEpSet  epSet;
+} SCorEpSet;
+
+int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp);
+void    addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port);
+
+bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2);
+
+void   updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet);
+SEpSet getEpSet_s(SCorEpSet* pEpSet);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TMISCE_H

include/common/tmsg.h

@@ -649,34 +649,6 @@ int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pR
 int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp);
 void    tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp);
 
-typedef struct {
-  int16_t lowerRelOptr;
-  int16_t upperRelOptr;
-  int16_t filterstr;  // denote if current column is char(binary/nchar)
-
-  union {
-    struct {
-      int64_t lowerBndi;
-      int64_t upperBndi;
-    };
-    struct {
-      double lowerBndd;
-      double upperBndd;
-    };
-    struct {
-      int64_t pz;
-      int64_t len;
-    };
-  };
-} SColumnFilterInfo;
-
-typedef struct {
-  int16_t numOfFilters;
-  union {
-    int64_t            placeholder;
-    SColumnFilterInfo* filterInfo;
-  };
-} SColumnFilterList;
 /*
  * for client side struct, only column id, type, bytes are necessary
  * But for data in vnode side, we need all the following information.
@@ -687,10 +659,10 @@ typedef struct {
     int16_t  slotId;
   };
 
-  int8_t  type;
-  int32_t bytes;
   uint8_t precision;
   uint8_t scale;
+  int32_t bytes;
+  int8_t  type;
 } SColumnInfo;
 
 typedef struct STimeWindow {

include/libs/executor/executor.h

@@ -142,8 +142,11 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table
  */
 
 int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bool* hasMore, SLocalFetch* pLocal);
+
 int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pBlock, uint64_t* useconds);
 
+void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo);
+
 /**
  * kill the ongoing query asynchronously
  * @param tinfo  qhandle

include/libs/function/function.h

@@ -115,7 +115,7 @@ typedef struct SInputColumnInfoData {
   int32_t           startRowIndex;    // handle started row index
   int32_t           numOfRows;        // the number of rows needs to be handled
   int32_t           numOfInputCols;   // PTS is not included
-  bool              colDataAggIsSet;  // if agg is set or not
+  bool              colDataSMAIsSet;  // if agg is set or not
   SColumnInfoData  *pPTS;             // primary timestamp column
   SColumnInfoData **pData;
   SColumnDataAgg  **pColumnDataAgg;

include/os/os.h

@@ -43,6 +43,7 @@ extern "C" {
 #include <sys/utsname.h>
 #include <sys/wait.h>
 #include <termios.h>
+#include <cpuid.h>
 
 #if defined(DARWIN)
 #else
@@ -82,6 +83,12 @@ extern "C" {
 #include <wchar.h>
 #include <wctype.h>
 
+#if __AVX__
+#include <immintrin.h>
+#elif __SSE4_2__
+#include <nmmintrin.h>
+#endif
+
 #include "osThread.h"
 
 #include "osAtomic.h"

include/os/osDef.h

@@ -168,22 +168,22 @@ void syslog(int unused, const char *format, ...);
     }                            \
   } while (0)
 
-#define DEFAULT_DOUBLE_COMP(x, y) \
-  do {                            \
-    if (isnan(x) && isnan(y)) {   \
-      return 0;                   \
-    }                             \
-    if (isnan(x)) {               \
-      return -1;                  \
-    }                             \
-    if (isnan(y)) {               \
-      return 1;                   \
-    }                             \
-    if ((x) == (y)) {             \
-      return 0;                   \
-    } else {                      \
-      return (x) < (y) ? -1 : 1;  \
-    }                             \
+#define DEFAULT_DOUBLE_COMP(x, y)         \
+  do {                                    \
+    if (isnan(x) && isnan(y)) {           \
+      return 0;                           \
+    }                                     \
+    if (isnan(x)) {                       \
+      return -1;                          \
+    }                                     \
+    if (isnan(y)) {                       \
+      return 1;                           \
+    }                                     \
+    if (fabs((x) - (y)) <= DBL_EPSILON) { \
+      return 0;                           \
+    } else {                              \
+      return (x) < (y) ? -1 : 1;          \
+    }                                     \
   } while (0)
 
 #define DEFAULT_FLOAT_COMP(x, y) DEFAULT_DOUBLE_COMP(x, y)

include/os/osEnv.h

@@ -36,6 +36,11 @@ extern int64_t         tsStreamMax;
 extern float           tsNumOfCores;
 extern int64_t         tsTotalMemoryKB;
 extern char           *tsProcPath;
+extern char            tsSIMDEnable;
+extern char            tsSSE42Enable;
+extern char            tsAVXEnable;
+extern char            tsAVX2Enable;
+extern char            tsFMAEnable;
 
 extern char configDir[];
 extern char tsDataDir[];

include/os/osMemory.h

@@ -37,6 +37,7 @@ void    taosMemoryFree(void *ptr);
 int64_t taosMemorySize(void *ptr);
 void    taosPrintBackTrace();
 void    taosMemoryTrim(int32_t size);
+void   *taosMemoryMallocAlign(uint32_t alignment, int64_t size);
 
 #define taosMemoryFreeClear(ptr)   \
   do {                             \

include/os/osSysinfo.h

@@ -40,6 +40,7 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen);
 int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores);
 int32_t taosGetCpuCores(float *numOfCores);
 void    taosGetCpuUsage(double *cpu_system, double *cpu_engine);
+int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma);
 int32_t taosGetTotalMemory(int64_t *totalKB);
 int32_t taosGetProcMemory(int64_t *usedKB);
 int32_t taosGetSysMemory(int64_t *usedKB);

include/util/tarray.h

@@ -104,6 +104,15 @@ static FORCE_INLINE void* taosArrayPush(SArray* pArray, const void* pData) {
   return taosArrayAddBatch(pArray, pData, 1);
 }
 
+/**
+ * @brief reserve the capacity of the array
+ *
+ * @param pArray
+ * @param num
+ * @return void* the start position of the reserved memory
+ */
+void* taosArrayReserve(SArray* pArray, int32_t num);
+
 /**
  *
  * @param pArray

include/util/tlist.h

@@ -225,7 +225,7 @@ void      *tdListFree(SList *list);
 void       tdListPrependNode(SList *list, SListNode *node);
 void       tdListAppendNode(SList *list, SListNode *node);
 int32_t    tdListPrepend(SList *list, void *data);
-int32_t    tdListAppend(SList *list, void *data);
+int32_t    tdListAppend(SList *list, const void *data);
 SListNode *tdListPopHead(SList *list);
 SListNode *tdListPopTail(SList *list);
 SListNode *tdListGetHead(SList *list);