MDEV-37463 - minimize recompilation required for tpool_min

On Linux, introduce object library tpool_common, which contains non-AIO
code.It is linked to both full tpool library (with AIO),
and tpool_min(without AIO).

As a result, building tpool_min now requires only compiling a single
small source file.

aio::synchronous(), aio::flush_synchronous() are moved into aio specific
source file, from common threadpool_generic.cc, in order to avoid ABI
breakage (size of aiocb depends on HAVE_LIBAIO etc compile options on Linux)

Author: vaintroub

tpool/CMakeLists.txt

@@ -1,6 +1,5 @@
 
-SET(SOURCES
-  aio_simulated.cc
+SET(COMMON_SOURCES
   tpool_structs.h
   tpool.h
   tpool_generic.cc
@@ -9,14 +8,21 @@ SET(SOURCES
   wait_notification.cc
 )
 
-ADD_LIBRARY(tpool OBJECT ${SOURCES})
 
-TARGET_INCLUDE_DIRECTORIES(tpool PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}"
-  ${PROJECT_SOURCE_DIR}/include)
-IF(WIN32)
-  TARGET_SOURCES(tpool PRIVATE tpool_win.cc aio_win.cc)
-ELSEIF(CMAKE_SYSTEM_NAME STREQUAL "Linux")
-  TARGET_SOURCES(tpool PRIVATE aio_linux.cc)
+IF(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")
+  ADD_LIBRARY(tpool OBJECT ${COMMON_SOURCES}  aio_simulated.cc)
+  IF(WIN32)
+    TARGET_SOURCES(tpool PRIVATE tpool_win.cc aio_win.cc)
+  ENDIF()
+  TARGET_INCLUDE_DIRECTORIES(tpool PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/include)
+  ADD_LIBRARY(tpool_min ALIAS tpool)
+ELSE() # Linux
+  ADD_LIBRARY(tpool_common OBJECT ${COMMON_SOURCES})
+  # Ensure that aiocb used as opaque in common code, or ABI will be broken
+  TARGET_COMPILE_DEFINITIONS(tpool_common PRIVATE -DTPOOL_OPAQUE_AIOCB)
+  TARGET_INCLUDE_DIRECTORIES(tpool_common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/include)
+  ADD_LIBRARY(tpool STATIC aio_linux.cc aio_simulated.cc)
+  TARGET_LINK_LIBRARIES(tpool PUBLIC tpool_common)
   OPTION(WITH_URING "Require that io_uring be used" OFF)
   OPTION(WITH_LIBAIO "Require that libaio is used" OFF)
   IF(WITH_URING)
@@ -57,12 +63,10 @@ ELSEIF(CMAKE_SYSTEM_NAME STREQUAL "Linux")
     # Create a "light" version of the library that does not link
     # to additional io libs, and is not compiled with
     # -DHAVE_LIBAIO or -DHAVE_URING flags.
-    ADD_LIBRARY(tpool_min STATIC ${SOURCES} aio_linux.cc)
-    TARGET_INCLUDE_DIRECTORIES(tpool_min
-      PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}  ${PROJECT_SOURCE_DIR}/include)
+    ADD_LIBRARY(tpool_min STATIC aio_linux.cc)
+    TARGET_LINK_LIBRARIES(tpool_min PUBLIC tpool_common)
+  ELSE()
+    ADD_LIBRARY(tpool_min ALIAS tpool)
   ENDIF()
 ENDIF()
 
-IF(NOT TARGET tpool_min)
-  ADD_LIBRARY(tpool_min ALIAS tpool)
-ENDIF()

tpool/aio_simulated.cc

@@ -162,4 +162,50 @@ aio *create_simulated_aio(thread_pool *tp)
   return new simulated_aio(tp);
 }
 
+void aio::finish_synchronous(aiocb *cb)
+{
+  if (!cb->m_err && cb->m_ret_len != cb->m_len)
+  {
+    /* partial read/write */
+    cb->m_buffer= (char *) cb->m_buffer + cb->m_ret_len;
+    cb->m_len-= (unsigned int) cb->m_ret_len;
+    cb->m_offset+= cb->m_ret_len;
+    synchronous(cb);
+  }
+}
+
+/**
+  Process the cb synchronously
+*/
+void aio::synchronous(aiocb *cb)
+{
+  intptr_t ret_len;
+  int err= 0;
+  switch (cb->m_opcode)
+  {
+  case aio_opcode::AIO_PREAD:
+    ret_len= pread(cb->m_fh, cb->m_buffer, cb->m_len, cb->m_offset);
+    break;
+  case aio_opcode::AIO_PWRITE:
+    ret_len= pwrite(cb->m_fh, cb->m_buffer, cb->m_len, cb->m_offset);
+    break;
+  default:
+    abort();
+  }
+
+  if (ret_len < 0)
+  {
+#ifdef _WIN32
+    err= GetLastError();
+#else
+    err= errno;
+#endif
+    ret_len= 0;
+  }
+  cb->m_ret_len = ret_len;
+  cb->m_err = err;
+  if (ret_len)
+    finish_synchronous(cb);
+}
+
 } // namespace tpool

tpool/tpool.h

@@ -138,6 +138,9 @@ enum class aio_opcode
 };
 constexpr size_t MAX_AIO_USERDATA_LEN= 4 * sizeof(void*);
 
+#ifdef TPOOL_OPAQUE_AIOCB
+struct aiocb;
+#else
 /** IO control block, includes parameters for the IO, and the callback*/
 
 struct aiocb
@@ -181,6 +184,7 @@ struct aiocb
   }
 };
 
+#endif /* TPOOL_OPAQUE_AIOCB */
 
 /**
  AIO interface
@@ -202,17 +206,7 @@ class aio
 protected:
   static void synchronous(aiocb *cb);
   /** finish a partial read/write callback synchronously */
-  static inline void finish_synchronous(aiocb *cb)
-  {
-    if (!cb->m_err && cb->m_ret_len != cb->m_len)
-    {
-      /* partial read/write */
-      cb->m_buffer= (char *) cb->m_buffer + cb->m_ret_len;
-      cb->m_len-= (unsigned int) cb->m_ret_len;
-      cb->m_offset+= cb->m_ret_len;
-      synchronous(cb);
-    }
-  }
+  static void finish_synchronous(aiocb *cb);
 };
 
 class timer

tpool/tpool_generic.cc

@@ -46,43 +46,6 @@ namespace tpool
   static const std::chrono::milliseconds LONG_TASK_DURATION = std::chrono::milliseconds(500);
   static const int  OVERSUBSCRIBE_FACTOR = 2;
 
-/**
-  Process the cb synchronously
-*/
-void aio::synchronous(aiocb *cb)
-{
-#ifdef _WIN32
-  size_t ret_len;
-#else
-  ssize_t ret_len;
-#endif
-  int err= 0;
-  switch (cb->m_opcode)
-  {
-  case aio_opcode::AIO_PREAD:
-    ret_len= pread(cb->m_fh, cb->m_buffer, cb->m_len, cb->m_offset);
-    break;
-  case aio_opcode::AIO_PWRITE:
-    ret_len= pwrite(cb->m_fh, cb->m_buffer, cb->m_len, cb->m_offset);
-    break;
-  default:
-    abort();
-  }
-#ifdef _WIN32
-  if (static_cast<int>(ret_len) < 0)
-    err= GetLastError();
-#else
-  if (ret_len < 0)
-  {
-    err= errno;
-    ret_len= 0;
-  }
-#endif
-  cb->m_ret_len = ret_len;
-  cb->m_err = err;
-  if (ret_len)
-    finish_synchronous(cb);
-}
 
 
 /**