Merge pull request #22870 from taosdata/fix/xsren/TS-3957/readLineOnW…

…indows_main Fix/xsren/ts 3957/read line on windows main
taosdata · Sep 12, 2023 · 4f6f953 · 4f6f953
2 parents 4d33387 + 16dee56
commit 4f6f953
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 8 deletions.
diff --git a/cmake/cmake.platform b/cmake/cmake.platform
@@ -176,6 +176,14 @@ IF(APPLE)
     set(THREADS_PREFER_PTHREAD_FLAG ON)
 ENDIF()
 
+IF(TD_WINDOWS)
+  IF(MSVC)
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /STACK:8388608")
+  ELSEIF(MINGW)
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--stack,8388608")
+  ENDIF()
+ENDIF()
+
 MESSAGE(STATUS "Platform arch:" ${PLATFORM_ARCH_STR})
 
 set(TD_DEPS_DIR "x86")

diff --git a/include/os/osFile.h b/include/os/osFile.h
@@ -112,6 +112,8 @@ int32_t taosGetErrorFile(TdFilePtr pFile);
 
 int32_t taosCompressFile(char *srcFileName, char *destFileName);
 
+int32_t taosSetFileHandlesLimit();
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
@@ -1598,6 +1598,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
     if (taosSetS3Cfg(tsCfg) != 0) return -1;
   }
   taosSetSystemCfg(tsCfg);
+  if (taosSetFileHandlesLimit() != 0) return -1;
 
   cfgDumpCfg(tsCfg, tsc, false);
 

diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c
@@ -819,14 +819,38 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) {
     return -1;
   }
 #ifdef WINDOWS
-  *ptrBuf = taosMemoryMalloc(1024);
+  size_t bufferSize = 512;
+  *ptrBuf = taosMemoryMalloc(bufferSize);
   if (*ptrBuf == NULL) return -1;
-  if (fgets(*ptrBuf, 1023, pFile->fp) == NULL) {
-    taosMemoryFreeClear(*ptrBuf);
-    return -1;
+
+  size_t bytesRead = 0;
+  size_t totalBytesRead = 0;
+
+  while (1) {
+    char *result = fgets(*ptrBuf + totalBytesRead, bufferSize - totalBytesRead, pFile->fp);
+    if (result == NULL) {
+      taosMemoryFreeClear(*ptrBuf);
+      return -1;
+    }
+    bytesRead = strlen(*ptrBuf + totalBytesRead);
+    totalBytesRead += bytesRead;
+
+    if (totalBytesRead < bufferSize - 1 || (*ptrBuf)[totalBytesRead - 1] == '\n') {
+      break;
+    }
+
+    bufferSize += 512;
+    void* newBuf = taosMemoryRealloc(*ptrBuf, bufferSize);
+    if (newBuf == NULL) {
+      taosMemoryFreeClear(*ptrBuf);
+      return -1;
+    }
+
+    *ptrBuf = newBuf;
   }
-  (*ptrBuf)[1023] = 0;
-  return strlen(*ptrBuf);
+
+  (*ptrBuf)[totalBytesRead] = '\0';
+  return totalBytesRead;
 #else
   size_t len = 0;
   return getline(ptrBuf, &len, pFile->fp);
@@ -904,10 +928,16 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) {
     goto cmp_end;
   }
 
-  dstFp = gzdopen(pFile->fd, "wb6f");
+  // Both gzclose() and fclose() will close the associated fd, so they need to have different fds.
+  FileFd gzFd = dup(pFile->fd);
+  if (gzFd < 0) {
+    ret = -4;
+    goto cmp_end;
+  }
+  dstFp = gzdopen(gzFd, "wb6f");
   if (dstFp == NULL) {
     ret = -3;
-    taosCloseFile(&pFile);
+    close(gzFd);
     goto cmp_end;
   }
 
@@ -932,3 +962,12 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) {
 
   return ret;
 }
+
+int32_t taosSetFileHandlesLimit() {
+#ifdef WINDOWS
+  const int max_handles = 8192;
+  int       res = _setmaxstdio(max_handles);
+  return res == max_handles ? 0 : -1;
+#endif
+  return 0;
+}