Add daemon watch mode to ecompile (#630)

* Add libefsw at 3419347

* saving work

* some cleanup

* move to fs::paths

* saving work.. maybe hitting a limit with number of watched dirs?

* migrate to watching only ecompile.cfg specified folders, cleanup/doc

* Display summary between compilations in watch mode; fix cache reuse

* Fix inf loop; some cmake, format changes...

* some windows compilation fixes... tweaks to windows cmake?

* Update Efsw.cmake for build lib changes

* removed debug flag

* silence efsw build
dont add antlr project if it already exists

* Address review comments:
- use std::move on efswwatcher add_*

* add docs

* Add WatchModeByDefault ecompile.cfg option

* iniital work on tests

* Add CI actions

* some changes (passes locally on win, mac, linux)

* Fix fixtures dependency; use ecompile verbose mode

* Add core-changes

* Send ctrl+c instead of killing process

* Proper Python file naming

* Kill ecompile process on windows, SIGINT otherwise

---------

Co-authored-by: turleypol <turley@polserver.com>

.github/workflows/build.yml

@@ -158,6 +158,13 @@ jobs:
           cmake --build . --config ${{ matrix.config.build-type }} --target test_pol_only
           #env:
           #POLCORE_TEST_FILTER: testitems
+      - name: Test (Ecompile Watch)
+        if: matrix.config.nopch == 0
+        run: |
+          cd bin-build
+          cmake --build . --config ${{ matrix.config.build-type }} --target test_ecompile_watch
+          #env:
+          #POLCORE_TEST_FILTER: testitems
       - name: Upload coretest folder
         if: matrix.config.nopch == 0 && always()
         uses: actions/upload-artifact@v3

.github/workflows/buildmacos.yml

@@ -98,6 +98,10 @@ jobs:
         run: |
           cd bin-build
           cmake --build . --config ${{ matrix.config.build-type }} --target test_pol_only
+      - name: Test (Ecompile Watch)
+        run: |
+          cd bin-build
+          cmake --build . --config ${{ matrix.config.build-type }} --target test_ecompile_watch
       - name: Upload shard test log
         uses: actions/upload-artifact@v3
         with:

.github/workflows/buildsan.yml

@@ -136,6 +136,10 @@ jobs:
           cmake --build . --config ${{ matrix.config.build-type }} --target test_pol_only
           #env:
           #POLCORE_TEST_FILTER: testitems
+      - name: Test (Ecompile Watch)
+        run: |
+          cd bin-build
+          cmake --build . --config ${{ matrix.config.build-type }} --target test_ecompile_watch
       - name: Upload shard test log
         uses: actions/upload-artifact@v3
         with:

.github/workflows/coverage.yml

@@ -115,6 +115,10 @@ jobs:
         run: |
           cd bin-build
           cmake --build . --config ${{ matrix.config.build-type }} --target test_pol_only
+      - name: Test (Ecompile Watch)
+        run: |
+          cd bin-build
+          cmake --build . --config ${{ matrix.config.build-type }} --target test_ecompile_watch
       - name: Coverage
         run: |
           cd bin-build

.gitignore

@@ -9,4 +9,5 @@ Makefile
 /CMakeSettings.json
 __pycache__
 gen/
-build/
+build/
+.vscode

CMakeLists.txt

@@ -130,6 +130,7 @@ include(cmake/Boost.cmake)
 include(cmake/Benchmark.cmake)
 include(cmake/Cppdap.cmake)
 include(cmake/Curl.cmake)
+include(cmake/Efsw.cmake)
 include(cmake/EscriptGrammarLib.cmake)
 include(cmake/escript_grammar.cmake)
 include(cmake/Fmt.cmake)
@@ -229,7 +230,7 @@ set_target_properties(test_pol PROPERTIES FOLDER !BuildTargets)
 message("  test_pol_fixture")
 add_custom_target(test_pol_fixture
   COMMAND ${CMAKE_CTEST_COMMAND}
-    -j2 --output-on-failure -E "(escript_|shard_test_|unittest_)" -C ${CMAKE_BUILD_TYPE}
+    -j2 --output-on-failure -E "(escript_|shard_test_|unittest_|ecompile_watch_test)" -C ${CMAKE_BUILD_TYPE}
   COMMENT "Run pol fixture tests"
   VERBATIM
   USES_TERMINAL
@@ -249,6 +250,18 @@ set_target_properties(test_pol_only PROPERTIES EXCLUDE_FROM_ALL TRUE)
 set_target_properties(test_pol_only PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD TRUE)
 set_target_properties(test_pol_only PROPERTIES FOLDER !BuildTargets)
 
+message("  test_ecompile_watch")
+add_custom_target(test_ecompile_watch
+  COMMAND ${CMAKE_CTEST_COMMAND}
+    --verbose -R ecompile_watch_test -C ${CMAKE_BUILD_TYPE}
+  COMMENT "Run ecompile watch tests"
+  VERBATIM
+  USES_TERMINAL
+)
+set_target_properties(test_ecompile_watch PROPERTIES EXCLUDE_FROM_ALL TRUE)
+set_target_properties(test_ecompile_watch PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD TRUE)
+set_target_properties(test_ecompile_watch PROPERTIES FOLDER !BuildTargets)
+
 message("  unittest_pol")
 add_custom_target(unittest_pol
   COMMAND ${CMAKE_CTEST_COMMAND}

bin-build/.gitignore

@@ -29,3 +29,4 @@ ipch
 module_defs
 coretest
 fmt
+efsw

cmake/Antlr.cmake

@@ -14,34 +14,35 @@ else()
   set(ANTLR_LIB "${ANTLR_INSTALL_DIR}/lib/antlr4-runtime-static.lib")
 endif()
 
-ExternalProject_Add(libantlr_ext
-  SOURCE_DIR  ${ANTLR_SOURCE_DIR}
-  PREFIX antlr
-  LIST_SEPARATOR |
-  CMAKE_ARGS 
-    -DCMAKE_BUILD_TYPE=Release
-    -DCMAKE_INSTALL_PREFIX=${ANTLR_INSTALL_DIR}
-    -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-    -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-    -DWITH_LIBCXX=Off
-    -DANTLR_BUILD_SHARED=Off
-    -DANTLR_BUILD_CPP_TESTS=Off
-    -DCMAKE_OSX_ARCHITECTURES=${PIPED_OSX_ARCHITECTURES}
-    -DCMAKE_POSITION_INDEPENDENT_CODE=ON
-  BINARY_DIR ${ANTLR_SOURCE_DIR}/build
-  BUILD_COMMAND ${CMAKE_COMMAND} --build . --config Release
-  INSTALL_COMMAND ${CMAKE_COMMAND} --build . --config Release --target install
-
-  BUILD_BYPRODUCTS ${ANTLR_LIB}
-  LOG_DOWNLOAD 1
-  LOG_CONFIGURE 1
-  LOG_BUILD 1
-  LOG_INSTALL 1
-  LOG_OUTPUT_ON_FAILURE 1
-  EXCLUDE_FROM_ALL 1
-)
 if (NOT EXISTS ${ANTLR_LIB})
-  file(MAKE_DIRECTORY ${ANTLR_INCLUDE_DIR}) #directory has to exist during configure
+  ExternalProject_Add(libantlr_ext
+    SOURCE_DIR  ${ANTLR_SOURCE_DIR}
+    PREFIX antlr
+    LIST_SEPARATOR |
+    CMAKE_ARGS
+      -DCMAKE_BUILD_TYPE=Release
+      -DCMAKE_INSTALL_PREFIX=${ANTLR_INSTALL_DIR}
+      -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+      -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+      -DWITH_LIBCXX=Off
+      -DANTLR_BUILD_SHARED=Off
+      -DANTLR_BUILD_CPP_TESTS=Off
+      -DCMAKE_OSX_ARCHITECTURES=${PIPED_OSX_ARCHITECTURES}
+      -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+    BINARY_DIR ${ANTLR_SOURCE_DIR}/build
+    BUILD_COMMAND ${CMAKE_COMMAND} --build . --config Release
+    INSTALL_COMMAND ${CMAKE_COMMAND} --build . --config Release --target install
+
+    BUILD_BYPRODUCTS ${ANTLR_LIB}
+    LOG_DOWNLOAD 1
+    LOG_CONFIGURE 1
+    LOG_BUILD 1
+    LOG_INSTALL 1
+    LOG_OUTPUT_ON_FAILURE 1
+    EXCLUDE_FROM_ALL 1
+  )
+else()
+  message("  - already built")
 endif()
 
 # imported target to add include/lib dir and additional dependencies
@@ -53,4 +54,7 @@ set_target_properties(libantlr PROPERTIES
   INTERFACE_COMPILE_DEFINITIONS ANTLR4CPP_STATIC
   FOLDER 3rdParty
 )
-add_dependencies(libantlr libantlr_ext)
+if (NOT EXISTS ${ANTLR_LIB})
+  file(MAKE_DIRECTORY ${ANTLR_INCLUDE_DIR}) #directory has to exist during configure
+  add_dependencies(libantlr libantlr_ext)
+endif()

cmake/Efsw.cmake

@@ -0,0 +1,71 @@
+message("* libefsw")
+
+set (EFSW_REPOSITORY "https://github.com/SpartanJ/efsw.git")
+set (EFSW_VERSION_REF "3419347")
+
+set (EFSW_SOURCE_DIR "${POL_EXT_LIB_DIR}/efsw-${EFSW_VERSION_REF}")
+set (EFSW_INSTALL_DIR "${EFSW_SOURCE_DIR}/build")
+
+if (${windows})
+  set (EFSW_LIB "${EFSW_INSTALL_DIR}/lib/efsw-static.lib" )
+else()
+  set (EFSW_LIB "${EFSW_INSTALL_DIR}/lib/libefsw-static.a" )
+endif()
+
+set (EFSW_ARGS ${EFSW_FLAGS}
+  -DCMAKE_INSTALL_PREFIX=${EFSW_INSTALL_DIR}
+  -DCMAKE_BUILD_TYPE=Release
+  -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+  -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+  -DCMAKE_OSX_ARCHITECTURES=${PIPED_OSX_ARCHITECTURES}
+  -DBUILD_TEST_APP=OFF
+  -DBUILD_SHARED_LIBS=ON # The -static file isn't installed unless this is enabled?
+  -DBUILD_STATIC_LIBS=ON
+  -DCMAKE_USER_MAKE_RULES_OVERRIDE_CXX=${CMAKE_CURRENT_LIST_DIR}/cxx_flag_overrides.cmake
+  -DCMAKE_USER_MAKE_RULES_OVERRIDE=${CMAKE_CURRENT_LIST_DIR}/c_flag_overrides.cmake
+  -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded
+)
+
+if(NOT EXISTS ${EFSW_LIB})
+  ExternalProject_Add(libefsw_ext
+    GIT_REPOSITORY   ${EFSW_REPOSITORY}
+    GIT_TAG          ${EFSW_VERSION_REF}
+    GIT_SHALLOW      TRUE
+    PREFIX           efsw
+    LIST_SEPARATOR |
+    CMAKE_ARGS       ${EFSW_ARGS}
+    SOURCE_DIR       "${EFSW_SOURCE_DIR}"
+    BUILD_IN_SOURCE  1
+    BUILD_COMMAND ${CMAKE_COMMAND} --build . --config Release
+    INSTALL_COMMAND  ${CMAKE_COMMAND} --build . --config Release --target install
+    BUILD_BYPRODUCTS ${EFSW_LIB}
+    EXCLUDE_FROM_ALL 1
+    LOG_DOWNLOAD 1
+    LOG_CONFIGURE 1
+    LOG_BUILD 1
+    LOG_INSTALL 1
+    LOG_OUTPUT_ON_FAILURE 1
+  )
+else()
+  message("  - already built")
+endif()
+
+add_library(libefsw STATIC IMPORTED)
+set_target_properties(libefsw PROPERTIES
+  IMPORTED_LOCATION ${EFSW_LIB}
+  IMPORTED_IMPLIB ${EFSW_LIB}
+  INTERFACE_INCLUDE_DIRECTORIES ${EFSW_INSTALL_DIR}/include
+  FOLDER 3rdParty
+)
+if(APPLE)
+  find_library(CoreFoundation_Library CoreFoundation)
+  find_library(CoreServices_Library CoreServices)
+  set_property(TARGET libefsw APPEND
+    PROPERTY INTERFACE_LINK_LIBRARIES
+      ${CoreFoundation_Library} ${CoreServices_Library})
+endif()
+
+if(NOT EXISTS ${EFSW_LIB})
+  file(MAKE_DIRECTORY ${EFSW_INSTALL_DIR}/include) #directory has to exist during configure
+  add_dependencies(libefsw libefsw_ext)
+endif()

cmake/core_tests.cmake

@@ -105,16 +105,16 @@ set_tests_properties(shard_ecompile_cfg PROPERTIES FIXTURES_SETUP shard)
 # uoconvert part
 
 add_test(NAME uoconvert_map
-  COMMAND uoconvert map realm=britannia 
-    width=192 height=192 
+  COMMAND uoconvert map realm=britannia
+    width=192 height=192
     uodata=client
   WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/coretest
 )
 set_tests_properties( uoconvert_map PROPERTIES FIXTURES_REQUIRED client)
 set_tests_properties( uoconvert_map PROPERTIES FIXTURES_SETUP uoconvert)
 
 add_test(NAME uoconvert_statics
-  COMMAND uoconvert statics realm=britannia 
+  COMMAND uoconvert statics realm=britannia
     uodata=client
   WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/coretest
 )
@@ -123,7 +123,7 @@ set_tests_properties( uoconvert_statics PROPERTIES FIXTURES_REQUIRED client)
 set_tests_properties( uoconvert_statics PROPERTIES FIXTURES_SETUP uoconvert)
 
 add_test(NAME uoconvert_maptile
-  COMMAND uoconvert maptile realm=britannia 
+  COMMAND uoconvert maptile realm=britannia
     uodata=client
   WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/coretest
 )
@@ -207,6 +207,14 @@ if (${Python3_FOUND})
       -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/core_tests_start.cmake
     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/coretest
   )
+  add_test(NAME ecompile_watch_test
+    COMMAND ${CMAKE_COMMAND}
+      -Decompile=$<TARGET_FILE:ecompile>
+      -Dtestdir=${CMAKE_CURRENT_SOURCE_DIR}/testsuite
+      -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/core_tests_ecompile_watch.cmake
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/coretest
+    )
+    set_tests_properties(ecompile_watch_test PROPERTIES FIXTURES_REQUIRED shard)
 else()
   message(" - core test without testclient python3 not found")
   add_test(NAME shard_test_1

cmake/core_tests_ecompile_watch.cmake

@@ -0,0 +1,11 @@
+find_package(Python3 COMPONENTS Interpreter REQUIRED QUIET)
+execute_process(
+  COMMAND ${Python3_EXECUTABLE} ${testdir}/ecompile_watch_test.py ${ecompile}
+  COMMAND_ECHO STDOUT
+  WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+  RESULT_VARIABLE res
+  TIMEOUT 600
+)
+if(NOT "${res}" STREQUAL "0")
+  message(SEND_ERROR "${res}")
+endif()

docs/docs.polserver.com/pol100/configfiles.xml

@@ -119,10 +119,10 @@ Listener
     <explain>MaxSkillID allows you to use new client skills, but setting it too high can crash older clients. You must define attributes.cfg/uoskills.cfg entries for each skillID up to MaxSkillID (default 57).</explain>
     <explain>The Protocol and Listener sections are optional. You need at least one Listener to accept client connections.</explain>
     <explain>Encryption: valid values are 'major.minor.build', '2.0.0x', 'ignition', 'uorice', and 'none'.</explain>
-    <explain> 
+    <explain>
         Encryptions 'ignition', 'uorice', and 'none' all mean 'no encryption'.
         If the 'major.minor.build' is given, the encryption key will be calculated for
-        that client version. 
+        that client version.
         Example, '7.0.4' is a listener for client 7.0.4.1 (no patch)
     </explain>
     <explain>Each Listener section allows you to listen for different client encryption types on different ports.
@@ -1027,6 +1027,7 @@ PackageRoot (path to package root)
 [DisplayWarnings (0/1 {default 0})]           //same as -w flag
 [CompileAspPages (0/1 {default 0})]           //same as -a flag
 [OnlyCompileUpdatedScripts (0/1 {default 0})] //same as -u flag
+[WatchModeByDefault (0/1 {default 0})]        //Keep ecompile running to re-compile files that have changed.
 [GenerateDependencyInfo (0/1 {default 0})]    //Generate .dep files whether or not updating
 [DisplayUpToDateScripts (0/1 {default 0})]    //Display the "xxx/script.ecl is up-to-date" message, or not.
 [AutoCompileByDefault (0/1 {default 0})]      //If set, and you don't pass any script names or -r or -A,

docs/docs.polserver.com/pol100/corechanges.xml

@@ -2,9 +2,21 @@
 <ESCRIPT>
 	<header>
 		<topic>Latest Core Changes</topic>
-		<datemodified>03-10-2024</datemodified>
+		<datemodified>03-14-2024</datemodified>
 	</header>
 	<version name="POL100.2.0">
+		<entry>
+			<date>03-14-2024</date>
+			<author>Kevin:</author>
+			<change type="Added">ecompile option -W (watch mode) / ecompile.cfg option WatchModeByDefault<br/>
+Ecompile now includes a daemon watch mode! When using this mode,<br/>
+ecompile will stay running and watch for file system changes,<br/>
+recompiling scripts if they (or their dependent includes or modules)<br/>
+are changed. This works in conjunction with other options, such as -A<br/>
+for watching all scripts in the distro, or -r for watching all<br/>
+scripts in a specific folder.</change>
+			<change type="Note">When using watch mode, threaded-compilation is disabled.</change>
+		</entry>
 		<entry>
 			<date>03-10-2024</date>
 			<author>Turley:</author>

lib/.gitignore

@@ -2,3 +2,4 @@ curl-*
 boost_*
 zlib-*
 cppdap-*
+efsw-*

pol-core/bscript/compiler/astbuilder/SourceFileProcessor.cpp

@@ -42,8 +42,8 @@ void SourceFileProcessor::use_module( const std::string& module_name,
                                       SourceLocation& including_location,
                                       long long* micros_counted )
 {
-  std::string pathname = compilercfg.ModuleDirectory + module_name + ".em";
-
+  std::string pathname =
+      Clib::FullPath( ( compilercfg.ModuleDirectory + module_name + ".em" ).c_str() );
   if ( workspace.source_files.find( pathname ) != workspace.source_files.end() )
     return;
 

pol-core/bscript/compiler/file/SourceFileCache.cpp

@@ -87,4 +87,15 @@ void SourceFileCache::keep_some()
   profile.prune_cache_delete_micros += delete_timer.ellapsed().count();
 }
 
+void SourceFileCache::clear()
+{
+  std::unique_lock<std::mutex> lock( mutex, std::try_to_lock_t() );
+  if ( !lock.owns_lock() )
+    return;
+
+  Pol::Tools::HighPerfTimer delete_timer;
+  files.clear();
+  profile.prune_cache_delete_micros += delete_timer.ellapsed().count();
+}
+
 }  // namespace Pol::Bscript::Compiler

pol-core/bscript/compiler/file/SourceFileCache.h

@@ -23,6 +23,7 @@ class SourceFileCache
   std::shared_ptr<SourceFile> load( const SourceFileIdentifier&, Report& report );
 
   void keep_some();
+  void clear();
 
 private:
   unsigned keep;

pol-core/bscript/compilercfg.cpp

@@ -53,6 +53,7 @@ void CompilerConfig::Read( const std::string& path )
   AutoCompileByDefault = elem.remove_bool( "AutoCompileByDefault", false );
   UpdateOnlyOnAutoCompile = elem.remove_bool( "UpdateOnlyOnAutoCompile", false );
   OnlyCompileUpdatedScripts = elem.remove_bool( "OnlyCompileUpdatedScripts", false );
+  WatchModeByDefault = elem.remove_bool( "WatchModeByDefault", false );
   DisplaySummary = elem.remove_bool( "DisplaySummary", false );
   OptimizeObjectMembers = elem.remove_bool( "OptimizeObjectMembers", true );
   ErrorOnWarning = elem.remove_bool( "ErrorOnWarning", false );

pol-core/bscript/compilercfg.h

@@ -30,6 +30,7 @@ struct CompilerConfig
   bool AutoCompileByDefault;
   bool UpdateOnlyOnAutoCompile;
   bool OnlyCompileUpdatedScripts;
+  bool WatchModeByDefault;
   bool DisplaySummary;
   bool DisplayUpToDateScripts;
   bool OptimizeObjectMembers;