refs #4559 fixed FsUtil to behave more consistently regarding target …

…directories when copying (#4560)

refs #4557 implemented an event-based data provider for the FtpPoller module

CMakeLists.txt

@@ -692,7 +692,7 @@ if (APPLE)
     set_target_properties(libqore PROPERTIES SOVERSION 12)
     set_target_properties(libqore PROPERTIES INSTALL_NAME_DIR ${CMAKE_INSTALL_FULL_LIBDIR})
 else (APPLE)
-    set_target_properties(libqore PROPERTIES VERSION 12.0.2)
+    set_target_properties(libqore PROPERTIES VERSION 12.0.3)
     set_target_properties(libqore PROPERTIES SOVERSION 12)
 endif (APPLE)
 
@@ -848,6 +848,7 @@ qore_user_module("qlib/TextWrap.qm" "Util")
 
 # base 3 modules
 qore_user_module("qlib/FilePoller.qm" "Util;DataProvider")
+qore_user_module("qlib/FileDataProvider" "FsUtil;DataProvider")
 qore_user_module("qlib/SqlUtil" "Util;DataProvider")
 qore_user_module("qlib/ConnectionProvider" "Util;DataProvider")
 qore_user_module("qlib/CsvUtil" "Util;DataProvider")

Makefile.am

@@ -28,6 +28,7 @@ DOX_SRC_MODULES = qlib/Mime.qm \
 	qlib/DebugProgramControl.qm \
 	qlib/WebSocketHandler.qm \
     qlib/FtpPollerUtil.qm \
+	qlib/FsUtil.qm \
 	qlib/WebSocketClient.qm
 
 DOX_SRC_SPLIT_MODULES = \
@@ -46,7 +47,6 @@ DOX_MODULES = qlib/DatasourceProvider.qm \
 	qlib/ServiceNowRestClient.qm \
 	qlib/FilePoller.qm \
 	qlib/FreetdsSqlUtil.qm \
-	qlib/FsUtil.qm \
 	qlib/FtpPoller.qm \
 	qlib/OracleSqlUtil.qm \
 	qlib/PgsqlSqlUtil.qm \
@@ -81,7 +81,8 @@ DOX_SPLIT_MODULES = \
     qlib/SwaggerDataProvider \
     qlib/SalesforceRestDataProvider \
     qlib/CdsRestDataProvider \
-    qlib/ServiceNowRestDataProvider
+    qlib/ServiceNowRestDataProvider \
+    qlib/FileDataProvider
 
 SqlUtil_modverdir = $(modverdir)/SqlUtil
 ConnectionProvider_modverdir = $(modverdir)/ConnectionProvider
@@ -1082,6 +1083,9 @@ doxygen/qlib/Doxyfile.WebSocketHandler: doxygen/qlib/Doxyfile.tmpl $(QDX_DEP)
 doxygen/qlib/Doxyfile.FtpPollerUtil: doxygen/qlib/Doxyfile.tmpl $(QDX_DEP)
 	$(QDX) -M=qlib/FtpPollerUtil.qm:doxygen/qlib/FtpPollerUtil.qm -tDataProvider.tag=../../DataProvider/html $< $@
 
+doxygen/qlib/Doxyfile.FsUtil: doxygen/qlib/Doxyfile.tmpl $(QDX_DEP)
+	$(QDX) -M=qlib/FsUtil.qm:doxygen/qlib/FsUtil.qm $< $@
+
 doxygen/modules/Doxyfile.astparser: doxygen/modules/Doxyfile.in
 	sed -e s/@QORE_MOD_NAME@/astparser/ \
         -e "s/@VERSION_MAJOR@.@VERSION_MINOR@.@VERSION_SUB@/${PACKAGE_VERSION}/" \

bin/qdp

@@ -52,33 +52,34 @@ class QdpCmd {
 
         #! commands
         const Cmds = {
+            "create": \QdpCmd::create(),
+            "delete": \QdpCmd::del(),
+            "dorequest": \QdpCmd::doRequest(),
             "errors": \QdpCmd::errors(),
+            "event": \QdpCmd::event(),
+            "fadd": \QdpCmd::fieldAdd(),
+            "fdelete": \QdpCmd::fieldDelete(),
+            "field-add": \QdpCmd::fieldAdd(),
+            "field-del": \QdpCmd::fieldDelete(),
+            "field-update": \QdpCmd::fieldUpdate(),
+            "fupdate": \QdpCmd::fieldUpdate(),
             "info": \QdpCmd::getInfo(),
+            "ldetails": \QdpCmd::listChildDetails(),
             "list": \QdpCmd::listChildren(),
             "list-details": \QdpCmd::listChildDetails(),
-            "ldetails": \QdpCmd::listChildDetails(),
             "listen": \QdpCmd::listen(),
+            "pcreate": \QdpCmd::providerCreate(),
+            "pdelete": \QdpCmd::providerDelete(),
+            "provider-create": \QdpCmd::providerCreate(),
+            "provider-delete": \QdpCmd::providerDelete(),
             "record": \QdpCmd::getRecord(),
-            "search": \QdpCmd::search(),
-            "update": \QdpCmd::update(),
-            "create": \QdpCmd::create(),
-            "upsert": \QdpCmd::upsert(),
-            "delete": \QdpCmd::del(),
+            "reply": \QdpCmd::response(),
             "request": \QdpCmd::request(),
             "response": \QdpCmd::response(),
-            "reply": \QdpCmd::response(),
-            "dorequest": \QdpCmd::doRequest(),
             "rsearch": \QdpCmd::doRequestSearch(),
-            "provider-create": \QdpCmd::providerCreate(),
-            "pcreate": \QdpCmd::providerCreate(),
-            "provider-delete": \QdpCmd::providerDelete(),
-            "pdelete": \QdpCmd::providerDelete(),
-            "field-add": \QdpCmd::fieldAdd(),
-            "fadd": \QdpCmd::fieldAdd(),
-            "field-update": \QdpCmd::fieldUpdate(),
-            "fupdate": \QdpCmd::fieldUpdate(),
-            "field-del": \QdpCmd::fieldDelete(),
-            "fdelete": \QdpCmd::fieldDelete(),
+            "search": \QdpCmd::search(),
+            "update": \QdpCmd::update(),
+            "upsert": \QdpCmd::upsert(),
         };
 
         #! valid field keys
@@ -149,6 +150,10 @@ class QdpCmd {
         }
     }
 
+    static event(AbstractDataProvider provider) {
+        QdpCmd::showType(provider.getEventType());
+    }
+
     static errors(AbstractDataProvider provider) {
         *hash<string, AbstractDataProviderType> errs = provider.getErrorResponseTypes();
         *string err = QdpCmd::getString("errors");
@@ -543,6 +548,8 @@ class QdpCmd {
         executes a request against the given provider (if supported)
     errors [<code>]
         lists all error replies
+    event
+        shows the event type (if supported)
     info
         show information about the data provider
     list

doxygen/lang/120_modules.dox.tmpl

@@ -71,6 +71,8 @@ ModuleName/
         develop debug server
     |user|<a href="../../modules/DebugHandler/html/index.html">DebugHandler</a>|Provides a websocket for debug server
     |user|<a href="../../modules/DebugUtil/html/index.html">DebugUtil</a>|Provides a general stuff for debugger
+    |user|<a href="../../modules/FileDataProvider/html/index.html">FileDataProvider</a>|Provides a \
+        <a href="../../modules/DataProvider/html/index.html">data provider</a> API for the local filesystem
     |user|<a href="../../modules/FileLocationHandler/html/index.html">FileLocationHandler</a>|Provides an API for \
         retrieving file data based on a URL-like location string
     |user|<a href="../../modules/FilePoller/html/index.html">FilePoller</a>|Provides an API for polling files on the \

doxygen/lang/900_release_notes.dox.tmpl

@@ -7,14 +7,24 @@
     @par Release Summary
     Bugfix release with minor new features in user modules; see below for more information
 
-    @subsection qore_1_10_0_bug_fixes Bug Fixes in Qore
+    @subsection qore_1_10_0_new_features New Features in Qore
     - <a href="../../modules/DataProvider/html/index.html">DataProvider</a> module
       - implemented support for the observer pattern / event API
         (<a href="https://github.com/qorelanguage/qore/issues/4557">issue 4557</a>)
+    - <a href="../../modules/FileDataProvider/html/index.html">FileDataProvider</a> module
+      - new module
+    - <a href="../../modules/FtpPoller/html/index.html">FtpPoller</a> module
+      - added support for the event-based DataProvider API
+        (<a href="https://github.com/qorelanguage/qore/issues/4557">issue 4557</a>)
     - <a href="../../modules/FilePoller/html/index.html">FilePoller</a> module
       - added support for the event-based DataProvider API
         (<a href="https://github.com/qorelanguage/qore/issues/4557">issue 4557</a>)
 
+    @subsection qore_1_10_0_bug_fixes Bug Fixes in Qore
+    - <a href="../../modules/FsUtil/html/index.html">FsUtil</a> module
+      - fixed inconsistencies handling target directories in copy operations
+        (<a href="https://github.com/qorelanguage/qore/issues/4559">issue 4559</a>)
+
     @section qore_1_9_1 Qore 1.9.1
 
     @par Release Summary

examples/test/qlib/FileDataProvider/FileDataProvider.qtest

@@ -0,0 +1,69 @@
+#!/usr/bin/env qore
+# -*- mode: qore; indent-tabs-mode: nil -*-
+
+%require-types
+%enable-all-warnings
+%new-style
+%strict-args
+%allow-injection
+
+%requires ../../../../qlib/Util.qm
+%requires ../../../../qlib/QUnit.qm
+%requires ../../../../qlib/Logger.qm
+%requires ../../../../qlib/DataProvider
+%requires ../../../../qlib/FsUtil.qm
+%requires ../../../../qlib/FileDataProvider
+
+%exec-class FileDataProviderTest
+
+public class FileDataProviderTest inherits QUnit::Test {
+    constructor() : Test("FileDataProvider Test", "1.0") {
+        addTestCase("test", \fileDataProviderTest());
+
+        # Return for compatibility with test harness that checks return value.
+        set_return_value(main());
+    }
+
+    fileDataProviderTest() {
+        # create test file
+        TmpFile tmp("FileDataProviderTest-");
+        string str = get_random_string();
+        tmp.file.write(str);
+
+        FileDataProvider fdp();
+
+        AbstractDataProvider prov = fdp.getChildProviderEx("stat");
+        hash<auto> h = prov.doRequest({
+            "path": tmp.path,
+        });
+        assertEq(tmp.path, h.filepath);
+        assertEq("REGULAR", h.type);
+
+        prov = fdp.getChildProviderEx("copy");
+        string targ = join_paths(tmp_location(), get_random_string());
+        h = prov.doRequest({
+            "source": tmp.path,
+            "target": targ,
+        });
+        assertEq(targ, h.target);
+        assertTrue(is_file(targ));
+
+        prov = fdp.getChildProviderEx("move");
+        string new_targ = join_paths(tmp_location(), get_random_string());
+        assertNeq(new_targ, targ);
+        h = prov.doRequest({
+            "source": targ,
+            "target": new_targ,
+        });
+        assertEq(new_targ, h.target);
+        assertFalse(is_file(targ));
+        assertTrue(is_file(new_targ));
+
+        prov = fdp.getChildProviderEx("delete");
+        h = prov.doRequest({
+            "path": new_targ,
+        });
+        assertEq(new_targ, h.path);
+        assertFalse(is_file(new_targ));
+    }
+}

examples/test/qlib/FilePoller/FilePoller.qtest

@@ -8,6 +8,8 @@
 
 %requires ../../../../qlib/Util.qm
 %requires ../../../../qlib/QUnit.qm
+%requires ../../../../qlib/FsUtil.qm
+%requires ../../../../qlib/DataProvider
 %requires ../../../../qlib/FilePoller.qm
 
 %exec-class FilePollerTest
@@ -34,11 +36,12 @@ public class FilePollerTest inherits QUnit::Test {
             "file1",
             "file2",
             "file3",
-            );
+        );
     }
 
     constructor() : Test("FilePoller", "1.0") {
         addTestCase("FilePoller", \filePollerTest());
+        addTestCase("FilePollerDataProvider", \filePollerDataProviderTest());
 
         # Return for compatibility with test harness that checks return value
         set_return_value(main());
@@ -60,4 +63,89 @@ public class FilePollerTest inherits QUnit::Test {
         fp.stop();
         assertEq(Files, fp.fl);
     }
+
+    private filePollerDataProviderTest() {
+        Logger logger();
+        {
+            LoggerLevel level;
+            if (m_options.verbose > 3) {
+                level = LoggerLevel::getLevelDebug();
+            } else if (m_options.verbose > 2) {
+                level = LoggerLevel::getLevelInfo();
+            } else {
+                level = LoggerLevel::getLevelError();
+            }
+            logger = new Logger("test", level);
+        }
+        logger.addAppender(new TestAppender());
+
+        TmpDir dir("FilePollerDataProvider-");
+
+        MyObserver observer();
+        FilePollerDataProvider fp({
+            "path": dir.path,
+            "poll_interval": 1,
+        });
+        fp.setLogger(logger);
+        fp.registerObserver(observer);
+
+        File f();
+        map f.open2(dir.path + "/" + $1, O_CREAT|O_TRUNC|O_WRONLY), Files;
+        on_exit map unlink($1), dir.path + "/" + Files;
+
+        observer.wait();
+        assertEq("file1", observer.h.file1.name);
+        assertEq("file2", observer.h.file2.name);
+        assertEq("file3", observer.h.file3.name);
+        assertTrue(True);
+    }
+}
+
+class MyObserver inherits Observer {
+    public {
+        hash<string, hash<auto>> h();
+    }
+
+    private {
+        Mutex m();
+        Condition cond();
+    }
+
+    update(string event_id, hash<auto> data_) {
+        h{data_.name} = data_;
+        if (h.size() == 3) {
+            m.lock();
+            on_exit m.unlock();
+
+            cond.broadcast();
+        }
+    }
+
+    wait() {
+        date start = now_us();
+
+        m.lock();
+        on_exit m.unlock();
+
+        while (h.size() != 3) {
+            cond.wait(m, 1s);
+            if ((now_us() - start) > 5s) {
+                throw "ERROR", "no files in timeout";
+            }
+        }
+    }
+}
+
+class TestAppender inherits LoggerAppenderWithLayout {
+    constructor() : LoggerAppenderWithLayout("test", new LoggerLayoutPattern("%d T%t [%p]: %m%n")) {
+        open();
+    }
+
+    processEventImpl(int type, auto params) {
+        switch (type) {
+            case EVENT_LOG:
+                print(params);
+                break;
+        }
+    }
 }