diff --git a/CMakeLists.txt b/CMakeLists.txt index 80c95537f8..5f8ac8a796 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -692,7 +692,7 @@ if (APPLE) set_target_properties(libqore PROPERTIES SOVERSION 12) set_target_properties(libqore PROPERTIES INSTALL_NAME_DIR ${CMAKE_INSTALL_FULL_LIBDIR}) else (APPLE) - set_target_properties(libqore PROPERTIES VERSION 12.0.2) + set_target_properties(libqore PROPERTIES VERSION 12.0.3) set_target_properties(libqore PROPERTIES SOVERSION 12) endif (APPLE) @@ -848,6 +848,7 @@ qore_user_module("qlib/TextWrap.qm" "Util") # base 3 modules qore_user_module("qlib/FilePoller.qm" "Util;DataProvider") +qore_user_module("qlib/FileDataProvider" "FsUtil;DataProvider") qore_user_module("qlib/SqlUtil" "Util;DataProvider") qore_user_module("qlib/ConnectionProvider" "Util;DataProvider") qore_user_module("qlib/CsvUtil" "Util;DataProvider") diff --git a/Makefile.am b/Makefile.am index 4a422af099..62fd164a43 100644 --- a/Makefile.am +++ b/Makefile.am @@ -28,6 +28,7 @@ DOX_SRC_MODULES = qlib/Mime.qm \ qlib/DebugProgramControl.qm \ qlib/WebSocketHandler.qm \ qlib/FtpPollerUtil.qm \ + qlib/FsUtil.qm \ qlib/WebSocketClient.qm DOX_SRC_SPLIT_MODULES = \ @@ -46,7 +47,6 @@ DOX_MODULES = qlib/DatasourceProvider.qm \ qlib/ServiceNowRestClient.qm \ qlib/FilePoller.qm \ qlib/FreetdsSqlUtil.qm \ - qlib/FsUtil.qm \ qlib/FtpPoller.qm \ qlib/OracleSqlUtil.qm \ qlib/PgsqlSqlUtil.qm \ @@ -81,7 +81,8 @@ DOX_SPLIT_MODULES = \ qlib/SwaggerDataProvider \ qlib/SalesforceRestDataProvider \ qlib/CdsRestDataProvider \ - qlib/ServiceNowRestDataProvider + qlib/ServiceNowRestDataProvider \ + qlib/FileDataProvider SqlUtil_modverdir = $(modverdir)/SqlUtil ConnectionProvider_modverdir = $(modverdir)/ConnectionProvider @@ -1082,6 +1083,9 @@ doxygen/qlib/Doxyfile.WebSocketHandler: doxygen/qlib/Doxyfile.tmpl $(QDX_DEP) doxygen/qlib/Doxyfile.FtpPollerUtil: doxygen/qlib/Doxyfile.tmpl $(QDX_DEP) $(QDX) -M=qlib/FtpPollerUtil.qm:doxygen/qlib/FtpPollerUtil.qm -tDataProvider.tag=../../DataProvider/html $< $@ +doxygen/qlib/Doxyfile.FsUtil: doxygen/qlib/Doxyfile.tmpl $(QDX_DEP) + $(QDX) -M=qlib/FsUtil.qm:doxygen/qlib/FsUtil.qm $< $@ + doxygen/modules/Doxyfile.astparser: doxygen/modules/Doxyfile.in sed -e s/@QORE_MOD_NAME@/astparser/ \ -e "s/@VERSION_MAJOR@.@VERSION_MINOR@.@VERSION_SUB@/${PACKAGE_VERSION}/" \ diff --git a/bin/qdp b/bin/qdp index bdee3dc80c..d2a6105601 100755 --- a/bin/qdp +++ b/bin/qdp @@ -52,33 +52,34 @@ class QdpCmd { #! commands const Cmds = { + "create": \QdpCmd::create(), + "delete": \QdpCmd::del(), + "dorequest": \QdpCmd::doRequest(), "errors": \QdpCmd::errors(), + "event": \QdpCmd::event(), + "fadd": \QdpCmd::fieldAdd(), + "fdelete": \QdpCmd::fieldDelete(), + "field-add": \QdpCmd::fieldAdd(), + "field-del": \QdpCmd::fieldDelete(), + "field-update": \QdpCmd::fieldUpdate(), + "fupdate": \QdpCmd::fieldUpdate(), "info": \QdpCmd::getInfo(), + "ldetails": \QdpCmd::listChildDetails(), "list": \QdpCmd::listChildren(), "list-details": \QdpCmd::listChildDetails(), - "ldetails": \QdpCmd::listChildDetails(), "listen": \QdpCmd::listen(), + "pcreate": \QdpCmd::providerCreate(), + "pdelete": \QdpCmd::providerDelete(), + "provider-create": \QdpCmd::providerCreate(), + "provider-delete": \QdpCmd::providerDelete(), "record": \QdpCmd::getRecord(), - "search": \QdpCmd::search(), - "update": \QdpCmd::update(), - "create": \QdpCmd::create(), - "upsert": \QdpCmd::upsert(), - "delete": \QdpCmd::del(), + "reply": \QdpCmd::response(), "request": \QdpCmd::request(), "response": \QdpCmd::response(), - "reply": \QdpCmd::response(), - "dorequest": \QdpCmd::doRequest(), "rsearch": \QdpCmd::doRequestSearch(), - "provider-create": \QdpCmd::providerCreate(), - "pcreate": \QdpCmd::providerCreate(), - "provider-delete": \QdpCmd::providerDelete(), - "pdelete": \QdpCmd::providerDelete(), - "field-add": \QdpCmd::fieldAdd(), - "fadd": \QdpCmd::fieldAdd(), - "field-update": \QdpCmd::fieldUpdate(), - "fupdate": \QdpCmd::fieldUpdate(), - "field-del": \QdpCmd::fieldDelete(), - "fdelete": \QdpCmd::fieldDelete(), + "search": \QdpCmd::search(), + "update": \QdpCmd::update(), + "upsert": \QdpCmd::upsert(), }; #! valid field keys @@ -149,6 +150,10 @@ class QdpCmd { } } + static event(AbstractDataProvider provider) { + QdpCmd::showType(provider.getEventType()); + } + static errors(AbstractDataProvider provider) { *hash errs = provider.getErrorResponseTypes(); *string err = QdpCmd::getString("errors"); @@ -543,6 +548,8 @@ class QdpCmd { executes a request against the given provider (if supported) errors [] lists all error replies + event + shows the event type (if supported) info show information about the data provider list diff --git a/doxygen/lang/120_modules.dox.tmpl b/doxygen/lang/120_modules.dox.tmpl index 06e1a3beef..d06e879a61 100644 --- a/doxygen/lang/120_modules.dox.tmpl +++ b/doxygen/lang/120_modules.dox.tmpl @@ -71,6 +71,8 @@ ModuleName/ develop debug server |user|DebugHandler|Provides a websocket for debug server |user|DebugUtil|Provides a general stuff for debugger + |user|FileDataProvider|Provides a \ + data provider API for the local filesystem |user|FileLocationHandler|Provides an API for \ retrieving file data based on a URL-like location string |user|FilePoller|Provides an API for polling files on the \ diff --git a/doxygen/lang/900_release_notes.dox.tmpl b/doxygen/lang/900_release_notes.dox.tmpl index 6528b6502b..3a61e0613b 100644 --- a/doxygen/lang/900_release_notes.dox.tmpl +++ b/doxygen/lang/900_release_notes.dox.tmpl @@ -7,14 +7,24 @@ @par Release Summary Bugfix release with minor new features in user modules; see below for more information - @subsection qore_1_10_0_bug_fixes Bug Fixes in Qore + @subsection qore_1_10_0_new_features New Features in Qore - DataProvider module - implemented support for the observer pattern / event API (issue 4557) + - FileDataProvider module + - new module + - FtpPoller module + - added support for the event-based DataProvider API + (issue 4557) - FilePoller module - added support for the event-based DataProvider API (issue 4557) + @subsection qore_1_10_0_bug_fixes Bug Fixes in Qore + - FsUtil module + - fixed inconsistencies handling target directories in copy operations + (issue 4559) + @section qore_1_9_1 Qore 1.9.1 @par Release Summary diff --git a/examples/test/qlib/FileDataProvider/FileDataProvider.qtest b/examples/test/qlib/FileDataProvider/FileDataProvider.qtest new file mode 100755 index 0000000000..a68aeec6fa --- /dev/null +++ b/examples/test/qlib/FileDataProvider/FileDataProvider.qtest @@ -0,0 +1,69 @@ +#!/usr/bin/env qore +# -*- mode: qore; indent-tabs-mode: nil -*- + +%require-types +%enable-all-warnings +%new-style +%strict-args +%allow-injection + +%requires ../../../../qlib/Util.qm +%requires ../../../../qlib/QUnit.qm +%requires ../../../../qlib/Logger.qm +%requires ../../../../qlib/DataProvider +%requires ../../../../qlib/FsUtil.qm +%requires ../../../../qlib/FileDataProvider + +%exec-class FileDataProviderTest + +public class FileDataProviderTest inherits QUnit::Test { + constructor() : Test("FileDataProvider Test", "1.0") { + addTestCase("test", \fileDataProviderTest()); + + # Return for compatibility with test harness that checks return value. + set_return_value(main()); + } + + fileDataProviderTest() { + # create test file + TmpFile tmp("FileDataProviderTest-"); + string str = get_random_string(); + tmp.file.write(str); + + FileDataProvider fdp(); + + AbstractDataProvider prov = fdp.getChildProviderEx("stat"); + hash h = prov.doRequest({ + "path": tmp.path, + }); + assertEq(tmp.path, h.filepath); + assertEq("REGULAR", h.type); + + prov = fdp.getChildProviderEx("copy"); + string targ = join_paths(tmp_location(), get_random_string()); + h = prov.doRequest({ + "source": tmp.path, + "target": targ, + }); + assertEq(targ, h.target); + assertTrue(is_file(targ)); + + prov = fdp.getChildProviderEx("move"); + string new_targ = join_paths(tmp_location(), get_random_string()); + assertNeq(new_targ, targ); + h = prov.doRequest({ + "source": targ, + "target": new_targ, + }); + assertEq(new_targ, h.target); + assertFalse(is_file(targ)); + assertTrue(is_file(new_targ)); + + prov = fdp.getChildProviderEx("delete"); + h = prov.doRequest({ + "path": new_targ, + }); + assertEq(new_targ, h.path); + assertFalse(is_file(new_targ)); + } +} diff --git a/examples/test/qlib/FilePoller/FilePoller.qtest b/examples/test/qlib/FilePoller/FilePoller.qtest index 239c28d4cb..934f2ddbfe 100755 --- a/examples/test/qlib/FilePoller/FilePoller.qtest +++ b/examples/test/qlib/FilePoller/FilePoller.qtest @@ -8,6 +8,8 @@ %requires ../../../../qlib/Util.qm %requires ../../../../qlib/QUnit.qm +%requires ../../../../qlib/FsUtil.qm +%requires ../../../../qlib/DataProvider %requires ../../../../qlib/FilePoller.qm %exec-class FilePollerTest @@ -34,11 +36,12 @@ public class FilePollerTest inherits QUnit::Test { "file1", "file2", "file3", - ); + ); } constructor() : Test("FilePoller", "1.0") { addTestCase("FilePoller", \filePollerTest()); + addTestCase("FilePollerDataProvider", \filePollerDataProviderTest()); # Return for compatibility with test harness that checks return value set_return_value(main()); @@ -60,4 +63,89 @@ public class FilePollerTest inherits QUnit::Test { fp.stop(); assertEq(Files, fp.fl); } + + private filePollerDataProviderTest() { + Logger logger(); + { + LoggerLevel level; + if (m_options.verbose > 3) { + level = LoggerLevel::getLevelDebug(); + } else if (m_options.verbose > 2) { + level = LoggerLevel::getLevelInfo(); + } else { + level = LoggerLevel::getLevelError(); + } + logger = new Logger("test", level); + } + logger.addAppender(new TestAppender()); + + TmpDir dir("FilePollerDataProvider-"); + + MyObserver observer(); + FilePollerDataProvider fp({ + "path": dir.path, + "poll_interval": 1, + }); + fp.setLogger(logger); + fp.registerObserver(observer); + + File f(); + map f.open2(dir.path + "/" + $1, O_CREAT|O_TRUNC|O_WRONLY), Files; + on_exit map unlink($1), dir.path + "/" + Files; + + observer.wait(); + assertEq("file1", observer.h.file1.name); + assertEq("file2", observer.h.file2.name); + assertEq("file3", observer.h.file3.name); + assertTrue(True); + } +} + +class MyObserver inherits Observer { + public { + hash> h(); + } + + private { + Mutex m(); + Condition cond(); + } + + update(string event_id, hash data_) { + h{data_.name} = data_; + if (h.size() == 3) { + m.lock(); + on_exit m.unlock(); + + cond.broadcast(); + } + } + + wait() { + date start = now_us(); + + m.lock(); + on_exit m.unlock(); + + while (h.size() != 3) { + cond.wait(m, 1s); + if ((now_us() - start) > 5s) { + throw "ERROR", "no files in timeout"; + } + } + } +} + +class TestAppender inherits LoggerAppenderWithLayout { + constructor() : LoggerAppenderWithLayout("test", new LoggerLayoutPattern("%d T%t [%p]: %m%n")) { + open(); + } + + processEventImpl(int type, auto params) { + switch (type) { + case EVENT_LOG: + print(params); + break; + } + } } diff --git a/examples/test/qlib/FsUtil/copy_path.qtest b/examples/test/qlib/FsUtil/copy_path.qtest index 97b685d551..57248c2780 100755 --- a/examples/test/qlib/FsUtil/copy_path.qtest +++ b/examples/test/qlib/FsUtil/copy_path.qtest @@ -381,41 +381,34 @@ public class CopyPathTest inherits QUnit::Test { remove_tree(dst_path); assertFalse(path_exists(dst_path)); - # let's try again but this time use an existing directory as destination - mkdir(dst_path); # first without overwrite parameter - assertThrows("PATH-EXISTS-ERROR", \copy_path(), (source_path, dst_path)); - # and then properly with overwrite flag - cp_dest_path = copy_path(source_path, dst_path, NOTHING, True); - assertEq(dst_path, cp_dest_path); - assertTrue(path_exists(dst_path)); - assertTrue(is_dir(dst_path)); - assertTrue(path_exists(dst_dir_path)); - assertTrue(is_dir(dst_dir_path)); - assertTrue(path_exists(dst_file_path)); - assertTrue(is_file(dst_file_path)); - assertTrue(path_exists(dst_link_path)); - assertTrue(is_link(dst_link_path)); - assertEq(file_path, readlink(dst_link_path)); # follow_symlinks=False - # let's clean it - remove_tree(dst_path); - assertFalse(path_exists(dst_path)); + { + { + File f(); + f.open2(dst_path, O_CREAT | O_TRUNC | O_WRONLY); + } + on_exit unlink(dst_path); + assertThrows("PATH-EXISTS-ERROR", \copy_path(), (source_path, dst_path)); + } - # let's try again with follow_symlinks=True + # let's try again but this time use an existing directory as destination mkdir(dst_path); - # first without overwrite parameter - assertThrows("PATH-EXISTS-ERROR", \copy_path(), (source_path, dst_path)); # and then properly with overwrite flag - cp_dest_path = copy_path(source_path, dst_path, True, True); - assertEq(dst_path, cp_dest_path); - assertTrue(path_exists(dst_path)); - assertTrue(is_dir(dst_path)); - assertTrue(path_exists(dst_dir_path)); - assertTrue(is_dir(dst_dir_path)); - assertTrue(path_exists(dst_file_path)); - assertTrue(is_file(dst_file_path)); - assertTrue(path_exists(dst_link_path)); - assertTrue(is_file(dst_link_path)); # follow_symlinks=True + cp_dest_path = copy_path(source_path, dst_path); + string new_dst_path = join_paths(dst_path, basename(source_path)); + string new_dst_dir_path = join_paths(new_dst_path, "dir"); + string new_dst_file_path = join_paths(new_dst_path, "file"); + string new_dst_link_path = join_paths(new_dst_dir_path, "link"); + assertEq(new_dst_path, cp_dest_path); + assertTrue(path_exists(new_dst_path)); + assertTrue(is_dir(new_dst_path)); + assertTrue(path_exists(new_dst_dir_path)); + assertTrue(is_dir(new_dst_dir_path)); + assertTrue(path_exists(new_dst_file_path)); + assertTrue(is_file(new_dst_file_path)); + assertTrue(path_exists(new_dst_link_path)); + assertTrue(is_link(new_dst_link_path)); + assertEq(file_path, readlink(new_dst_link_path)); # follow_symlinks=False # let's clean it remove_tree(dst_path); assertFalse(path_exists(dst_path)); @@ -478,16 +471,13 @@ public class CopyPathTest inherits QUnit::Test { assertTrue(path_exists(dst_file_path)); assertTrue(is_file(dst_file_path)); - # it shouldn't be able to copy it again to the same location without overwrite - assertThrows("PATH-EXISTS-ERROR", \copy_path(), (source_path, dst_path)); - assertThrows("PATH-EXISTS-ERROR", \copy_path(), (source_path, dst_path, NOTHING, False)); - # it will be OK with overwrite though - cp_dest_path = copy_path(source_path, dst_path, NOTHING, True); - assertEq(dst_path, cp_dest_path); + cp_dest_path = copy_path(source_path, dst_path); + string new_dst_path = join_paths(dst_path, basename(source_path)); + assertEq(new_dst_path, cp_dest_path); # everything is copied anew -> let's check it - dst_dir_path = join_paths(dst_path, "dir"); - dst_file_path = join_paths(dst_path, "file"); + dst_dir_path = join_paths(new_dst_path, "dir"); + dst_file_path = join_paths(new_dst_path, "file"); assertTrue(path_exists(dst_path)); assertTrue(is_dir(dst_path)); assertTrue(path_exists(dst_dir_path)); @@ -589,8 +579,6 @@ public class CopyPathTest inherits QUnit::Test { # now let's make tmp_root/src/file unreadable -> copying it should fail chmod(file_path, 0000); - # without overwrite it will complain about destination path existing - assertThrows("PATH-EXISTS-ERROR", \copy_path(), (source_path, dest_dir)); # with overwrite it will fail because of the permissions assertThrows("FILE-OPEN2-ERROR", \copy_path(), (source_path, dest_dir, NOTHING, True)); diff --git a/examples/test/qlib/FsUtil/copy_tree.qtest b/examples/test/qlib/FsUtil/copy_tree.qtest index 66d2b8f31b..c1470e6d47 100755 --- a/examples/test/qlib/FsUtil/copy_tree.qtest +++ b/examples/test/qlib/FsUtil/copy_tree.qtest @@ -77,29 +77,27 @@ public class CopyTreeTest inherits QUnit::Test { # let's try again but this time use an existing directory as destination mkdir(dst_path); - # first without overwrite parameter - assertThrows("PATH-EXISTS-ERROR", \copy_tree(), (source_path, dst_path)); - # and then properly with overwrite flag - cp_dest_path = copy_tree(source_path, dst_path, NOTHING, True); - assertEq(dst_path, cp_dest_path); - assertTrue(path_exists(dst_path)); - assertTrue(is_dir(dst_path)); - assertTrue(path_exists(dst_dir_path)); - assertTrue(is_dir(dst_dir_path)); - assertTrue(path_exists(dst_file_path)); - assertTrue(is_file(dst_file_path)); - assertTrue(path_exists(dst_link_path)); - assertTrue(is_link(dst_link_path)); - assertEq(file_path, readlink(dst_link_path)); # follow_symlinks=False + cp_dest_path = copy_tree(source_path, dst_path); + string new_dst_path = join_paths(dst_path, basename(source_path)); + string new_dst_dir_path = join_paths(new_dst_path, "dir"); + string new_dst_file_path = join_paths(new_dst_path, "file"); + string new_dst_link_path = join_paths(new_dst_dir_path, "link"); + assertEq(new_dst_path, cp_dest_path); + assertTrue(path_exists(new_dst_path)); + assertTrue(is_dir(new_dst_path)); + assertTrue(path_exists(new_dst_dir_path)); + assertTrue(is_dir(new_dst_dir_path)); + assertTrue(path_exists(new_dst_file_path)); + assertTrue(is_file(new_dst_file_path)); + assertTrue(path_exists(new_dst_link_path)); + assertTrue(is_link(new_dst_link_path)); + assertEq(file_path, readlink(new_dst_link_path)); # follow_symlinks=False # let's clean it remove_tree(dst_path); assertFalse(path_exists(dst_path)); # let's try again with follow_symlinks=True mkdir(dst_path); - # first without overwrite parameter - assertThrows("PATH-EXISTS-ERROR", \copy_tree(), (source_path, dst_path, True)); - # and then properly with overwrite flag cp_dest_path = copy_tree(source_path, dst_path, True, True); assertEq(dst_path, cp_dest_path); assertTrue(path_exists(dst_path)); @@ -172,11 +170,7 @@ public class CopyTreeTest inherits QUnit::Test { assertTrue(path_exists(dst_file_path)); assertTrue(is_file(dst_file_path)); - # it shouldn't be able to copy it again to the same location without overwrite - assertThrows("PATH-EXISTS-ERROR", \copy_tree(), (source_path, dst_path)); - assertThrows("PATH-EXISTS-ERROR", \copy_tree(), (source_path, dst_path, NOTHING, False)); - - # it will be OK with overwrite though + # copy with overwrite cp_dest_path = copy_tree(source_path, dst_path, NOTHING, True); assertEq(dst_path, cp_dest_path); # everything is copied anew -> let's check it @@ -289,8 +283,6 @@ public class CopyTreeTest inherits QUnit::Test { # now let's make tmp_root/src/file unreadable -> copying it should fail chmod(file_path, 0000); bool exception_thrown = False; - # without overwrite it will complain about destination path existing - assertThrows("PATH-EXISTS-ERROR", \copy_tree(), (source_path, dest_dir)); # with overwrite it will fail because of the permissions assertThrows("FILE-OPEN2-ERROR", \copy_tree(), (source_path, dest_dir, NOTHING, True)); diff --git a/examples/test/qlib/FsUtil/move_path.qtest b/examples/test/qlib/FsUtil/move_path.qtest new file mode 100755 index 0000000000..84af60d114 --- /dev/null +++ b/examples/test/qlib/FsUtil/move_path.qtest @@ -0,0 +1,423 @@ +#!/usr/bin/env qore + +%requires Util + +%requires ../../../../qlib/FsUtil.qm +%requires ../../../../qlib/QUnit.qm + +%new-style +%require-types +%enable-all-warnings + +%exec-class MovePathTest + +public class MovePathTest inherits QUnit::Test { + + constructor() : Test ("MovePathTest", "1.0") { + addTestCase("move_path file test", \test_move_path()); + addTestCase("move_path file test - exceptions", \test_move_path_exceptions()); + addTestCase("move_path file test - symlink in source", \test_move_path_symlink_in_src()); + addTestCase("move_path tree test", \test_move_path_tree()); + addTestCase("move_path tree overwrite test", \test_move_path_tree_overwrite()); + addTestCase("move_path tree exception test", \test_move_path_tree_exception()); + set_return_value(main()); + } + + set_content_to_file(string path, string content) { + File f(); + f.open2(path, O_TRUNC | O_WRONLY); + f.write(content); + f.close(); + } + + assert_file_contains(string path, string content) { + File f(); + f.open2(path); + string actual = f.read(-1); + assertEq(content, actual); + f.close(); + } + + test_move_path() { + string tmp_path = make_tmp_dir(); + on_exit { + remove_tree(tmp_path); + } + + string content = "Hello World!"; + + hash original = make_tmp_file(NOTHING, NOTHING, tmp_path); + original.file.write(content); + original.file.close(); + assert_file_contains(original.path, content); + hash orig_info = hstat(original.path); + + string move0_path = join_paths(tmp_path, "move"); + assertFalse(path_exists(move0_path)); + string dest_path = move_path(original.path, move0_path); + assertEq(move0_path, dest_path); + assertTrue(path_exists(move0_path)); + assertEq(orig_info.mode, hstat(move0_path).mode); + assert_file_contains(move0_path, content); + + # now check that permission bits are copied too + move_path(dest_path, original.path); + assertEq(0100600, orig_info.mode); + chmod(original.path, 0640); + orig_info = hstat(original.path); + assertEq(0100640, orig_info.mode); + dest_path = move_path(original.path, move0_path); + assertEq(move0_path, dest_path); + assertEq(0100640, hstat(move0_path).mode); + + # now overwrite the file with different content + copy_path(dest_path, original.path); + string content2 = "Goodbye..."; + set_content_to_file(original.path, content2); + assert_file_contains(original.path, content2); + assert_file_contains(move0_path, content); + orig_info = hstat(original.path); + # first without the overwrite flag + assertThrows("PATH-EXISTS-ERROR", \move_path(), (original.path, move0_path)); + # now with overwrite + dest_path = move_path(original.path, move0_path, True); + assertEq(move0_path, dest_path); + assertEq(orig_info.mode, hstat(move0_path).mode); + assert_file_contains(move0_path, content2); + + # now overwrite the file with different permissions + copy_path(dest_path, original.path); + orig_info = hstat(original.path); + assertEq(0100640, orig_info.mode); + assertEq(0100640, hstat(move0_path).mode); + chmod(original.path, 0600); + orig_info = hstat(original.path); + assertEq(0100600, orig_info.mode); + # first without the overwrite flag + assertThrows("PATH-EXISTS-ERROR", \move_path(), (original.path, move0_path)); + # now with overwrite + dest_path = move_path(original.path, move0_path, True); + assertEq(move0_path, dest_path); + assertEq(0100600, hstat(move0_path).mode); + } + + test_move_path_exceptions() { + string tmp_path = make_tmp_dir(); + on_exit { + remove_tree(tmp_path); + } + + # prepare FS structure + hash file1 = make_tmp_file(NOTHING, NOTHING, tmp_path); + string content = "Hello World!"; + file1.file.write(content); + file1.file.close(); + assert_file_contains(file1.path, content); + string link_file1 = join_paths(tmp_path, "link_file1"); + symlink(file1.path, link_file1); + assertTrue(is_link(link_file1)); + assertEq(file1.path, readlink(link_file1)); + string subdir = make_tmp_dir(NOTHING, NOTHING, tmp_path); + + # non-existent source should always fail + string non_existent = join_paths(tmp_path, "non_existent.txt"); + assertThrows("FILE-STAT-ERROR", \move_path(), (non_existent, file1.path)); + + # existing destination + hash file2 = make_tmp_file(NOTHING, NOTHING, tmp_path); + file1.file.close(); + assertThrows("PATH-EXISTS-ERROR", \move_path(), (file2.path, file1.path)); + + # existing destination via a link + assertThrows("PATH-EXISTS-ERROR", \move_path(), (file2.path, link_file1)); + + # it's possible to move to a directory though + hash orig_info = hstat(file1.path); + string move0_path = join_paths(subdir, basename(file1.path)); + assertFalse(path_exists(move0_path)); + string dest_path = move_path(file1.path, subdir); + assertEq(move0_path, dest_path); + assertTrue(path_exists(move0_path)); + assertEq(orig_info.mode, hstat(move0_path).mode); + assert_file_contains(move0_path, content); + } + + test_move_path_symlink_in_src() { + string tmp_path = make_tmp_dir(); + on_exit { + remove_tree(tmp_path); + } + + string content = "Qore is great"; + + # get a file + hash original = make_tmp_file(NOTHING, NOTHING, tmp_path); + original.file.write(content); + original.file.close(); + assert_file_contains(original.path, content); + + # create a link to that file + string link_original = join_paths(tmp_path, "link1"); + symlink(original.path, link_original); + assertTrue(is_link(link_original)); + assertEq(original.path, readlink(link_original)); + + # create a target path + string move = join_paths(tmp_path, "move"); + + hash orig_info = hstat(link_original); + + # move original via symlink with follow_symlinks + string dest_path = move_path(link_original, move); + assertEq(move, dest_path); + assertTrue(path_exists(move)); + assertEq(orig_info.mode, hstat(move).mode); + assertTrue(is_link(move)); + assert_file_contains(move, content); + unlink(move); + } + + test_move_path_tree() { + # create a temporary directory and a test directory tree in it + string tmp_root = make_tmp_dir(); + on_exit { + remove_tree(tmp_root); + } + string source_path = join_paths(tmp_root, "src"); + mkdir(source_path); + string dir_path = join_paths(source_path, "dir"); + mkdir(dir_path); + File tmp_file = new File(); + string file_path = join_paths(source_path, "file"); + tmp_file.open(file_path, O_CREAT, 0600); + tmp_file.close(); + string link_path = join_paths(dir_path, "link"); + symlink(file_path, link_path); + + # now we should have something like... + # tmp_root + # └── src + # ├── dir + # │   └── link -> tmp_root/src/file + # └── file + # ... let's check it + assertTrue(path_exists(source_path)); + assertTrue(is_dir(source_path)); + assertTrue(path_exists(dir_path)); + assertTrue(is_dir(dir_path)); + assertTrue(path_exists(file_path)); + assertTrue(is_file(file_path)); + assertTrue(path_exists(link_path)); + assertTrue(is_link(link_path)); + assertEq(file_path, readlink(link_path)); + + # let's move the source directory tree and check the result too + string dst_path = join_paths(tmp_root, "dst"); + string cp_dest_path = move_path(source_path, dst_path); + assertEq(dst_path, cp_dest_path); + # this is what we expect + string dst_dir_path = join_paths(dst_path, "dir"); + string dst_file_path = join_paths(dst_path, "file"); + string dst_link_path = join_paths(dst_dir_path, "link"); + assertTrue(path_exists(dst_path)); + assertTrue(is_dir(dst_path)); + assertTrue(path_exists(dst_dir_path)); + assertTrue(is_dir(dst_dir_path)); + assertTrue(path_exists(dst_file_path)); + assertTrue(is_file(dst_file_path)); + assertTrue(path_exists(dst_link_path)); + assertTrue(is_link(dst_link_path)); + assertEq(file_path, readlink(dst_link_path)); # follow_symlinks=False + # let's clean it + move_path(dst_path, source_path); + assertFalse(path_exists(dst_path)); + + # let's try again but this time use an existing directory as destination + mkdir(dst_path); + cp_dest_path = move_path(source_path, dst_path, True); + assertEq(dst_path, cp_dest_path); + assertTrue(path_exists(dst_path)); + assertTrue(is_dir(dst_path)); + assertTrue(path_exists(dst_dir_path)); + assertTrue(is_dir(dst_dir_path)); + assertTrue(path_exists(dst_file_path)); + assertTrue(is_file(dst_file_path)); + assertTrue(path_exists(dst_link_path)); + assertTrue(is_link(dst_link_path)); + assertEq(file_path, readlink(dst_link_path)); # follow_symlinks=False + # let's clean it + move_path(dst_path, source_path); + assertFalse(path_exists(dst_path)); + + # now let's move it via a link to the source + # let's try again with follow_symlinks=True + string src_link_path = join_paths(tmp_root, "src_link"); + assertFalse(path_exists(src_link_path)); + symlink(source_path, src_link_path); + assertTrue(path_exists(src_link_path)); + assertTrue(is_link(src_link_path)); + cp_dest_path = move_path(source_path, dst_path); + assertEq(dst_path, cp_dest_path); + assertTrue(path_exists(dst_path)); + assertTrue(is_dir(dst_path)); + assertTrue(path_exists(dst_dir_path)); + assertTrue(is_dir(dst_dir_path)); + assertTrue(path_exists(dst_file_path)); + assertTrue(is_file(dst_file_path)); + assertTrue(path_exists(dst_link_path)); + assertFalse(is_file(dst_link_path)); + } + + test_move_path_tree_overwrite() { + # create a temporary directory and a test directory tree in it + TmpDir tmp_root = new TmpDir(); + string source_path = join_paths(tmp_root.path, "src"); + mkdir(source_path); + string dir_path = join_paths(source_path, "dir"); + mkdir(dir_path); + File tmp_file = new File(); + string file_path = join_paths(source_path, "file"); + tmp_file.open(file_path, O_CREAT, 0600); + tmp_file.close(); + + # now we should have something like... + # tmp_root + # └── src + # ├── dir + # └── file + # ... let's check it + assertTrue(path_exists(source_path)); + assertTrue(is_dir(source_path)); + assertTrue(path_exists(dir_path)); + assertTrue(is_dir(dir_path)); + assertTrue(path_exists(file_path)); + assertTrue(is_file(file_path)); + + # let's move the source directory tree and check the result too + string dst_path = join_paths(tmp_root.path, "dst"); + string cp_dest_path = move_path(source_path, dst_path); + assertEq(dst_path, cp_dest_path); + # this is what we expect + string dst_dir_path = join_paths(dst_path, "dir"); + string dst_file_path = join_paths(dst_path, "file"); + assertTrue(path_exists(dst_path)); + assertTrue(is_dir(dst_path)); + assertTrue(path_exists(dst_dir_path)); + assertTrue(is_dir(dst_dir_path)); + assertTrue(path_exists(dst_file_path)); + assertTrue(is_file(dst_file_path)); + + copy_path(dst_path, source_path); + + # it will be OK with overwrite though + cp_dest_path = move_path(source_path, dst_path, True); + assertEq(dst_path, cp_dest_path); + # everything is copied anew -> let's check it + dst_dir_path = join_paths(dst_path, "dir"); + dst_file_path = join_paths(dst_path, "file"); + assertTrue(path_exists(dst_path)); + assertTrue(is_dir(dst_path)); + assertTrue(path_exists(dst_dir_path)); + assertTrue(is_dir(dst_dir_path)); + assertTrue(path_exists(dst_file_path)); + assertTrue(is_file(dst_file_path)); + + copy_path(dst_path, source_path); + + # now let's add something to the destination path and then overwrite with a new move + string new_dir_dst = join_paths(dst_path, "new_dir"); + mkdir(new_dir_dst); + assertTrue(path_exists(new_dir_dst)); + assertTrue(is_dir(new_dir_dst)); + # let's overwrite destination now + cp_dest_path = move_path(source_path, dst_path, True); + assertEq(dst_path, cp_dest_path); + # everything is copied anew -> let's check it + dst_dir_path = join_paths(dst_path, "dir"); + dst_file_path = join_paths(dst_path, "file"); + assertTrue(path_exists(dst_path)); + assertTrue(is_dir(dst_path)); + assertTrue(path_exists(dst_dir_path)); + assertTrue(is_dir(dst_dir_path)); + assertTrue(path_exists(dst_file_path)); + assertTrue(is_file(dst_file_path)); + # and check that the new dir is not there + assertFalse(path_exists(new_dir_dst)); + assertFalse(is_dir(new_dir_dst)); + + copy_path(dst_path, source_path); + + # now let's add something to the source path and then overwrite destination again + string new_dir_src = join_paths(source_path, "new_dir"); + mkdir(new_dir_src); + assertTrue(path_exists(new_dir_src)); + assertTrue(is_dir(new_dir_src)); + assertFalse(path_exists(new_dir_dst)); + assertFalse(is_dir(new_dir_dst)); + + # let's overwrite destination now + cp_dest_path = move_path(source_path, dst_path, True); + assertEq(dst_path, cp_dest_path); + # everything is copied anew -> let's check it + dst_dir_path = join_paths(dst_path, "dir"); + dst_file_path = join_paths(dst_path, "file"); + assertTrue(path_exists(dst_path)); + assertTrue(is_dir(dst_path)); + assertTrue(path_exists(dst_dir_path)); + assertTrue(is_dir(dst_dir_path)); + assertTrue(path_exists(dst_file_path)); + assertTrue(is_file(dst_file_path)); + # and check that the new dir is there as well + assertTrue(path_exists(new_dir_dst)); + assertTrue(is_dir(new_dir_dst)); + } + + test_move_path_tree_exception() { + # create a temporary directory and a test directory tree in it + string tmp_root = make_tmp_dir(); + on_exit { + remove_tree(tmp_root); + } + string source_path = join_paths(tmp_root, "src"); + mkdir(source_path); + string dir_path = join_paths(source_path, "dir"); + mkdir(dir_path); + File tmp_file = new File(); + string file_path = join_paths(source_path, "file"); + tmp_file.open(file_path, O_CREAT, 0600); + tmp_file.close(); + string link_path = join_paths(dir_path, "link"); + symlink(file_path, link_path); + string dest_dir = join_paths(tmp_root, "dest"); + mkdir(dest_dir); + + # now we should have something like... + # tmp_root + # ├── dest + # └── src + # ├── dir + # │   └── link -> tmp_root/src/file + # └── file + # ... let's check it + assertTrue(path_exists(source_path)); + assertTrue(is_dir(source_path)); + assertTrue(path_exists(dir_path)); + assertTrue(is_dir(dir_path)); + assertTrue(path_exists(file_path)); + assertTrue(is_file(file_path)); + assertTrue(path_exists(link_path)); + assertTrue(is_link(link_path)); + assertEq(file_path, readlink(link_path)); + assertTrue(path_exists(dest_dir)); + assertTrue(is_dir(dest_dir)); + + # try and move non-existing directory + string non_existing = join_paths(tmp_root, "non-existing"); + assertThrows("FILE-STAT-ERROR", \move_path(), (non_existing, dest_dir)); + + # try and move a directory into itself + assertThrows("SAME-PATH-ERROR", \move_path(), (dest_dir, dest_dir)); + + remove_tree(dest_dir); + } +} \ No newline at end of file diff --git a/examples/test/qlib/FtpPoller/FtpPoller.qtest b/examples/test/qlib/FtpPoller/FtpPoller.qtest index 4b3771fcab..3fc8d00060 100755 --- a/examples/test/qlib/FtpPoller/FtpPoller.qtest +++ b/examples/test/qlib/FtpPoller/FtpPoller.qtest @@ -11,6 +11,8 @@ %requires ../../../../qlib/Util.qm %requires ../../../../qlib/FsUtil.qm %requires ../../../../qlib/QUnit.qm +%requires ../../../../qlib/DataProvider +%requires ../../../../qlib/FtpPollerUtil.qm %requires ../../../../qlib/FtpPoller.qm %exec-class FtpPollerTest @@ -36,7 +38,7 @@ class MyFtpPoller inherits FtpPoller { } } -public class FtpPollerTest inherits QUnit::Test { +class FtpPollerTest inherits QUnit::Test { public { } @@ -50,7 +52,8 @@ public class FtpPollerTest inherits QUnit::Test { } constructor() : Test("FtpPollerTest", "1.0") { - addTestCase("main test", \main_test()); + addTestCase("main test", \mainTest()); + addTestCase("data provider test", \dataProviderTest()); # Return for compatibility with test harness that checks return value set_return_value(main()); @@ -69,12 +72,12 @@ public class FtpPollerTest inherits QUnit::Test { return port; } - private main_test() { + private mainTest() { int port = getPort(); # PORT only supports IPv4, so we can't use "localhost", which may resolve # to an IPv6 address - FtpServer serv(port, m_options.verbose, "127.0.0.1"); + FtpServer serv(port, m_options.verbose - 1, "127.0.0.1"); on_exit serv.shutdown(); port = serv.getPort(); @@ -101,9 +104,112 @@ public class FtpPollerTest inherits QUnit::Test { assertTrue(poller.runOnce()); foreach hash file in (poller.files) { - printf("check file: %y\n", file.name); assertTrue(file.name != NOTHING); assertTrue(file_paths.hasKey(file.filepath)); } } + + private dataProviderTest() { + Logger logger(); + { + LoggerLevel level; + if (m_options.verbose > 3) { + level = LoggerLevel::getLevelDebug(); + } else if (m_options.verbose > 2) { + level = LoggerLevel::getLevelInfo(); + } else { + level = LoggerLevel::getLevelError(); + } + logger = new Logger("test", level); + } + logger.addAppender(new TestAppender()); + + int port = getPort(); + + # PORT only supports IPv4, so we can't use "localhost", which may resolve + # to an IPv6 address + FtpServer serv(port, m_options.verbose - 1, "127.0.0.1"); + on_exit serv.shutdown(); + + TmpDir tmp_dir(); + string dir = tmp_dir.path; + serv.setPwd(dir); + serv.setDirect(); + + port = serv.getPort(); + + TmpDir local_dir(); + MyObserver observer(); + FtpDelayedPollerDataProvider prov({ + "url": sprintf("ftp://user:pass@127.0.0.1:%d%s", port, tmp_dir.path), + "poll_interval": 1, + "local_dir": local_dir.path, + }); + prov.setLogger(logger); + prov.registerObserver(observer); + + list files; + map files += new TmpFile("ftp_test_" + $1, NOTHING, dir), Files; + map $1.file.write(get_random_string()), files; + + prov.observersReady(); + + observer.wait(); + map assertEq($1, observer.h{$1}.name), (map basename($1.path), files); + + assertTrue(True); + + map assertEq(File::readBinaryFile($1.path), + File::readBinaryFile(local_dir.path + DirSep + basename($1.path)), basename($1.path)), + files; + } +} + +class MyObserver inherits Observer { + public { + hash> h(); + } + + private { + Mutex m(); + Condition cond(); + } + + update(string event_id, hash data_) { + h{data_.name} = data_; + if (h.size() == 3) { + m.lock(); + on_exit m.unlock(); + + cond.broadcast(); + } + } + + wait() { + date start = now_us(); + + m.lock(); + on_exit m.unlock(); + + while (h.size() != 4) { + cond.wait(m, 1s); + if ((now_us() - start) > 5s) { + throw "ERROR", sprintf("%d/4 files received in timeout: %y", h.size(), keys h); + } + } + } +} + +class TestAppender inherits LoggerAppenderWithLayout { + constructor() : LoggerAppenderWithLayout("test", new LoggerLayoutPattern("%d T%t [%p]: %m%n")) { + open(); + } + + processEventImpl(int type, auto params) { + switch (type) { + case EVENT_LOG: + print(params); + break; + } + } } diff --git a/examples/test/qore/classes/FtpClient/FtpClient.qtest b/examples/test/qore/classes/FtpClient/FtpClient.qtest index e0102cad65..1e2f0ed10f 100755 --- a/examples/test/qore/classes/FtpClient/FtpClient.qtest +++ b/examples/test/qore/classes/FtpClient/FtpClient.qtest @@ -203,8 +203,7 @@ class FtpTest inherits QUnit::Test { try { fc.put(local_path); assertTrue(False); - } - catch (hash ex1) { + } catch (hash ex1) { ex = ex1; } assertTrue(ex.err == "SOCKET-TIMEOUT" || ex.err == "FTP-RECEIVE-ERROR"); @@ -215,7 +214,7 @@ class FtpTest inherits QUnit::Test { assertNothing(fc.put(local_path)); assertNothing(fc.get(local_path, local_path)); - assertEq(True, ReadOnlyFile::readTextFile(local_path) =~ /abc/); + assertRegex("abc", ReadOnlyFile::readTextFile(local_path)); serv.setBroken("retr", 2s); fc.setTimeout(1ms); diff --git a/examples/test/qore/classes/FtpServer.qc b/examples/test/qore/classes/FtpServer.qc index c470b8775f..4d1e3082f3 100644 --- a/examples/test/qore/classes/FtpServer.qc +++ b/examples/test/qore/classes/FtpServer.qc @@ -17,7 +17,9 @@ class FtpServer { TmpDir t; - *string pwd; + string pwd = "/"; + + bool direct; const PollInterval = 250ms; } @@ -39,6 +41,16 @@ class FtpServer { t = new TmpDir(); } + setDirect() { + direct = True; + } + + *string getLocalDir() { + if (t) { + return t.path; + } + } + shutdown() { quit = True; cnt.waitForZero(); @@ -213,15 +225,22 @@ class FtpServer { if (t) { string path = t.path + DirSep + file; + if (hstat(path)) { - data.sendFromInputStream(new FileInputStream(path), -1); + data.sendFromInputStream(new FileInputStream(path)); data.close(); ftpSend(ns, 226, "Transfer completed."); continue; } } - #printf("get: empty %y\n", get_empty); + if (direct && hstat(file)) { + data.sendFromInputStream(new FileInputStream(file)); + data.close(); + ftpSend(ns, 226, "Transfer completed."); + continue; + } + if (!get_empty) { # write a bunch of data and then exit for (int i = 0; i < 200; ++i) diff --git a/include/qore/intern/qore_socket_private.h b/include/qore/intern/qore_socket_private.h index 0c8c372e5c..6a94f9dc3f 100644 --- a/include/qore/intern/qore_socket_private.h +++ b/include/qore/intern/qore_socket_private.h @@ -2556,7 +2556,8 @@ struct qore_socket_private { return rc < 0 || sock == QORE_INVALID_SOCKET ? rc : 0; } - DLLLOCAL void sendFromInputStream(InputStream *is, int64 size, int64 timeout, ExceptionSink *xsink, QoreThreadLock* l) { + DLLLOCAL void sendFromInputStream(InputStream *is, int64 size, int64 timeout, ExceptionSink *xsink, + QoreThreadLock* l) { if (sock == QORE_INVALID_SOCKET) { se_not_open("Socket", "sendFromInputStream", xsink); return; diff --git a/lib/Makefile.am b/lib/Makefile.am index 2b98edf639..85094230a2 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -11,7 +11,7 @@ dummy: echo "Build started!" EXTRA_INCLUDES = -I$(top_srcdir)/include -I$(top_builddir)/include -I$(top_builddir)/lib -libqore_la_LDFLAGS = -version-info 12:2:0 -no-undefined ${QORE_LIB_LDFLAGS} +libqore_la_LDFLAGS = -version-info 12:3:0 -no-undefined ${QORE_LIB_LDFLAGS} AM_CPPFLAGS = $(EXTRA_INCLUDES) ${QORE_LIB_CPPFLAGS} AM_CXXFLAGS = ${QORE_LIB_CXXFLAGS} AM_YFLAGS = -d diff --git a/lib/QoreFtpClient.cpp b/lib/QoreFtpClient.cpp index 2278d9de60..794ac6b848 100644 --- a/lib/QoreFtpClient.cpp +++ b/lib/QoreFtpClient.cpp @@ -59,10 +59,37 @@ enum qore_ftp_mode { //FTP_MODE_LPSV }; -class FtpResp { +class TmpLocalName { +public: + DLLLOCAL TmpLocalName(const char* name1, const char* name2) : str(name1) { + if (!name1) { + tmp_str = q_basename(name2); + } + } + + DLLLOCAL ~TmpLocalName() { + if (tmp_str) { + free(tmp_str); + } + } + + DLLLOCAL const char* operator*() const { + return str ? str : tmp_str; + } + + DLLLOCAL void discard() { + if (tmp_str) { + free(tmp_str); + tmp_str = nullptr; + } + } + private: - QoreStringNode* str = nullptr; + const char* str; + char* tmp_str = nullptr; +}; +class FtpResp { public: DLLLOCAL FtpResp() {} @@ -70,13 +97,15 @@ class FtpResp { } DLLLOCAL ~FtpResp() { - if (str) + if (str) { str->deref(); + } } DLLLOCAL QoreStringNode* assign(QoreStringNode* s) { - if (str) + if (str) { str->deref(); + } str = s; return s; } @@ -88,6 +117,9 @@ class FtpResp { DLLLOCAL QoreStringNode* getStr() { return str; } + +private: + QoreStringNode* str = nullptr; }; struct qore_ftp_private { @@ -912,87 +944,83 @@ QoreStringNode* QoreFtpClient::list(const char* path, bool long_list, ExceptionS // public locked int QoreFtpClient::put(const char* localpath, const char* remotename, ExceptionSink* xsink) { - printd(5, "QoreFtpClient::put(%s, %s)\n", localpath, remotename ? remotename : "NULL"); - - SafeLocker sl(priv->m); - if (priv->checkConnectedUnlocked(xsink)) - return -1; + printd(5, "QoreFtpClient::put(%s, %s)\n", localpath, remotename ? remotename : "NULL"); - int fd = open(localpath, O_RDONLY, 0); - if (fd < 0) { - xsink->raiseErrnoException("FTP-FILE-OPEN-ERROR", errno, "%s", localpath); - return -1; - } - ON_BLOCK_EXIT(close, fd); - - // set binary mode and establish data connection - if (priv->setBinaryMode(true, xsink) || priv->connectData(xsink)) { - return -1; - } + SafeLocker sl(priv->m); + if (priv->checkConnectedUnlocked(xsink)) + return -1; - // get file size - struct stat file_info; - if (fstat(fd, &file_info) == -1) { - int en = errno; - xsink->raiseErrnoException("FTP-FILE-PUT-ERROR", en, "could not get file size"); - return -1; - } + int fd = open(localpath, O_RDONLY, 0); + if (fd < 0) { + xsink->raiseErrnoException("FTP-FILE-OPEN-ERROR", errno, "%s", localpath); + return -1; + } + ON_BLOCK_EXIT(close, fd); - // get remote file name - char* rn = remotename ? (char*)remotename : q_basename(localpath); + // set binary mode and establish data connection + if (priv->setBinaryMode(true, xsink) || priv->connectData(xsink)) { + return -1; + } - // transfer file - int code; + // get file size + struct stat file_info; + if (fstat(fd, &file_info) == -1) { + int en = errno; + xsink->raiseErrnoException("FTP-FILE-PUT-ERROR", en, "could not get file size"); + return -1; + } - QoreStringNode* mr = priv->sendMsg(code, "STOR", rn, xsink); - if (rn != remotename) - free(rn); - if (!mr) { - assert(*xsink); - priv->data.close(); - return -1; - } + // get remote file name + TmpLocalName rn(remotename, localpath); + // transfer file + int code; + QoreStringNode* mr = priv->sendMsg(code, "STOR", *rn, xsink); + rn.discard(); + if (!mr) { + assert(*xsink); + priv->data.close(); + return -1; + } - FtpResp resp(mr); - if (*xsink) { - priv->data.close(); - return -1; - } - //printf("%s", resp->c_str()); + FtpResp resp(mr); + if (*xsink) { + priv->data.close(); + return -1; + } + //printf("%s", resp->c_str()); - if ((code / 100) != 1) { - priv->data.close(); - xsink->raiseException("FTP-PUT-ERROR", "could not put file, FTP server replied: %s", resp.c_str()); - return -1; - } + if ((code / 100) != 1) { + priv->data.close(); + xsink->raiseException("FTP-PUT-ERROR", "could not put file, FTP server replied: %s", resp.c_str()); + return -1; + } - if ((priv->mode == FTP_MODE_PORT && priv->acceptDataConnection(xsink)) || *xsink) { - priv->data.close(); - return -1; - } - else if (priv->secure_data && priv->data.upgradeClientToSSL(0, 0, priv->timeout_ms, xsink)) { - return -1; - } + if ((priv->mode == FTP_MODE_PORT && priv->acceptDataConnection(xsink)) || *xsink) { + priv->data.close(); + return -1; + } else if (priv->secure_data && priv->data.upgradeClientToSSL(0, 0, priv->timeout_ms, xsink)) { + return -1; + } - int rc = priv->data.send(fd, file_info.st_size ? file_info.st_size : -1, priv->timeout_ms, xsink); - priv->data.close(); + int rc = priv->data.send(fd, file_info.st_size ? file_info.st_size : -1, priv->timeout_ms, xsink); + priv->data.close(); - resp.assign(priv->getResponse(code, xsink)); - sl.unlock(); - if (*xsink) - return -1; + resp.assign(priv->getResponse(code, xsink)); + sl.unlock(); + if (*xsink) + return -1; - //printf("PUT: %s", resp->c_str()); - if ((code / 100 != 2)) { - xsink->raiseException("FTP-PUT-ERROR", "FTP server returned an error to the STOR command: %s", resp.c_str()); - return -1; - } + //printf("PUT: %s", resp->c_str()); + if ((code / 100 != 2)) { + xsink->raiseException("FTP-PUT-ERROR", "FTP server returned an error to the STOR command: %s", resp.c_str()); + return -1; + } - if (rc) { - xsink->raiseException("FTP-PUT-ERROR", "error sending file, may not be complete on target"); - return -1; - } - return 0; + if (rc) { + xsink->raiseException("FTP-PUT-ERROR", "error sending file, may not be complete on target"); + return -1; + } + return 0; } // public locked @@ -1124,56 +1152,51 @@ int QoreFtpClient::putData(const void *data, size_t len, const char* remotename, // public locked int QoreFtpClient::get(const char* remotepath, const char* localname, ExceptionSink* xsink) { - printd(5, "QoreFtpClient::get(%s, %s)\n", remotepath, localname ? localname : "NULL"); + printd(5, "QoreFtpClient::get(%s, %s)\n", remotepath, localname ? localname : "NULL"); - SafeLocker sl(priv->m); - if (priv->checkConnectedUnlocked(xsink)) - return -1; + SafeLocker sl(priv->m); + if (priv->checkConnectedUnlocked(xsink)) { + return -1; + } - // get local file name - char* ln = localname ? (char* )localname : q_basename(remotepath); + // get local file name + TmpLocalName ln(localname, remotepath); + printd(FTPDEBUG, "QoreFtpClient::get(%s) %s\n", remotepath, *ln); + // open local file + int fd = open(*ln, O_WRONLY|O_CREAT|O_TRUNC, 0644); + if (fd < 0) { + xsink->raiseErrnoException("FTP-FILE-OPEN-ERROR", errno, "%s", *ln); + return -1; + } - printd(FTPDEBUG, "QoreFtpClient::get(%s) %s\n", remotepath, ln); - // open local file - int fd = open(ln, O_WRONLY|O_CREAT|O_TRUNC, 0644); - if (fd < 0) { - xsink->raiseErrnoException("FTP-FILE-OPEN-ERROR", errno, "%s", ln); - if (ln != localname) - free(ln); - return -1; - } + FtpResp resp; + { + ON_BLOCK_EXIT(close, fd); + if (priv->pre_get(resp, remotepath, xsink)) { + // delete temporary file + unlink(*ln); + return -1; + } + ln.discard(); - FtpResp resp; - { - ON_BLOCK_EXIT(close, fd); - if (priv->pre_get(resp, remotepath, xsink)) { - // delete temporary file - unlink(ln); - if (ln != localname) - free(ln); - return -1; - } - - if (ln != localname) - free(ln); - - priv->data.recv(fd, -1, priv->timeout_ms, xsink); - priv->data.close(); - } + priv->data.recv(fd, -1, priv->timeout_ms, xsink); + priv->data.close(); + } - int code; - resp.assign(priv->getResponse(code, xsink)); - sl.unlock(); - if (*xsink) - return -1; + int code; + resp.assign(priv->getResponse(code, xsink)); + sl.unlock(); + if (*xsink) { + return -1; + } - //printf("PUT: %s", resp->c_str()); - if ((code / 100 != 2)) { - xsink->raiseException("FTP-GET-ERROR", "FTP server returned an error to the RETR command: %s", - resp.c_str()); - return -1; - } - return 0; + //printf("PUT: %s", resp->c_str()); + if ((code / 100 != 2)) { + xsink->raiseException("FTP-GET-ERROR", "FTP server returned an error to the RETR command: %s", + resp.c_str()); + return -1; + } + return 0; } // public locked diff --git a/lib/QoreSocket.cpp b/lib/QoreSocket.cpp index caa1e79cf5..2f3667153e 100644 --- a/lib/QoreSocket.cpp +++ b/lib/QoreSocket.cpp @@ -736,17 +736,18 @@ int qore_socket_private::recv(int fd, qore_offset_t size, int timeout_ms, Except while (true) { // calculate bytes needed int bn; - if (size == -1) + if (size == -1) { bn = DEFAULT_SOCKET_BUFSIZE; - else { + } else { bn = size - br; if (bn > DEFAULT_SOCKET_BUFSIZE) bn = DEFAULT_SOCKET_BUFSIZE; } rc = brecv(xsink, "recv", buf, bn, 0, timeout_ms); - if (rc <= 0) + if (rc <= 0) { break; + } br += rc; do_data_event(QORE_EVENT_SOCKET_DATA_READ, QORE_SOURCE_SOCKET, buf, rc); @@ -766,7 +767,8 @@ int qore_socket_private::recv(int fd, qore_offset_t size, int timeout_ms, Except } // write(2) should not return 0, but in case it does, it's treated as an error if (errno != EINTR) { - xsink->raiseErrnoException("FILE-READ-ERROR", errno, "error writing file after " QSD " bytes read in Socket::send()", br); + xsink->raiseErrnoException("FILE-READ-ERROR", errno, "error writing file after " QSD + " bytes read in Socket::send()", br); break; } } diff --git a/qlib/CdsRestDataProvider/CdsRestDataProviderFactory.qc b/qlib/CdsRestDataProvider/CdsRestDataProviderFactory.qc index 21e00557db..568f668e38 100644 --- a/qlib/CdsRestDataProvider/CdsRestDataProviderFactory.qc +++ b/qlib/CdsRestDataProvider/CdsRestDataProviderFactory.qc @@ -34,6 +34,7 @@ public class CdsRestDataProviderFactory inherits DataProvider::AbstractDataProvi const FactoryInfo = { "name": "cdsrest", "desc": "Microsoft CDS REST data provider factory", + "children_can_support_records": True, }; } diff --git a/qlib/CsvUtil/CsvReadDataProviderFactory.qc b/qlib/CsvUtil/CsvReadDataProviderFactory.qc index 5d2aff7d49..d65d0914fb 100644 --- a/qlib/CsvUtil/CsvReadDataProviderFactory.qc +++ b/qlib/CsvUtil/CsvReadDataProviderFactory.qc @@ -34,6 +34,7 @@ public class CsvReadDataProviderFactory inherits AbstractDataProviderFactory { const FactoryInfo = { "name": "csvread", "desc": "CSV reader data provider factory", + "children_can_support_records": True, }; } diff --git a/qlib/CsvUtil/CsvWriteDataProviderFactory.qc b/qlib/CsvUtil/CsvWriteDataProviderFactory.qc index 5ce7a999b8..51f90166de 100644 --- a/qlib/CsvUtil/CsvWriteDataProviderFactory.qc +++ b/qlib/CsvUtil/CsvWriteDataProviderFactory.qc @@ -34,6 +34,7 @@ public class CsvWriteDataProviderFactory inherits AbstractDataProviderFactory { const FactoryInfo = { "name": "csvwrite", "desc": "CSV writer data provider factory", + "children_can_support_records": True, }; } diff --git a/qlib/DataProvider/AbstractDataProvider.qc b/qlib/DataProvider/AbstractDataProvider.qc index 77ad670d0a..15454f47d2 100644 --- a/qlib/DataProvider/AbstractDataProvider.qc +++ b/qlib/DataProvider/AbstractDataProvider.qc @@ -1457,6 +1457,18 @@ public class AbstractDataProvider { return getErrorResponseTypeImpl(error_code); } + #! Returns the description of an event, if any + /** @return the event type for this provider + + @throw INVALID-OPERATION the data provider does not support the observer pattern / event API + + @since DataProvider 2.5 + */ + *AbstractDataProviderType getEventType() { + checkObservable(); + return getEventTypeImpl(); + } + #! Return data provider summary info /** @note This creates an AbstractDataProvider object for each child; for cases when this is expensive, this method must be overridden / reimplemented in the child class for performance reasons @@ -2764,6 +2776,17 @@ public class AbstractDataProvider { return type; } + #! Returns the description of an event, if any + /** @return the event type for this provider + + @note only called if the provider supports the observer pattern / event API + + @since DataProvider 2.5 + */ + private *AbstractDataProviderType getEventTypeImpl() { + throwUnimplementedException(); + } + #! Throws an \c INVALID-OPERATION exception /** @throw INVALID-OPERATION this exception is thrown unconditionally by this method */ diff --git a/qlib/DataProvider/AbstractDataProviderFactory.qc b/qlib/DataProvider/AbstractDataProviderFactory.qc index bf5b5e66b8..c90620837d 100644 --- a/qlib/DataProvider/AbstractDataProviderFactory.qc +++ b/qlib/DataProvider/AbstractDataProviderFactory.qc @@ -48,6 +48,10 @@ public hashdecl DataProviderFactoryInfo { bool api_management = False; #! Can any child data providers offer API services (request - response data providers)? bool children_can_support_apis = False; + #! Can any child data providers offer record-based providers? + bool children_can_support_records = False; + #! Can any child data providers support the observer pattern / event API? + bool children_can_support_observers = False; #! Which server profiles are supported for API management (if any)? *softlist api_profiles; #! Info about providers created from this factory (without the \a name and \a children attributes) diff --git a/qlib/DataProvider/DataProvider.qc b/qlib/DataProvider/DataProvider.qc index aa3ee8da33..87f7e03adb 100644 --- a/qlib/DataProvider/DataProvider.qc +++ b/qlib/DataProvider/DataProvider.qc @@ -32,18 +32,20 @@ public class DataProvider { public { #! Map of known data provider factory names to modules const FactoryMap = { - "db": "DbDataProvider", - "swagger": "SwaggerDataProvider", "cdsrest": "CdsRestDataProvider", "csvread": "CsvUtil", "csvwrite": "CsvUtil", + "db": "DbDataProvider", + "file": "FileDataProvider", "filepoller": "FilePoller", "fixedlengthread": "FixedLengthUtil", "fixedlengthwrite": "FixedLengthUtil", + "ftppoller": "FtpPoller", "salesforcerest": "SalesforceRestDataProvider", "servicenowrest": "ServiceNowRestDataProvider", # provided by the xml module "soap": "SoapDataProvider", + "swagger": "SwaggerDataProvider", }; #! Map of known type path prefixes to modules @@ -136,8 +138,8 @@ public class DataProvider { static AbstractDataProviderFactory getFactoryEx(string name) { *AbstractDataProviderFactory factory = DataProvider::getFactory(name); if (!factory) { - throw "PROVIDER-ERROR", sprintf("data provider factory %y is unknown; known data provider factories: %y", name, - keys factory_cache); + throw "PROVIDER-ERROR", sprintf("data provider factory %y is unknown; known data provider factories: %y", + name, keys factory_cache); } return factory; } @@ -159,7 +161,8 @@ public class DataProvider { } #! Returns a data provider object from the given factory string - /** Factory options are given as string-formatted hash in curly brackets and child data providers are separated by forward slashes + /** Factory options are given as string-formatted hash in curly brackets and child data providers are separated by + forward slashes @par Example: @code{.py} @@ -176,7 +179,8 @@ DbDataProvider db = DataProvider::getFactoryObjectFromString("db{oracle:user/pas } #! Returns a data provider object from the given factory string using environment variables to find the factory - /** Factory options are given as string-formatted hash in curly brackets and child data providers are separated by forward slashes + /** Factory options are given as string-formatted hash in curly brackets and child data providers are separated by + forward slashes @par Example: @code{.py} diff --git a/qlib/DataProvider/DataProvider.qm b/qlib/DataProvider/DataProvider.qm index 2ef2dc6685..ecb88036df 100644 --- a/qlib/DataProvider/DataProvider.qm +++ b/qlib/DataProvider/DataProvider.qm @@ -58,11 +58,12 @@ module DataProvider { @section dataproviderintro Introduction to the DataProvider Module The %DataProvider module provides APIs for hierarchical data structures from arbitrary sources to be described, - queried, introspected, and updated. It also supports data providers with request-reply semantics such as REST - schemas or with SOAP messaging. + queried, introspected, and updated. It supports data providers supporting record-based APIs as well as + request-reply semantics such as REST schemas or with SOAP messaging, as well as event-based data providers. - The data provider module supports high-performance reading (searching) and writing as well as record creation and - upserting and transaction management if supported by the underlying data provider implementation as well. + The data provider module supports high-performance reading (native or simulated searching, also with advanced + search expression support) and writing as well as record creation, upserting, and transaction management for + record-based data providers if supported by the underlying data provider implementation as well. The %Qore command-line program \c qdp provides a user-friendly interface to data provider functionality. diff --git a/qlib/DataProvider/Observerable.qc b/qlib/DataProvider/Observable.qc similarity index 100% rename from qlib/DataProvider/Observerable.qc rename to qlib/DataProvider/Observable.qc diff --git a/qlib/DbDataProvider/DbDataProviderFactory.qc b/qlib/DbDataProvider/DbDataProviderFactory.qc index 2837487928..9286b56663 100644 --- a/qlib/DbDataProvider/DbDataProviderFactory.qc +++ b/qlib/DbDataProvider/DbDataProviderFactory.qc @@ -34,6 +34,7 @@ public class DbDataProviderFactory inherits AbstractDataProviderFactory { const FactoryInfo = { "name": "db", "desc": "Database data provider factory", + "children_can_support_records": True, }; } diff --git a/qlib/FileDataProvider/FileCopyDataProvider.qc b/qlib/FileDataProvider/FileCopyDataProvider.qc new file mode 100644 index 0000000000..cc7910501c --- /dev/null +++ b/qlib/FileDataProvider/FileCopyDataProvider.qc @@ -0,0 +1,89 @@ +# -*- mode: qore; indent-tabs-mode: nil -*- +#! Qore FileCopyDataProvider module definition + +/** FileCopyDataProvider.qc Copyright 2019 - 2022 Qore Technologies, s.r.o. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#! contains all public definitions in the FileCopyDataProvider module +public namespace FileDataProvider { +#! The File data provider class +public class FileCopyDataProvider inherits AbstractDataProvider { + public { + #! Provider info + const ProviderInfo = { + "type": "FileCopyDataProvider", + "supports_request": True, + }; + + #! Request type + const RequestType = new FileCopyRequestDataType(); + + #! Response type + const ResponseType = new FileTargetResponseDataType(); + } + + #! Returns the data provider name + string getName() { + return "copy"; + } + + #! Returns the data provider description + *string getDesc() { + return "File copy data provider; copies files or entire directory trees on the filesystem given source and " + "target paths as arguments"; + } + + #! Makes a request and returns the response + /** @param req req the request info + @param request_options the request options; will be processed by validateRequestOptions() + + @return the response to the request + + @throws SAME-DIR-ERROR If both paths point to the same file + @throws DIR-STAT-ERROR If stat call for the source path fails + @throws PATH-EXISTS-ERROR If destination path exists and overwrite flag is not set. + */ + private auto doRequestImpl(auto req, *hash request_options) { + return { + "target": copy_path(req.source, req.target, req.follow_symlinks, req.overwrite), + }; + } + + #! Returns the description of a successful request message, if any + /** @return the request type for this provider + */ + private *AbstractDataProviderType getRequestTypeImpl() { + return RequestType; + } + + #! Returns the description of a response message, if this object represents a response message + /** @return the response type for this response message + */ + private *AbstractDataProviderType getResponseTypeImpl() { + return ResponseType; + } + + #! Returns data provider static info + hash getStaticInfoImpl() { + return ProviderInfo; + } +} +} diff --git a/qlib/FileDataProvider/FileCopyRequestDataType.qc b/qlib/FileDataProvider/FileCopyRequestDataType.qc new file mode 100644 index 0000000000..fb7da8f7d9 --- /dev/null +++ b/qlib/FileDataProvider/FileCopyRequestDataType.qc @@ -0,0 +1,45 @@ +# -*- mode: qore; indent-tabs-mode: nil -*- + +/** FileCopyRequestDataType.qc Copyright 2022 Qore Technologies, s.r.o. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#! contains all public definitions in the FileCopyDataProvider module +public namespace FileDataProvider { +#! Data type for copy file request calls +public class FileCopyRequestDataType inherits FileMoveRequestDataType { + private { + #! Field descriptions + const Fields = { + "follow_symlinks": { + "type": BoolType, + "desc": "Should symbolic links be followed?", + "default_value": False, + }, + }; + } + + #! Creates the object + constructor() { + map addField(new QoreDataField($1.key, $1.value.desc, $1.value.type, $1.value.default_value)), + Fields.pairIterator(); + } +} +} diff --git a/qlib/FileDataProvider/FileDataProvider.qc b/qlib/FileDataProvider/FileDataProvider.qc new file mode 100644 index 0000000000..2ed36aa5b6 --- /dev/null +++ b/qlib/FileDataProvider/FileDataProvider.qc @@ -0,0 +1,93 @@ +# -*- mode: qore; indent-tabs-mode: nil -*- +#! Qore FileDataProvider class definition + +/** FileDataProvider.qc Copyright 2019 - 2022 Qore Technologies, s.r.o. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#! contains all public definitions in the FileDataProvider module +public namespace FileDataProvider { +#! The database data provider class, provides tables as children +public class FileDataProvider inherits AbstractDataProvider { + public { + #! Provider info + const ProviderInfo = { + "type": "FileDataProvider", + "supports_children": True, + }; + } + + private { + const ChildMap = { + "copy": Class::forName("FileDataProvider::FileCopyDataProvider"), + "move": Class::forName("FileDataProvider::FileMoveDataProvider"), + "delete": Class::forName("FileDataProvider::FileDeleteDataProvider"), + "stat": Class::forName("FileDataProvider::FileStatDataProvider"), + }; + } + + #! Creates the object from constructor options + constructor(*hash options) { + checkOptions("CONSTRUCTOR-ERROR", NOTHING, options); + } + + #! Returns the data provider name + string getName() { + return "file"; + } + + #! Returns the data provider description + *string getDesc() { + return "Data provider for the local filesystem"; + } + + #! Return data provider summary info + *list> getChildProviderSummaryInfo() { + return map $1.getStaticMember("StaticInfo").getValue(), ChildMap.iterator(); + } + + #! Returns a list of child data provider names, if any + /** @return a list of child data provider names, if any + */ + private *list getChildProviderNamesImpl() { + return keys ChildMap; + } + + #! Returns the given child provider or @ref nothing if the given child is unknown + /** @return the given child provider or @ref nothing if the given child is unknown + + @throw CHILD-PROVIDER-ERROR error acquiring child provider + + @see getChildProviderEx() + */ + private *AbstractDataProvider getChildProviderImpl(string name) { + *Class cls = ChildMap{name}; + if (!cls) { + return; + } + return cls.newObject(); + } + + #! Returns data provider static info + private hash getStaticInfoImpl() { + return ProviderInfo; + } +} +} diff --git a/qlib/FileDataProvider/FileDataProvider.qm b/qlib/FileDataProvider/FileDataProvider.qm new file mode 100644 index 0000000000..9275f0a324 --- /dev/null +++ b/qlib/FileDataProvider/FileDataProvider.qm @@ -0,0 +1,75 @@ +# -*- mode: qore; indent-tabs-mode: nil -*- +#! Qore FileDataProvider module definition + +/* FileDataProvider.qm Copyright 2019 - 2022 Qore Technologies, s.r.o. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +# minimum required Qore version +%requires qore >= 1.10 +# assume local scope for variables, do not use "$" signs +%new-style +# require type definitions everywhere +%require-types +# strict argument handling +%strict-args +# enable all warnings +%enable-all-warnings + +%requires(reexport) DataProvider +%requires FsUtil + +module FileDataProvider { + version = "1.0"; + desc = "user module providing a data provider API for local filesystems"; + author = "David Nichols "; + url = "http://qore.org"; + license = "MIT"; + init = sub () { + # register the data provider factory + DataProvider::registerFactory(new FileDataProviderFactory()); + }; +} + +/** @mainpage FileDataProvider Module + + @tableofcontents + + @section filedataproviderintro Introduction to the FileDataProvider Module + + The %FileDataProvider module provides a data provider API for the local filesystem through the + @ref dataproviderintro "DataProvider" API. + + The following classes are provided by this module: + - @ref FileDataProvider::FileCopyDataProvider "FileCopyDataProvider" + - @ref FileDataProvider::FileDeleteDataProvider "FileDeleteDataProvider" + - @ref FileDataProvider::FileDataProvider "FileDataProvider" + - @ref FileDataProvider::FileDataProviderFactory "FileDataProviderFactory" + - @ref FileDataProvider::FileMoveDataProvider "FileMoveDataProvider" + + @section filedataprovider_relnotes Release Notes + + @subsection filedataprovider_v1_0 FileDataProvider v1.0 + - initial release of the module +*/ + +#! contains all public definitions in the FileDataProvider module +public namespace FileDataProvider { +} diff --git a/qlib/FileDataProvider/FileDataProviderFactory.qc b/qlib/FileDataProvider/FileDataProviderFactory.qc new file mode 100644 index 0000000000..cf53bfa021 --- /dev/null +++ b/qlib/FileDataProvider/FileDataProviderFactory.qc @@ -0,0 +1,59 @@ +# -*- mode: qore; indent-tabs-mode: nil -*- + +/* FileDataProviderFactory.qc Copyright (C) 2014 - 2022 Qore Technologies s.r.o. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#! Main module namespace +public namespace FileDataProviderFactory { +#! The file poller data provider factory +public class FileDataProviderFactory inherits AbstractDataProviderFactory { + private { + #! Data provider type info + static Class cls = new Class("FileDataProvider"); + + #! Factory info + const FactoryInfo = { + "name": "file", + "desc": "File data provider factory", + "children_can_support_apis": True, + }; + } + + #! Returns static factory information without \a provider_info + /** @return static factory information without \a provider_info which is provided by @ref getProviderInfo() + */ + private hash getInfoImpl() { + return FactoryInfo; + } + + #! Returns static provider information + /** @note the \c name and \c children attributes are not returned as they are dynamic attributes + */ + private hash getProviderInfoImpl() { + return FileDataProvider::ProviderInfo; + } + + #! Returns the class for the data provider object + private Class getClassImpl() { + return cls; + } +} +} diff --git a/qlib/FileDataProvider/FileDeleteDataProvider.qc b/qlib/FileDataProvider/FileDeleteDataProvider.qc new file mode 100644 index 0000000000..17080875cd --- /dev/null +++ b/qlib/FileDataProvider/FileDeleteDataProvider.qc @@ -0,0 +1,87 @@ +# -*- mode: qore; indent-tabs-mode: nil -*- +#! Qore FileDeleteDataProvider module definition + +/** FileDeleteDataProvider.qc Copyright 2019 - 2022 Qore Technologies, s.r.o. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#! contains all public definitions in the FileDeleteDataProvider module +public namespace FileDataProvider { +#! The File data provider class +public class FileDeleteDataProvider inherits AbstractDataProvider { + public { + #! Provider info + const ProviderInfo = { + "type": "FileDeleteDataProvider", + "supports_request": True, + }; + + #! Request type + const RequestType = new FileDeleteRequestDataType(); + } + + #! Returns the data provider name + string getName() { + return "delete"; + } + + #! Returns the data provider description + *string getDesc() { + return "File delete data provider; deletes files or entire directory trees on the filesystem given a path " + "as an argument"; + } + + #! Makes a request and returns the response + /** @param req the request info + @param request_options the request options; will be processed by validateRequestOptions() + + @return the response to the request + + @throws SAME-FILE-ERROR If both paths point to the same file. + @throws FILE-STAT-ERROR If stat call for the source path fails. + @throws PATH-EXISTS-ERROR If destination path exists and overwrite flag is False + */ + private auto doRequestImpl(auto req, *hash request_options) { + remove_path(req.path); + return { + "path": req.path, + }; + } + + #! Returns the description of a successful request message, if any + /** @return the request type for this provider + */ + private *AbstractDataProviderType getRequestTypeImpl() { + return RequestType; + } + + #! Returns the description of a response message, if this object represents a response message + /** @return the response type for this response message + */ + private *AbstractDataProviderType getResponseTypeImpl() { + return RequestType; + } + + #! Returns data provider static info + hash getStaticInfoImpl() { + return ProviderInfo; + } +} +} diff --git a/qlib/FileDataProvider/FileDeleteRequestDataType.qc b/qlib/FileDataProvider/FileDeleteRequestDataType.qc new file mode 100644 index 0000000000..dddb5b47ba --- /dev/null +++ b/qlib/FileDataProvider/FileDeleteRequestDataType.qc @@ -0,0 +1,44 @@ +# -*- mode: qore; indent-tabs-mode: nil -*- + +/** FileDeleteRequestDataType.qc Copyright 2022 Qore Technologies, s.r.o. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR DELETERIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#! contains all public definitions in the FileDeleteDataProvider module +public namespace FileDataProvider { +#! Data type for delete file request calls +public class FileDeleteRequestDataType inherits HashDataType { + private { + #! Field descriptions + const Fields = { + "path": { + "type": StringType, + "desc": "The file path", + }, + }; + } + + #! Creates the object + constructor() { + map addField(new QoreDataField($1.key, $1.value.desc, $1.value.type, $1.value.default_value)), + Fields.pairIterator(); + } +} +} diff --git a/qlib/FileDataProvider/FileMoveDataProvider.qc b/qlib/FileDataProvider/FileMoveDataProvider.qc new file mode 100644 index 0000000000..166f158749 --- /dev/null +++ b/qlib/FileDataProvider/FileMoveDataProvider.qc @@ -0,0 +1,89 @@ +# -*- mode: qore; indent-tabs-mode: nil -*- +#! Qore FileMoveDataProvider module definition + +/** FileMoveDataProvider.qc Copyright 2019 - 2022 Qore Technologies, s.r.o. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#! contains all public definitions in the FileMoveDataProvider module +public namespace FileDataProvider { +#! The File data provider class +public class FileMoveDataProvider inherits AbstractDataProvider { + public { + #! Provider info + const ProviderInfo = { + "type": "FileMoveDataProvider", + "supports_request": True, + }; + + #! Request type + const RequestType = new FileMoveRequestDataType(); + + #! Response type + const ResponseType = new FileTargetResponseDataType(); + } + + #! Returns the data provider name + string getName() { + return "move"; + } + + #! Returns the data provider description + *string getDesc() { + return "File move data provider; moves files or entire directory trees on the filesystem given source and " + "target paths as arguments"; + } + + #! Makes a request and returns the response + /** @param req the request info + @param request_options the request options; will be processed by validateRequestOptions() + + @return the response to the request + + @throws SAME-FILE-ERROR If both paths point to the same file. + @throws FILE-STAT-ERROR If stat call for the source path fails. + @throws PATH-EXISTS-ERROR If destination path exists and overwrite flag is False + */ + private auto doRequestImpl(auto req, *hash request_options) { + return { + "target": move_path(req.source, req.target, req.overwrite), + }; + } + + #! Returns the description of a successful request message, if any + /** @return the request type for this provider + */ + private *AbstractDataProviderType getRequestTypeImpl() { + return RequestType; + } + + #! Returns the description of a response message, if this object represents a response message + /** @return the response type for this response message + */ + private *AbstractDataProviderType getResponseTypeImpl() { + return ResponseType; + } + + #! Returns data provider static info + hash getStaticInfoImpl() { + return ProviderInfo; + } +} +} diff --git a/qlib/FileDataProvider/FileMoveRequestDataType.qc b/qlib/FileDataProvider/FileMoveRequestDataType.qc new file mode 100644 index 0000000000..2cc9301a47 --- /dev/null +++ b/qlib/FileDataProvider/FileMoveRequestDataType.qc @@ -0,0 +1,53 @@ +# -*- mode: qore; indent-tabs-mode: nil -*- + +/** FileMoveRequestDataType.qc Copyright 2022 Qore Technologies, s.r.o. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR MOVERIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#! contains all public definitions in the FileMoveDataProvider module +public namespace FileDataProvider { +#! Data type for move file request calls +public class FileMoveRequestDataType inherits HashDataType { + private { + #! Field descriptions + const Fields = { + "source": { + "type": StringType, + "desc": "The source file path", + }, + "target": { + "type": StringType, + "desc": "The target file path", + }, + "overwrite": { + "type": BoolType, + "desc": "Should existing target files be overwritten?", + "default_value": False, + }, + }; + } + + #! Creates the object + constructor() { + map addField(new QoreDataField($1.key, $1.value.desc, $1.value.type, $1.value.default_value)), + Fields.pairIterator(); + } +} +} diff --git a/qlib/FileDataProvider/FileStatDataProvider.qc b/qlib/FileDataProvider/FileStatDataProvider.qc new file mode 100644 index 0000000000..ab48ac3bef --- /dev/null +++ b/qlib/FileDataProvider/FileStatDataProvider.qc @@ -0,0 +1,95 @@ +# -*- mode: qore; indent-tabs-mode: nil -*- +#! Qore FileStatDataProvider module definition + +/** FileStatDataProvider.qc Copyright 2019 - 2022 Qore Technologies, s.r.o. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#! contains all public definitions in the FileStatDataProvider module +public namespace FileDataProvider { +#! The File data provider class +public class FileStatDataProvider inherits AbstractDataProvider { + public { + #! Provider info + const ProviderInfo = { + "type": "FileStatDataProvider", + "supports_request": True, + }; + + #! Request type + const RequestType = new FileStatRequestDataType(); + + #! Response type + const ResponseType = new FileStatResponseDataType(); + } + + #! Returns the data provider name + string getName() { + return "stat"; + } + + #! Returns the data provider description + *string getDesc() { + return "File stat data provider; returns information about the path given as an argument"; + } + + #! Makes a request and returns the response + /** @param req the request info + @param request_options the request options; will be processed by validateRequestOptions() + + @return the response to the request + + @throw STAT-ERROR if the path cannot be read + */ + private auto doRequestImpl(auto req, *hash request_options) { + # += to ensure that "rv" stays "hash" + string path = normalize_dir(req.path); + hash rv += (req.symlink ? hlstat(path) : hstat(path)); + + if (!rv) { + throw "STAT-ERROR", sprintf("%s: cannot stat(): %s", path, strerror()), errno(); + } + rv += { + "name": basename(path), + "filepath": path, + }; + return rv; + } + + #! Returns the description of a successful request message, if any + /** @return the request type for this provider + */ + private *AbstractDataProviderType getRequestTypeImpl() { + return RequestType; + } + + #! Returns the description of a response message, if this object represents a response message + /** @return the response type for this response message + */ + private *AbstractDataProviderType getResponseTypeImpl() { + return ResponseType; + } + + #! Returns data provider static info + hash getStaticInfoImpl() { + return ProviderInfo; + } +} +} diff --git a/qlib/FileDataProvider/FileStatRequestDataType.qc b/qlib/FileDataProvider/FileStatRequestDataType.qc new file mode 100644 index 0000000000..c1a540024e --- /dev/null +++ b/qlib/FileDataProvider/FileStatRequestDataType.qc @@ -0,0 +1,50 @@ +# -*- mode: qore; indent-tabs-mode: nil -*- + +/** FileStatRequestDataType.qc Copyright 2022 Qore Technologies, s.r.o. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR DELETERIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#! contains all public definitions in the FileStatDataProvider module +public namespace FileDataProvider { +#! Data type for delete file request calls +public class FileStatRequestDataType inherits HashDataType { + private { + #! Field descriptions + const Fields = { + "path": { + "type": StringType, + "desc": "The file path", + }, + "symlink": { + "type": BoolType, + "desc": "If True, then if the target path is a symbolic link, information about the link itself will " + "be returned instead of the target", + "default_value": False, + }, + }; + } + + #! Creates the object + constructor() { + map addField(new QoreDataField($1.key, $1.value.desc, $1.value.type, $1.value.default_value)), + Fields.pairIterator(); + } +} +} diff --git a/qlib/FileDataProvider/FileStatResponseDataType.qc b/qlib/FileDataProvider/FileStatResponseDataType.qc new file mode 100644 index 0000000000..c874db9af6 --- /dev/null +++ b/qlib/FileDataProvider/FileStatResponseDataType.qc @@ -0,0 +1,63 @@ +# -*- mode: qore; indent-tabs-mode: nil -*- + +/** FileStatResponseDataType.qc Copyright 2022 Qore Technologies, s.r.o. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR DELETERIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#! contains all public definitions in the FileStatDataProvider module +public namespace FileDataProvider { +#! Data type for delete file request calls +public class FileStatResponseDataType inherits HashDataType { + public { + #! Markdown descriptions for hashdecl members + const FieldDescriptions = { + "dev": "The device inode number the file is on", + "inode": "The inode of the file", + "mode": "The file protection mode; a bitfield of file permissions", + "nlink": "The number of hard links to this file", + "uid": "The UID of the owner of the file", + "gid": "The GID of the owner of the file", + "rdev": "The device type number", + "size": "The size of the file in bytes", + "atime": "The last accessed date/time of the file", + "mtime": "The last modified date/time of the file", + "ctime": "The created date/time of the file", + "blksize": "Block size; is zero if the file is zero length or if the platform's internal `stat()` (2) " + "function does not provide this info", + "blocks": "Blocks allocated for the file; is zero if the file is zero length or if the platform's " + "internal `stat()` (2) function does not provide this info", + "type": "The type of file; one of:\n- `REGULAR`\n- `DIRECTORY`\n- `SYMBOLIC-LINK`\n" + "- `BLOCK-DEVICE`\n- `CHARACTER-DEVICE`\n- `FIFO`\n- `SYMBOLIC-LINK`\n- `SOCKET`\n" + "- `UNKNOWN`", + "perm": "A string giving UNIX-style permissions for the file (ex: `-rwxr-xr-x`)", + "name": "The name of the file, link, or directory", + "filepath": "The full filepath", + "link": "Symbolic link target (if present)", + }; + } + + #! Creates the type + constructor() { + # add members for base hashdecl + map addField(new QoreDataField($1.getName(), FieldDescriptions{$1.getName()}, $1.getType(), + $1.getDefaultValue())), TypedHash::forName("Qore::StatInfo").getMembers(); + } +} +} diff --git a/qlib/FileDataProvider/FileTargetResponseDataType.qc b/qlib/FileDataProvider/FileTargetResponseDataType.qc new file mode 100644 index 0000000000..d3c1e5e941 --- /dev/null +++ b/qlib/FileDataProvider/FileTargetResponseDataType.qc @@ -0,0 +1,44 @@ +# -*- mode: qore; indent-tabs-mode: nil -*- + +/** FileTargetResponseDataType.qc Copyright 2022 Qore Technologies, s.r.o. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#! contains all public definitions in the FileCopyDataProvider module +public namespace FileDataProvider { +#! Data type for file response APIs with a single "target" key +public class FileTargetResponseDataType inherits HashDataType { + private { + #! Field descriptions + const Fields = { + "target": { + "type": StringType, + "desc": "The target file path", + }, + }; + } + + #! Creates the object + constructor() { + map addField(new QoreDataField($1.key, $1.value.desc, $1.value.type, $1.value.default_value)), + Fields.pairIterator(); + } +} +} diff --git a/qlib/FilePoller.qm b/qlib/FilePoller.qm index ea9c8baa86..211d64ebb8 100644 --- a/qlib/FilePoller.qm +++ b/qlib/FilePoller.qm @@ -33,6 +33,8 @@ module FilePoller { init = sub () { # register the data provider factory DataProvider::registerFactory(new FilePollerDataProviderFactory()); + # register the event data type + DataProvider::registerType("qore/file/event", new FilePollerFileEventInfoDataType()); }; } @@ -77,9 +79,83 @@ module FilePoller { - initial release */ -#! main module namespace +#! Main module namespace public namespace FilePoller { -#! file polling class +#! FilePoller file event hash +public hashdecl FilePollerFileEventInfo { + #! The device inode number the file is on + int dev; + + #! The inode of the file + int inode; + + #! The file protection mode + /** A bitfield of file permissions + */ + int mode; + + #! The number of hard links to this file + int nlink; + + #! The UID of the owner of the file + int uid; + + #! The GID of the owner of the file + int gid; + + #! The device type number + int rdev; + + #! The size of the file in bytes + int size; + + #! The last accessed date/time of the file + date atime; + + #! The last modified date/time of the file + date mtime; + + #! The created date/time of the file + date ctime; + + #! Block size + /** may be zero if the platform's internal %stat() (2) function does not provide this info + */ + int blksize; + + #! Blocks allocated for the file + /** may be zero if the platform's internal %stat() (2) function does not provide this info + */ + int blocks; + + #! the type of file + /** one of: + - \c "REGULAR" + - \c "DIRECTORY" + - \c "SYMBOLIC-LINK" + - \c "BLOCK-DEVICE" + - \c "CHARACTER-DEVICE" + - \c "FIFO" + - \c "SYMBOLIC-LINK" + - \c "SOCKET" + - \c "UNKNOWN" + */ + string type; + + #! a string giving UNIX-style permissions for the file (ex: "-rwxr-xr-x") + string perm; + + #! The name of the file, link, or directory + string name; + + #! The entire path of the file + string filepath; + + #! symbolic link target (if present) + *string link; +} + +#! File polling class public class FilePoller { public { #! ascending sort order @@ -178,13 +254,22 @@ public class FilePoller { /** @param n_path the path to poll @param n_mask the regular expression mask to use to match the files @param n_opts a hash with the following optional keys: - - \c "log_info": a @ref closure "closure" or @ref call_reference "call reference" taking a single string argument as an information message for logging - - \c "log_detail": a @ref closure "closure" or @ref call_reference "call reference" taking a single string argument as a detail information message for logging - - \c "log_debug": a @ref closure "closure" or @ref call_reference "call reference" taking a single string argument as a debug information message for logging - - \c "minage": the minimum file age in seconds as calculated from the file's "last modified" timestamp (\c mtime attribute) before a file will be acquired (default: 0); use this option if files could be otherwise read while being written - - \c "poll_interval": an integer poll interval in seconds; if this option is not supplied, then the default \c poll_inteval is 10 seconds - - \c "reopt": regular expression options; see @ref regex_constants for possible values (ex @ref Qore::RE_Caseless for case-insensitive matches) - - \c "sleep": (required when imported into a context where @ref Qore::PO_NO_PROCESS_CONTROL is set) a @ref closure "closure" or @ref call_reference "call reference" to use instead of @ref Qore::sleep() (if not set then @ref Qore::sleep() will be used) + - \c "log_info": a @ref closure "closure" or @ref call_reference "call reference" taking a single string + argument as an information message for logging + - \c "log_detail": a @ref closure "closure" or @ref call_reference "call reference" taking a single string + argument as a detail information message for logging + - \c "log_debug": a @ref closure "closure" or @ref call_reference "call reference" taking a single string + argument as a debug information message for logging + - \c "minage": the minimum file age in seconds as calculated from the file's "last modified" timestamp + (\c mtime attribute) before a file will be acquired (default: 0); use this option if files could be + otherwise read while being written + - \c "poll_interval": an integer poll interval in seconds; if this option is not supplied, then the default + \c poll_inteval is 10 seconds + - \c "reopt": regular expression options; see @ref regex_constants for possible values (ex + @ref Qore::RE_Caseless for case-insensitive matches) + - \c "sleep": (required when imported into a context where @ref Qore::PO_NO_PROCESS_CONTROL is set) a + @ref closure "closure" or @ref call_reference "call reference" to use instead of @ref Qore::sleep() (if not + set then @ref Qore::sleep() will be used) - \c "sort_order": an integer constant giving the sort order; valid options are: - @ref OrderAsc "OrderAsc": ascending (the default) - @ref OrderDesc "OrderDesc": descending @@ -192,17 +277,20 @@ public class FilePoller { - @ref SortNone "SortNone": no sort - @ref SortName "SortName": sort by file name (the default) - @ref SortDate "SortDate": sort by the file's last modified date - - \c "start_thread": (required when imported into a context where @ref Qore::PO_NO_THREAD_CONTROL is set) a @ref closure "closure" or @ref call_reference "call reference" for starting threads; must return the integer thread ID (if not set then @ref background will be used) + - \c "start_thread": (required when imported into a context where @ref Qore::PO_NO_THREAD_CONTROL is set) a + @ref closure "closure" or @ref call_reference "call reference" for starting threads; must return the integer + thread ID (if not set then @ref background will be used) @throw FILEPOLLER-CONSTRUCTOR-ERROR invalid option */ - constructor(string n_path, string n_mask, *hash n_opts) { + constructor(string n_path, string n_mask, *hash n_opts) { path = n_path; mask = n_mask; foreach string k in (RequiredKeys) { if (!exists n_opts{k}) - throw "FILEPOLLER-CONSTRUCTOR-ERROR", sprintf("required key %y missing from constructor hash argument)", k); + throw "FILEPOLLER-CONSTRUCTOR-ERROR", sprintf("required key %y missing from constructor hash " + "argument)", k); } foreach hash h in (n_opts.pairIterator()) { @@ -222,13 +310,15 @@ public class FilePoller { case "minage": { minage = h.value; if (minage < 0) - throw "FILEPOLLER-CONSTRUCTOR-ERROR", sprintf("invalid %y = %d; must be a non-negative number", h.key, h.value); + throw "FILEPOLLER-CONSTRUCTOR-ERROR", sprintf("invalid %y = %d; must be a non-negative " + "number", h.key, h.value); break; } case "poll_interval": { poll_interval = h.value; if (poll_interval <= 0) - throw "FILEPOLLER-CONSTRUCTOR-ERROR", sprintf("invalid %y = %d; must be a positive number", h.key, h.value); + throw "FILEPOLLER-CONSTRUCTOR-ERROR", sprintf("invalid %y = %d; must be a positive number", + h.key, h.value); break; } case "reopt": { @@ -253,7 +343,8 @@ public class FilePoller { } default: - throw "FILEPOLLER-CONSTRUCTOR-ERROR", sprintf("unknown option %y; known options: %y", h.key, Options); + throw "FILEPOLLER-CONSTRUCTOR-ERROR", sprintf("unknown option %y; known options: %y", h.key, + Options); } } } @@ -267,30 +358,22 @@ public class FilePoller { /** @param sort the sort option for the list returned @param order the ordering of sorted data returned - @return a list of regular file hashes with the following keys in each list element: - - \c name: the name of the file, link, or directory - - \c filepath: the complete file path - - \c size: the size of the file in bytes - - \c uid: the UID of the owner of the file - - \c gid: the GID of the owner of the file - - \c mode: the permissions / mode of the file - - \c atime: the last accessed date/time of the file - - \c mtime: the last modified date/time of the file - - \c type: the type of file is always \c "REGULAR" - - \c perm: a string giving UNIX-style permissions for the file (ex: "-rwxr-xr-x") + @return a list of @ref FilePollerFileEventInfo hashes for each matched file */ - list getFiles(int sort = FilePoller::SortNone, int order = FilePoller::OrderAsc) { + list> getFiles(int sort = FilePoller::SortNone, int order = FilePoller::OrderAsc) { Dir d(); d.chdir(path); # file of file hashes - list ret = map $1 + ("filepath": path + DirSep + $1.name), d.listFiles(mask, reopt, True); + list> ret = + map cast>($1 + ("filepath": path + DirSep + $1.name)), + d.listFiles(mask, reopt, True); # remove all files that aren't old enough if (minage) { date now = Qore::now(); - list n = (); - foreach hash h in (ret) { + list> n = (); + foreach hash h in (ret) { if ((now - h.mtime).durationSeconds() < minage) { logDebug("file %y is not old enough (minage: %d, current age: %d)", h.name, minage, (now - h.mtime).durationSeconds()); @@ -306,14 +389,30 @@ public class FilePoller { case FilePoller::SortName: # sort by file name return order == FilePoller::OrderAsc - ? Qore::sort(ret, int sub (hash l, hash r) { return l.name <=> r.name; }) - : sort_descending(ret, int sub (hash l, hash r) { return l.name <=> r.name; }); + ? Qore::sort(ret, + int sub (hash l, hash r) { + return l.name <=> r.name; + } + ) + : sort_descending(ret, + int sub (hash l, hash r) { + return l.name <=> r.name; + } + ); case FilePoller::SortDate: # sort by last modification date return order == FilePoller::OrderAsc - ? Qore::sort(ret, int sub (hash l, hash r) { return l.mtime <=> r.mtime; }) - : sort_descending(ret, int sub (hash l, hash r) { return l.name <=> r.name; }); + ? Qore::sort(ret, + int sub (hash l, hash r) { + return l.mtime <=> r.mtime; + } + ) + : sort_descending(ret, + int sub (hash l, hash r) { + return l.mtime <=> r.mtime; + } + ); } return ret; @@ -345,7 +444,8 @@ public class FilePoller { #! stops the polling operation, returns when the polling operation has been stopped /** if polling was not in progress then this method returns immediately - @throw THREAD-ERROR this exception is thrown if this method is called from the polling thread since it would result in a deadlock + @throw THREAD-ERROR this exception is thrown if this method is called from the polling thread since it would + result in a deadlock @see - stopNoWait() @@ -375,9 +475,11 @@ public class FilePoller { runflag = False; } - #! waits indefinitely for the polling operation to stop; if polling was not in progress then this method returns immediately - /** - @throw THREAD-ERROR this exception is thrown if this method is called from the event thread since it would result in a deadlock + #! Waits indefinitely for the polling operation to stop + /** If polling was not in progress then this method returns immediately + + @throw THREAD-ERROR this exception is thrown if this method is called from the event thread since it would + result in a deadlock @see - stop() @@ -398,8 +500,9 @@ public class FilePoller { m.lock(); on_exit m.unlock(); - if (runflag) + if (runflag) { throw "FILEPOLLER-ERROR", sprintf("the polling thread is already running in TID %d", tid); + } runflag = True; tid = gettid(); @@ -412,7 +515,7 @@ public class FilePoller { bool runOnce() { ++pollcnt; - list files = getFiles(sort_type, sort_order); + list> files = getFiles(sort_type, sort_order); if (files) { logDebug("got files in %y: %y", path, files); fileEvent(files); @@ -454,17 +557,21 @@ public class FilePoller { logInfo("polling finished"); } - #! called for each poll event with a list of all files matched; calls singleFileEvent() on each file hash in the list - fileEvent(list files) { - foreach hash fih in (files) { - logInfo("got file: %y", fih.name); - logDebug("file info: %y", fih); - singleFileEvent(fih); + #! Called for each poll event with a list of all files matched + /** Calls singleFileEvent() on each file hash in the list + */ + fileEvent(list> files) { + foreach hash event in (files) { + logInfo("got file: %y", event.name); + logDebug("file info: %y", event); + singleFileEvent(event); } } - #! called for each matching file individually whenever matching files are polled with the list of matching file names; if any error occurs here, the error is logged and the polling operation is retried - /** @param fih a hash of file data and information with the following keys: + #! called for each matching file individually when matching files are polled + /** If any error occurs here, the error is logged and the polling operation is retried + + @param event a hash of file data and information with the following keys: - \c name: the name of the file, link, or directory - \c filepath: the complete path to the file including the directory - \c size: the size of the file in bytes @@ -473,17 +580,27 @@ public class FilePoller { - \c mode: the permissions / mode of the file - \c atime: the last accessed date/time of the file - \c mtime: the last modified date/time of the file - - \c type: the type of file; one of: \c "REGULAR", \c "DIRECTORY", \c "SYMBOLIC-LINK", \c "BLOCK-DEVICE", \c "CHARACTER-DEVICE", \c "FIFO", \c "SYMBOLIC-LINK", \c "SOCKET", or \c "UNKNOWN" + - \c type: the type of file; one of: + - \c "REGULAR" + - \c "DIRECTORY" + - \c "SYMBOLIC-LINK" + - \c "BLOCK-DEVICE" + - \c "CHARACTER-DEVICE" + - \c "FIFO" + - \c "SYMBOLIC-LINK" + - \c "SOCKET" + - \c "UNKNOWN" - \c perm: a string giving UNIX-style permissions for the file (ex: "-rwxr-xr-x") */ - abstract singleFileEvent(hash fih); + abstract singleFileEvent(hash event); #! checks a path on the local file system /** - @throw DIR-ERROR this exception is thrown if the local path does not exist, is not readable, is not a directory, or should be writable and is not + @throw DIR-ERROR this exception is thrown if the local path does not exist, is not readable, is not a + directory, or should be writable and is not */ static checkPath(string path, string type, bool write = False) { - *hash h = hstat(path); + *hash h = hstat(path); if (!exists h) throw "DIR-ERROR", sprintf("%y: %s path does not exist", path, type); if (h.type != "DIRECTORY") @@ -503,19 +620,19 @@ public class FilePoller { return pollcnt; } - #! calls the \c "log_info" @ref closure "closure" or @ref call_reference "call reference" with important information + #! calls \c "log_info" with important information, if set private logInfo(string fmt) { if (log_info) log_info(vsprintf(fmt, argv)); } - #! calls the \c "log_detail" @ref closure "closure" or @ref call_reference "call reference" with detail information + #! calls \c "log_detail" with detail information; if set private logDetail(string fmt) { if (log_detail) log_detail(vsprintf(fmt, argv)); } - #! calls the \c "log_debug" @ref closure "closure" or @ref call_reference "call reference" with verbose debugging information + #! calls \c "log_debug" with verbose debugging information; if set private logDebug(string fmt) { if (log_debug) log_debug(vsprintf(fmt, argv)); @@ -527,7 +644,7 @@ public class FilePoller { In the assumed scenario, matches files are removed by a single observer in the first notification event */ -class FilePollerDataProvider inherits DataProvider::AbstractDataProvider, DataProvider::Observable { +public class FilePollerDataProvider inherits DataProvider::AbstractDataProvider, DataProvider::Observable { public { #! Provider info const ProviderInfo = { @@ -582,18 +699,6 @@ class FilePollerDataProvider inherits DataProvider::AbstractDataProvider, DataPr "type": AbstractDataProviderType::get(StringType), "desc": "Either `name` or `date` for the data to use for sorting", }, - -/* -%ifdef PO_NO_PROCESS_CONTROL - "sleep": { - }, -%endif - -%ifdef PO_NO_THREAD_CONTROL - "start_thread": { - }, -%endif -*/ }; } @@ -606,7 +711,7 @@ class FilePollerDataProvider inherits DataProvider::AbstractDataProvider, DataPr constructor(*hash options) { hash copts = checkOptions("CONSTRUCTOR-ERROR", ConstructorOptions, options); string path = remove copts.path; - string mask = remove copts.mask; + string mask = remove copts.mask ?? ConstructorOptions.mask.default_value; bool regex = remove copts.regex ?? False; if (*string sort_type = remove copts.sort_type) { if (sort_type == "name") { @@ -661,6 +766,51 @@ class FilePollerDataProvider inherits DataProvider::AbstractDataProvider, DataPr private hash getStaticInfoImpl() { return ProviderInfo; } + + #! Returns the description of an event, if any + /** @return the event type for this provider + */ + private *AbstractDataProviderType getEventTypeImpl() { + return new FilePollerFileEventInfoDataType(); + } +} + +#! File poller event data description +public class FilePollerFileEventInfoDataType inherits HashDataType { + public { + #! Markdown descriptions for hashdecl members + const FieldDescriptions = { + "dev": "The device inode number the file is on", + "inode": "The inode of the file", + "mode": "The file protection mode; a bitfield of file permissions", + "nlink": "The number of hard links to this file", + "uid": "The UID of the owner of the file", + "gid": "The GID of the owner of the file", + "rdev": "The device type number", + "size": "the size of the file in bytes", + "atime": "the last accessed date/time of the file", + "mtime": "the last modified date/time of the file", + "ctime": "the created date/time of the file", + "blksize": "Block size; is zero if the file is zero length or if the platform's internal `stat()` (2) " + "function does not provide this info", + "blocks": "Blocks allocated for the file; is zero if the file is zero length or if the platform's " + "internal `stat()` (2) function does not provide this info", + "type": "the type of file; one of:\n- `REGULAR`\n- `DIRECTORY`\n- `SYMBOLIC-LINK`\n" + "- `BLOCK-DEVICE`\n- `CHARACTER-DEVICE`\n- `FIFO`\n- `SYMBOLIC-LINK`\n- `SOCKET`\n" + "- `UNKNOWN`", + "perm": "a string giving UNIX-style permissions for the file (ex: `-rwxr-xr-x`)", + "name": "the name of the file, link, or directory", + "filepath": "the remote filepath relative to SFTP root directory", + "link": "symbolic link target (if present)", + }; + } + + #! Creates the type + constructor() { + # add members for base hashdecl + map addField(new QoreDataField($1.getName(), FieldDescriptions{$1.getName()}, $1.getType(), + $1.getDefaultValue())), TypedHash::forName("FilePoller::FilePollerFileEventInfo").getMembers(); + } } #! The file poller data provider factory @@ -673,6 +823,7 @@ public class FilePollerDataProviderFactory inherits AbstractDataProviderFactory const FactoryInfo = { "name": "filepoller", "desc": "File poller data provider factory", + "children_can_support_observers": True, }; } @@ -713,8 +864,10 @@ class EmbeddedFilePoller inherits FilePoller { self.provider := provider; } - singleFileEvent(hash fih) { - provider.notifyObservers((++id).toString(), fih); + singleFileEvent(hash event) { + provider.notifyObservers((++id).toString(), event); } + + } } diff --git a/qlib/FixedLengthUtil/FixedLengthReadDataProviderFactory.qc b/qlib/FixedLengthUtil/FixedLengthReadDataProviderFactory.qc index 76f3720047..0eff6dfd2b 100644 --- a/qlib/FixedLengthUtil/FixedLengthReadDataProviderFactory.qc +++ b/qlib/FixedLengthUtil/FixedLengthReadDataProviderFactory.qc @@ -34,6 +34,7 @@ public class FixedLengthReadDataProviderFactory inherits AbstractDataProviderFac const FactoryInfo = { "name": "fixedlengthread", "desc": "Fixed length data reader data provider factory", + "children_can_support_records": True, }; } diff --git a/qlib/FixedLengthUtil/FixedLengthWriteDataProviderFactory.qc b/qlib/FixedLengthUtil/FixedLengthWriteDataProviderFactory.qc index 5ce8811de4..dba3f16397 100644 --- a/qlib/FixedLengthUtil/FixedLengthWriteDataProviderFactory.qc +++ b/qlib/FixedLengthUtil/FixedLengthWriteDataProviderFactory.qc @@ -34,6 +34,7 @@ public class FixedLengthWriteDataProviderFactory inherits AbstractDataProviderFa const FactoryInfo = { "name": "fixedlengthwrite", "desc": "Fixed length data writer data provider factory", + "children_can_support_records": True, }; } diff --git a/qlib/FsUtil.qm b/qlib/FsUtil.qm index 1c519a0c62..2e97232424 100644 --- a/qlib/FsUtil.qm +++ b/qlib/FsUtil.qm @@ -34,7 +34,7 @@ %enable-all-warnings module FsUtil { - version = "1.2"; + version = "1.3"; desc = "user module providing file system related functionality"; author = "Tomas Heger"; url = "http://qore.org"; @@ -83,6 +83,11 @@ module FsUtil { @section fsutil_relnotes Release Notes + @subsection fsutil_v1_3 Version 1.3 + - added @ref FsUtil::move_path() "move_path()" + - fixed inconsistencies handling target directories in copy operations + (issue 4559) + @subsection fsutil_v1_2 Version 1.2 - fixed join_paths() to handle an arbitrary number of paths as arguments (the most common use case) (issue 4495) @@ -112,6 +117,9 @@ namespace Init { #! the FsUtil namespace contains all the objects in the FsUtil module public namespace FsUtil { + #! Default file block buffer size + const DefaultBufferSize = 16 * 1024; + #! generic path handler implementing functionality common for both platforms class PathHandler { public { @@ -157,7 +165,7 @@ public namespace FsUtil { return ("", path); } - #! Returns path resulting from joining the given paths + #! Returns the path resulting from joining the given paths /** @param paths a list of paths to join @return all paths joined into one valid path (i.e. delimiters are added/removed as needed) except for cases @@ -207,7 +215,7 @@ public namespace FsUtil { delimiters = ("/",); } - #! Returns path resulting from joining the given paths + #! Returns the path resulting from joining the given paths /** @param paths a list of paths to join @return all paths joined into one valid path (i.e. delimiters are added/removed as needed) except @@ -258,7 +266,7 @@ public namespace FsUtil { } if (!string_ends_with(bn, extension)) { - throw "EXTENSION-NOT-FOUND", sprintf("Extension '%s' not found in basename '%s'.", extension, bn); + throw "EXTENSION-NOT-FOUND", sprintf("Extension %y not found in basename %y", extension, bn); } # return everything without the extension @@ -267,7 +275,7 @@ public namespace FsUtil { } - #! Returns path resulting from joining the given paths + #! Returns the path resulting from joining the given paths /** @param path1 the first part of the path @param paths the other parts of the path @@ -281,7 +289,7 @@ public namespace FsUtil { return local_handler.joinPaths(cast>((path1,) + paths)); } - #! Returns path resulting from joining the given paths + #! Returns the path resulting from joining the given paths /** @param path1 the first part of the path @param ... the remaining parts of the path @@ -295,7 +303,7 @@ public namespace FsUtil { return local_handler.joinPaths(cast<*list>(argv)); } - #! Returns path resulting from joining the given paths + #! Returns the path resulting from joining the given paths /** @param paths the parts of the path to be joined into one @return all paths joined into one valid path (i.e. delimiters are added/removed as needed) except for cases when @@ -313,8 +321,8 @@ public namespace FsUtil { /** @param path the path to be checked @param follow_symlinks flag indicating whether target of a symlink should be checked instead - @return True if the given path exists, False otherwise. If follow_symlinks is True and path points to a symlink, - the link's target is checked instead of the link itself + @return True if the given path exists, False otherwise. If follow_symlinks is True and path points to a + symlink, the link's target is checked instead of the link itself */ public bool sub path_exists(string path, bool follow_symlinks = False) { *hash s = follow_symlinks ? hstat(path) : hlstat(path); @@ -340,7 +348,7 @@ public namespace FsUtil { public string sub make_tmp_dir(*string prefix, *string suffix, *string path) { if (exists path) { if (!is_writable(path)) { - throw "DIR-WRITE-ERROR", sprintf("'%s' is not a writable directory.", path); + throw "DIR-WRITE-ERROR", sprintf("%y is not a writable directory", path); } } else { # get standard tmp location (and check that it's writable) @@ -362,13 +370,13 @@ public namespace FsUtil { continue; } else { # another (unexpected) error - throw "DIR-WRITE-ERROR", sprintf("Directory '%s' couldn't be created: %s", tmp_path, strerror(err)); + throw "DIR-WRITE-ERROR", sprintf("Directory %y couldn't be created: %s", tmp_path, strerror(err)); } } } # even after TMP_MAX_ATTEMPTS we didn't succeed in finding a unique name - throw "DIR-WRITE-ERROR", "Impossible to find a unique name."; + throw "DIR-WRITE-ERROR", "Impossible to find a unique name"; } public hashdecl TmpFileHash { @@ -380,10 +388,10 @@ public namespace FsUtil { /** @param prefix prefix to be used in the newly created file name @param suffix suffix to be used in the newly created file name @param path path to a dir in which the caller wants to create the new file; if not provided, standard temp - location will be used + location will be used @return hash with absolute path to the newly created file and its File object - (keys "path" and "file" respectively) + (keys "path" and "file" respectively) @throws FILE-WRITE-ERROR The file couldn't be created. @@ -394,7 +402,7 @@ public namespace FsUtil { public hash sub make_tmp_file(*string prefix, *string suffix, *string path) { if (exists path) { if (!is_writable(path)) { - throw "FILE-WRITE-ERROR", sprintf("'%s' is not a writable directory.", path); + throw "FILE-WRITE-ERROR", sprintf("%y is not a writable directory", path); } } else { # get standard tmp location (and check that it's writable) @@ -420,18 +428,20 @@ public namespace FsUtil { continue; } else { # another (unexpected) error - throw "FILE-WRITE-ERROR", sprintf("File '%s' couldn't be created: %s", + throw "FILE-WRITE-ERROR", sprintf("File %y couldn't be created: %s", tmp_file.path, strerror(err)); } } } # even after TMP_MAX_ATTEMPTS we didn't succeed in finding a unique name - throw "FILE-WRITE-ERROR", "Impossible to find a unique name."; + throw "FILE-WRITE-ERROR", "Impossible to find a unique name"; } - #! Class implementing a user friendly temporary directory creation; the directory and all its contents are removed in the destructor - /** @since %FsUtil 1.1 inherits @ref Qore::Dir + #! Class implementing a user friendly temporary directory creation + /** The directory and all its contents are removed in the destructor + + @since %FsUtil 1.1 inherits @ref Qore::Dir */ public class TmpDir inherits Qore::Dir { public { @@ -442,7 +452,8 @@ public namespace FsUtil { #! Creates a unique temporary directory and returns its absolute path /** @param prefix prefix to be used in the newly created directory name @param suffix suffix to be used in the newly created directory name - @param path path to a dir in which the caller wants to create the new directory; if not provided, the standard temp location will be used + @param path path to a dir in which the caller wants to create the new directory; if not provided, the + standard temp location will be used */ public constructor(*string prefix, *string suffix, *string path) { self.path = make_tmp_dir(prefix, suffix, path); @@ -494,8 +505,8 @@ public namespace FsUtil { #! Removes the filesystem tree specified by path /** @param path path to a filesystem tree root which should be removed - @param fail_immediately flag indicating whether the function is supposed to fail with an exception immediately - on the first error instead of accumulating all exceptions (True by default) + @param fail_immediately flag indicates if the function is supposed to fail with an exception immediately + on the first error instead of accumulating all exceptions (True by default) @throws REMOVE-TREE-ERROR If something goes wrong in the recursive calls. In such case the rest of the tree is removed anyway and this exception's description contains a list of exceptions thrown @@ -507,7 +518,7 @@ public namespace FsUtil { parts of the tree can't be removed (e.g. due to permissions) and if fail_immediately is False, the tree is partially removed anyway and REMOVE-TREE-ERROR is thrown and describes what exactly went wrong for each error. */ - public sub remove_tree(string path, bool fail_immediately=True) { + public sub remove_tree(string path, bool fail_immediately = True) { Dir dir(); dir.chdir(path); @@ -571,7 +582,7 @@ public namespace FsUtil { without taking care of possible NOTHINGs. In that case the paths can probably be considered not pointing to the same file. */ - public bool sub same_file_stat(*hash stat1, *hash stat2, bool ignore_errors=True) { + public bool sub same_file_stat(*hash stat1, *hash stat2, bool ignore_errors = True) { # handle special case when one of the stat hashes doesn't exist (or both) if (!exists stat1 || !exists stat2) { if (ignore_errors) { @@ -604,7 +615,7 @@ public namespace FsUtil { because the most common error here is that one of the paths doesn't exist (or both). In that case the paths indeed don't point to the same file. */ - public bool sub same_file(string path1, string path2, bool follow_symlinks=True, bool ignore_errors=True) { + public bool sub same_file(string path1, string path2, bool follow_symlinks = True, bool ignore_errors = True) { hash stat1; hash stat2; @@ -634,17 +645,17 @@ public namespace FsUtil { @param follow_symlinks flag indicating whether symlink should be followed in source (False by default) @param overwrite flag indicating whether destination should be overwritten if exists (False by default) @param merge flag indicating whether source should be merged into destination if the latter exists - (False by default) + (False by default) @param fail_immediately flag indicating whether the function is supposed to fail with an exception immediately - on the first error instead of accumulating all exceptions (True by default) + on the first error instead of accumulating all exceptions (True by default) @param depth determines how many levels of subdirectories and files should be copied @throws SAME-DIR-ERROR If both paths point to the same file. @throws DIR-STAT-ERROR If stat call for the source path fails. @throws PATH-EXISTS-ERROR If destination path exists and overwrite flag is not set. @throws COPY-TREE-ERROR If something goes wrong in the recursive calls. In such case the rest of the tree is - copied anyway and this exception's description contains a list of exceptions thrown - in the recursive calls unless fail_immediately flag is True. + copied anyway and this exception's description contains a list of exceptions thrown + in the recursive calls unless fail_immediately flag is True. This is an internal implementation only and is not intended to be called from outside of FsUtil module. Source must be a directory (or a symlink pointing to a directory if follow_symlinks is True). Destination must @@ -659,33 +670,37 @@ public namespace FsUtil { description of COPY-TREE-ERROR exception, if only some parts of the tree can't be copied (e.g. due to permissions) and if fail_immediately is False, the tree is partially copied anyway and COPY-TREE-ERROR is thrown and describes what exactly went wrong for each error. If the whole tree is copied correctly, the function - returns path to the new copy. + returns the path to the new copy. */ - string sub copy_tree_internal(string source, string destination, bool follow_symlinks=False, bool overwrite=False, - bool merge=False, bool fail_immediately=True, *int depth) { + string sub copy_tree_internal(string source, string destination, bool follow_symlinks = False, + bool overwrite = False, bool merge = False, bool fail_immediately = True, *int depth) { if (same_file(source, destination, follow_symlinks, True)) { - throw "SAME-DIR-ERROR", sprintf("'%s' and '%s' point to the same directory.", source, destination); + throw "SAME-DIR-ERROR", sprintf("%y and %y point to the same directory", source, destination); } - list errors = (); + list errors = (); # prepare the source Dir src_dir(); src_dir.chdir(source); *hash src_stat = follow_symlinks ? hstat(source) : hlstat(source); if (!exists src_stat) { - throw "DIR-STAT-ERROR", sprintf("Couldn't stat source path '%s': %s", source, strerror()); + throw "DIR-STAT-ERROR", sprintf("Couldn't stat source path %y: %s", source, strerror()); } # check and prepare the destination directory + if (is_dir(destination) && (src_stat.type != "DIRECTORY" || !overwrite) && !merge) { # works even for symlinks pointing to directories + destination = join_paths(destination, basename(source)); + } if (path_exists(destination)) { if (overwrite && !merge) { remove_path(destination); mkdir(destination, 0700); } else { if (!merge) { - throw "PATH-EXISTS-ERROR", sprintf("Destination already '%s' exists.", destination); + throw "PATH-EXISTS-ERROR", sprintf("Destination already %y exists (file type %y)", destination, + hstat(destination).type); } } } else { @@ -749,7 +764,7 @@ public namespace FsUtil { By default this function does not close the file objects after the copy is made. This can be changed using the close parameter. */ - public sub copy_file_obj(File source, File destination, bool close=False, int buf_size=4*1024) { + public sub copy_file_obj(File source, File destination, bool close = False, int buf_size = DefaultBufferSize) { *binary buf; while (True) { buf = source.readBinary(buf_size); @@ -779,17 +794,18 @@ public namespace FsUtil { Destination can be a directory. If that's the case, this function will attempt to create a copy of source using the same filename in the destination directory. Symlinks in destination are always followed (even dangling links). Mode (permissions) bits are preserved but UID/GID, access time and modification time are not. - If the file is copied correctly, the function returns path to the new copy. + If the file is copied correctly, the function returns the path to the new copy. */ - public string sub copy_file(string source, string destination, bool follow_symlinks=False, bool overwrite=False) { + public string sub copy_file(string source, string destination, bool follow_symlinks = False, + bool overwrite = False) { if (same_file(source, destination, follow_symlinks, True)) { - throw "SAME-FILE-ERROR", sprintf("'%s' and '%s' point to the same file.", source, destination); + throw "SAME-FILE-ERROR", sprintf("%y and %y point to the same file", source, destination); } # prepare the source *hash src_stat = follow_symlinks ? hstat(source) : hlstat(source); if (!exists src_stat) { - throw "FILE-STAT-ERROR", sprintf("Couldn't stat source path '%s': %s", source, strerror()); + throw "FILE-STAT-ERROR", sprintf("Couldn't stat source path %y: %s", source, strerror()); } # prepare the destination @@ -798,13 +814,13 @@ public namespace FsUtil { } *hash dst_stat = hstat(destination); if (!overwrite && exists dst_stat) { - throw "PATH-EXISTS-ERROR", sprintf("Destination path '%s' already exists and overwriting is forbidden.", - destination); + throw "PATH-EXISTS-ERROR", sprintf("Destination path %y already exists and overwriting is forbidden", + destination); } foreach *hash stat in (src_stat, dst_stat) { if (exists stat && stat.type != "REGULAR" && stat.type != "SYMBOLIC-LINK") { - throw "UNSUPPORTED-TYPE-ERROR", sprintf("Unsupported file type: '%s'.", stat.type); + throw "UNSUPPORTED-TYPE-ERROR", sprintf("Unsupported file type: %y", stat.type); } } @@ -816,13 +832,13 @@ public namespace FsUtil { } symlink(readlink(source), destination); } else { - File src_obj(); - File dst_obj(); - src_obj.open2(source, O_RDONLY); - dst_obj.open2(destination, O_CREAT | O_TRUNC | O_WRONLY, 0600); - copy_file_obj(src_obj, dst_obj); - src_obj.close(); - dst_obj.close(); + { + File src_obj(); + File dst_obj(); + src_obj.open2(source, O_RDONLY); + dst_obj.open2(destination, O_CREAT | O_TRUNC | O_WRONLY, 0600); + copy_file_obj(src_obj, dst_obj); + } chmod(destination, src_stat.mode); } @@ -835,24 +851,24 @@ public namespace FsUtil { @param follow_symlinks flag indicating whether symlink should be followed in source (False by default) @param overwrite flag indicating whether destination should be overwritten if exists (False by default) @param fail_immediately flag indicating whether the function is supposed to fail with an exception immediately - on the first error instead of accumulating all exceptions (True by default) + on the first error instead of accumulating all exceptions (True by default) - @throws SAME-DIR-ERROR If both paths point to the same file. - @throws DIR-STAT-ERROR If stat call for the source path fails. - @throws PATH-EXISTS-ERROR If destination path exists and overwrite flag is not set. + @throws SAME-DIR-ERROR If both paths point to the same file + @throws DIR-STAT-ERROR If stat call for the source path fails + @throws PATH-EXISTS-ERROR If destination path exists and overwrite flag is not set @throws COPY-TREE-ERROR If something goes wrong in the recursive calls. In such case the rest of the tree is - copied anyway and this exception's description contains a list of exceptions thrown - in the recursive calls unless fail_immediately flag is True. + copied anyway and this exception's description contains a list of exceptions thrown + in the recursive calls unless fail_immediately flag is True Source must be a directory (or a symlink pointing to a directory if follow_symlinks is True). Destination must not exist unless the function is called with the overwrite flag. Mode (permissions) bits are preserved but UID/GID, access time and modification time are not. As mentioned in description of COPY-TREE-ERROR exception, if only some parts of the tree can't be copied (e.g. due to permissions) and if fail_immediately is False, the tree is partially copied anyway and COPY-TREE-ERROR is thrown and describes what exactly went wrong for each error. - If the whole tree is copied correctly, the function returns path to the new copy. + If the whole tree is copied correctly, the function returns the path to the new copy. */ - public string sub copy_tree(string source, string destination, bool follow_symlinks=False, bool overwrite=False, - bool fail_immediately=True) { + public string sub copy_tree(string source, string destination, bool follow_symlinks = False, + bool overwrite = False, bool fail_immediately = True) { return copy_tree_internal(source, destination, follow_symlinks, overwrite, False, fail_immediately); } @@ -861,17 +877,27 @@ public namespace FsUtil { @param destination path where the copy should be created @param follow_symlinks flag indicating whether symlink should be followed in source (False by default) @param overwrite flag indicating whether destination should be overwritten if exists (False by default) + @param fail_immediately flag indicates if the function is supposed to fail with an exception immediately + on the first error instead of accumulating all exceptions (True by default) @param depth determines how many levels of subdirectories and files should be copied + @throws SAME-DIR-ERROR If both paths point to the same file + @throws DIR-STAT-ERROR If stat call for the source path fails + @throws PATH-EXISTS-ERROR If destination path exists and overwrite flag is not set + @throws COPY-TREE-ERROR If something goes wrong in the recursive calls. In such case the rest of the tree is + copied anyway and this exception's description contains a list of exceptions thrown + in the recursive calls unless fail_immediately flag is True + This wrapper function exists for convenience. Source can be a regular file, symlink or a directory, - the respective functions for copying files/links or directories will be called. Destination can either be a path - that doesn't exist on the filesystem yet (and then it will be created as a copy of source) or it can be - an existing path if overwrite flag is set (and then the original contents will be replaced by a copy of source). + the respective functions for copying files/links or directories will be called. Destination can either be a + path that doesn't exist on the filesystem yet (and then it will be created as a copy of source) or it can be + an existing path if overwrite flag is set (and then the original contents will be replaced by a copy of + source). Mode (permissions) bits are preserved but UID/GID, access time and modification time are not. If the source is - copied correctly, the function returns path to the new copy. + copied correctly, the function returns the path to the new copy. */ - public string sub copy_path(string source, string destination, bool follow_symlinks=False, bool overwrite=False, - bool fail_immediately=True, *int depth) { + public string sub copy_path(string source, string destination, bool follow_symlinks = False, + bool overwrite = False, bool fail_immediately = True, *int depth) { if (is_dir(source)) { return copy_tree(source, destination, follow_symlinks, overwrite, fail_immediately); } else { @@ -884,7 +910,7 @@ public namespace FsUtil { @param destination path the copy should be merged into (must be a directory or must not exist) @param follow_symlinks flag indicating whether symlink should be followed in source (False by default) @param fail_immediately flag indicating whether the function is supposed to fail with an exception immediately - on the first error instead of accumulating all exceptions (True by default) + on the first error instead of accumulating all exceptions (True by default) @param depth determines how many levels of subdirectories and files should be copied @throws SAME-DIR-ERROR If both paths point to the same file. @@ -898,20 +924,71 @@ public namespace FsUtil { modification time are not. As mentioned in description of COPY-TREE-ERROR exception, if only some parts of the tree can't be copied and merged (e.g. due to permissions) and if fail_immediately is False, the tree is partially copied anyway and COPY-TREE-ERROR is thrown and describes what exactly went wrong for each error. - If the whole tree is copied correctly, the function returns path to the new copy. + If the whole tree is copied correctly, the function returns the path to the new copy. */ - public string sub merge_tree(string source, string destination, bool follow_symlinks=False, bool overwrite=False, - bool fail_immediately=True, *int depth) { + public string sub merge_tree(string source, string destination, bool follow_symlinks = False, + bool overwrite = False, bool fail_immediately = True, *int depth) { return copy_tree_internal(source, destination, follow_symlinks, overwrite, True, fail_immediately, depth); } + #! Moves a path from source to destination. + /** @param source path to the file to be copied + @param destination path where the copy should be created + @param overwrite flag indicating whether destination should be overwritten if exists (False by default) + + @throws SAME-PATH-ERROR If both paths point to the same location + @throws FILE-STAT-ERROR If stat call for the source path fails + @throws PATH-EXISTS-ERROR If destination path exists and overwrite flag is False + + The Destination can be a directory, in which case, this function will attempt to move the source to + the same filename in the destination directory unless \a overwrite is @ref True. + Symlinks in destination are always followed (even dangling + links). Mode (permissions) bits are preserved but UID/GID, access time and modification time are not. + If the file is copied correctly, the function returns the path to the new location of the path. + */ + public string sub move_path(string source, string destination, bool overwrite = False) { + if (same_file(source, destination, True, True)) { + throw "SAME-PATH-ERROR", sprintf("%y and %y point to the same file", source, destination); + } + + # prepare the source + *hash src_stat = hlstat(source); + if (!exists src_stat) { + throw "FILE-STAT-ERROR", sprintf("Couldn't stat source path %y: %s", source, strerror()); + } + + # prepare the destination + if (is_dir(destination) && (!overwrite || src_stat.type != "DIRECTORY")) { # works even for symlinks pointing to directories + destination = join_paths(destination, basename(source)); + } + *hash dst_stat = hstat(destination); + if (!overwrite && exists dst_stat) { + throw "PATH-EXISTS-ERROR", sprintf("Destination path %y already exists and overwriting is forbidden", + destination); + } + + if (exists dst_stat && dst_stat.type == "DIRECTORY") { + # check if directory is empty + if (overwrite) { + remove_path(destination); + } else { + throw "UNSUPPORTED-TYPE-ERROR", sprintf("Cannot overwrite target %y of type %y", destination, + dst_stat.type); + } + } + + rename(source, destination); + + return destination; + } + #! Universal merge function /** @param source path to be copied @param destination path the copy should be merged into @param follow_symlinks flag indicating whether symlink should be followed in source (False by default) @param overwrite flag indicating whether destination should be overwritten if exists (False by default) @param fail_immediately flag indicating whether the function is supposed to fail with an exception immediately - on the first error instead of accumulating all exceptions (True by default) + on the first error instead of accumulating all exceptions (True by default) @param depth determines how many levels of subdirectories and files should be copied This wrapper function exists for convenience. Source can be a regular file, symlink or a directory, @@ -921,10 +998,10 @@ public namespace FsUtil { set, then common parts of the subtree will be replaced by the parts from source, otherwise the original contents will be kept there. Mode (permissions) bits are preserved but UID/GID, access time and modification time are not. If the source is - copied correctly, the function returns path to the new copy. + copied correctly, the function returns the path to the new copy. */ - public string sub merge_path(string source, string destination, bool follow_symlinks=False, bool overwrite=False, - bool fail_immediately=True, *int depth) { + public string sub merge_path(string source, string destination, bool follow_symlinks = False, bool overwrite = False, + bool fail_immediately = True, *int depth) { if (is_dir(source)) { return merge_tree(source, destination, follow_symlinks, overwrite, fail_immediately, depth); } else { @@ -945,24 +1022,24 @@ public namespace FsUtil { Source must be a directory, destination must not exist. Mode (permissions) bits are preserved but UID/GID, access time and modification time are not. If the whole tree - (to the specified depth) is copied correctly, the function returns path to the new copy. + (to the specified depth) is copied correctly, the function returns the path to the new copy. */ public string sub copy_dir_structure(string source, string destination, *int depth) { # prepare the source Dir src_dir(); if (!src_dir.chdir(source)) { - throw "NOT-DIRECTORY-ERROR", sprintf("Source path '%s' is not a directory.", source); + throw "NOT-DIRECTORY-ERROR", sprintf("Source path %y is not a directory", source); } else if (is_link(source)) { - throw "NOT-DIRECTORY-ERROR", sprintf("Source path '%s' is not a directory but a link.", source); + throw "NOT-DIRECTORY-ERROR", sprintf("Source path %y is not a directory but a link", source); } *hash src_stat = hlstat(source); if (!exists src_stat) { - throw "DIR-STAT-ERROR", sprintf("Couldn't stat source path '%s': %s", source, strerror()); + throw "DIR-STAT-ERROR", sprintf("Couldn't stat source path %y: %s", source, strerror()); } # create destination directory if (path_exists(destination)) { - throw "PATH-EXISTS-ERROR", sprintf("Destination path '%s' already exists.", destination); + throw "PATH-EXISTS-ERROR", sprintf("Destination path %y already exists", destination); } mkdir(destination, 0700); diff --git a/qlib/FtpPoller.qm b/qlib/FtpPoller.qm index 76b4acd175..1e4a6e9baf 100644 --- a/qlib/FtpPoller.qm +++ b/qlib/FtpPoller.qm @@ -26,37 +26,40 @@ %strict-args %require-types %enable-all-warnings +%allow-weak-references # make sure we have the required qore version -%requires qore >= 0.9.4 +%requires qore >= 1.10 + %requires(reexport) FtpPollerUtil -%requires DataProvider +%requires(reexport) DataProvider module FtpPoller { - version = "1.0"; + version = "2.0"; desc = "user module providing FTP polling functionality"; author = "Alzhan Turlybekov "; url = "http://qore.org"; license = "MIT"; + init = sub () { + # register the data provider factory + DataProvider::registerFactory(new FtpPollerDataProviderFactory()); + }; } -/* Version History - * 2019-10-16 v1.0: Alzhan Turlybekov -*/ - /** @mainpage FtpPoller Module @section ftppollerintro Introduction to the FtpPoller Module - The FtpPoller module implements an abstract class that will poll a remote directory with the FTP protocol and return - matching files. + The FtpPoller module implements an abstract class that will poll a remote directory with the FTP protocol and + return matching files. - To use this class, subclass the @ref FtpPoller::FtpPoller "FtpPoller" class and implement the @ref FtpPoller::FtpPoller::singleFileEvent() and - @ref FtpPoller::FtpPoller::postSingleFileEvent() methods. + To use this class, subclass the @ref FtpPoller::FtpPoller "FtpPoller" class and implement the + @ref FtpPoller::FtpPoller::singleFileEvent() and @ref FtpPoller::FtpPoller::postSingleFileEvent() methods. @section ftppollerexamples FtpPoller Module Examples - The following simple example will poll for files and then print out information for the files polled (as well as all info, detail, and debug messages) and exit immediately: + The following simple example will poll for files and then print out information for the files polled (as well as + all info, detail, and debug messages) and exit immediately: @code %requires FtpPoller @@ -66,7 +69,8 @@ class MyFtpPoller inherits FtpPoller { singleFileEvent(hash file_info) { printf("GOT FILE: %y\n", file_info); - # in this case, the polling stop operation will take effect after all the singleFileEvent() calls are made for the polling operation + # in this case, the polling stop operation will take effect after all the singleFileEvent() calls are made for + # the polling operation stopNoWait(); } @@ -90,20 +94,30 @@ FtpPoller poller(ftp_client, opts); poller.waitStop(); @endcode - Note that @ref FtpPoller::FtpPoller::stopNoWait() "FtpPoller::stopNoWait()" was called in the event thread because calling - @ref FtpPoller::FtpPoller::stop() "FtpPoller::stop()" in the event thread would cause an exception to be thrown. + Note that @ref FtpPoller::FtpPoller::stopNoWait() "FtpPoller::stopNoWait()" was called in the event thread because + calling @ref FtpPoller::FtpPoller::stop() "FtpPoller::stop()" in the event thread would cause an exception to be + thrown. - A useful poller class would implement the @ref FtpPoller::FtpPoller::singleFileEvent() "FtpPoller::singleFileEvent()" method which process already-transferred - files and the @ref FtpPoller::FtpPoller::postSingleFileEvent() "FtpPoller::postSingleFileEvent()" by deleting / moving / renaming the files so that they would not be acquired on the next poll. + A useful poller class would implement the + @ref FtpPoller::FtpPoller::singleFileEvent() "FtpPoller::singleFileEvent()" method which process already- + transferred files and the @ref FtpPoller::FtpPoller::postSingleFileEvent() "FtpPoller::postSingleFileEvent()" by + deleting / moving / renaming the files so that they would not be acquired on the next poll. @section ftpollersandbox FtpPoller Module in Sandboxed Programs - The @ref FtpPoller::FtpPoller "FtpPoller" class includes support for running in sandboxed Program objects with the following parse options set: - - \c PO_NO_THREAD_CONTROL: in this case the \c "start_thread" option is required in @ref FtpPoller::FtpPoller::constructor() "FtpPoller::constructor()" - - \c PO_NO_PROCESS_CONTROL: in this case the \c "sleep" option is required in @ref FtpPoller::FtpPoller::constructor() "FtpPoller::constructor()" + The @ref FtpPoller::FtpPoller "FtpPoller" class includes support for running in sandboxed Program objects with the + following parse options set: + - \c PO_NO_THREAD_CONTROL: in this case the \c "start_thread" option is required in + @ref FtpPoller::FtpPoller::constructor() "FtpPoller::constructor()" + - \c PO_NO_PROCESS_CONTROL: in this case the \c "sleep" option is required in + @ref FtpPoller::FtpPoller::constructor() "FtpPoller::constructor()" @section ftppollerrelnotes FtpPoller Module Release Notes + @section ftppollerv2_0 Version 2.0 + - added support for the event-based DataProvider API + (issue 4557) + @section ftppollerv1_0 Version 1.0 - initial release */ @@ -122,7 +136,7 @@ public class FtpPoller { const SortNone = 0; #! sort by name const SortName = 1; - #const SortDate = 2; + const SortDate = 2; #! minimum required keys for all constructors const RequiredKeys = ( @@ -139,17 +153,19 @@ public class FtpPoller { #! default values for constructor hash argument const Defaults = { + "protocol": "ftp", "port": 21, "mask": "*", "poll_interval": 10, "reopts": 0, - "binary": False, + "tempfile_template": ".tmp.%s.part", + "atomic_transfer": False, }; #! optional constructor hash keys const OptionalKeys = ( "path", "user", "pass", "regex_mask", "minage", - "encoding", "log_info", "log_detail", "log_debug", "binary", + "log_info", "log_detail", "log_debug", "local_dir", %ifndef PO_NO_PROCESS_CONTROL "sleep", %endif @@ -159,13 +175,16 @@ public class FtpPoller { ); #! all keys - const AllKeys = RequiredKeysWithHost + Defaults.keys() + OptionalKeys; + const AllKeys = RequiredKeysWithHost + keys Defaults + OptionalKeys; #! pause when Ftp errors are detected const ErrorDelay = 1m; } private { + #! Protocol ("ftp" or "ftps") + string protocol; + #! host or address name string host; @@ -196,6 +215,15 @@ public class FtpPoller { #! poll interval in seconds int poll_interval; + #! Local directory to transfer file + string local_dir; + + #! The temporary filename template when \c local_dir is set + string tempfile_template; + + #! Atomic transfer flag for use with \c local_dir + bool atomic_transfer; + #! run flag bool runflag = False; @@ -229,9 +257,6 @@ public class FtpPoller { #! minimum file age *softint minage; - #! file encoding for text files - *string encoding; - #! optional info log closure *code log_info; @@ -246,35 +271,45 @@ public class FtpPoller { #! optional sleep closure *code sleep; - - #! binary transfer flag (for singleFileEvent()) - bool binary; } #! creates the FtpPoller object from the @ref Qore::FtpClient "FtpClient" argument and configuration hash argument passed /** @param n_ftp the new @ref Qore::FtpClient "FtpClient" object - @param nconf a hash with the following optional keys: - - \c poll_interval: the integer polling interval in seconds (default: 10 seconds; must be > 0 if given) - - \c mask: the file glob mask to use (default: \c "*", ignored if \c "regex_mask" is also present) - - \c path: the remote path(s) for retrieving the files; if a list of strings is given then each path will be polled for matching files according to the \c "mask" or \c "regex_mask" option - - \c regex_mask: a regular expression to use as a mask (overrides any \c "mask" value) - - \c reopts: regular expression match options (ex RE_Caseless for case-insensitive matches) - - \c minage: the minimum file age in seconds before a file will be acquired (default: 0) - - \c encoding: the encoding for any text files received - - \c binary: if set to @ref True "True" then files are transferred in binary mode by default (with singleFileEvent() usage only), otherwise file data is returned in text format - - \c log_info: a @ref closure "closure" or @ref call_reference "call reference" for logging important information; must accept a single string giving the log message - - \c log_detail: a @ref closure "closure" or @ref call_reference "call reference" for logging detailed information; must accept a single string giving the log message - - \c log_debug: a @ref closure "closure" or @ref call_reference "call reference" for logging verbose debgugging information; must accept a single string giving the log message - - \c start_thread: (required when imported into a context where @ref Qore::PO_NO_THREAD_CONTROL is set) a @ref closure "closure" or @ref call_reference "call reference" for starting threads; must return the integer thread ID (if not set then @ref background will be used) - - \c sleep: (required when imported into a context where @ref Qore::PO_NO_PROCESS_CONTROL is set) a @ref closure "closure" or @ref call_reference "call reference" to use instead of @ref Qore::sleep() (if not set then @ref Qore::sleep() will be used) - - @throw FTPPOLLER-CONSTRUCTOR-ERROR missing required key, invalid port or poll_interval given - @throw FTPCLIENT-PARAMETER-ERROR empty hostname passed - @throw SOCKET-CONNECT-ERROR error establishing socket connection (no listener, port blocked, etc); timeout establishing socket connection - @throw SSH2CLIENT-CONNECT-ERROR no user name set; ssh2 or libssh2 error - @throw SSH2-ERROR error initializing or establishing ssh2 session - @throw SSH2CLIENT-AUTH-ERROR no proper authentication method found - @throw FTPCLIENT-CONNECT-ERROR error initializing ftp session or getting remote path + @param nconf a hash with the following optional keys: + - \c atomic_transfer: if @ref True and \c local_dir is set, then \c tempfile_template is used to write FTP + files to a temporary filename during transfer + - \c local_dir: a local directory to use to transfer remote files so large file data does not appear in events + - \c log_debug: a @ref closure "closure" or @ref call_reference "call reference" for logging verbose + debgugging information; must accept a single string giving the log message + - \c log_detail: a @ref closure "closure" or @ref call_reference "call reference" for logging detailed + information; must accept a single string giving the log message + - \c log_info: a @ref closure "closure" or @ref call_reference "call reference" for logging important + information; must accept a single string giving the log message + - \c mask: the file glob mask to use (default: \c "*", ignored if \c "regex_mask" is also present) + - \c minage: the minimum file age in seconds before a file will be acquired (default: 0) + - \c path: the remote path(s) for retrieving the files; if a list of strings is given then each path will be + polled for matching files according to the \c "mask" or \c "regex_mask" option + - \c poll_interval: the integer polling interval in seconds (default: 10 seconds; must be > 0 if given) + - \c regex_mask: a regular expression to use as a mask (overrides any \c "mask" value) + - \c reopts: regular expression match options (ex RE_Caseless for case-insensitive matches) + - \c sleep: (required when imported into a context where @ref Qore::PO_NO_PROCESS_CONTROL is set) a + @ref closure "closure" or @ref call_reference "call reference" to use instead of @ref Qore::sleep() (if not + set then @ref Qore::sleep() will be used) + - \c start_thread: (required when imported into a context where @ref Qore::PO_NO_THREAD_CONTROL is set) a + @ref closure "closure" or @ref call_reference "call reference" for starting threads; must return the integer + thread ID (if not set then @ref background will be used) + - \c tempfile_template: (used when \c local_dir and \c atomic_transfer are set) the temporary filename prefix + to use when transferring files to the local directory specified by \c local_dir; use \c "%s" as the + placeholder for the target filename + + @throw FTPPOLLER-CONSTRUCTOR-ERROR missing required key, invalid port or poll_interval given + @throw FTPCLIENT-PARAMETER-ERROR empty hostname passed + @throw SOCKET-CONNECT-ERROR error establishing socket connection (no listener, port blocked, etc); timeout + establishing socket connection + @throw SSH2CLIENT-CONNECT-ERROR no user name set; ssh2 or libssh2 error + @throw SSH2-ERROR error initializing or establishing ssh2 session + @throw SSH2CLIENT-AUTH-ERROR no proper authentication method found + @throw FTPCLIENT-CONNECT-ERROR error initializing ftp session or getting remote path */ constructor(Qore::FtpClient n_ftp, hash nconf) { # create hash of keys with only valid options in argument hash @@ -287,6 +322,11 @@ public class FtpPoller { } } + if (exists conf.local_dir && (!is_dir(conf.local_dir) || !is_writable(conf.local_dir))) { + throw "FTPPOLLER-CONSTRUCTOR-ERROR", sprintf("invalid local directory %y; directory does not exist or " + "cannot be written to", conf.local_dir); + } + conf.poll_interval = int(conf.poll_interval); if (conf.poll_interval <= 0) { @@ -325,7 +365,7 @@ public class FtpPoller { port = ftp.getPort(); - url = sprintf("%s@%s:%d", user ?? "", ftp.getHostName(), port); + url = sprintf("%s://%s@%s:%d", protocol, user ?? "", ftp.getHostName(), port); urlh = parse_url(ftp.getURL()); @@ -345,30 +385,43 @@ public class FtpPoller { #! creates the FtpPoller object from the configuration hash argument passed /** @param nconf a hash with the following keys: - - \c host: (required) the hostname or address to connect to - - \c port: the integer port number to connect to (default 21; must be > 0 if given) - - \c user: the username to use for the connection - - \c pass: the password to use for the connection - - \c path: the remote path(s) for retrieving the files - - \c poll_interval: the integer polling interval in seconds (default: 10 seconds; must be > 0 if given) - - \c mask: the file glob mask to use (default: \c "*", ignored if \c "regex_mask" is also present) - - \c regex_mask: a regular expression to use as a mask (overrides any \c "mask" value) - - \c reopts: regular expression match options (ex RE_Caseless for case-insensitive matches) - - \c minage: the minimum file age in seconds before a file will be acquired (default: 0) - - \c encoding: the encoding for any text files received - - \c log_info: a @ref closure "closure" or @ref call_reference "call reference" for logging important information; must accept a single string giving the log message - - \c log_detail: a @ref closure "closure" or @ref call_reference "call reference" for logging detailed information; must accept a single string giving the log message - - \c log_debug: a @ref closure "closure" or @ref call_reference "call reference" for logging verbose debgugging information; must accept a single string giving the log message - - \c start_thread: (required when imported into a context where @ref Qore::PO_NO_THREAD_CONTROL is set) a @ref closure "closure" or @ref call_reference "call reference" for starting threads; must return the integer thread ID (if not set then @ref background will be used) - - \c sleep: (required when imported into a context where @ref Qore::PO_NO_PROCESS_CONTROL is set) a @ref closure "closure" or @ref call_reference "call reference" to use instead of @ref Qore::sleep() (if not set then @ref Qore::sleep() will be used) - - @throw FTPPOLLER-CONSTRUCTOR-ERROR missing required key, invalid port or poll_interval given - @throw FTPCLIENT-PARAMETER-ERROR empty hostname passed - @throw SOCKET-CONNECT-ERROR error establishing socket connection (no listener, port blocked, etc) - @throw SSH2CLIENT-CONNECT-ERROR no user name set; ssh2 or libssh2 error - @throw SSH2-ERROR error initializing or establishing ssh2 session - @throw SSH2CLIENT-AUTH-ERROR no proper authentication method found - @throw FTPCLIENT-CONNECT-ERROR error initializing ftp session or getting remote path + - \c atomic_transfer: if @ref True and \c local_dir is set, then \c tempfile_template is used to write FTP + files to a temporary filename during transfer + - \c host: (required) the hostname or address to connect to + - \c local_dir: a local directory to use to transfer remote files so large file data does not appear in events + - \c log_debug: a @ref closure "closure" or @ref call_reference "call reference" for logging verbose + debgugging information; must accept a single string giving the log message + - \c log_detail: a @ref closure "closure" or @ref call_reference "call reference" for logging detailed + information; must accept a single string giving the log message + - \c log_info: a @ref closure "closure" or @ref call_reference "call reference" for logging important + information; must accept a single string giving the log message + - \c mask: the file glob mask to use (default: \c "*", ignored if \c "regex_mask" is also present) + - \c minage: the minimum file age in seconds before a file will be acquired (default: 0) + - \c pass: the password to use for the connection + - \c path: the remote path(s) for retrieving the files + - \c poll_interval: the integer polling interval in seconds (default: 10 seconds; must be > 0 if given) + - \c port: the integer port number to connect to (default 21; must be > 0 if given) + - \c protocol: either \c "ftp" (the detault) or \c "ftps" for secure FTP + - \c regex_mask: a regular expression to use as a mask (overrides any \c "mask" value) + - \c reopts: regular expression match options (ex RE_Caseless for case-insensitive matches) + - \c start_thread: (required when imported into a context where @ref Qore::PO_NO_THREAD_CONTROL is set) a + @ref closure "closure" or @ref call_reference "call reference" for starting threads; must return the integer + thread ID (if not set then @ref background will be used) + - \c sleep: (required when imported into a context where @ref Qore::PO_NO_PROCESS_CONTROL is set) a + @ref closure "closure" or @ref call_reference "call reference" to use instead of @ref Qore::sleep() (if not + set then @ref Qore::sleep() will be used) + - \c tempfile_template: (used when \c local_dir and \c atomic_transfer are set) the temporary filename prefix + to use when transferring files to the local directory specified by \c local_dir; use \c "%s" as the + placeholder for the target filename + - \c user: the username to use for the connection + + @throw FTPPOLLER-CONSTRUCTOR-ERROR missing required key, invalid port, poll_interval, or protocol given + @throw FTPCLIENT-PARAMETER-ERROR empty hostname passed + @throw SOCKET-CONNECT-ERROR error establishing socket connection (no listener, port blocked, etc) + @throw SSH2CLIENT-CONNECT-ERROR no user name set; ssh2 or libssh2 error + @throw SSH2-ERROR error initializing or establishing ssh2 session + @throw SSH2CLIENT-AUTH-ERROR no proper authentication method found + @throw FTPCLIENT-CONNECT-ERROR error initializing ftp session or getting remote path */ constructor(hash nconf) { # create hash of keys with only valid options in argument hash @@ -376,16 +429,27 @@ public class FtpPoller { foreach string k in (RequiredKeysWithHost) { if (!exists conf{k}) { - throw "FTPPOLLER-CONSTRUCTOR-ERROR", sprintf("required key %y missing from constructor hash argument)", - k); + throw "FTPPOLLER-CONSTRUCTOR-ERROR", sprintf("required key %y missing from constructor hash " + "argument)", k); } } + if (conf.protocol != "ftp" && conf.protocol != "ftps") { + throw "FTPPOLLER-CONSTRUCTOR-ERROR", sprintf("invalid URL scheme %y; expecting \"ftp\" or \"ftps\"", + conf.protocol); + } + + if (exists conf.local_dir && (!is_dir(conf.local_dir) || !is_writable(conf.local_dir))) { + throw "FTPPOLLER-CONSTRUCTOR-ERROR", sprintf("invalid local directory %y; directory does not exist or " + "cannot be written to", conf.local_dir); + } + conf.poll_interval = int(conf.poll_interval); conf.port = int(conf.port); if (conf.poll_interval <= 0) { - throw "FTPPOLLER-CONSTRUCTOR-ERROR", sprintf("poll_interval cannot be <= 0 (val: %d)", conf.poll_interval); + throw "FTPPOLLER-CONSTRUCTOR-ERROR", sprintf("poll_interval cannot be <= 0 (val: %d)", + conf.poll_interval); } if (conf.port <= 0) { throw "FTPPOLLER-CONSTRUCTOR-ERROR", sprintf("port cannot be <= 0 (val: %d)", conf.port); @@ -407,7 +471,7 @@ public class FtpPoller { } # create FtpClient object - ftp = new FtpClient("ftp://" + host + ":" + port); + ftp = new FtpClient(sprintf("%s://%s:%d", protocol, host, port)); if (user) { ftp.setUserName(user); } @@ -415,7 +479,7 @@ public class FtpPoller { ftp.setPassword(pass); } - url = sprintf("%s@%s:%d", user ? user : "", host, port); + url = sprintf("%s://%s@%s:%d", protocol, user ? user : "", host, port); if (mask) { logDetail("%s: file regex mask: %s", url, mask); @@ -481,13 +545,13 @@ public class FtpPoller { # replace ? -> . mask =~ s/\?/./g; # replace * -> .* - mask =~ s/\*/.*/g; #//; # previous comment needed only for broken emacs qore-mode syntax highlighting + mask =~ s/\*/.*/g; mask = sprintf("^%s$", mask); #"); } #! retrieves a remote file and stores it to a local path /** @param remote_path the remote file path - @param local_path the local file path + @param local_path the local file path */ getStoreFile(string remote_path, string local_path) { ftp.get(remote_path, local_path); @@ -524,7 +588,8 @@ public class FtpPoller { - \c size: the size of the file in bytes - \c mtime: the last modified date/time of the file */ - list> getFiles(string subdir, int sort = FtpPoller::SortNone, int order = FtpPoller::OrderAsc) { + list> getFiles(string subdir, int sort = FtpPoller::SortNone, + int order = FtpPoller::OrderAsc) { ftp.cwd(subdir); list fl; { @@ -585,7 +650,18 @@ public class FtpPoller { # sort by file name case FtpPoller::SortName: { # sort closure - code sorter = int sub (hash lt, hash rt) { return lt.name <=> rt.name; }; + code sorter = int sub (hash lt, hash rt) { + return lt.name <=> rt.name; + }; + l = (order == FtpPoller::OrderAsc ? Qore::sort(l, sorter) : sort_descending(l, sorter)); + break; + } + + case FtpPoller::SortDate: { + # sort by last modification date + code sorter = int sub (hash l, hash r) { + return l.mtime <=> r.mtime; + }; l = (order == FtpPoller::OrderAsc ? Qore::sort(l, sorter) : sort_descending(l, sorter)); break; } @@ -600,7 +676,7 @@ public class FtpPoller { #! starts polling in the background; returns the thread ID of the polling thread /** if polling had already been started, then the thread ID of the polling thread is - returned immediately + returned immediately */ int start() { m.lock(); @@ -635,9 +711,10 @@ public class FtpPoller { #! stops the polling operation, returns when the polling operation has been stopped /** if polling was not in progress then this method returns immediately - @throw THREAD-ERROR this exception is thrown if this method is called from the event thread since it would result in a deadlock + @throw THREAD-ERROR this exception is thrown if this method is called from the event thread since it would + result in a deadlock - @see stopNoWait() + @see stopNoWait() */ stop() { if (gettid() == tid && sc.getCount()) { @@ -651,9 +728,11 @@ public class FtpPoller { sc.waitForZero(); } - #! waits indefinitely for the polling operation to stop; if polling was not in progress then this method returns immediately - /** - @throw THREAD-ERROR this exception is thrown if this method is called from the event thread since it would result in a deadlock + #! waits indefinitely for the polling operation to stop + /** If polling was not in progress then this method returns immediately + + @throw THREAD-ERROR this exception is thrown if this method is called from the event thread since it would + result in a deadlock */ waitStop() { if (gettid() == tid) { @@ -664,7 +743,7 @@ public class FtpPoller { #! starts the polling operation inline (not in a background thread) /** - @throw FTPPOLLER-ERROR this exception is thrown if polling is already in progress + @throw FTPPOLLER-ERROR this exception is thrown if polling is already in progress */ startInline() { { @@ -701,30 +780,27 @@ public class FtpPoller { logDetail("got new files in %y: %y", pwd, files); if (fileEvent(files)) { - foreach hash fh in (files) { - # transfer file from server - logInfo("%y: retrieving %s file data", fh.name, binary ? "binary" : "text"); - date t1 = now_us(); - + foreach hash event in (files) { get_files = True; - if (fh.size) { - fh.data = binary ? getFile(fh.name) : getTextFile(fh.name); + event.filepath = subdir == "." ? event.name : (subdir + DirSep + event.name); + if (local_dir) { + if (atomic_transfer) { + event = retrieveTempFile(event); + } else { + event = retrieveFile(event); + } } else { - fh.data = binary ? Qore::binary() : ""; + event = getRemoteFileData(event); } - logInfo("%y: retrieved %d bytes in %y", fh.name, fh.data.size(), now_us() - t1); - # make sure any errors after this point cause the polling operation to stop get_files = False; - fh.filepath = subdir == "." ? fh.name : (subdir + DirSep + fh.name); - singleFileEvent(fh); - - try { - postSingleFileEvent(fh); - } catch () { - fatal = True; - rethrow; - } + + singleFileEvent(event); + + # set the "fatal" flag if an exception is thrown in postSingleFileEvent() + on_error fatal = True; + + postSingleFileEvent(event); } } ret = True; @@ -733,13 +809,97 @@ public class FtpPoller { return ret; } + #! Retrieves remote file data and adds it to the event data + hash getRemoteFileData(hash event) { + # get file to binary stream + BinaryOutputStream binstream(); + + logInfo("retrieving %y (%d bytes) to memory", event.filepath, event.size); + date start = now_us(); + ftp.get(event.filepath, binstream); + event.transfer_time = now_us() - start; + logInfo("retrieved %y in %y", event.filepath, event.transfer_time); + + # write binary data to event hash + event += { + "data": binstream.getData(), + }; + + return event; + } + + #! Retrieves the remote file to local_dir using a temporary file + private hash retrieveTempFile(hash event) { + # add filename + string tmp_name = sprintf(tempfile_template, event.name); + string tmp_path = normalize_dir(local_dir + DirSep + tmp_name); + string local_path = normalize_dir(local_dir + DirSep + event.name); + + logInfo("using local dir %y to stream %y -> %y (%d bytes)", local_dir, event.filepath, + tmp_path, event.size); + date start = now_us(); + if (event.size) { + # stream remote file to local file + ftp.get(event.filepath, tmp_path); + } else { + File f(); + f.open2(tmp_path, O_CREAT | O_WRONLY | O_TRUNC); + } + + # move to target filename after transfer to ensure atomicity + if (tmp_path != local_path) { + # NOTE: uses a simple rename() which may cause problems on some OSes if moving between filesystems + Qore::rename(tmp_path, local_path); + logInfo("renamed %y -> %y", tmp_path, local_path); + } + + event += { + "transfer_time": now_us() - start, + "local_path": local_path, + }; + logInfo("streamed %d byte%s from %y -> %y in %y", event.size, event.size == 1 ? "" : "s", event.filepath, + local_path, event.transfer_time); + return event; + } + + #! Retrieves the remote file to local_dir directly + private hash retrieveFile(hash event) { + # add filename + string local_path = normalize_dir(local_dir + DirSep + event.name); + + logInfo("using local dir %y to stream %y -> %y (%d bytes)", local_dir, event.filepath, + local_path, event.size); + date start = now_us(); + if (event.size) { + # stream remote file to local file + ftp.get(event.filepath, local_path); + } else { + File f(); + f.open2(local_path, O_CREAT | O_WRONLY | O_TRUNC); + } + + event += { + "transfer_time": now_us() - start, + "local_path": local_path, + }; + logInfo("streamed %d byte%s from %y -> %y in %y", event.size, event.size == 1 ? "" : "s", event.filepath, + local_path, event.transfer_time); + return event; + } + #! sleeps for the specificed number of seconds private ftpSleep(softint secs) { + date end = now_us() + seconds(secs); + while (runflag) { %ifdef PO_NO_PROCESS_CONTROL - call_function(sleep, secs); + call_function(sleep, 1); %else - sleep ? call_function(sleep, secs) : Qore::sleep(secs); + sleep ? call_function(sleep, 1) : Qore::usleep(250ms); %endif + if (!runflag || (now_us() >= end)) { + break; + } + } } #! starts the polling operation @@ -751,15 +911,11 @@ public class FtpPoller { while (runflag) { try { runOnce(); - if (runflag) { - ftpSleep(poll_interval); - } + ftpSleep(poll_interval); } catch (hash ex) { if (get_files) { logInfo("FTP error in %y: %s: %s; waiting %y for next try", path, ex.err, ex.desc, ErrorDelay); - if (runflag) { - ftpSleep(ErrorDelay); - } + ftpSleep(ErrorDelay); } else { # error thrown in "post transfer" code logInfo("error in post transfer code; polling terminated: %s: %s", ex.err, ex.desc); @@ -771,38 +927,36 @@ public class FtpPoller { logInfo("polling finished"); } - #! called for each poll with a list of all files matched before transfer; if this method returns False or @ref nothing, then the singleFileEvent method is not called - *bool fileEvent(list l) { + #! called for each poll with a list of all files matched before transfer + /** if this method returns False or @ref nothing, then the singleFileEvent method is not called + */ + *bool fileEvent(list> l) { return True; } - #! called for each matching file individually whenever matching files are polled with the list of matching file names; if any error occurs here, the error is logged and the polling operation is retried - /** @param fih a hash of file data and information with the following keys: - - \c name: the name of the file, link, or directory - - \c size: the size of the file in bytes - - \c mtime: the last modified date/time of the file - - \c data: the file's data; this will be a string unless the \a "binary" option is set to @ref True "True", in which case this key is assigned to the files binary data - - \c filepath: the remote filepath relative to FTP root directory + #! called for each matching file individually whenever matching files are polled + /** If any error occurs here, the error is logged and the polling operation is retried + + @param event a hash of file data */ - abstract singleFileEvent(hash fih); + abstract singleFileEvent(hash event); - #! called after singleFileEvent() for each matching file individually whenever matching files are polled with the list of matching file names; if any error occurs here, the polling operation stops - /** This method would normally delete / rename / move files processed by singleFileEvent() so that they would not be polled a second time. - If an error occurs in this operation, then the polling event will stop since continuing after failing to delete, rename, or move a file already processed - would cause the file to be processed more than once. + #! called after singleFileEvent() for each matching file individually + /** If any error occurs here, the polling operation stops - @param fih a hash of file data and information with the following keys: - - \c name: the name of the file, link, or directory - - \c size: the size of the file in bytes - - \c mtime: the last modified date/time of the file - - \c data: the file's data; this will be a string unless the \a "binary" option is set to @ref True "True", in which case this key is assigned to the files binary data - - \c filepath: the remote filepath relative to FTP root directory + This method would normally delete / rename / move files processed by singleFileEvent() so that they would not + be polled a second time. + If an error occurs in this operation, then the polling event will stop since continuing after failing to + delete, rename, or move a file already processed would cause the file to be processed more than once. + + @param event a hash of file data */ - abstract postSingleFileEvent(hash fih); + abstract postSingleFileEvent(hash event); #! checks a path on the local file system /** - @throw DIR-ERROR this exception is thrown if the local path does not exist, is not readable, is not a directory, or should be writable and is not + @throw DIR-ERROR this exception is thrown if the local path does not exist, is not readable, is not a + directory, or should be writable and is not */ static checkPath(string path, string type, bool write = False) { *hash h = hstat(path); @@ -824,4 +978,244 @@ public class FtpPoller { } } } + +#! Event-based data provider for FTP polling events +/** When using the \c local_dir option, the local file must be removed / moved / archived by the event handler +*/ +public class FtpPollerDataProviderBase inherits DataProvider::AbstractDataProvider { + public { + #! Provider info + const ProviderInfo = { + "type": "FtpPollerDataProvider", + "supports_observable": True, + "constructor_options": ConstructorOptions, + }; + + #! Constructor options + const ConstructorOptions = { + "atomic_transfer": { + "type": AbstractDataProviderType::get(BoolType), + "desc": "Use an atomic transfer mechanism with `local_dir` where remote files are first moved to a " + "temporary location and then moved to the final location when they have been fully transferred", + "default_value": True, + }, + + "local_dir": { + "type": AbstractDataProviderType::get(StringOrNothingType), + "desc": "A local directory that will be used to retrieve files", + }, + + "mask": { + "type": AbstractDataProviderType::get(StringType), + "desc": "The glob mask to use; will be treated as a regex if `regex` is `true`", + "default_value": "*", + }, + + "minage": { + "type": AbstractDataProviderType::get(IntType), + "desc": "An integer giving the minimum file age in seconds before the file will be polled; this is " + "meant to work around non-atomic file transfer operations", + }, + + "poll_interval": { + "type": AbstractDataProviderType::get(IntType), + "desc": "The interval in seconds between polling for files", + "default_value": 10, + }, + + "regex": { + "type": AbstractDataProviderType::get(BoolType), + "desc": "If `true` then `mask` is treated as a regular expression instead of a glob pattern", + }, + + "reopt": { + "type": AbstractDataProviderType::get(IntType), + "desc": "A bitfield of regular expression options (`1` = ignore case, `2` = treat EOL as a regular " + "character); ignored if `regex` is not `true`", + }, + + "sort_desc": { + "type": AbstractDataProviderType::get(BoolType), + "desc": "Sort descending; if not given then an ascending sort is assumed if a `sort_type` is given", + }, + + "sort_type": { + "type": AbstractDataProviderType::get(StringType), + "desc": "Either `name` or `date` for the data to use for sorting", + }, + + "url": { + "type": AbstractDataProviderType::get(StringType), + "desc": "A URL for an FTP connection", + "required": True, + }, + }; + } + + private { + #! The file poller itself + EmbeddedFtpPoller poller; + } + + #! Creates the object from constructor options + constructor(*hash options) { + hash copts = checkOptions("CONSTRUCTOR-ERROR", ConstructorOptions, options); + string url = remove copts.url; + hash url_info = parse_url(url); + + copts += url_info; + if (copts.username) { + copts.user = remove copts.username; + } + if (copts.password) { + copts.pass = remove copts.password; + } + + bool regex = remove copts.regex ?? False; + if (!copts.mask) { + copts.mask = regex ? ".*" : ConstructorOptions.mask.default_value; + } + if (*string sort_type = remove copts.sort_type) { + if (sort_type == "name") { + copts.sort_type = FtpPoller::SortName; + } else if (sort_type == "date") { + copts.sort_type = FtpPoller::SortDate; + } else { + throw "CONSTRUCTOR-ERROR", sprintf("expecting \"name\" or \"date\" for the \"sort_type\" option " + "value; got %y instead", sort_type); + } + if (remove copts.sort_desc) { + copts.sort_order = FtpPoller::OrderDesc; + } + } else { + remove copts.sort_desc; + } + + if (regex) { + copts.regex_mask = remove copts.mask; + } + + copts.log_info = sub (string msg) { + if (logger) { + logger.info("%s", msg); + } + }; + copts.log_detail = sub (string msg) { + if (logger) { + logger.info("%s", msg); + } + }; + copts.log_debug = sub (string msg) { + if (logger) { + logger.debug("%s", msg); + } + }; + + poller = new EmbeddedFtpPoller(self, copts); + } + + destructor() { + delete poller; + } + + string getName() { + return "ftppoller"; + } + + private hash getStaticInfoImpl() { + return ProviderInfo; + } + + #! Returns the description of an event, if any + /** @return the event type for this provider + */ + private *AbstractDataProviderType getEventTypeImpl() { + return new FtpPollerFileEventInfoDataType(); + } +} + +#! Event-based data provider for FTP polling events +/** When using the \c local_dir option, the local file must be removed / moved / archived by the event handler +*/ +public class FtpPollerDataProvider inherits FtpPollerDataProviderBase, DataProvider::Observable { + #! Creates the object from constructor options + constructor(*hash options) : FtpPollerDataProviderBase(options) { + poller.start(); + } +} + +#! Event-based data provider for FTP polling events +/** When using the \c local_dir option, the local file must be removed / moved / archived by the event handler +*/ +public class FtpDelayedPollerDataProvider inherits FtpPollerDataProviderBase, DataProvider::DelayedObservable { + #! Creates the object from constructor options + constructor(*hash options) : FtpPollerDataProviderBase(options) { + } + + #! Called when all observers have been added to the object + /** This method is meant to trigger event generation + */ + observersReady() { + poller.start(); + } +} + +#! The file poller data provider factory +public class FtpPollerDataProviderFactory inherits AbstractDataProviderFactory { + private { + #! Data provider type info + static Class cls = new Class("FtpPollerDataProvider"); + + #! Factory info + const FactoryInfo = { + "name": "ftppoller", + "desc": "FTP poller data provider factory", + "children_can_support_observers": True, + }; + } + + #! Returns static factory information without \a provider_info + /** @return static factory information without \a provider_info which is provided by @ref getProviderInfo() + */ + private hash getInfoImpl() { + return FactoryInfo; + } + + #! Returns static provider information + /** @note the \c name and \c children attributes are not returned as they are dynamic attributes + */ + private hash getProviderInfoImpl() { + return FtpPollerDataProvider::ProviderInfo; + } + + #! Returns the class for the data provider object + private Class getClassImpl() { + return cls; + } +} +} + +# private namespace; not exported +namespace Priv { +class EmbeddedFtpPoller inherits FtpPoller { + private { + #! The parent observer object + Observable observable; + + #! The unique ID counter + int id = 0; + } + + constructor(FtpPollerDataProviderBase provider, hash options) + : FtpPoller(options) { + observable := cast(provider); + } + + singleFileEvent(hash event) { + observable.notifyObservers((++id).toString(), event); + } + + postSingleFileEvent(hash event) { + } +} } diff --git a/qlib/FtpPollerUtil.qm b/qlib/FtpPollerUtil.qm index fc9610dbae..973e75d452 100644 --- a/qlib/FtpPollerUtil.qm +++ b/qlib/FtpPollerUtil.qm @@ -66,12 +66,16 @@ public class FtpPollerFileEventInfoDataType inherits HashDataType { public { #! Markdown descriptions for hashdecl members const FieldDescriptions = { - "name": "the name of the file, link, or directory", - "size": "the size of the file in bytes", - "mtime": "the last modified date/time of the file", - "data": "the file's data; this will be a string unless the `binary` option is set to `True`, in " - "which case this key is assigned to the files binary data", - "filepath": "the remote filepath relative to SFTP root directory", + "name": "The name of the file, link, or directory", + "size": "The size of the file in bytes", + "mtime": "The last modified date/time of the file", + "data": "The file's data if `local_dir` is not set; this will be a string unless the `binary` option is " + "set to `True`, in which case this key is assigned to the file's binary data; when `local_dir` is " + "set, file data is not included in the event but is rather transferred to the local directory", + "filepath": "The remote filepath relative to SFTP root directory", + "transfer_time": "The relative date/time value giving the time taken to transfer the file", + "local_path": "Only included when `local_dir` is set; the target path for the file where the file will " + "be moved to in the post event action", }; } @@ -105,7 +109,13 @@ public hashdecl FtpPollerFileEventInfo { */ data data; - #! The remote filepath relative to FTP root directory + #! The remote filepath relative to the FTP root directory string filepath; + + #! The relative date/time value giving the time taken to transfer the file + date transfer_time; + + #! Only included when `local_dir` is set; the target path for the file + *string local_path; } } diff --git a/qlib/SalesforceRestDataProvider/SalesforceRestDataProviderFactory.qc b/qlib/SalesforceRestDataProvider/SalesforceRestDataProviderFactory.qc index 8326f2444f..7bba9297a0 100644 --- a/qlib/SalesforceRestDataProvider/SalesforceRestDataProviderFactory.qc +++ b/qlib/SalesforceRestDataProvider/SalesforceRestDataProviderFactory.qc @@ -34,6 +34,7 @@ public class SalesforceRestDataProviderFactory inherits AbstractDataProviderFact const FactoryInfo = { "name": "salesforcerest", "desc": "Salesforce REST data provider factory", + "children_can_support_records": True, }; } diff --git a/qlib/ServiceNowRestDataProvider/ServiceNowRestDataProviderFactory.qc b/qlib/ServiceNowRestDataProvider/ServiceNowRestDataProviderFactory.qc index d898301d00..423ed1311c 100644 --- a/qlib/ServiceNowRestDataProvider/ServiceNowRestDataProviderFactory.qc +++ b/qlib/ServiceNowRestDataProvider/ServiceNowRestDataProviderFactory.qc @@ -34,6 +34,7 @@ public class ServiceNowRestDataProviderFactory inherits DataProvider::AbstractDa const FactoryInfo = { "name": "servicenowrest", "desc": "ServiceNow REST data provider factory", + "children_can_support_records": True, }; } diff --git a/qore.spec-fedora b/qore.spec-fedora index 821ac2ecd1..018ed19c6b 100644 --- a/qore.spec-fedora +++ b/qore.spec-fedora @@ -43,7 +43,7 @@ This package provides the qore library required for all clients using qore functionality. %files -n libqore -%{_libdir}/libqore.so.12.0.2 +%{_libdir}/libqore.so.12.0.3 %{_libdir}/libqore.so.12 %license COPYING.LGPL COPYING.GPL COPYING.MIT README-LICENSE %doc README.md README-MODULES RELEASE-NOTES AUTHORS ABOUT @@ -175,6 +175,7 @@ make check %changelog * Sat Jul 16 2022 David Nichols 1.10.0-1 - updated version to 1.10.0-1 +- updated libqore version to 12.0.3 * Tue Jul 5 2022 David Nichols 1.9.1-1 - updated version to 1.9.1-1 diff --git a/qore.spec-multi b/qore.spec-multi index 7416224a0e..1ff596a49c 100644 --- a/qore.spec-multi +++ b/qore.spec-multi @@ -112,7 +112,7 @@ functionality. %files -n %{libname} %defattr(-,root,root,-) -%{_libdir}/libqore.so.12.0.2 +%{_libdir}/libqore.so.12.0.3 %{_libdir}/libqore.so.12 %doc COPYING.LGPL COPYING.GPL COPYING.MIT README.md README-LICENSE README-MODULES RELEASE-NOTES AUTHORS ABOUT @@ -280,6 +280,7 @@ rm -rf $RPM_BUILD_ROOT %changelog * Sat Jul 16 2022 David Nichols 1.10.0 - updated version to 1.10.0 +- updated libqore version to 12.0.3 * Tue Jul 5 2022 David Nichols 1.9.1 - updated version to 1.9.1 diff --git a/qore.spec-opensuse b/qore.spec-opensuse index aaf3a70a97..5d9bf13850 100644 --- a/qore.spec-opensuse +++ b/qore.spec-opensuse @@ -68,7 +68,7 @@ functionality. %files -n libqore12 %defattr(-,root,root,-) -%{_libdir}/libqore.so.12.0.2 +%{_libdir}/libqore.so.12.0.3 %{_libdir}/libqore.so.12 %doc COPYING.LGPL COPYING.GPL COPYING.MIT README-LICENSE