From 89199b955ea815209b6cd927b6fdb1807818fb44 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Wed, 10 Sep 2025 17:43:36 +0200 Subject: [PATCH 1/8] Move tests to version 3.7 and fixed some of them with temp files to make it compatible with older pythons --- .github/workflows/autotests.yml | 2 +- mergin/client_pull.py | 10 +++++----- mergin/client_push.py | 7 ++++--- mergin/utils.py | 18 +++++++++++++++++- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/.github/workflows/autotests.yml b/.github/workflows/autotests.yml index fc658a58..41c8d7a6 100644 --- a/.github/workflows/autotests.yml +++ b/.github/workflows/autotests.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.7' - name: Install python package dependencies run: | diff --git a/mergin/client_pull.py b/mergin/client_pull.py index c3eb422f..1e0f8e8a 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -22,7 +22,7 @@ from .common import CHUNK_SIZE, ClientError from .merginproject import MerginProject -from .utils import save_to_file +from .utils import cleanup_tmp_dir, save_to_file # status = download_project_async(...) @@ -145,7 +145,7 @@ def download_project_async(mc, project_path, directory, project_version=None): mp.log.info("--- version: " + mc.user_agent_info()) mp.log.info(f"--- start download {project_path}") - tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-", ignore_cleanup_errors=True, delete=True) + tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") try: # check whether we download the latest version or not @@ -250,7 +250,7 @@ def download_project_finalize(job): # final update of project metadata job.mp.update_metadata(job.project_info) - job.tmp_dir.cleanup() + cleanup_tmp_dir(job.mp, job.tmp_dir) def download_project_cancel(job): @@ -424,7 +424,7 @@ def pull_project_async(mc, directory): # then we just download the whole file _pulling_file_with_diffs = lambda f: "diffs" in f and len(f["diffs"]) != 0 - tmp_dir = tempfile.TemporaryDirectory(prefix="mm-pull-", ignore_cleanup_errors=True, delete=True) + tmp_dir = tempfile.TemporaryDirectory(prefix="mm-pull-") pull_changes = mp.get_pull_changes(server_info["files"]) mp.log.debug("pull changes:\n" + pprint.pformat(pull_changes)) fetch_files = [] @@ -646,7 +646,7 @@ def pull_project_finalize(job: PullJob): else: job.mp.log.info("--- pull finished -- at version " + job.mp.version()) - job.tmp_dir.cleanup() # delete our temporary dir and all its content + cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content return conflicts diff --git a/mergin/client_push.py b/mergin/client_push.py index 90fbd19a..315046c0 100644 --- a/mergin/client_push.py +++ b/mergin/client_push.py @@ -19,6 +19,7 @@ from .common import UPLOAD_CHUNK_SIZE, ClientError from .merginproject import MerginProject from .editor import filter_changes +from .utils import cleanup_tmp_dir class UploadJob: @@ -124,7 +125,7 @@ def push_project_async(mc, directory): changes = filter_changes(mc, project_info, changes) mp.log.debug("push changes:\n" + pprint.pformat(changes)) - tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-", ignore_cleanup_errors=True, delete=True) + tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") # If there are any versioned files (aka .gpkg) that are not updated through a diff, # we need to make a temporary copy somewhere to be sure that we are uploading full content. @@ -275,7 +276,7 @@ def push_project_finalize(job): f"Upload details: {len(job.upload_queue_items)} file chunks, total size {job.total_size} bytes." ) # server returns various error messages with filename or something generic - # it would be better if it returned list of failed files (and reasons) whenever possible + # it would be better if it returned list of failed files (and reasons) whenever possible job.mp.log.error("--- push finish failed! " + str(err)) # if push finish fails, the transaction is not killed, so we @@ -296,7 +297,7 @@ def push_project_finalize(job): job.mp.log.info("--- push aborted") raise ClientError("Failed to apply push changes: " + str(e)) - job.tmp_dir.cleanup() # delete our temporary dir and all its content + cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content remove_diff_files(job) diff --git a/mergin/utils.py b/mergin/utils.py index 7b6c9f9f..6ffd732d 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -6,6 +6,7 @@ import sqlite3 from datetime import datetime from pathlib import Path +import tempfile from .common import ClientError @@ -201,7 +202,7 @@ def conflicted_copy_file_name(path, user, version): head, tail = os.path.split(os.path.normpath(path)) ext = "".join(Path(tail).suffixes) file_name = tail.replace(ext, "") - # in case of QGIS project files we have to add "~" (tilde) to suffix + # in case of QGIS project files we TemporaryDirectoryhave to add "~" (tilde) to suffix # to avoid having several QGIS project files inside Mergin Maps project. # See https://github.com/lutraconsulting/qgis-mergin-plugin/issues/382 # for more details @@ -294,3 +295,18 @@ def bytes_to_human_size(bytes: int): return f"{round( bytes / 1024.0 / 1024.0 / 1024.0, precision )} GB" else: return f"{round( bytes / 1024.0 / 1024.0 / 1024.0 / 1024.0, precision )} TB" + +def cleanup_tmp_dir(mp, tmp_dir: tempfile.TemporaryDirectory): + """ + Remove temporary from tempfile.TemporaryDirectory instance + This is needed beacause ignore_clanup_errors is not accepted under < Python 3.10 + """ + + try: + tmp_dir.cleanup() + except PermissionError: + mp.log.warning(f"Permission error during tmp dir cleanup: {tmp_dir.name}") + pass # Ignore the error and continue + except Exception as e: + mp.log.error(f"Error during tmp dir cleanup: {tmp_dir.name}: {e}") + pass # Ignore the error and continue \ No newline at end of file From 4f8b455f12cb39d6c90e59bc06e657d1461967d4 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Wed, 10 Sep 2025 17:45:35 +0200 Subject: [PATCH 2/8] proper version --- .github/workflows/autotests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autotests.yml b/.github/workflows/autotests.yml index 41c8d7a6..bdf1a21b 100644 --- a/.github/workflows/autotests.yml +++ b/.github/workflows/autotests.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: '3.7' + python-version: '3.7.17' - name: Install python package dependencies run: | From d0ed9a0c56a97607bd5bc8afd1df122d52aa2635 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Wed, 10 Sep 2025 17:48:02 +0200 Subject: [PATCH 3/8] ubuntu --- .github/workflows/autotests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/autotests.yml b/.github/workflows/autotests.yml index bdf1a21b..9bbb7263 100644 --- a/.github/workflows/autotests.yml +++ b/.github/workflows/autotests.yml @@ -13,13 +13,13 @@ concurrency: jobs: tests: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '3.7.17' + python-version: "3.7.17" - name: Install python package dependencies run: | From a018e9726dfe4dfbe728c4859c9255c8ee3124cc Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Wed, 10 Sep 2025 17:48:42 +0200 Subject: [PATCH 4/8] black --- mergin/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mergin/utils.py b/mergin/utils.py index 6ffd732d..e02e3f4b 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -296,6 +296,7 @@ def bytes_to_human_size(bytes: int): else: return f"{round( bytes / 1024.0 / 1024.0 / 1024.0 / 1024.0, precision )} TB" + def cleanup_tmp_dir(mp, tmp_dir: tempfile.TemporaryDirectory): """ Remove temporary from tempfile.TemporaryDirectory instance @@ -309,4 +310,4 @@ def cleanup_tmp_dir(mp, tmp_dir: tempfile.TemporaryDirectory): pass # Ignore the error and continue except Exception as e: mp.log.error(f"Error during tmp dir cleanup: {tmp_dir.name}: {e}") - pass # Ignore the error and continue \ No newline at end of file + pass # Ignore the error and continue From 86c40c6265754fb64dd6c5779a1b831b71ee5bc9 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Wed, 10 Sep 2025 18:10:24 +0200 Subject: [PATCH 5/8] Fix tests for older version --- mergin/test/test_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index be13ffb1..58278b52 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -2266,8 +2266,11 @@ def test_clean_diff_files(mc): shutil.copy(mp.fpath("inserted_1_A.gpkg"), mp.fpath(f_updated)) mc.push_project(project_dir) - diff_files = glob.glob("*-diff-*", root_dir=os.path.split(mp.fpath_meta("inserted_1_A.gpkg"))[0]) + # Get the directory path + directory = os.path.split(mp.fpath_meta("inserted_1_A.gpkg"))[0] + diff_files = [f for f in os.listdir(directory) if "-diff-" in f] + # Assert that no matching files are found assert diff_files == [] From fd7c771f29bbc57678f155a5ccd24c769f671a2d Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Thu, 11 Sep 2025 15:45:07 +0200 Subject: [PATCH 6/8] update tests for 3.10 and address comments @wonder-sk --- .github/workflows/autotests.yml | 4 ++-- mergin/client_pull.py | 3 +++ mergin/client_push.py | 6 +++++- mergin/utils.py | 2 -- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/autotests.yml b/.github/workflows/autotests.yml index 9bbb7263..99cddbaa 100644 --- a/.github/workflows/autotests.yml +++ b/.github/workflows/autotests.yml @@ -13,13 +13,13 @@ concurrency: jobs: tests: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: "3.7.17" + python-version: "3.10" - name: Install python package dependencies run: | diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 1e0f8e8a..264f4708 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -263,6 +263,7 @@ def download_project_cancel(job): job.is_cancelled = True job.executor.shutdown(wait=True) job.mp.log.info("--- download cancelled") + cleanup_tmp_dir(job.mp, job.tmp_dir) class UpdateTask: @@ -550,6 +551,7 @@ def pull_project_cancel(job): job.is_cancelled = True job.executor.shutdown(wait=True) job.mp.log.info("--- pull cancelled") + cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content class FileToMerge: @@ -637,6 +639,7 @@ def pull_project_finalize(job: PullJob): except Exception as e: job.mp.log.error("Failed to apply pull changes: " + str(e)) job.mp.log.info("--- pull aborted") + cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content raise ClientError("Failed to apply pull changes: " + str(e)) job.mp.update_metadata(job.project_info) diff --git a/mergin/client_push.py b/mergin/client_push.py index 315046c0..95eb8394 100644 --- a/mergin/client_push.py +++ b/mergin/client_push.py @@ -261,6 +261,7 @@ def push_project_finalize(job): job.transferred_size, job.total_size ) job.mp.log.error("--- push finish failed! " + error_msg) + cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content raise ClientError("Upload error: " + error_msg) if with_upload_of_files: @@ -287,6 +288,7 @@ def push_project_finalize(job): job.mp.log.info("cancel response: " + resp_cancel.msg) except ClientError as err2: job.mp.log.info("cancel response: " + str(err2)) + cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content raise err job.mp.update_metadata(job.server_resp) @@ -295,10 +297,10 @@ def push_project_finalize(job): except Exception as e: job.mp.log.error("Failed to apply push changes: " + str(e)) job.mp.log.info("--- push aborted") + cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content raise ClientError("Failed to apply push changes: " + str(e)) cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content - remove_diff_files(job) job.mp.log.info("--- push finished - new project version " + job.server_resp["version"]) @@ -321,6 +323,8 @@ def push_project_cancel(job): except ClientError as err: job.mp.log.error("--- push cancelling failed! " + str(err)) raise err + finally: + cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content job.mp.log.info("--- push cancel response: " + str(job.server_resp)) diff --git a/mergin/utils.py b/mergin/utils.py index e02e3f4b..c9b480a0 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -307,7 +307,5 @@ def cleanup_tmp_dir(mp, tmp_dir: tempfile.TemporaryDirectory): tmp_dir.cleanup() except PermissionError: mp.log.warning(f"Permission error during tmp dir cleanup: {tmp_dir.name}") - pass # Ignore the error and continue except Exception as e: mp.log.error(f"Error during tmp dir cleanup: {tmp_dir.name}: {e}") - pass # Ignore the error and continue From 694d886812ed73ad6feefd9001194908030e92ae Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Fri, 12 Sep 2025 11:53:36 +0200 Subject: [PATCH 7/8] get rid of finnally --- mergin/client_push.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mergin/client_push.py b/mergin/client_push.py index 95eb8394..54afdd40 100644 --- a/mergin/client_push.py +++ b/mergin/client_push.py @@ -322,9 +322,9 @@ def push_project_cancel(job): job.server_resp = resp_cancel.msg except ClientError as err: job.mp.log.error("--- push cancelling failed! " + str(err)) + cleanup_tmp_dir(job.mp, job.tmp_dir) raise err - finally: - cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content + cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content job.mp.log.info("--- push cancel response: " + str(job.server_resp)) From ee66f237ca423d7dcd18280345796f4039a6ca6d Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Fri, 12 Sep 2025 11:58:08 +0200 Subject: [PATCH 8/8] Downgrade version of python for tests --- .github/workflows/autotests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autotests.yml b/.github/workflows/autotests.yml index 99cddbaa..c292728a 100644 --- a/.github/workflows/autotests.yml +++ b/.github/workflows/autotests.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.8" - name: Install python package dependencies run: |