diff --git a/docs/configuration.rst b/docs/configuration.rst index 8702d93543..27efbf549f 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -3975,6 +3975,19 @@ Description Only compare file sizes. Do not read and compare their content. +exec.archive +------------ +Type + |Path|_ +Description + File to store IDs of executed commands in, + similar to `extractor.*.archive`_. + + ``archive-format`` and ``archive-prefix`` options, + akin to `extractor.*.archive-format`_ and `extractor.*.archive-prefix`_, + are supported as well. + + exec.async ---------- Type diff --git a/gallery_dl/postprocessor/common.py b/gallery_dl/postprocessor/common.py index ef211e6e55..ee18398849 100644 --- a/gallery_dl/postprocessor/common.py +++ b/gallery_dl/postprocessor/common.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2018-2020 Mike Fährmann +# Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -8,13 +8,40 @@ """Common classes and constants used by postprocessor modules.""" +from .. import util, formatter + class PostProcessor(): """Base class for postprocessors""" def __init__(self, job): - name = self.__class__.__name__[:-2].lower() - self.log = job.get_logger("postprocessor." + name) + self.name = self.__class__.__name__[:-2].lower() + self.log = job.get_logger("postprocessor." + self.name) def __repr__(self): return self.__class__.__name__ + + def _init_archive(self, job, options, prefix=None): + archive = options.get("archive") + if archive: + extr = job.extractor + archive = util.expand_path(archive) + if not prefix: + prefix = "_" + self.name.upper() + "_" + archive_format = ( + options.get("archive-prefix", extr.category) + + options.get("archive-format", prefix + extr.archive_fmt)) + try: + if "{" in archive: + archive = formatter.parse(archive).format_map( + job.pathfmt.kwdict) + self.archive = util.DownloadArchive( + archive, archive_format, "_archive_" + self.name) + except Exception as exc: + self.log.warning( + "Failed to open %s archive at '%s' ('%s: %s')", + self.name, archive, exc.__class__.__name__, exc) + else: + self.log.debug("Using %s archive '%s'", self.name, archive) + else: + self.archive = None diff --git a/gallery_dl/postprocessor/exec.py b/gallery_dl/postprocessor/exec.py index cc217c3779..e81c6cfe27 100644 --- a/gallery_dl/postprocessor/exec.py +++ b/gallery_dl/postprocessor/exec.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2018-2021 Mike Fährmann +# Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -43,11 +43,18 @@ def __init__(self, job, options): events = events.split(",") job.register_hooks({event: execute for event in events}, options) + self._init_archive(job, options) + def exec_list(self, pathfmt, status=None): if status: return + archive = self.archive kwdict = pathfmt.kwdict + + if archive and archive.check(kwdict): + return + kwdict["_directory"] = pathfmt.realdirectory kwdict["_filename"] = pathfmt.filename kwdict["_path"] = pathfmt.realpath @@ -55,10 +62,17 @@ def exec_list(self, pathfmt, status=None): args = [arg.format_map(kwdict) for arg in self.args] self._exec(args, False) + if archive: + archive.add(kwdict) + def exec_string(self, pathfmt, status=None): if status: return + archive = self.archive + if archive and archive.check(pathfmt.kwdict): + return + if status is None and pathfmt.realpath: args = self.args.replace("{}", quote(pathfmt.realpath)) else: @@ -66,6 +80,9 @@ def exec_string(self, pathfmt, status=None): self._exec(args, True) + if archive: + archive.add(pathfmt.kwdict) + def _exec(self, args, shell): self.log.debug("Running '%s'", args) retcode = subprocess.Popen(args, shell=shell).wait() diff --git a/gallery_dl/postprocessor/metadata.py b/gallery_dl/postprocessor/metadata.py index 2ee1cf8715..09f54fbe07 100644 --- a/gallery_dl/postprocessor/metadata.py +++ b/gallery_dl/postprocessor/metadata.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2022 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -83,28 +83,7 @@ def __init__(self, job, options): events = events.split(",") job.register_hooks({event: self.run for event in events}, options) - archive = options.get("archive") - if archive: - extr = job.extractor - archive = util.expand_path(archive) - archive_format = ( - options.get("archive-prefix", extr.category) + - options.get("archive-format", "_MD_" + extr.archive_fmt)) - try: - if "{" in archive: - archive = formatter.parse(archive).format_map( - job.pathfmt.kwdict) - self.archive = util.DownloadArchive( - archive, archive_format, "_archive_metadata") - except Exception as exc: - self.log.warning( - "Failed to open download archive at '%s' ('%s: %s')", - archive, exc.__class__.__name__, exc) - else: - self.log.debug("Using download archive '%s'", archive) - else: - self.archive = None - + self._init_archive(job, options, "_MD_") self.mtime = options.get("mtime") self.omode = options.get("open", omode) self.encoding = options.get("encoding", "utf-8")