[postprocessor:exec] implement archive options (#3584)

docs/configuration.rst

@@ -3975,6 +3975,19 @@ Description
     Only compare file sizes. Do not read and compare their content.
 
 
+exec.archive
+------------
+Type
+    |Path|_
+Description
+    File to store IDs of executed commands in,
+    similar to `extractor.*.archive`_.
+
+    ``archive-format`` and ``archive-prefix`` options,
+    akin to `extractor.*.archive-format`_ and `extractor.*.archive-prefix`_,
+    are supported as well.
+
+
 exec.async
 ----------
 Type

gallery_dl/postprocessor/common.py

@@ -1,20 +1,47 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2018-2020 Mike Fährmann
+# Copyright 2018-2023 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
 """Common classes and constants used by postprocessor modules."""
 
+from .. import util, formatter
+
 
 class PostProcessor():
     """Base class for postprocessors"""
 
     def __init__(self, job):
-        name = self.__class__.__name__[:-2].lower()
-        self.log = job.get_logger("postprocessor." + name)
+        self.name = self.__class__.__name__[:-2].lower()
+        self.log = job.get_logger("postprocessor." + self.name)
 
     def __repr__(self):
         return self.__class__.__name__
+
+    def _init_archive(self, job, options, prefix=None):
+        archive = options.get("archive")
+        if archive:
+            extr = job.extractor
+            archive = util.expand_path(archive)
+            if not prefix:
+                prefix = "_" + self.name.upper() + "_"
+            archive_format = (
+                options.get("archive-prefix", extr.category) +
+                options.get("archive-format", prefix + extr.archive_fmt))
+            try:
+                if "{" in archive:
+                    archive = formatter.parse(archive).format_map(
+                        job.pathfmt.kwdict)
+                self.archive = util.DownloadArchive(
+                    archive, archive_format, "_archive_" + self.name)
+            except Exception as exc:
+                self.log.warning(
+                    "Failed to open %s archive at '%s' ('%s: %s')",
+                    self.name, archive, exc.__class__.__name__, exc)
+            else:
+                self.log.debug("Using %s archive '%s'", self.name, archive)
+        else:
+            self.archive = None

gallery_dl/postprocessor/exec.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2018-2021 Mike Fährmann
+# Copyright 2018-2023 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -43,29 +43,46 @@ def __init__(self, job, options):
             events = events.split(",")
         job.register_hooks({event: execute for event in events}, options)
 
+        self._init_archive(job, options)
+
     def exec_list(self, pathfmt, status=None):
         if status:
             return
 
+        archive = self.archive
         kwdict = pathfmt.kwdict
+
+        if archive and archive.check(kwdict):
+            return
+
         kwdict["_directory"] = pathfmt.realdirectory
         kwdict["_filename"] = pathfmt.filename
         kwdict["_path"] = pathfmt.realpath
 
         args = [arg.format_map(kwdict) for arg in self.args]
         self._exec(args, False)
 
+        if archive:
+            archive.add(kwdict)
+
     def exec_string(self, pathfmt, status=None):
         if status:
             return
 
+        archive = self.archive
+        if archive and archive.check(pathfmt.kwdict):
+            return
+
         if status is None and pathfmt.realpath:
             args = self.args.replace("{}", quote(pathfmt.realpath))
         else:
             args = self.args.replace("{}", quote(pathfmt.realdirectory))
 
         self._exec(args, True)
 
+        if archive:
+            archive.add(pathfmt.kwdict)
+
     def _exec(self, args, shell):
         self.log.debug("Running '%s'", args)
         retcode = subprocess.Popen(args, shell=shell).wait()

gallery_dl/postprocessor/metadata.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2019-2022 Mike Fährmann
+# Copyright 2019-2023 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -83,28 +83,7 @@ def __init__(self, job, options):
             events = events.split(",")
         job.register_hooks({event: self.run for event in events}, options)
 
-        archive = options.get("archive")
-        if archive:
-            extr = job.extractor
-            archive = util.expand_path(archive)
-            archive_format = (
-                options.get("archive-prefix", extr.category) +
-                options.get("archive-format", "_MD_" + extr.archive_fmt))
-            try:
-                if "{" in archive:
-                    archive = formatter.parse(archive).format_map(
-                        job.pathfmt.kwdict)
-                self.archive = util.DownloadArchive(
-                    archive, archive_format, "_archive_metadata")
-            except Exception as exc:
-                self.log.warning(
-                    "Failed to open download archive at '%s' ('%s: %s')",
-                    archive, exc.__class__.__name__, exc)
-            else:
-                self.log.debug("Using download archive '%s'", archive)
-        else:
-            self.archive = None
-
+        self._init_archive(job, options, "_MD_")
         self.mtime = options.get("mtime")
         self.omode = options.get("open", omode)
         self.encoding = options.get("encoding", "utf-8")