doc: Add autogen plugin hook/event function interfaces

[ci skip]

doc/changelog.rst

@@ -15,8 +15,19 @@ Unreleased
 * Changed: IP addresses are normalized to a standard notation to avoid fetching duplicates such as IPv4 addresses written in hexadecimal or long-hand IPv6 addresses.
 * Changed: Scripting is now done using plugin interface via ``--plugin-script``.
 
-This release contains backwards incompatible changes to the scripting
-interface.
+
+Backwards incompatibility
++++++++++++++++++++++++++
+
+This release contains backwards incompatible changes to the database
+schema and scripting interface.
+
+If you use ``--database``, the database created by older versions in
+Wpull cannot be used in this version.
+
+Scripting hook code will need to be rewritten to use the new API. See
+the new documentation for scripting for the new style of interfacing
+with Wpull.
 
 
 1.2.3 (2016-02-03)

doc/gen_plugin_interface.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import inspect
+import os
+
+import wpull.application.builder
+import wpull.application.plugin
+
+
+def main():
+    assert wpull.application.builder
+
+    path = os.path.join(
+        os.path.dirname(os.path.abspath(__file__)),
+        'scripting_interfaces_include.rst'
+    )
+    print(path)
+
+    with open(path, 'w') as out_file:
+        out_file.write('.. This document was automatically generated.\n')
+        out_file.write('   DO NOT EDIT!\n\n')
+
+        for hook_name, function in sorted(wpull.application.plugin.global_interface_registry.items(), key=lambda item: item[0].value):
+            hook_name_str = inspect.getmodule(hook_name).__name__ + '.' + str(hook_name)
+            function_name_str = inspect.getmodule(function).__name__ + '.' + function.__qualname__
+
+            out_file.write(':py:attr:`{} <{}>`\n'.format(hook_name, hook_name_str))
+            out_file.write('    Interface: :py:meth:`{} <{}>`\n\n'.format(function.__qualname__, function_name_str))
+
+
+if __name__ == '__main__':
+    main()

doc/scripting.rst

@@ -42,11 +42,7 @@ Interfaces
 The global hooks and events constants are located at
 :py:class:`wpull.application.plugin.PluginFunctions`.
 
-TODO: document the hooks available
-
-The module providing the interface for user plugins is located
-at :py:mod:`wpull.application.plugin` and the interface for code
-is located at :py:mod:`wpull.application.hook`.
+.. include:: scripting_interfaces_include.rst
 
 Example
 +++++++
@@ -62,6 +58,3 @@ Here is a example Python script. It refuses to download anything with the word "
         def my_accept_func(self, item_session: ItemSession, verdict: bool, reasons: dict) -> bool:
             return 'dog' not in item_session.request.url
 
-
-For an example, see `ArchiveBot's scripting hook file <https://github.com/ArchiveTeam/ArchiveBot/blob/065b0cc2549224f72a16cd3611fffb2050962c74/pipeline/wpull_hooks.py>`_.
-

doc/scripting_interfaces_include.rst

@@ -0,0 +1,39 @@
+.. This document was automatically generated.
+   DO NOT EDIT!
+
+:py:attr:`PluginFunctions.accept_url <wpull.application.plugin.PluginFunctions.accept_url>`
+    Interface: :py:meth:`FetchRule.plugin_accept_url <wpull.processor.rule.FetchRule.plugin_accept_url>`
+
+:py:attr:`PluginFunctions.dequeued_url <wpull.application.plugin.PluginFunctions.dequeued_url>`
+    Interface: :py:meth:`URLTableHookWrapper.dequeued_url <wpull.database.wrap.URLTableHookWrapper.dequeued_url>`
+
+:py:attr:`PluginFunctions.exit_status <wpull.application.plugin.PluginFunctions.exit_status>`
+    Interface: :py:meth:`AppStopTask.plugin_exit_status <wpull.application.tasks.shutdown.AppStopTask.plugin_exit_status>`
+
+:py:attr:`PluginFunctions.finishing_statistics <wpull.application.plugin.PluginFunctions.finishing_statistics>`
+    Interface: :py:meth:`StatsStopTask.plugin_finishing_statistics <wpull.application.tasks.stats.StatsStopTask.plugin_finishing_statistics>`
+
+:py:attr:`PluginFunctions.get_urls <wpull.application.plugin.PluginFunctions.get_urls>`
+    Interface: :py:meth:`ProcessingRule.plugin_get_urls <wpull.processor.rule.ProcessingRule.plugin_get_urls>`
+
+:py:attr:`PluginFunctions.handle_error <wpull.application.plugin.PluginFunctions.handle_error>`
+    Interface: :py:meth:`ResultRule.plugin_handle_error <wpull.processor.rule.ResultRule.plugin_handle_error>`
+
+:py:attr:`PluginFunctions.handle_pre_response <wpull.application.plugin.PluginFunctions.handle_pre_response>`
+    Interface: :py:meth:`ResultRule.plugin_handle_pre_response <wpull.processor.rule.ResultRule.plugin_handle_pre_response>`
+
+:py:attr:`PluginFunctions.handle_response <wpull.application.plugin.PluginFunctions.handle_response>`
+    Interface: :py:meth:`ResultRule.plugin_handle_response <wpull.processor.rule.ResultRule.plugin_handle_response>`
+
+:py:attr:`PluginFunctions.queued_url <wpull.application.plugin.PluginFunctions.queued_url>`
+    Interface: :py:meth:`URLTableHookWrapper.queued_url <wpull.database.wrap.URLTableHookWrapper.queued_url>`
+
+:py:attr:`PluginFunctions.resolve_dns <wpull.application.plugin.PluginFunctions.resolve_dns>`
+    Interface: :py:meth:`Resolver.resolve_dns <wpull.network.dns.Resolver.resolve_dns>`
+
+:py:attr:`PluginFunctions.resolve_dns_result <wpull.application.plugin.PluginFunctions.resolve_dns_result>`
+    Interface: :py:meth:`Resolver.resolve_dns_result <wpull.network.dns.Resolver.resolve_dns_result>`
+
+:py:attr:`PluginFunctions.wait_time <wpull.application.plugin.PluginFunctions.wait_time>`
+    Interface: :py:meth:`ResultRule.plugin_wait_time <wpull.processor.rule.ResultRule.plugin_wait_time>`
+