diff --git a/.github/workflows/postrun.yml b/.github/workflows/postrun.yml new file mode 100644 index 0000000..0cf9091 --- /dev/null +++ b/.github/workflows/postrun.yml @@ -0,0 +1,35 @@ +name: Unit test postrun update +on: + push: + + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + # Not sure it's worth testing on multiple Pythons + python-version: [3.8] + steps: + - uses: actions/checkout@master + with: + persist-credentials: false + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pelican[Markdown]==4.5.4 + pip install -r requirements.txt + pip install 'MarkupSafe<2.1.0' # needed for Pelican 4.5.4 + - name: Run test + run: | + cd tests/postrun + bash runtest.sh diff --git a/bin/buildsite.py b/bin/buildsite.py index 5bdda95..daf911a 100755 --- a/bin/buildsite.py +++ b/bin/buildsite.py @@ -381,11 +381,12 @@ def generate_settings(source_yaml, settings_path, builtin_p_paths=None, sourcepa tdata['uses_run'] = 'yes' # ezt.boolean tdata['run'] = sdata['run'] tdata['use'].append('asfrun') # add the plugin - # Run the included scripts with the asfpostrun plugin during finalize + # Run the included scripts with the asfrun plugin during finalize if 'postrun' in sdata: tdata['uses_postrun'] = 'yes' # ezt.boolean tdata['postrun'] = sdata['postrun'] - tdata['use'].append('asfpostrun') # add the plugin + if not 'run' in sdata: + tdata['use'].append('asfrun') # add the plugin (if not already added) # Ignore files avoids copying these files to output. if 'ignore' in sdata: tdata['uses_ignore'] = 'yes' # ezt.boolean diff --git a/bin/pelican.auto.ezt b/bin/pelican.auto.ezt index 68e5dfa..7b8bbd8 100644 --- a/bin/pelican.auto.ezt +++ b/bin/pelican.auto.ezt @@ -106,11 +106,11 @@ ASF_DATA = { } [end] [if-any uses_run] -# Configure the asfrun plugin +# Configure the asfrun plugin (initialization) ASF_RUN = [ [for run]'[run]', [end] ] [end] [if-any uses_postrun] -# Configure the asfpostrun plugin +# Configure the asfrun plugin (finalization) ASF_POSTRUN = [ [for postrun]'[postrun]', [end] ] [end] [if-any uses_ignore] diff --git a/plugins/PLUGIN_ARCH.md b/plugins/PLUGIN_ARCH.md index accfb8d..aef7731 100644 --- a/plugins/PLUGIN_ARCH.md +++ b/plugins/PLUGIN_ARCH.md @@ -71,12 +71,12 @@ At a high level consider the following sequence of events: ] ``` - - The [asfpostrun plugin](./asfpostrun.py) runs shell scripts during finalization + - The [asfrun plugin](./asfrun.py) also runs shell scripts during finalization ```yaml setup: postrun: - - /bin/bash shell.sh + - /bin/bash postshell.sh ``` ```python diff --git a/plugins/asfpostrun.py b/plugins/asfpostrun.py deleted file mode 100644 index 25fff29..0000000 --- a/plugins/asfpostrun.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/python -B -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# -# asfpostrun.py - Pelican plugin that runs shell scripts during finalization -# - -import os -import sys -import subprocess -import shlex -import traceback - -import pelican.plugins.signals -import pelican.settings - - -# open a subprocess -def os_run(args, env=None): - return subprocess.Popen(args, env=env, stdout=subprocess.PIPE, universal_newlines=True) - -# run shell -def postrun_script(pel_ob): - asf_postrun = pel_ob.settings.get('ASF_POSTRUN') - if asf_postrun: - print('-----\nasfpostrun') - # copy the pelican environment into the OS env - my_env = os.environ.copy() - for k,v in sorted(pel_ob.settings.items()): - if k != 'ASF_DATA': # rather large; not needed - my_env['PELICAN_'+k] = str(v) - for command in asf_postrun: - print(f'-----\n{command}') - args = shlex.split(command) - print(args) - with os_run(args, my_env) as s: - for line in s.stdout: - line = line.strip() - print(f'{line}') - - -def tb_finalized(pel_ob): - """ Print any exception, before Pelican chews it into nothingness.""" - try: - postrun_script(pel_ob) - except Exception: - print('-----', file=sys.stderr) - traceback.print_exc() - # exceptions here stop the build - raise - - -def register(): - pelican.plugins.signals.finalized.connect(tb_finalized) diff --git a/plugins/asfrun.py b/plugins/asfrun.py index 53724ca..1fab6d4 100644 --- a/plugins/asfrun.py +++ b/plugins/asfrun.py @@ -17,9 +17,10 @@ # under the License. # # -# asfrun.py - Pelican plugin that runs shell scripts during initialization +# asfrun.py - Pelican plugin that runs shell scripts during initialization or finalization # +import os import sys import subprocess import shlex @@ -30,20 +31,27 @@ # open a subprocess -def os_run(args): - return subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True) - +def os_run(args, env=None): + return subprocess.Popen(args, env=env, stdout=subprocess.PIPE, universal_newlines=True) # run shell -def run_script(pel_ob): - asf_run = pel_ob.settings.get('ASF_RUN') - if asf_run: - print('-----\nasfrun') - for command in asf_run: +def run_script(pel_ob, command_source, env=False): + commands = pel_ob.settings.get(command_source) + if commands: + print(f'-----\nasfrun {command_source}') + if env: + # copy the pelican environment into the OS env + my_env = os.environ.copy() + for k, v in sorted(pel_ob.settings.items()): + if k != 'ASF_DATA': # rather large; not needed + my_env['PELICAN_'+k] = str(v) + else: + my_env = None + for command in commands: print(f'-----\n{command}') args = shlex.split(command) print(args) - with os_run(args) as s: + with os_run(args, my_env) as s: for line in s.stdout: line = line.strip() print(f'{line}') @@ -52,7 +60,17 @@ def run_script(pel_ob): def tb_initialized(pel_ob): """ Print any exception, before Pelican chews it into nothingness.""" try: - run_script(pel_ob) + run_script(pel_ob, 'ASF_RUN') + except Exception: + print('-----', file=sys.stderr) + traceback.print_exc() + # exceptions here stop the build + raise + +def tb_finalized(pel_ob): + """ Print any exception, before Pelican chews it into nothingness.""" + try: + run_script(pel_ob, 'ASF_POSTRUN', env=True) except Exception: print('-----', file=sys.stderr) traceback.print_exc() @@ -62,3 +80,4 @@ def tb_initialized(pel_ob): def register(): pelican.plugins.signals.initialized.connect(tb_initialized) + pelican.plugins.signals.finalized.connect(tb_finalized) diff --git a/tests/postrun/content/empty.md b/tests/postrun/content/empty.md new file mode 100644 index 0000000..027ae71 --- /dev/null +++ b/tests/postrun/content/empty.md @@ -0,0 +1,3 @@ +title: Dummy + +Dummy file diff --git a/tests/postrun/pelican.auto.py b/tests/postrun/pelican.auto.py new file mode 100644 index 0000000..fc23fde --- /dev/null +++ b/tests/postrun/pelican.auto.py @@ -0,0 +1,52 @@ + +# Basic information about the site. +SITENAME = 'Apache Software Foundation' +SITEDESC = 'Test run command' +SITEDOMAIN = 'test.apache.org' +SITEURL = 'https://test.apache.org' +SITELOGO = 'https://test.apache.org/None' +SITEREPOSITORY = 'None' +CURRENTYEAR = 2023 +TRADEMARKS = 'Apache, the Apache feather logo are trademarks' +TIMEZONE = 'UTC' +# Theme includes templates and possibly static files +THEME = './theme/apache' +# Specify location of plugins, and which to use +PLUGIN_PATHS = [ '/private/var/sebb/git/infra/pelican/bin/../plugins', ] +PLUGINS = [ 'gfm', 'asfrun', ] +# All content is located at '.' (aka content/ ) +PAGE_PATHS = [ '.' ] +STATIC_PATHS = [ '.', ] +# Where to place/link generated pages + +PATH_METADATA = '(?P.*)\\..*' + +PAGE_SAVE_AS = '{path_no_ext}.html' +# Don't try to translate +PAGE_TRANSLATION_ID = None +# Disable unused Pelican features +# N.B. These features are currently unsupported, see https://github.com/apache/infrastructure-pelican/issues/49 +FEED_ALL_ATOM = None +INDEX_SAVE_AS = '' +TAGS_SAVE_AS = '' +CATEGORIES_SAVE_AS = '' +AUTHORS_SAVE_AS = '' +ARCHIVES_SAVE_AS = '' +# Disable articles by pointing to a (should-be-absent) subdir +ARTICLE_PATHS = [ 'blog' ] +# needed to create blogs page +ARTICLE_URL = 'blog/{slug}.html' +ARTICLE_SAVE_AS = 'blog/{slug}.html' +# Disable all processing of .html files +READERS = { 'html': None, } + + + + + +# Configure the asfrun plugin (finalization) +ASF_POSTRUN = [ '/bin/bash postrun.sh', ] + + + + diff --git a/tests/postrun/pelicanconf.yaml b/tests/postrun/pelicanconf.yaml new file mode 100644 index 0000000..bd3a9fb --- /dev/null +++ b/tests/postrun/pelicanconf.yaml @@ -0,0 +1,15 @@ +site: + name: Apache Software Foundation + domain: test.apache.org + logo: None + repository: None + description: Test run command + trademarks: Apache, the Apache feather logo are trademarks + +theme: theme/apache + +plugins: [] + +setup: + postrun: + - /bin/bash postrun.sh diff --git a/tests/postrun/pelicanconf1.yaml b/tests/postrun/pelicanconf1.yaml new file mode 100644 index 0000000..0a30f80 --- /dev/null +++ b/tests/postrun/pelicanconf1.yaml @@ -0,0 +1,15 @@ +site: + name: Apache Software Foundation + domain: test.apache.org + logo: None + repository: None + description: Test run command + trademarks: Apache, the Apache feather logo are trademarks + +theme: theme/apache + +plugins: [] + +setup: + run: + - /bin/bash run.sh diff --git a/tests/postrun/pelicanconf2.yaml b/tests/postrun/pelicanconf2.yaml new file mode 100644 index 0000000..bd3a9fb --- /dev/null +++ b/tests/postrun/pelicanconf2.yaml @@ -0,0 +1,15 @@ +site: + name: Apache Software Foundation + domain: test.apache.org + logo: None + repository: None + description: Test run command + trademarks: Apache, the Apache feather logo are trademarks + +theme: theme/apache + +plugins: [] + +setup: + postrun: + - /bin/bash postrun.sh diff --git a/tests/postrun/pelicanconf3.yaml b/tests/postrun/pelicanconf3.yaml new file mode 100644 index 0000000..34d1e2d --- /dev/null +++ b/tests/postrun/pelicanconf3.yaml @@ -0,0 +1,17 @@ +site: + name: Apache Software Foundation + domain: test.apache.org + logo: None + repository: None + description: Test run command + trademarks: Apache, the Apache feather logo are trademarks + +theme: theme/apache + +plugins: [] + +setup: + run: + - /bin/bash run.sh + postrun: + - /bin/bash postrun.sh diff --git a/tests/postrun/postrun.sh b/tests/postrun/postrun.sh new file mode 100755 index 0000000..a0ff421 --- /dev/null +++ b/tests/postrun/postrun.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# Script to test postrun command + +cd temp +cp date2.txt date2.tmp + +if [ -z "$PELICAN_OUTPUT_PATH" ] +then + echo "PELICAN_OUTPUT_PATH is not defined!" + exit 1 +fi diff --git a/tests/postrun/run.sh b/tests/postrun/run.sh new file mode 100755 index 0000000..76cef6a --- /dev/null +++ b/tests/postrun/run.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# Script to test run command + +cd temp +cp date1.txt date1.tmp diff --git a/tests/postrun/runtest.sh b/tests/postrun/runtest.sh new file mode 100755 index 0000000..7bb45ff --- /dev/null +++ b/tests/postrun/runtest.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# Script to set up and run a test + +setup() { + + rm -rf site-generated + rm -f pelicanconf.yaml + rm -f pelican.auto.py + rm -rf temp + mkdir temp + date >temp/date1.txt + date >temp/date2.txt + + cp "$1" pelicanconf.yaml + + ../../bin/buildsite.py dir + + grep 'ASF_RUN\|ASF_POSTRUN' pelican.auto.py + +} + +#============================= + +setup pelicanconf1.yaml # This tests run script + +echo "date1.tmp should exist as copy of date1.txt" +cmp temp/date1.txt temp/date1.tmp +echo "date2.tmp should not exist" +test -f temp/date2.tmp && { echo "date2.tmp is present!"; exit 1; } + +#============================= + +setup pelicanconf2.yaml # This tests postrun script + +echo "date2.tmp should exist as copy of date2.txt" +cmp temp/date2.txt temp/date2.tmp +echo "date1.tmp should not exist" +test -f temp/date1.tmp && { echo "date1.tmp is present!"; exit 1; } + +#============================= + +setup pelicanconf3.yaml # This tests run and postrun script + +echo "date1.tmp should exist as copy of date1.txt" +cmp temp/date1.txt temp/date1.tmp +echo "date2.tmp should exist as copy of date2.txt" +cmp temp/date2.txt temp/date2.tmp + +#============================= diff --git a/tests/postrun/site-generated/empty.html b/tests/postrun/site-generated/empty.html new file mode 100644 index 0000000..754ad02 --- /dev/null +++ b/tests/postrun/site-generated/empty.html @@ -0,0 +1,31 @@ + + + + Apache Software Foundation - Dummy + + + + + + + + + +

Dummy

+ + +

Dummy file

+ + + + + \ No newline at end of file diff --git a/tests/postrun/temp/date1.txt b/tests/postrun/temp/date1.txt new file mode 100644 index 0000000..b89f707 --- /dev/null +++ b/tests/postrun/temp/date1.txt @@ -0,0 +1 @@ +Wed 28 Jun 2023 12:50:15 BST diff --git a/tests/postrun/temp/date2.tmp b/tests/postrun/temp/date2.tmp new file mode 100644 index 0000000..b89f707 --- /dev/null +++ b/tests/postrun/temp/date2.tmp @@ -0,0 +1 @@ +Wed 28 Jun 2023 12:50:15 BST diff --git a/tests/postrun/temp/date2.txt b/tests/postrun/temp/date2.txt new file mode 100644 index 0000000..b89f707 --- /dev/null +++ b/tests/postrun/temp/date2.txt @@ -0,0 +1 @@ +Wed 28 Jun 2023 12:50:15 BST diff --git a/tests/postrun/theme/apache/base.html b/tests/postrun/theme/apache/base.html new file mode 100644 index 0000000..e69de29