From fd5f0b4da4826e2cd5bec88edc572ed5bca4a3bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Re=C3=A9?= <95078727+reekitconcept@users.noreply.github.com> Date: Tue, 10 Oct 2023 16:41:47 +0200 Subject: [PATCH] Support Plone 5.2 / also refactor reindex helpers (#17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Support Plone 5.2 - remove Plone dependencies so the package can work with Plone 5.2 - make constraints work in an alternate way between versions - unpin plone.restapi - test Plone 5.2 on the CI - currently install only, as test framework conflicts with 5.2 * Refactor reindex helpers in an importable way * Github action updates for 5.2 --------- Co-authored-by: Balázs Reé --- .github/workflows/test-plone-5.2.yml | 48 +++++++++++++++ Makefile | 16 ++++- README.md | 64 +++++++++++++++++++- constraints-5.2.txt | 1 + constraints-6.0.txt | 2 + constraints.txt | 1 + mx.ini | 4 +- requirements-5.2.txt | 4 ++ requirements-6.0.txt | 2 + scripts/solr_activate_and_reindex.py | 82 ++++---------------------- setup.py | 5 +- src/kitconcept/solr/reindex_helpers.py | 68 +++++++++++++++++++++ 12 files changed, 219 insertions(+), 78 deletions(-) create mode 100644 .github/workflows/test-plone-5.2.yml create mode 100644 constraints-5.2.txt create mode 100644 constraints-6.0.txt create mode 100644 requirements-5.2.txt create mode 100644 requirements-6.0.txt create mode 100644 src/kitconcept/solr/reindex_helpers.py diff --git a/.github/workflows/test-plone-5.2.yml b/.github/workflows/test-plone-5.2.yml new file mode 100644 index 0000000..027294b --- /dev/null +++ b/.github/workflows/test-plone-5.2.yml @@ -0,0 +1,48 @@ +name: Plone 5.2 install - tests are currently not run +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + - main + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.8"] + steps: + # git checkout + - uses: actions/checkout@v4 + + # pin plone version (happens from make, but to be on the safe side) + - name: Pin plone 5.2 + run: | + cp constraints-5.2.txt constraints.txt + cp requirements-5.2.txt requirements.txt + + # python setup + - name: Set up Python ${{ matrix.python-version }} with Plone 5.2.5 + uses: plone/setup-plone@v1.0.0 + with: + python-version: ${{ matrix.python-version }} + plone-version: "5.2.5" + + # python cache + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + # test + # XXX cannot run pytests, version conflict. + # - name: Run tests + # run: bin/test diff --git a/Makefile b/Makefile index 853abbc..305e267 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ ifeq ($(PYTHON_VERSION_OK),0) $(error "Need python $(PYTHON_VERSION) >= $(PYTHON_VERSION_MIN)") endif -all: build +all: install # Add the following 'help' target to your Makefile # And add help text after each target name starting with '\#\#' @@ -55,6 +55,8 @@ clean-instance: ## remove existing instance .PHONY: clean-venv clean-venv: ## remove virtual environment rm -fr bin include lib lib64 env pyvenv.cfg .tox .pytest_cache requirements-mxdev.txt + cp constraints-6.0.txt constraints.txt + cp requirements-6.0.txt requirements.txt .PHONY: clean-build clean-build: ## remove build artifacts @@ -93,6 +95,18 @@ config: bin/pip ## Create instance configuration .PHONY: install-plone-6.0 install-plone-6.0: bin/mxdev config ## pip install Plone packages @echo "$(GREEN)==> Setup Build$(RESET)" + cp constraints-6.0.txt constraints.txt + cp requirements-6.0.txt requirements.txt + bin/tox -e init + bin/mxdev -c mx.ini + bin/pip install -r requirements-mxdev.txt + +.PHONY: install-plone-5.2 +install-plone-5.2: bin/mxdev config ## pip install Plone packages + @echo "$(GREEN)==> Setup Build$(RESET)" + cp constraints-5.2.txt constraints.txt + cp requirements-5.2.txt requirements.txt + bin/tox -e init bin/mxdev -c mx.ini bin/pip install -r requirements-mxdev.txt diff --git a/README.md b/README.md index e00e5ef..ff273aa 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,12 @@ Also, add `kitconcept.solr` to your package's `configure.zcml` (or `dependencies ``` +#### Remark with Plone 6.0 + +With Plone 6.0 you must add an additional dependency `"plone.restapi>=8.40.0"`. + +The package also keeps support with Plone 5.2 where `"plone.restapi>=8.21.2"` is a working version. It will not support image scales, but the package will work gracefully without displaying image previews in the search result list. + ### Generic Setup To automatically enable this package when your add-on is installed, add the following line inside the package's `profiles/default/metadata.xml` `dependencies` element: @@ -195,6 +201,63 @@ Example value: If needed, the default [`kitconcept.solr.interfaces.IKitconceptSolrSettings`](./src/kitconcept/solr/profiles/default/registry/kitconcept.solr.interfaces.IKitconceptSolrSettings.xml) can be customized in the registry via GenericSetup. +### Using reindex helpers + +Helpers for activate and reindex solr are importable from the package. + +Example for a reindex script that can be called from Makefile: + +```py +from kitconcept.solr.reindex_helpers import activate_and_reindex +from Testing.makerequest import makerequest +from zope.site.hooks import setSite + +import sys +import transaction + + +if __name__ == "__main__": + app = makerequest(app) # noQA + + # Set site to Plone + site_id = "Plone" + portal = app.unrestrictedTraverse(site_id) + setSite(portal) + + activate_and_reindex(portal, clear="--clear" in sys.argv) + + transaction.commit() + app._p_jar.sync() +``` + +Example for an upgrade step that adds the `kitconcept.solr` package, and one that does the solr activation for the first time: + +```py +from kitconcept.solr.reindex_helpers import activate_and_reindex +from plone import api + +import logging + + +logger = logging.getLogger("your_package_name_here") + + +# We suggest to add two distinct upgrade step for the package installation +# and the solr activation, in case of a failure this allows to +# identify the problem easier. + + +def install_kitconcept_solr(context): + st = api.portal.get_tool("portal_setup") + st.runAllImportStepsFromProfile("kitconcept.solr:default") + logger.info("Installed kitconcept.solr") + + +def activate_and_reindex_solr(context): + activate_and_reindex(context) + logger.info("Activated and reindexed solr") +``` + ### Update translations ```bash @@ -242,7 +305,6 @@ The development of this add-on has been kindly sponsored by [German Aerospace Ce German Aerospace Center (DLR) Forschungszentrum Jülich - Developed by [kitconcept](https://www.kitconcept.com/) ## License diff --git a/constraints-5.2.txt b/constraints-5.2.txt new file mode 100644 index 0000000..22e7324 --- /dev/null +++ b/constraints-5.2.txt @@ -0,0 +1 @@ +-c https://dist.plone.org/release/5.2.5/constraints.txt diff --git a/constraints-6.0.txt b/constraints-6.0.txt new file mode 100644 index 0000000..404cebd --- /dev/null +++ b/constraints-6.0.txt @@ -0,0 +1,2 @@ +-c https://dist.plone.org/release/6.0.6/constraints.txt +plone.restapi>=8.40.0 diff --git a/constraints.txt b/constraints.txt index 9120178..404cebd 100644 --- a/constraints.txt +++ b/constraints.txt @@ -1 +1,2 @@ -c https://dist.plone.org/release/6.0.6/constraints.txt +plone.restapi>=8.40.0 diff --git a/mx.ini b/mx.ini index c9bae05..94f8369 100644 --- a/mx.ini +++ b/mx.ini @@ -5,8 +5,8 @@ [settings] ; example how to override a package version -version-overrides = - plone.restapi>=8.40.0 +; version-overrides = +; plone.restapi>=8.40.0 ; example section to use packages from git ; [example.contenttype] diff --git a/requirements-5.2.txt b/requirements-5.2.txt new file mode 100644 index 0000000..8c3fcef --- /dev/null +++ b/requirements-5.2.txt @@ -0,0 +1,4 @@ +-c constraints.txt + +# Cannot install new testing framework with 5.2 +# -e ".[test]" diff --git a/requirements-6.0.txt b/requirements-6.0.txt new file mode 100644 index 0000000..9dd6dd4 --- /dev/null +++ b/requirements-6.0.txt @@ -0,0 +1,2 @@ +-c constraints.txt +-e ".[test]" diff --git a/scripts/solr_activate_and_reindex.py b/scripts/solr_activate_and_reindex.py index 4ecf605..96ff3c2 100644 --- a/scripts/solr_activate_and_reindex.py +++ b/scripts/solr_activate_and_reindex.py @@ -1,82 +1,20 @@ -from collective.solr.interfaces import ISolrConnectionManager -from plone.registry.interfaces import IRegistry +from kitconcept.solr.reindex_helpers import activate_and_reindex from Testing.makerequest import makerequest -from zope.component import getUtility -from zope.component import queryUtility from zope.site.hooks import setSite -import logging import sys import transaction -logger = logging.getLogger("kitconcept.solr") -logger.setLevel(logging.DEBUG) +if __name__ == "__main__": + app = makerequest(app) # noQA -indexer_logger = logging.getLogger("collective.solr.indexer") + # Set site to Plone + site_id = "Plone" + portal = app.unrestrictedTraverse(site_id) + setSite(portal) + activate_and_reindex(portal, clear="--clear" in sys.argv) -def solr_is_running(portal): - manager = queryUtility(ISolrConnectionManager, context=portal) - schema = manager.getSchema() - return schema is not None - - -def solr_must_be_running(portal): - if not solr_is_running(portal): - logger.fatal("*** Solr must be running! (make solr-start) ***") - sys.exit(1) - - -def activate(active=True): - """(de)activate the solr integration""" - registry = getUtility(IRegistry) - registry["collective.solr.active"] = active - - -def silence_logger(): - orig_logger_exception = indexer_logger.exception - - def new_logger_exception(msg): - if msg != "Error occured while getting data for indexing!": - orig_logger_exception(msg) - - indexer_logger.exception = new_logger_exception - - def reactivate_logger(): - indexer_logger.exception = orig_logger_exception - - return reactivate_logger - - -def reindex(portal): - """reindex the existing content in solr""" - maintenance = portal.unrestrictedTraverse("@@solr-maintenance") - if "--clear" in sys.argv: - logger.info("Clearing solr...") - maintenance.clear() - # Avoid throwing a lot of errors which are actually not errors, - # but the indexer keeps throwing them when it tries to traverse everything. - reactivate_logger = silence_logger() - logger.info("Reindexing solr...") - maintenance.reindex() - reactivate_logger() - - -app = makerequest(app) # noQA - -# Set site to Plone -site_id = "Plone" -portal = app.unrestrictedTraverse(site_id) -setSite(portal) - -# Activate before confirming solr is running, -# because the confirmation only works if solr is enabled in the registry. -# If solr isn't running, we'll exit -# before committing the transaction with the activation. -activate() -solr_must_be_running(portal) -reindex(portal) - -transaction.commit() -app._p_jar.sync() + transaction.commit() + app._p_jar.sync() diff --git a/setup.py b/setup.py index 409319a..0276b5a 100644 --- a/setup.py +++ b/setup.py @@ -47,8 +47,9 @@ "Tracker": "https://github.com/kitconcept/kitconcept.portal/issues", }, install_requires=[ - "Plone>=6.0.0", - "plone.restapi>=8.40.0", + "Plone>=5.2.0", + "plone.restapi", + # "plone.restapi>=8.40.0", # for Plone 6.0 "plone.api", "setuptools", "collective.solr>=9.0.1", diff --git a/src/kitconcept/solr/reindex_helpers.py b/src/kitconcept/solr/reindex_helpers.py new file mode 100644 index 0000000..c311274 --- /dev/null +++ b/src/kitconcept/solr/reindex_helpers.py @@ -0,0 +1,68 @@ +from collective.solr.interfaces import ISolrConnectionManager +from plone import api +from zope.component import queryUtility + +import logging + + +logger = logging.getLogger("kitconcept.solr") +logger.setLevel(logging.DEBUG) + +indexer_logger = logging.getLogger("collective.solr.indexer") + + +def solr_is_running(portal): + manager = queryUtility(ISolrConnectionManager, context=portal) + schema = manager.getSchema() + return schema is not None + + +def solr_must_be_running(portal): + if not solr_is_running(portal): + logger.fatal("*** Solr must be running! (make solr-start) ***") + return False + return True + + +def activate(active=True): + """(de)activate the solr integration""" + api.portal.set_registry_record("collective.solr.active", active) + + +def silence_logger(): + orig_logger_exception = indexer_logger.exception + + def new_logger_exception(msg): + if msg != "Error occured while getting data for indexing!": + orig_logger_exception(msg) + + indexer_logger.exception = new_logger_exception + + def reactivate_logger(): + indexer_logger.exception = orig_logger_exception + + return reactivate_logger + + +def reindex(portal, clear=False): + """reindex the existing content in solr""" + maintenance = portal.unrestrictedTraverse("@@solr-maintenance") + if clear: + logger.info("Clearing solr...") + maintenance.clear() + # Avoid throwing a lot of errors which are actually not errors, + # but the indexer keeps throwing them when it tries to traverse everything. + reactivate_logger = silence_logger() + logger.info("Reindexing solr...") + maintenance.reindex() + reactivate_logger() + + +def activate_and_reindex(portal, clear=False): + # Activate before confirming solr is running, + # because the confirmation only works if solr is enabled in the registry. + # If solr isn't running, we'll exit + # before committing the transaction with the activation. + activate() + if solr_must_be_running(portal): + reindex(portal, clear=clear)