trim large registers down to reasonable sizes

.circleci/config.yml

@@ -5,14 +5,15 @@ version: 2.1
 restore_cache: &restore_cache
   restore_cache:
     keys:
+      - v17-dependencies-{{ .Environment.CIRCLE_BRANCH }}-{{ checksum "requirements.txt" }}-{{ .Environment.CIRCLE_WORKFLOW_WORKSPACE_ID }}
       - v17-dependencies-{{ .Environment.CIRCLE_BRANCH }}-{{ checksum "requirements.txt" }}
       - v17-dependencies-{{ .Environment.CIRCLE_BRANCH }}-
 
 save_cache: &save_cache
   save_cache:
     paths:
       - ./venv
-    key: v17-dependencies-{{ .Environment.CIRCLE_BRANCH }}-{{ checksum "requirements.txt" }}
+    key: v17-dependencies-{{ .Environment.CIRCLE_BRANCH }}-{{ checksum "requirements.txt" }}-{{ .Environment.CIRCLE_WORKFLOW_WORKSPACE_ID }}
 
 install_dependencies: &install_dependencies
   run:

Makefile

@@ -7,22 +7,20 @@ clean-pyc :
 
 # dependency management
 
-install_pip :
+install_pip_tools :
 	echo "########## UPDATE PIP ##########"
-	pip install --upgrade pip
+	pip install --upgrade pip pip-tools setuptools wheel
 
-pip3 : install_pip
+pip3 : install_pip_tools
 	echo "##### INSTALL REQUIREMENTS #####"
 	pip3 install -r requirements.txt
 
-pip3-dev : install_pip
+pip3-dev : install_pip_tools
 	echo "##### INSTALL REQUIREMENTS #####"
 	pip3 install -r requirements.txt -r requirements-dev.txt
 
-update_pip3 : install_pip
+update_pip3 : install_pip_tools
 	echo "##### UPDATE REQUIREMENTS ######"
-	pip install pip-tools -U
-	pip install wheel -U
 	rm requirements.txt requirements-dev.txt || true
 	pip-compile --output-file requirements.txt requirements.in
 	pip-compile --output-file requirements-dev.txt requirements-dev.in

requirements-dev.txt

@@ -1,38 +1,38 @@
 #
-# This file is autogenerated by pip-compile with python 3.11
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
 #
 #    pip-compile --output-file=requirements-dev.txt requirements-dev.in
 #
-astroid==2.12.13
+astroid==2.13.3
     # via pylint
 bandit==1.7.4
     # via -r requirements-dev.in
 beautifulsoup4==4.11.1
     # via
     #   -c requirements.txt
     #   -r requirements-dev.in
-boto3==1.26.19
+boto3==1.26.57
     # via
     #   -c requirements.txt
     #   moto
-boto3-stubs[essential]==1.26.19
+boto3-stubs[essential]==1.26.57
     # via -r requirements-dev.in
-botocore==1.29.19
+botocore==1.29.57
     # via
     #   -c requirements.txt
     #   boto3
     #   moto
     #   s3transfer
-botocore-stubs==1.29.19
+botocore-stubs==1.29.57
     # via boto3-stubs
-certifi==2022.9.24
+certifi==2022.12.7
     # via
     #   -c requirements.txt
     #   requests
 cffi==1.15.1
     # via cryptography
-charset-normalizer==2.1.1
+charset-normalizer==3.0.1
     # via
     #   -c requirements.txt
     #   requests
@@ -42,11 +42,11 @@ codecov==2.1.12
     # via -r requirements-dev.in
 contextlib2==21.6.0
     # via schema
-coverage==6.5.0
+coverage==7.1.0
     # via
     #   -r requirements-dev.in
     #   codecov
-cryptography==38.0.4
+cryptography==39.0.0
     # via moto
 ddt==1.6.0
     # via -r requirements-dev.in
@@ -64,17 +64,17 @@ gitdb==4.0.10
     # via
     #   -c requirements.txt
     #   gitpython
-gitpython==3.1.29
+gitpython==3.1.30
     # via
     #   -c requirements.txt
     #   bandit
 idna==3.4
     # via
     #   -c requirements.txt
     #   requests
-internetarchive==3.0.2
+internetarchive==3.2.0
     # via -r requirements-dev.in
-isort==5.10.1
+isort==5.11.4
     # via pylint
 jinja2==3.1.2
     # via moto
@@ -87,30 +87,29 @@ jsonpatch==1.32
     # via internetarchive
 jsonpointer==2.3
     # via jsonpatch
-lazy-object-proxy==1.8.0
+lazy-object-proxy==1.9.0
     # via astroid
-markupsafe==2.1.1
+markupsafe==2.1.2
     # via
     #   jinja2
-    #   moto
     #   werkzeug
 mccabe==0.7.0
     # via
     #   flake8
     #   pylint
-moto==4.0.10
+moto==4.1.1
     # via -r requirements-dev.in
 mypy==0.991
     # via -r requirements-dev.in
-mypy-boto3-cloudformation==1.26.11.post1
+mypy-boto3-cloudformation==1.26.57
     # via boto3-stubs
-mypy-boto3-dynamodb==1.26.13.post16
+mypy-boto3-dynamodb==1.26.24
     # via boto3-stubs
-mypy-boto3-ec2==1.26.19
+mypy-boto3-ec2==1.26.57
     # via boto3-stubs
-mypy-boto3-lambda==1.26.18
+mypy-boto3-lambda==1.26.55
     # via boto3-stubs
-mypy-boto3-rds==1.26.17
+mypy-boto3-rds==1.26.47
     # via boto3-stubs
 mypy-boto3-s3==1.26.0.post1
     # via boto3-stubs
@@ -126,9 +125,9 @@ packaging==21.3
     #   safety
 patool==1.12
     # via -r requirements-dev.in
-pbr==5.11.0
+pbr==5.11.1
     # via stevedore
-platformdirs==2.5.4
+platformdirs==2.6.2
     # via pylint
 pycodestyle==2.10.0
     # via flake8
@@ -138,7 +137,7 @@ pyfiglet==0.8.post1
     # via -r requirements-dev.in
 pyflakes==3.0.1
     # via flake8
-pylint==2.15.7
+pylint==2.15.10
     # via -r requirements-dev.in
 pyparsing==3.0.9
     # via packaging
@@ -148,13 +147,11 @@ python-dateutil==2.8.2
     #   botocore
     #   freezegun
     #   moto
-pytz==2022.6
-    # via moto
 pyyaml==6.0
     # via
     #   -r requirements-dev.in
     #   bandit
-requests==2.28.1
+requests==2.28.2
     # via
     #   -c requirements.txt
     #   codecov
@@ -173,7 +170,7 @@ s3transfer==0.6.0
     # via
     #   -c requirements.txt
     #   boto3
-safety==2.3.3
+safety==2.3.5
     # via -r requirements-dev.in
 schema==0.7.5
     # via internetarchive
@@ -192,7 +189,7 @@ soupsieve==2.3.2.post1
     #   beautifulsoup4
 stevedore==4.1.1
     # via bandit
-testfixtures==7.0.3
+testfixtures==7.0.4
     # via -r requirements-dev.in
 toml==0.10.2
     # via
@@ -202,13 +199,13 @@ tomlkit==0.11.6
     # via pylint
 tqdm==4.64.1
     # via internetarchive
-types-awscrt==0.15.3
+types-awscrt==0.16.4
     # via
     #   botocore-stubs
     #   types-s3transfer
 types-freezegun==1.1.10
     # via -r requirements-dev.in
-types-requests==2.28.11.5
+types-requests==2.28.11.8
     # via -r requirements-dev.in
 types-s3transfer==0.6.0.post5
     # via boto3-stubs
@@ -227,7 +224,7 @@ typing-extensions==4.4.0
     #   mypy-boto3-rds
     #   mypy-boto3-s3
     #   mypy-boto3-sqs
-urllib3==1.26.13
+urllib3==1.26.14
     # via
     #   -c requirements.txt
     #   botocore

requirements.txt

@@ -1,46 +1,48 @@
 #
-# This file is autogenerated by pip-compile with python 3.11
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
 #
 #    pip-compile --output-file=requirements.txt requirements.in
 #
 beautifulsoup4==4.11.1
     # via -r requirements.in
-boto3==1.26.19
+boto3==1.26.57
     # via -r requirements.in
-botocore==1.29.19
+botocore==1.29.57
     # via
     #   boto3
     #   s3transfer
-certifi==2022.9.24
+certifi==2022.12.7
     # via requests
-charset-normalizer==2.1.1
+charset-normalizer==3.0.1
     # via requests
 defusedxml==0.7.1
     # via -r requirements.in
 dictdiffer==0.9.0
     # via -r requirements.in
 gitdb==4.0.10
     # via gitpython
-gitpython==3.1.29
+gitpython==3.1.30
     # via -r requirements.in
 idna==3.4
     # via requests
 jmespath==1.0.1
     # via
     #   boto3
     #   botocore
+mwparserfromhell==0.6.4
+    # via pywikibot
 oauthlib==3.2.2
     # via
     #   -r requirements.in
     #   requests-oauthlib
 python-dateutil==2.8.2
     # via botocore
-pywikibot==7.7.1
+pywikibot==8.0.0
     # via -r requirements.in
 regex==2022.10.31
     # via wikitextparser
-requests==2.28.1
+requests==2.28.2
     # via
     #   -r requirements.in
     #   pywikibot
@@ -57,11 +59,11 @@ smmap==5.0.0
     # via gitdb
 soupsieve==2.3.2.post1
     # via beautifulsoup4
-urllib3==1.26.13
+urllib3==1.26.14
     # via
     #   botocore
     #   requests
-wcwidth==0.2.5
+wcwidth==0.2.6
     # via wikitextparser
 wikitextparser==0.51.1
     # via -r requirements.in

service/ws_re/register/lemma.py

@@ -211,7 +211,7 @@ def is_valid(self) -> bool:
                     return False
         return True
 
-    def get_table_row(self, print_volume: bool = False) -> str:
+    def get_table_row(self, print_volume: bool = False, print_author: bool = True) -> str:
         row_string = ["|-"]
         multi_row = ""
         if len(self._chapters) > 1:
@@ -222,7 +222,8 @@ def get_table_row(self, print_volume: bool = False) -> str:
         if self._chapters:
             for idx, chapter in enumerate(self._chapters):
                 row_string.append(self._get_pages(chapter))
-                row_string.append(self._get_author_str(chapter))
+                if print_author:
+                    row_string.append(self._get_author_str(chapter))
                 if idx == 0:
                     row_string.append(f"{multi_row} style=\"background:{status[1]}\"|{status[0]}".strip())
                 row_string.append("-")

service/ws_re/register/printer.py

@@ -25,13 +25,11 @@ def _print_author(self):
                     "\n!Autor\n!Artikel\n!colspan=\"2\"|Erschließungsgrad"]
         for register in self.registers.author:
             if register.author.last_name:
-                self.logger.debug(register)
-                if register.author.name != "Hans Gärtner":
-                    save_if_changed(Page(self.wiki,
-                                         f"Paulys Realencyclopädie der classischen "
-                                         f"Altertumswissenschaft/Register/{register.author.name}"),
-                                    register.get_register_str(),
-                                    "Register aktualisiert")
+                save_if_changed(Page(self.wiki,
+                                     f"Paulys Realencyclopädie der classischen "
+                                     f"Altertumswissenschaft/Register/{register.author.name}"),
+                                register.get_register_str(print_details=(register.author.name != "Hans Gärtner")),
+                                "Register aktualisiert")
                 overview.append(register.overview_line)
         overview.append("|}")
         save_if_changed(Page(self.wiki,
@@ -73,13 +71,11 @@ def _print_short(self):
     def _print_volume(self):
         self.logger.info("Print volume register.")
         for register in self.registers.volumes.values():
-            if register.volume.name == "R":
-                continue
             self.logger.debug(register)
             save_if_changed(Page(self.wiki,
                                  f"Paulys Realencyclopädie der classischen "
                                  f"Altertumswissenschaft/Register/{register.volume.name}"),
-                            register.get_register_str(),
+                            register.get_register_str(print_details=(register.volume.name != "R")),
                             "Register aktualisiert")