Update

.coveragerc

@@ -1,6 +1,6 @@
 [run]
 branch=True
-source=observatory
+source=observatory_platform
 
 [report]
 exclude_lines =
@@ -12,10 +12,4 @@ exclude_lines =
     @abstract
 ignore_errors = True
 omit =
-    tests/*
-    observatory-api/observatory/api/client/api_client.py
-    observatory-api/observatory/api/client/configuration.py
-    observatory-api/observatory/api/client/exceptions.py
-    observatory-api/observatory/api/client/model_utils.py
-    observatory-api/observatory/api/client/rest.py
-    observatory-platform/observatory/platform/airflow/*
+    */tests*

.github/workflows/unit-tests.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Check licenses
         run: |
           # stop the build if there are licensing issues
-          # liccheck --sfile strategy.ini --rfile pyproject.toml --level CAUTIOUS --reporting liccheck-output.txt --no-deps
+          liccheck
 
       - name: Lint with flake8
         run: |

.gitignore

@@ -24,3 +24,4 @@ docs/schemas
 .env
 /observatory-platform/build
 /observatory-api/build
+liccheck-output.txt

observatory_platform/airflow/release.py

@@ -17,17 +17,12 @@
 from __future__ import annotations
 
 import logging
-
-try:
-    from typing import Protocol
-except ImportError:
-    from typing_extensions import Protocol
 import os
-import pendulum
 
+import pendulum
 from airflow.exceptions import AirflowException
-from observatory_platform.airflow.workflow import make_workflow_folder
 
+from observatory_platform.airflow.workflow import make_workflow_folder
 
 DATE_TIME_FORMAT = "YYYY-MM-DD_HH:mm:ss"
 

observatory_platform/google/bigquery.py

@@ -36,10 +36,7 @@
 from natsort import natsorted
 
 from observatory_platform.config import sql_templates_path
-from observatory_platform.jinja2_utils import (
-    make_sql_jinja2_filename,
-    render_template,
-)
+from observatory_platform.jinja2_utils import make_sql_jinja2_filename, render_template
 
 # BigQuery single query byte limit.
 # Daily limit is set in Terraform

pyproject.toml

@@ -27,66 +27,49 @@ classifiers = [
 ]
 dependencies = [
     # Airflow
-    "apache-airflow[redis,celery,slack,http]==2.7.3",
+    "apache-airflow[slack]==2.7.3",
     "apache-airflow-providers-cncf-kubernetes>=7.4.0",
     # Google Cloud
     "google-crc32c>=1.1.0,<2",
     "google-cloud-bigquery>=3,<4",
     "google-api-python-client>=2,<3",
     "google-cloud-storage>=2.7.0,<3",
-    "google-auth-oauthlib>=0.4.5,<1",
+    #"google-auth-oauthlib>=0.4.5,<1",
     "google-cloud-compute >=1.16.0,<2.0",
-    # AWS
-    "boto3>=1.15.0,<2",
-    # Command line interface, config files and templates
-    "click>=8,<9",
-    "pyyaml>=6,<7",
-    "cerberus>=1.3.4,<2",
+    # File manipulation, reading, writing
     "Jinja2>=3,<4",
-    "stringcase>=1.2.0,<2",
-    "pyhcl>=0.4.4,<1",
-    # Reading and writing jsonlines files
     "jsonlines>=2.0.0,<3", # Writing
     "json_lines>=0.5.0,<1", # Reading, including jsonl.gz
+    "pandas>=2,<3",
     # HTTP requests and URL cleaning
     "requests>=2.25.0,<3",
-    "tldextract>=3.1.1",
     "aiohttp>=3.7.0,<4",
-    "responses>=0.23.1,<1",
     # SFTP
-    "pysftp>=0.2.9,<1",
     "paramiko>=2.7.2,<3",
-    # FTP
-    "pyftpdlib>=1.5.7,<2",
+    "pysftp>=0.2.9,<1",
     # Utils
     "natsort>=7.1.1,<8",
     "backoff>=2,<3",
-    "timeout-decorator",
     "validators<=0.20.0",
     "xmltodict",
-    "pandas",
     "tenacity",
-    # Test utils
-    "time_machine>=2.0.0,<3",
-    "httpretty>=1.0.0,<2",
-    "sftpserver>=0.3,<1",
-    # Backports
-    "typing-extensions>=3.10.0.1,<4; python_version<'3.10'",
-    "oauth2client>=4.1.0,<5",
-    "deprecation>2,<3",
-    "virtualenv",
-    "deepdiff",
 ]
 
 [project.optional-dependencies]
 tests = [
     "liccheck>=0.4.9,<1",
     "flake8>=3.8.0,<4",
     "coverage>=5.2,<6",
-    "faker>=8.12.1,<9",
-    "redis>=3.5.3,<4",
+    "azure-storage-blob>=12.8.1,<13",
+    "click>=8,<9",
+    "httpretty>=1.0.0,<2",
+    "deepdiff>=6,<7",
+    "responses>=0.23.1,<1",
     "boto3>=1.15.0,<2",
-    "azure-storage-blob>=12.8.1,<13"
+    "timeout-decorator>=0,<1",
+    # SFTP & FTP
+    "sftpserver>=0.3,<1",
+    "pyftpdlib>=1.5.7,<2",
 ]
 
 [project.urls]
@@ -95,3 +78,71 @@ tests = [
 "Documentation" = "https://observatory-platform.readthedocs.io/en/latest/"
 "Source" = "https://github.com/The-Academic-Observatory/observatory-platform"
 "Repository" = "https://github.com/The-Academic-Observatory/observatory-platform"
+
+[tool.liccheck]
+authorized_licenses = [
+    # Unencumbered licenses: https://opensource.google/docs/thirdparty/licenses/#unencumbered
+    "public domain <http://unlicense.org>",
+    "cc0 1.0 universal (cc0 1.0) public domain dedication",
+    # Notice licenses: https://opensource.google/docs/thirdparty/licenses/#notice
+    "artistic",
+    "apache software license 2.0",
+    "apache license version 2.0",
+    "apache license, version 2.0",
+    "apache license 2.0",
+    "apache 2.0",
+    "apache-2.0",
+    "apache software",
+    "apache 2",
+    "apache-2",
+    "bsd",
+    "bsd-2-clause",
+    "bsd-3-clause",
+    "3-clause bsd",
+    "new bsd",
+    "bsd or apache license, version 2.0",
+    "bsd-2-clause or apache-2.0",
+    "isc license (iscl)",
+    "isc",
+#    "mit",
+    "python software foundation",
+    "psfl",
+    "zpl 2.1",
+    # Reciprocal licenses: https://opensource.google/docs/thirdparty/licenses/#reciprocal
+    "mozilla public license 1.1 (mpl 1.1)",
+    "mozilla public license 2.0 (mpl 2.0)",
+    "mpl-2.0",
+    # LGPL
+    "lgpl",
+    "gnu library or lesser general public license (lgpl)",
+    "gnu lesser general public license v3 or later (lgplv3+)",
+    "lgplv3+"
+]
+unauthorized_licenses = [
+    "mit", # just to check it is working
+
+    # Restricted licenses: https://opensource.google/docs/thirdparty/licenses/#restricted
+    "gpl v1",
+    "gpl v2",
+    "gpl v3",
+    # Banned licenses: https://opensource.google/docs/thirdparty/licenses/#banned
+    "agpl",
+    "affero gpl",
+    "agpl (affero gpl)",
+    "sspl"
+]
+level = "CAUTIOUS"
+reporting_txt_file = "liccheck-output.txt" # by default is None
+no_deps = false
+dependencies = true # to load [project.dependencies]
+optional_dependencies = ["tests"] # to load extras from [project.optional-dependencies]
+
+[tool.liccheck.authorized_packages]
+# MIT license: https://pypi.org/project/ordereddict/1.1/
+ordereddict = "1.1"
+
+# MIT license: https://pypi.org/project/pendulum/1.4.4/
+pendulum = ">=1.4.4"
+
+# Python Imaging Library (PIL) License: https://github.com/python-pillow/Pillow/blob/master/LICENSE
+Pillow = ">=7.2.0"