From 0271f57ad4a698dd5920c20ed8790ef12916ddf8 Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Thu, 17 Jul 2025 09:53:43 +0200 Subject: [PATCH 01/11] log directory --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c83b0858..9f5ff91a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,7 +49,7 @@ jobs: - name: Run tests run: | - sudo touch /var/log/alyx.log; sudo chmod 666 /var/log/alyx.log + sudo mdir -p /var/log/alyx; sudo chmod -fR 666 /var/log/alyx cd alyx cp ./alyx/environment_template.env ./alyx/.env cp ../deploy/docker/settings-deploy.py alyx/settings.py @@ -61,7 +61,7 @@ jobs: DJANGO_SETTINGS_MODULE: alyx.settings PYTHONPATH: $HOME/builds/cortexlab/alyx GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - APACHE_LOG_DIR: /var/log/alyx.log + APACHE_LOG_DIR: /var/log/alyx POSTGRES_HOST: localhost - name: Flake run: | From cdb878479d9e2fac94a1902bd95188657bd717fc Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Thu, 17 Jul 2025 09:57:47 +0200 Subject: [PATCH 02/11] typo --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9f5ff91a..fedfee73 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,7 +49,7 @@ jobs: - name: Run tests run: | - sudo mdir -p /var/log/alyx; sudo chmod -fR 666 /var/log/alyx + sudo mkdir -p /var/log/alyx; sudo chmod -fR 666 /var/log/alyx cd alyx cp ./alyx/environment_template.env ./alyx/.env cp ../deploy/docker/settings-deploy.py alyx/settings.py From 452d649f7eb308649f910e90e19ae7aee1df808e Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Thu, 17 Jul 2025 10:01:38 +0200 Subject: [PATCH 03/11] explicit creation of log folder and file --- .github/workflows/main.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fedfee73..cd90e050 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,11 +49,11 @@ jobs: - name: Run tests run: | - sudo mkdir -p /var/log/alyx; sudo chmod -fR 666 /var/log/alyx + sudo mkdir /var/log/alyx + sudo touch /var/log/alyx/django.log + sudo chmod 666 /var/log/alyx/django.log cd alyx - cp ./alyx/environment_template.env ./alyx/.env - cp ../deploy/docker/settings-deploy.py alyx/settings.py - cp ../deploy/docker/settings_lab-deploy.py alyx/settings_lab.py + cp alyx/settings_ci.py alyx/settings.py python manage.py collectstatic --noinput --link coverage run manage.py test -n coveralls --service=github @@ -61,8 +61,6 @@ jobs: DJANGO_SETTINGS_MODULE: alyx.settings PYTHONPATH: $HOME/builds/cortexlab/alyx GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - APACHE_LOG_DIR: /var/log/alyx - POSTGRES_HOST: localhost - name: Flake run: | cd alyx From 8dc64b45c2583dc4fad4c1110043ca1047630b77 Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Thu, 17 Jul 2025 10:05:25 +0200 Subject: [PATCH 04/11] add settings from deploy --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cd90e050..c0c0256a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -53,7 +53,8 @@ jobs: sudo touch /var/log/alyx/django.log sudo chmod 666 /var/log/alyx/django.log cd alyx - cp alyx/settings_ci.py alyx/settings.py + cp ./alyx/environment_template.env ./alyx/.env + cp ../deploy/docker/settings-deploy.py alyx/settings.py python manage.py collectstatic --noinput --link coverage run manage.py test -n coveralls --service=github From d7bcf5e217c626b6f0d9b45f3595dc743bd9649d Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Thu, 17 Jul 2025 10:31:45 +0200 Subject: [PATCH 05/11] add lab settings also --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c0c0256a..932e8c6f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -55,6 +55,7 @@ jobs: cd alyx cp ./alyx/environment_template.env ./alyx/.env cp ../deploy/docker/settings-deploy.py alyx/settings.py + cp ../deploy/docker/settings_lab-deploy.py alyx/settings_lab.py python manage.py collectstatic --noinput --link coverage run manage.py test -n coveralls --service=github From bf09221024beab67cdb2a4db76ad7298d5caf427 Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Thu, 17 Jul 2025 10:52:52 +0200 Subject: [PATCH 06/11] put environment variables --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 932e8c6f..8a3dc64f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -63,6 +63,8 @@ jobs: DJANGO_SETTINGS_MODULE: alyx.settings PYTHONPATH: $HOME/builds/cortexlab/alyx GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + APACHE_LOG_DIR: /var/log/alyx + POSTGRES_HOST: localhost - name: Flake run: | cd alyx From 25b5ec57de893c8557aee90db4466ee5cbbfc8de Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Thu, 17 Jul 2025 11:54:43 +0200 Subject: [PATCH 07/11] fix tests: remove struct log and all paths are strings --- deploy/docker/settings-deploy.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/deploy/docker/settings-deploy.py b/deploy/docker/settings-deploy.py index f9fcf425..ea6fed64 100644 --- a/deploy/docker/settings-deploy.py +++ b/deploy/docker/settings-deploy.py @@ -13,7 +13,6 @@ import dj_database_url import logging import dotenv -import structlog from pathlib import Path from django.conf.locale.en import formats as en_formats @@ -74,10 +73,6 @@ 'CRITICAL': 'bold_red', }, }, - 'json_formatter': { - '()': structlog.stdlib.ProcessorFormatter, - 'processor': structlog.processors.JSONRenderer(), - }, }, 'handlers': { 'file': { @@ -94,7 +89,7 @@ }, 'loggers': { 'django': { - 'handlers': ['file'], + 'handlers': ['file', 'console'], 'level': LOG_LEVEL, 'propagate': True, }, @@ -155,7 +150,6 @@ 'django.middleware.security.SecurityMiddleware', 'whitenoise.middleware.WhiteNoiseMiddleware', 'alyx.base.QueryPrintingMiddleware', - 'django_structlog.middlewares.RequestMiddleware', ) ROOT_URLCONF = 'alyx.urls' @@ -213,13 +207,13 @@ STATIC_ROOT = BASE_DIR.joinpath('static') # /var/www/alyx/alyx/static STATIC_URL = '/static/' -MEDIA_ROOT = os.getenv('DJANGO_MEDIA_ROOT', BASE_DIR.joinpath('uploaded')) +MEDIA_ROOT = os.getenv('DJANGO_MEDIA_ROOT', str(BASE_DIR.joinpath('uploaded'))) MEDIA_URL = '/uploaded/' UPLOADED_IMAGE_WIDTH = 800 # The location for saving and/or serving the cache tables. # May be a local path, http address or s3 uri (i.e. s3://) -TABLES_ROOT = os.getenv('DJANGO_TABLES_ROOT', BASE_DIR.joinpath('uploaded')) +TABLES_ROOT = os.getenv('DJANGO_TABLES_ROOT', str(BASE_DIR.joinpath('uploaded'))) # storage configurations STORAGES = { @@ -228,7 +222,7 @@ }, } -if str(MEDIA_ROOT).startswith('https://') and '.s3.' in str(MEDIA_ROOT): +if MEDIA_ROOT.startswith('https://') and '.s3.' in MEDIA_ROOT: _logger.warning('S3 backend enabled for uploads and tables') STORAGES['default'] = { "BACKEND": "storages.backends.s3.S3Storage", From 7b04e628f63d248fcf269701175b5aa30d5ab571 Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Thu, 17 Jul 2025 11:56:51 +0200 Subject: [PATCH 08/11] remove structlog --- alyx/actions/admin.py | 4 ++-- alyx/actions/models.py | 4 ++-- alyx/actions/notifications.py | 4 ++-- alyx/actions/water_control.py | 4 ++-- alyx/alyx/base.py | 4 ++-- alyx/data/models.py | 4 ++-- alyx/data/transfers.py | 4 ++-- alyx/data/views.py | 4 ++-- alyx/experiments/models.py | 4 ++-- alyx/subjects/models.py | 4 ++-- requirements.txt | 2 +- 11 files changed, 21 insertions(+), 21 deletions(-) diff --git a/alyx/actions/admin.py b/alyx/actions/admin.py index 5fada7e9..070960d3 100644 --- a/alyx/actions/admin.py +++ b/alyx/actions/admin.py @@ -1,6 +1,6 @@ import base64 import json -import structlog +import logging from django import forms from django.conf import settings @@ -27,7 +27,7 @@ from experiments.models import ProbeInsertion, FOV from jobs.models import Task -logger = structlog.get_logger(__name__) +logger = logging.getLogger(__name__) # Filters diff --git a/alyx/actions/models.py b/alyx/actions/models.py index 69c2e143..e6be347c 100644 --- a/alyx/actions/models.py +++ b/alyx/actions/models.py @@ -1,7 +1,7 @@ from datetime import timedelta from math import inf -import structlog +import logging from one.alf.spec import QC from django.conf import settings @@ -13,7 +13,7 @@ from misc.models import Lab, LabLocation, LabMember, Note -logger = structlog.get_logger(__name__) +logger = logging.getLogger(__name__) def _default_water_type(): diff --git a/alyx/actions/notifications.py b/alyx/actions/notifications.py index c0668ce3..409792ca 100644 --- a/alyx/actions/notifications.py +++ b/alyx/actions/notifications.py @@ -1,4 +1,4 @@ -import structlog +import logging from textwrap import dedent from django.utils import timezone @@ -6,7 +6,7 @@ from actions.models import create_notification -logger = structlog.get_logger(__name__) +logger = logging.getLogger(__name__) def responsible_user_changed(subject, old_user, new_user): diff --git a/alyx/actions/water_control.py b/alyx/actions/water_control.py index 6a9ebb77..6bbf9433 100644 --- a/alyx/actions/water_control.py +++ b/alyx/actions/water_control.py @@ -4,7 +4,7 @@ from dateutil.rrule import HOURLY import functools import io -import structlog +import logging from operator import attrgetter, itemgetter import os.path as op @@ -17,7 +17,7 @@ import numpy as np -logger = structlog.get_logger(__name__) +logger = logging.getLogger(__name__) PALETTE = { diff --git a/alyx/alyx/base.py b/alyx/alyx/base.py index 575ae961..e3f184c2 100644 --- a/alyx/alyx/base.py +++ b/alyx/alyx/base.py @@ -1,5 +1,5 @@ import json -import structlog +import logging import os import os.path as op from polymorphic.models import PolymorphicModel @@ -30,7 +30,7 @@ from reversion.admin import VersionAdmin from alyx import __version__ as version -logger = structlog.get_logger(__name__) +logger = logging.getLogger(__name__) DATA_DIR = op.abspath(op.join(op.dirname(__file__), '../../data')) DISABLE_MAIL = False # used for testing diff --git a/alyx/data/models.py b/alyx/data/models.py index 6b6674c0..106d899a 100644 --- a/alyx/data/models.py +++ b/alyx/data/models.py @@ -1,4 +1,4 @@ -import structlog +import logging from one.alf.spec import QC from django.core.validators import RegexValidator @@ -11,7 +11,7 @@ from actions.models import Session from alyx.base import BaseModel, modify_fields, BaseManager, CharNullField, BaseQuerySet, ALF_SPEC -logger = structlog.get_logger(__name__) +logger = logging.getLogger(__name__) def _related_string(field): diff --git a/alyx/data/transfers.py b/alyx/data/transfers.py index ff9eadf4..c17ebd3a 100644 --- a/alyx/data/transfers.py +++ b/alyx/data/transfers.py @@ -1,5 +1,5 @@ import json -import structlog +import logging import os import os.path as op import re @@ -18,7 +18,7 @@ from rest_framework.response import Response from actions.models import Session -logger = structlog.get_logger(__name__) +logger = logging.getLogger(__name__) # Login # ------------------------------------------------------------------------------------------------ diff --git a/alyx/data/views.py b/alyx/data/views.py index dc569b90..b65ce05f 100644 --- a/alyx/data/views.py +++ b/alyx/data/views.py @@ -1,4 +1,4 @@ -import structlog +import logging import re from django.contrib.auth import get_user_model @@ -37,7 +37,7 @@ _create_dataset_file_records, bulk_sync, _check_dataset_protected, _get_name_collection_revision) -logger = structlog.get_logger(__name__) +logger = logging.getLogger(__name__) # DataRepositoryType # ------------------------------------------------------------------------------------------------ diff --git a/alyx/experiments/models.py b/alyx/experiments/models.py index dfe04724..c4766c37 100644 --- a/alyx/experiments/models.py +++ b/alyx/experiments/models.py @@ -1,4 +1,4 @@ -import structlog +import logging import uuid from django.db import models @@ -13,7 +13,7 @@ from alyx.base import BaseModel, BaseManager from actions.models import ChronicRecording -logger = structlog.get_logger(__name__) +logger = logging.getLogger(__name__) X_HELP_TEXT = ("brain surface medio-lateral coordinate (um) of" "the insertion, right +, relative to Bregma") diff --git a/alyx/subjects/models.py b/alyx/subjects/models.py index ad61ed41..db2b948d 100644 --- a/alyx/subjects/models.py +++ b/alyx/subjects/models.py @@ -1,5 +1,5 @@ from datetime import datetime, timezone -import structlog +import logging from operator import attrgetter import urllib @@ -18,7 +18,7 @@ from actions.models import Surgery from misc.models import Lab, default_lab, Housing -logger = structlog.get_logger(__name__) +logger = logging.getLogger(__name__) # Zygosity constants diff --git a/requirements.txt b/requirements.txt index 64f07d05..b885d249 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,6 +23,7 @@ flake8 globus-cli globus-sdk gunicorn +ipython markdown matplotlib ONE-api>=3.0 @@ -33,7 +34,6 @@ python-magic pytz requests>=2.32.4 setuptools -structlog>=21.5.0 urllib3>=2.5.0 webdavclient3 whitenoise From 78edeee742760c4661b55ac5ec12907386df3900 Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Thu, 17 Jul 2025 11:57:01 +0200 Subject: [PATCH 09/11] move to pyproject.toml --- .gitignore | 3 + .readthedocs.yaml | 8 +-- pyproject.toml | 36 +++++++++++ setup.py | 160 ---------------------------------------------- 4 files changed, 41 insertions(+), 166 deletions(-) create mode 100644 pyproject.toml delete mode 100755 setup.py diff --git a/.gitignore b/.gitignore index 69a386ee..c101e416 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ alyx/alyx/settings.py alyx/static/*/* scripts/deployment_examples/docker-apache/settings* +# package if installed +alyx/alyx.egg-info + # other *.pyc .DS_Store diff --git a/.readthedocs.yaml b/.readthedocs.yaml index b29c3001..5e47bad9 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,13 +7,9 @@ version: 2 # Set the version of Python and other tools you might need build: - os: ubuntu-20.04 + os: ubuntu-24.04 tools: - python: "3.9" - # You can also specify other tool versions: - # nodejs: "16" - # rust: "1.55" - # golang: "1.17" + python: "3.12" # Build documentation in the docs/ directory with Sphinx sphinx: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..b31f8102 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,36 @@ +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[project] +name = "alyx" +description = "Database for experimental neuroscience laboratories" +readme = "README.md" +requires-python = ">=3.10" +license = {file = "LICENSE"} +authors = [ + {name = "IBL staff", email = "info@internationalbrainlab.org"}, +] +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] + +dynamic = ["dependencies", "version"] + + +[project.urls] +"Homepage" = "https://github.com/cortex-lab/alyx" +"Documentation" = "https://alyx.readthedocs.io/en/latest" +"ChangeLog" = "https://github.com/cortex-lab/alyx/blob/main/CHANGELOG.md" +"Bug Tracker" = "https://github.com/cortex-lab/alyx/issues" +"Example" = "https://test.alyx.internationalbrainlab.org/admin/login/?next=/admin/" + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} +version = {attr = "alyx.__version__"} + +[tool.setuptools] +include-package-data = true +package-dir = {"" = "alyx"} # This would map the root package to the src directory diff --git a/setup.py b/setup.py deleted file mode 100755 index ecba39ac..00000000 --- a/setup.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python3 - -from pathlib import Path -from getpass import getpass, getuser -import os -import os.path as op -import platform -from django.utils.crypto import get_random_string -import sys -from warnings import warn - -MACOS = platform.system() == 'Darwin' - - -def _secret_key(): - chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)' - return get_random_string(50, chars) - - -def _system(cmd): - res = os.popen(cmd).read().strip() - return res - - -def _psql(sql, **kwargs): - sql = sql.format(**kwargs) - if MACOS: - cmd = 'psql -tc "{}"'.format(sql) - else: - cmd = 'sudo su - postgres -c "psql -tc \\"{}\\""'.format(sql) - return _system(cmd) - - -def _replace_in_file(source_file, target_file, replacements=None, target_mode='w', chmod=None): - target_file = op.expanduser(target_file) - replacements = {} if replacements is None else replacements - #copy2(source_file, target_file) - with open(source_file, 'r') as f: - contents = f.read() - for key, value in replacements.items(): - contents = contents.replace(key, value) - with open(target_file, target_mode) as f: - f.write(contents) - if chmod: - os.chmod(target_file, chmod) - - -# Check if we are inside a virtual environment -if not MACOS and not hasattr(sys, 'real_prefix') and sys.base_prefix == sys.prefix: - warn('You are not currently in a virtual environment, ' - 'would you like to proceed anyway? (y/n): ', RuntimeWarning) - continue_anyway = input() - if continue_anyway not in ("y", 'yes'): - print('Create a virtual environment from the repository root with: ' - '"virtualenv alyxvenv --python=python3"') - print('Enter a virtual environment from the repository root with: ' - '"source alyxvenv/bin/activate"') - sys.exit() - - -# Prompt information. -DBNAME = input("Enter a database name [labdb]:") or 'labdb' -DBUSER = input("Enter a postgres username [labdbuser]:") or 'labdbuser' -DBPASSWORD = getpass("Enter a postgres password:") -if not DBPASSWORD: - print("A password is mandatory.") - exit(1) -if getpass("Enter a postgres password (again):") != DBPASSWORD: - print("The passwords don't match.") - exit(1) -SECRET_KEY = _secret_key() - - -# Make sure the database exists. -out = _psql("SELECT 1 FROM pg_database WHERE datname = '{DBNAME}'", DBNAME=DBNAME) -if not out: - out = _psql("CREATE DATABASE {DBNAME}", DBNAME=DBNAME) - -# Make sure the user exists. -out = _psql("SELECT 1 FROM pg_user WHERE usename = '{DBUSER}'", DBUSER=DBUSER) -if not out: - out = _psql("CREATE USER {DBUSER} WITH PASSWORD '{DBPASSWORD}'", - DBUSER=DBUSER, DBPASSWORD=DBPASSWORD) - - -# Set up the roles for the user. -try: - _psql("ALTER ROLE {DBUSER} SET client_encoding TO 'utf8';", DBUSER=DBUSER) - _psql("ALTER ROLE {DBUSER} SET default_transaction_isolation TO 'read committed';", - DBUSER=DBUSER) - _psql("ALTER ROLE {DBUSER} SET timezone TO 'UTC';", DBUSER=DBUSER) - _psql("GRANT ALL PRIVILEGES ON DATABASE {DBNAME} TO {DBUSER};", DBNAME=DBNAME, DBUSER=DBUSER) - _psql("ALTER USER {DBUSER} WITH CREATEROLE;", DBUSER=DBUSER) - _psql("ALTER USER {DBUSER} WITH SUPERUSER;", DBUSER=DBUSER) - _psql("ALTER USER {DBUSER} WITH CREATEDB;", DBUSER=DBUSER) - print('-----------------------------') - print('Database successfully created') -except Exception as e: - print('Could not create database, error message:\n') - raise e - - -file_log = Path('/var/log/alyx').joinpath(f"{DBNAME}.log") -file_log_json = Path('/var/log/alyx').joinpath(f"{DBNAME}_json.log") -repl = { - '%SECRET_KEY%': SECRET_KEY, - '%DBNAME%': DBNAME, - '%DBUSER%': DBUSER, - '%DBPASSWORD%': DBPASSWORD, - '%ALYX_JSON_LOG_FILE%': str(file_log_json), - '%ALYX_LOG_FILE%': str(file_log) -} - -try: - _replace_in_file('alyx/alyx/settings_template.py', - 'alyx/alyx/settings.py', replacements=repl) - _replace_in_file('alyx/alyx/settings_lab_template.py', - 'alyx/alyx/settings_lab.py') - # Set up the settings file. - _replace_in_file('alyx/alyx/settings_secret_template.py', - 'alyx/alyx/settings_secret.py', - replacements=repl, - ) - - # Set up the .pgpass file to avoid typing the postgres password. - _replace_in_file('scripts/templates/.pgpass_template', '~/.pgpass', - replacements=repl, target_mode='a', chmod=0o600) - - # Set up the maintainance scripts. - _replace_in_file('scripts/templates/load_db.sh', 'scripts/load_db.sh', - replacements=repl, chmod=0o755) - _replace_in_file('scripts/templates/dump_db.sh', 'scripts/dump_db.sh', - replacements=repl, chmod=0o755) - print('Configuration files successfully created from templates') -except Exception as e: - print('Could not create configuration from templates, error message:\n') - raise e - - -# Set up the database. -try: - _system(f'sudo mkdir -p {file_log_json.parent}') - _system(f'sudo mkdir -p {file_log.parent}') - _system(f'sudo chown {getuser()}:www-data -fR {file_log.parent}') - _system(f'sudo chown {getuser()}:www-data -fR {file_log_json.parent}') - _system(f'touch {file_log_json}') - _system(f'touch {file_log}') - _system('python3 alyx/manage.py makemigrations') - _system('python3 alyx/manage.py migrate') - - _system('''echo "from misc.models import LabMember;''' - '''LabMember.objects.create_superuser('admin', 'admin@example.com', 'admin')"''' - '''| python3 alyx/manage.py shell''') - print('Database successfully configured for Alyx') -except Exception as e: - print('Could not configure database for Alyx, error message:\n') - raise e - -print('------------------------') -print('Alyx setup successful <3') From e2f6e50aec58190e7d4632385d1159cfc4867e34 Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Thu, 17 Jul 2025 12:20:11 +0200 Subject: [PATCH 10/11] ruff check instead of flake --- .github/workflows/main.yml | 34 +++---------------- alyx/actions/tests.py | 4 +-- alyx/actions/tests_rest.py | 12 +++---- alyx/alyx/base.py | 4 +-- alyx/data/management/commands/files.py | 6 ++-- .../commands/transfers_integration.py | 2 +- .../management/commands/download_gsheets.py | 20 +++++------ alyx/misc/management/commands/report.py | 18 +++++----- .../management/commands/update_zygosities.py | 4 +-- alyx/subjects/models.py | 4 +-- deploy/docker/settings-deploy.py | 2 +- pyproject.toml | 9 +++++ 12 files changed, 52 insertions(+), 67 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8a3dc64f..d2338bc5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,6 +47,11 @@ jobs: env: PIP_USE_MIRRORS: true + - name: Ruff + uses: astral-sh/ruff-action@v3 + with: + src: "./alyx" + - name: Run tests run: | sudo mkdir /var/log/alyx @@ -65,35 +70,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} APACHE_LOG_DIR: /var/log/alyx POSTGRES_HOST: localhost - - name: Flake - run: | - cd alyx - flake8 . - - - name: Generate new requirements_frozen.txt if needed - # Only runs when branch pushed to directly OR when a PR is merged - if: ${{ github.event_name == 'push' }} - run: | - pip freeze > requirements_frozen_temp.txt - if diff requirements_frozen.txt requirements_frozen_temp.txt > /dev/null; then - echo "requirements_frozen.txt unchanged" - rm requirements_frozen_temp.txt - echo "GIT_PUSH_NEEDED=false" >> "$GITHUB_ENV" - else - echo "requirements_frozen.txt is different, git push needed" - mv requirements_frozen_temp.txt requirements_frozen.txt - echo "GIT_PUSH_NEEDED=true" >> "$GITHUB_ENV" - fi - - - name: Setup git/commit/push for requirements_frozen.txt if needed - # Only runs when requirements_frozen.txt is updated - if: env.GIT_PUSH_NEEDED == 'true' - run: | - git config user.name github-actions - git config user.email github-actions@github.com - git add requirements_frozen.txt - git commit -m "GitHub Actions generated requirements_frozen.txt" - git push # Docker steps only run when master branch pushed to directly OR when a PR is merged - name: Set Docker conditional value if needed diff --git a/alyx/actions/tests.py b/alyx/actions/tests.py index 3009b84d..46150ea0 100644 --- a/alyx/actions/tests.py +++ b/alyx/actions/tests.py @@ -219,8 +219,8 @@ def test_notif_water_1(self): def test_notif_water_2(self): # If the last water admin was on June 3 at 12pm, the notification # should be created after June 4 at 11am. - l = ((9, False), (10, False), (11, True), (12, True)) - for (h, r) in l: + teupeul = ((9, False), (10, False), (11, True), (12, True)) + for (h, r) in teupeul: date = timezone.datetime(2018, 6, 4, h, 0, 0) check_water_administration(self.subject, date=date) notif = Notification.objects.last() diff --git a/alyx/actions/tests_rest.py b/alyx/actions/tests_rest.py index 0596364b..60e52ffd 100644 --- a/alyx/actions/tests_rest.py +++ b/alyx/actions/tests_rest.py @@ -548,15 +548,15 @@ def test_list_retrieve_water_restrictions(self): def test_list_retrieve_lab_locations(self): # test list url = reverse("location-list") - l = self.ar(self.client.get(url)) - self.assertTrue(len(l) > 0) - self.assertEqual(set(l[0].keys()), {"name", "json", "lab"}) + reponse = self.ar(self.client.get(url)) + self.assertTrue(len(reponse) > 0) + self.assertEqual(set(reponse[0].keys()), {"name", "json", "lab"}) # test detail - url = reverse("location-detail", args=[l[0]["name"]]) + url = reverse("location-detail", args=[reponse[0]["name"]]) d = self.ar(self.client.get(url)) - self.assertEqual(d, l[0]) + self.assertEqual(d, reponse[0]) # test patch - url = reverse("location-detail", args=[l[0]["name"]]) + url = reverse("location-detail", args=[reponse[0]["name"]]) json_dict = { "string": "look at me! I'm a Json field", "integer": 15, diff --git a/alyx/alyx/base.py b/alyx/alyx/base.py index e3f184c2..3e9f0bfd 100644 --- a/alyx/alyx/base.py +++ b/alyx/alyx/base.py @@ -211,8 +211,8 @@ def __init__(self, *args, **kwargs): self.__dict__ = self -def flatten(l): - return [item for sublist in l for item in sublist] +def flatten(liste): + return [item for sublist in liste for item in sublist] def _show_change(date_time, old, new): diff --git a/alyx/data/management/commands/files.py b/alyx/data/management/commands/files.py index 61962d85..f0a5aec2 100644 --- a/alyx/data/management/commands/files.py +++ b/alyx/data/management/commands/files.py @@ -26,9 +26,9 @@ def _create_missing_file_records_main_globus(dry_run=False, lab=None): labs = Lab.objects.filter(name=lab) else: labs = Lab.objects.all() - for l in labs: - repos = l.repositories.filter(globus_is_personal=False) - dsets = Dataset.objects.filter(session__lab=l) + for lab in labs: + repos = lab.repositories.filter(globus_is_personal=False) + dsets = Dataset.objects.filter(session__lab=lab) for r in repos: dsr = dsets.annotate(rep_count=Count('file_records', filter=Q(file_records__data_repository=r))) diff --git a/alyx/data/management/commands/transfers_integration.py b/alyx/data/management/commands/transfers_integration.py index 6bc326c3..c5a2d778 100644 --- a/alyx/data/management/commands/transfers_integration.py +++ b/alyx/data/management/commands/transfers_integration.py @@ -289,7 +289,7 @@ def test_transfers(self): ls_local = self.gtc.operation_ls(self.local_endpoint_id, path=str(Path(exp_files[0]).parent)) ls_files = [ls['name'] for ls in ls_local['DATA']] - assert not Path(exp_files[0]).name in ls_files + assert Path(exp_files[0]).name not in ls_files dsets_to_del = Dataset.objects.filter(session__lab__name=self.lab_name, name='spikes.times.npy') diff --git a/alyx/misc/management/commands/download_gsheets.py b/alyx/misc/management/commands/download_gsheets.py index e148be57..d3cd4654 100644 --- a/alyx/misc/management/commands/download_gsheets.py +++ b/alyx/misc/management/commands/download_gsheets.py @@ -42,8 +42,8 @@ def pad(s): return re.sub(r'\_([0-9]+)$', lambda m: '_%04d' % int(m.group(1)), s) -def flatten(l): - return [item for sublist in l for item in sublist] +def flatten(line): + return [item for sublist in line for item in sublist] def get_username(initials): @@ -72,8 +72,8 @@ def parse(date_str, time=False): return '' try: ret = parse_(date_str) - except: - logger.warn("Could not parse date %s.", date_str) + except Exception: + logger.warning("Could not parse date %s.", date_str) return '' if not time: return ret.strftime("%Y-%m-%d") @@ -120,11 +120,11 @@ def sheet_to_table(wks, header_line=0, first_line=2): table = [] headers = rows[header_line] for row in rows[first_line:]: - l = {headers[i].strip(): row[i].strip() for i in range(len(headers))} + line = {headers[i].strip(): row[i].strip() for i in range(len(headers))} # Empty line = end of table. - if all(_ == '' for _ in l.values()): + if all(_ == '' for _ in line.values()): break - table.append(Bunch(l)) + table.append(Bunch(line)) return table @@ -366,9 +366,9 @@ def _get_subjects(self, line_tables): def _get_line(self, line): out = self.lines.get(line, None) if not out: - for l in self.lines.values(): - if l.auto_name == line: - return l + for line in self.lines.values(): + if line.auto_name == line: + return line return out def _get_litters(self, subjects): diff --git a/alyx/misc/management/commands/report.py b/alyx/misc/management/commands/report.py index b072b33c..6f1519b7 100644 --- a/alyx/misc/management/commands/report.py +++ b/alyx/misc/management/commands/report.py @@ -18,28 +18,28 @@ logger = logging.getLogger(__name__) -def _repr_log_entry(l): - if l.is_addition(): +def _repr_log_entry(log): + if log.is_addition(): action = 'Added' - elif l.is_change(): + elif log.is_change(): action = 'Changed' - elif l.is_deletion(): + elif log.is_deletion(): action = 'Deleted' - changed = json.loads(l.change_message or '[]') + changed = json.loads(log.change_message or '[]') if changed and changed[0].get('changed', {}): changed = ('(%s)' % (', '.join(changed[0].get('changed', {}).get('fields', {})))) else: changed = '' s = '%02d:%02d - %s <%s> %s' % ( - l.action_time.hour, - l.action_time.minute, + log.action_time.hour, + log.action_time.minute, action, # l.content_type, # NOTE: use this when debugging repr (the repr string is directly saved in the LogEntry) # str(l.get_edited_object()), - l.object_repr, + log.object_repr, changed, ) @@ -206,7 +206,7 @@ def make_past_changes(self, user): logs = LogEntry.objects.filter(user=user, action_time__date=yesterday, ).order_by('action_time') - return 'Your actions yesterday:\n\n' + '\n'.join('* ' + _repr_log_entry(l) for l in logs) + return 'Your actions yesterday:\n\n' + '\n'.join('* ' + _repr_log_entry(log) for log in logs) def make_training(self, user): """Send training report to the specified user.""" diff --git a/alyx/subjects/management/commands/update_zygosities.py b/alyx/subjects/management/commands/update_zygosities.py index 883b2b81..91c5c0d0 100644 --- a/alyx/subjects/management/commands/update_zygosities.py +++ b/alyx/subjects/management/commands/update_zygosities.py @@ -42,7 +42,7 @@ def handle(self, *args, **options): if options.get('migrate_rules'): from subjects.zygosities import ZYGOSITY_RULES - for l, a, rules in ZYGOSITY_RULES: + for line, a, rules in ZYGOSITY_RULES: for rule in rules: r = _parse_rule(rule) zygosity = ZYGOSITY_SYMBOLS.index(r.pop('res')) @@ -51,7 +51,7 @@ def handle(self, *args, **options): seq0 = k[0] res0 = r[seq0] kwargs = dict( - line=Line.objects.get(nickname=l), + line=Line.objects.get(nickname=line), allele=Allele.objects.get(nickname=a), sequence0=Sequence.objects.get_or_create(name=seq0)[0], sequence0_result=res0, diff --git a/alyx/subjects/models.py b/alyx/subjects/models.py index db2b948d..41581908 100644 --- a/alyx/subjects/models.py +++ b/alyx/subjects/models.py @@ -68,8 +68,8 @@ def save_old_fields(obj, fields): continue if field not in d: d[field] = [] - l = d[field] - l.append({'date_time': date_time, 'value': obj._original_fields[field]}) + list_field = d[field] + list_field.append({'date_time': date_time, 'value': obj._original_fields[field]}) # Update the new value. # obj._original_fields[field] = v # Set the object's JSON if necessary. diff --git a/deploy/docker/settings-deploy.py b/deploy/docker/settings-deploy.py index ea6fed64..b2ddf269 100644 --- a/deploy/docker/settings-deploy.py +++ b/deploy/docker/settings-deploy.py @@ -108,7 +108,7 @@ ALLOWED_HOSTS = ['localhost', '127.0.0.1', '.eu-west-2.compute.amazonaws.com'] if (web_host := os.getenv('APACHE_SERVER_NAME', '0.0.0.0')) is not None: ALLOWED_HOSTS.append(web_host) -CSRF_TRUSTED_ORIGINS = [f"http://{web_host}", f"https://{web_host}", f"https://*.internationalbrainlab.org"] +CSRF_TRUSTED_ORIGINS = [f"http://{web_host}", f"https://{web_host}", "https://*.internationalbrainlab.org"] CSRF_COOKIE_SECURE = True diff --git a/pyproject.toml b/pyproject.toml index b31f8102..4bd7bdd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,3 +34,12 @@ version = {attr = "alyx.__version__"} [tool.setuptools] include-package-data = true package-dir = {"" = "alyx"} # This would map the root package to the src directory + +[tool.ruff] +exclude = [ + "migrations", + ".venv", +] +indent-width = 4 +line-length = 99 +target-version = "py312" From d05ac640da7cb9fceceae6f299315df349d83eca Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Thu, 17 Jul 2025 12:30:51 +0200 Subject: [PATCH 11/11] more ruff --- .github/workflows/main.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d2338bc5..6778ef49 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,12 +46,9 @@ jobs: pip install coverage coveralls pyarrow pandas # for one_cache tests env: PIP_USE_MIRRORS: true - - name: Ruff - uses: astral-sh/ruff-action@v3 - with: - src: "./alyx" - + run: | + ruff check ./alyx - name: Run tests run: | sudo mkdir /var/log/alyx