diff --git a/ESSArch_Core/WorkflowEngine/util.py b/ESSArch_Core/WorkflowEngine/util.py
index 06c822612..02dd2c715 100644
--- a/ESSArch_Core/WorkflowEngine/util.py
+++ b/ESSArch_Core/WorkflowEngine/util.py
@@ -3,7 +3,7 @@
from celery import states as celery_states
from django.core.cache import cache
-# from django.db import transaction
+from django.db import OperationalError, transaction
from tenacity import (
RetryError,
Retrying,
@@ -161,51 +161,70 @@ def create_workflow(workflow_spec=None, ip=None, workflow_steps=None, name='', l
try:
for attempt in Retrying(stop=stop_after_delay(30),
wait=wait_random_exponential(multiplier=1, max=60),
- before_sleep=before_sleep_log(logging.getLogger('essarch'), logging.WARNING)):
+ before_sleep=before_sleep_log(logging.getLogger('essarch'), logging.WARNING),
+ retry_error_callback=lambda retry_state: logger.error(
+ f"Failed to create workflow for IP {ip} "
+ f"after retries: {retry_state.outcome.exception()}"), reraise=True):
with attempt:
try:
- # with transaction.atomic():
- # with ProcessStep.objects.delay_mptt_updates():
- if top_root_step:
- root_step = ProcessStep(
- name=name, eager=eager, information_package=ip, context=context,
- responsible=responsible, label=label, part_root=part_root,
- run_state=run_state)
- root_step.parent = top_root_step
- root_step.parent_pos = top_root_step.child_steps.count() + 1
- root_step.save()
- else:
- root_step = ProcessStep.objects.create(
- name=name, eager=eager, information_package=ip, context=context,
- responsible=responsible, label=label, part_root=part_root,
- run_state=run_state)
- on_error_tasks = list(_create_on_error_tasks(
- root_step, on_error, ip=ip, responsible=responsible, status=celery_states.SUCCESS))
- ProcessTask.objects.bulk_create(on_error_tasks)
- root_step.on_error.add(*on_error_tasks)
-
- if workflow_spec:
- _create_step(root_step, workflow_spec, ip, responsible)
-
- if workflow_steps:
- _add_steps(root_step, workflow_steps)
-
- # root_step.refresh_from_db()
- # with ProcessStep.objects.delay_mptt_updates():
- # remove steps without any tasks in any of its descendants
- empty_steps = root_step.get_descendants(include_self=True).filter(tasks=None).exists()
- while empty_steps:
- root_step.get_descendants(include_self=True).filter(
- child_steps__isnull=True,
- tasks=None,
- ).delete()
- empty_steps = root_step.get_descendants(
- include_self=True
- ).filter(tasks=None, child_steps__isnull=True).exists()
- except RuntimeError as e:
- logger.warning('Exception in create_workflow for ip: {}, error: {} - retry'.format(ip, e))
+ with transaction.atomic():
+ # with ProcessStep.objects.delay_mptt_updates():
+ # Create root step
+ if top_root_step:
+ root_step = ProcessStep(
+ name=name, eager=eager, information_package=ip, context=context,
+ responsible=responsible, label=label, part_root=part_root,
+ run_state=run_state)
+ root_step.parent = top_root_step
+ root_step.parent_pos = top_root_step.child_steps.count() + 1
+ root_step.save()
+ else:
+ root_step = ProcessStep.objects.create(
+ name=name, eager=eager, information_package=ip, context=context,
+ responsible=responsible, label=label, part_root=part_root,
+ run_state=run_state)
+
+ # Create on_error tasks
+ on_error_tasks = list(_create_on_error_tasks(
+ root_step, on_error, ip=ip, responsible=responsible, status=celery_states.SUCCESS))
+ ProcessTask.objects.bulk_create(on_error_tasks)
+ root_step.on_error.add(*on_error_tasks)
+
+ # Add workflow_spec steps
+ if workflow_spec:
+ _create_step(root_step, workflow_spec, ip, responsible)
+
+ # Add workflow_steps
+ if workflow_steps:
+ _add_steps(root_step, workflow_steps)
+
+ # root_step.refresh_from_db()
+ # with ProcessStep.objects.delay_mptt_updates():
+
+ # Clean up empty steps (no tasks & no children)
+ # remove steps without any tasks in any of its descendants
+ empty_steps = root_step.get_descendants(include_self=True).filter(tasks=None).exists()
+ while empty_steps:
+ root_step.get_descendants(include_self=True).filter(
+ child_steps__isnull=True,
+ tasks=None,
+ ).delete()
+ empty_steps = root_step.get_descendants(
+ include_self=True
+ ).filter(tasks=None, child_steps__isnull=True).exists()
+ # except RuntimeError as e:
+ # logger.warning('Exception in create_workflow for ip: {}, error: {} - retry'.format(ip, e))
+ # raise
+ except OperationalError as e:
+ # This is a transient DB error; trigger retry
+ logger.warning(f"OperationalError creating workflow for IP {ip}: {e} - retrying")
raise
- except RetryError:
- logger.warning('RetryError in create_workflow for ip: {}'.format(ip))
- raise
+ # except RetryError:
+ # logger.warning('RetryError in create_workflow for ip: {}'.format(ip))
+ # raise
+ except RetryError as e:
+ # Log and re-raise a simple exception that Celery can serialize
+ logger.error(f"RetryError in create_workflow for IP {ip}: {e}")
+ raise RuntimeError(f"Failed to create workflow for IP {ip} after retries") from None
+
return root_step
diff --git a/ESSArch_Core/auth/jwt_auth.py b/ESSArch_Core/auth/jwt_auth.py
new file mode 100644
index 000000000..73c1a1d06
--- /dev/null
+++ b/ESSArch_Core/auth/jwt_auth.py
@@ -0,0 +1,20 @@
+from rest_framework_simplejwt.authentication import JWTAuthentication
+
+
+class CookieJWTAuthentication(JWTAuthentication):
+
+ def authenticate(self, request):
+ raw_token = request.COOKIES.get("accessToken")
+
+ if not raw_token:
+ header = self.get_header(request)
+ if header is None:
+ return None
+
+ raw_token = self.get_raw_token(header)
+ if raw_token is None:
+ return None
+
+ validated_token = self.get_validated_token(raw_token)
+
+ return self.get_user(validated_token), validated_token
diff --git a/ESSArch_Core/auth/serializers.py b/ESSArch_Core/auth/serializers.py
index 65b43193d..412b60a44 100644
--- a/ESSArch_Core/auth/serializers.py
+++ b/ESSArch_Core/auth/serializers.py
@@ -31,6 +31,7 @@
from django.urls import reverse
from django.utils.translation import gettext_lazy as _
from rest_framework import exceptions, serializers
+from rest_framework_simplejwt.serializers import TokenObtainPairSerializer
from ESSArch_Core.auth.models import Group, Notification, UserProfile
from ESSArch_Core.auth.util import get_organization_groups
@@ -283,3 +284,16 @@ def validate(self, attrs):
attrs['user'] = user
return attrs
+
+
+class ESSArchTokenSerializer(TokenObtainPairSerializer):
+ @classmethod
+ def get_token(cls, user):
+ token = super().get_token(user)
+
+ token["username"] = user.username
+ token["first_name"] = user.first_name
+ token["last_name"] = user.last_name
+ token["email"] = user.email
+
+ return token
diff --git a/ESSArch_Core/auth/urls.py b/ESSArch_Core/auth/urls.py
index bcad0d63f..137ada16c 100644
--- a/ESSArch_Core/auth/urls.py
+++ b/ESSArch_Core/auth/urls.py
@@ -8,12 +8,23 @@
from knox import views as knox_views
from ESSArch_Core.auth.views import (
+ CookieTokenLogoutView,
+ CookieTokenObtainPairView,
+ CookieTokenObtainSSOCallbackView,
+ CookieTokenRefreshView,
LoginView,
LogoutView,
TokenLoginView,
login_services,
)
+# from rest_framework_simplejwt.views import (
+# TokenObtainPairView,
+# TokenRefreshView,
+# TokenVerifyView,
+# )
+
+
urlpatterns = [
# URLs that do not require a session or valid token
re_path(r'^password/reset/$', PasswordResetView.as_view(),
@@ -29,4 +40,14 @@
re_path(r'^token_login/$', TokenLoginView.as_view(), name='knox_login'),
re_path(r'^token_logout/$', knox_views.LogoutView.as_view(), name='knox_logout'),
re_path(r'^token_logoutall/$', knox_views.LogoutAllView.as_view(), name='knox_logoutall'),
+
+ # JWT Token
+ # re_path(r'^token/$', TokenObtainPairView.as_view(), name='jwt_token_obtain_pair'),
+ re_path(r'^token/$', CookieTokenObtainPairView.as_view(), name='jwt_token_obtain_pair'),
+ # re_path(r'^token/refresh/$', TokenRefreshView.as_view(), name='jwt_token_refresh'),
+ re_path(r'^token/refresh/$', CookieTokenRefreshView.as_view(), name='jwt_token_refresh'),
+ # re_path(r'^token/verify/$', TokenVerifyView.as_view(), name='jwt_token_verify'),
+ re_path(r'^token/logout/$', CookieTokenLogoutView.as_view(), name='jwt_token_logout'),
+ re_path(r'^saml2/jwt-api-callback/$', CookieTokenObtainSSOCallbackView.as_view(),
+ name='jwt_token_obtain_sso_callback'),
]
diff --git a/ESSArch_Core/auth/views.py b/ESSArch_Core/auth/views.py
index 7ad9ad344..c5d6c1e68 100644
--- a/ESSArch_Core/auth/views.py
+++ b/ESSArch_Core/auth/views.py
@@ -40,12 +40,18 @@
from rest_framework.authtoken.serializers import AuthTokenSerializer
from rest_framework.decorators import action, api_view, permission_classes
from rest_framework.generics import RetrieveUpdateAPIView
-from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
+from rest_framework.views import APIView
+from rest_framework_simplejwt.tokens import RefreshToken
+from rest_framework_simplejwt.views import (
+ TokenObtainPairView,
+ TokenRefreshView,
+)
from ESSArch_Core.api.filters import SearchFilter
from ESSArch_Core.auth.models import Group, Notification
from ESSArch_Core.auth.serializers import (
+ ESSArchTokenSerializer,
GroupDetailSerializer,
GroupSerializer,
LoginSerializer,
@@ -134,7 +140,7 @@ def get_serializer_class(self):
class MeView(RetrieveUpdateAPIView):
serializer_class = UserLoggedInSerializer
- permission_classes = (IsAuthenticated,)
+ permission_classes = (permissions.IsAuthenticated,)
def get_object(self):
return self.request.user
@@ -156,7 +162,7 @@ class PermissionViewSet(viewsets.ReadOnlyModelViewSet):
class NotificationViewSet(viewsets.ModelViewSet):
queryset = Notification.objects.all()
- permission_classes = (IsAuthenticated,)
+ permission_classes = (permissions.IsAuthenticated,)
filter_backends = (DjangoFilterBackend,)
filterset_fields = ('seen',)
@@ -235,3 +241,183 @@ def get(self, request, *args, **kwargs):
self.logout(request)
next_page = resolve_url(settings.LOGOUT_REDIRECT_URL)
return HttpResponseRedirect(next_page)
+
+
+class CookieTokenObtainPairView(TokenObtainPairView):
+ authentication_classes = []
+
+ def post(self, request, *args, **kwargs):
+ logger = logging.getLogger('essarch.auth')
+ serializer = self.get_serializer(data=request.data)
+
+ try:
+ if not serializer.is_valid():
+ logger.warning(serializer.errors)
+
+ return Response(
+ {"detail": "Invalid credentials"},
+ status=status.HTTP_401_UNAUTHORIZED,
+ )
+ except Exception as e:
+ logger.warning(e)
+ return Response(
+ {"detail": "Invalid refresh token"},
+ status=status.HTTP_401_UNAUTHORIZED,
+ )
+
+ access = serializer.validated_data["access"]
+ refresh = serializer.validated_data["refresh"]
+
+ response = Response(
+ {
+ # "access": access,
+ },
+ status=status.HTTP_200_OK,
+ )
+
+ response.set_cookie(
+ key="accessToken",
+ value=str(access),
+ max_age=int(settings.SIMPLE_JWT["ACCESS_TOKEN_LIFETIME"].total_seconds()),
+ **settings.JWT_AUTH_COOKIE,
+ )
+
+ response.set_cookie(
+ key="refreshToken",
+ value=str(refresh),
+ max_age=int(settings.SIMPLE_JWT["REFRESH_TOKEN_LIFETIME"].total_seconds()),
+ **settings.JWT_AUTH_COOKIE,
+ )
+
+ return response
+
+
+class CookieTokenRefreshView(TokenRefreshView):
+ authentication_classes = []
+
+ def post(self, request, *args, **kwargs):
+ logger = logging.getLogger('essarch.auth')
+ refresh = request.COOKIES.get("refreshToken")
+
+ if not refresh:
+ return Response(
+ {"detail": "Refresh token missing"},
+ status=status.HTTP_401_UNAUTHORIZED,
+ )
+
+ serializer = self.get_serializer(data={"refresh": refresh})
+
+ try:
+ if not serializer.is_valid():
+ logger.warning(serializer.errors)
+
+ return Response(
+ {"detail": "Invalid credentials"},
+ status=status.HTTP_401_UNAUTHORIZED,
+ )
+ except Exception as e:
+ logger.warning(e)
+ return Response(
+ {"detail": "Invalid refresh token"},
+ status=status.HTTP_401_UNAUTHORIZED,
+ )
+
+ access = serializer.validated_data["access"]
+
+ response = Response(
+ {
+ # "access": access,
+ },
+ status=status.HTTP_200_OK,
+ )
+
+ new_refresh = serializer.validated_data.get("refresh")
+
+ response.set_cookie(
+ key="accessToken",
+ value=str(access),
+ max_age=int(settings.SIMPLE_JWT["ACCESS_TOKEN_LIFETIME"].total_seconds()),
+ **settings.JWT_AUTH_COOKIE,
+ )
+
+ if new_refresh:
+ response.set_cookie(
+ key="refreshToken",
+ value=str(new_refresh),
+ max_age=int(settings.SIMPLE_JWT["REFRESH_TOKEN_LIFETIME"].total_seconds()),
+ **settings.JWT_AUTH_COOKIE,
+ )
+
+ return response
+
+
+class CookieTokenObtainSSOCallbackView(APIView):
+ permission_classes = [permissions.IsAuthenticated]
+ queryset = User.objects.all()
+
+ def get(self, request):
+ refresh = ESSArchTokenSerializer.get_token(request.user)
+
+ response = Response({}, status=status.HTTP_200_OK)
+
+ response.set_cookie(
+ key="accessToken",
+ value=str(refresh.access_token),
+ max_age=int(settings.SIMPLE_JWT["ACCESS_TOKEN_LIFETIME"].total_seconds()),
+ **settings.JWT_AUTH_COOKIE,
+ )
+
+ response.set_cookie(
+ key="refreshToken",
+ value=str(refresh),
+ max_age=int(settings.SIMPLE_JWT["REFRESH_TOKEN_LIFETIME"].total_seconds()),
+ **settings.JWT_AUTH_COOKIE,
+ )
+
+ return response
+
+
+class CookieTokenLogoutView(APIView):
+ permission_classes = [permissions.AllowAny]
+ authentication_classes = []
+
+ def post(self, request):
+ logger = logging.getLogger('essarch.auth')
+ refresh_token = request.COOKIES.get("refreshToken")
+
+ if not refresh_token:
+ refresh_token = request.data.get("refresh")
+
+ if refresh_token:
+ try:
+ token = RefreshToken(refresh_token)
+ token.blacklist()
+ except Exception as e:
+ logger.warning(f"Token blacklist failed: {e}")
+
+ if getattr(settings, 'ENABLE_SSO_LOGIN', False) or getattr(settings, 'ENABLE_ADFS_LOGIN', False):
+ try:
+ if _get_subject_id(request.saml_session):
+ saml2_logout().get(request)
+ except Exception:
+ logger.exception('Failed to logout using SAML, no active identity found')
+
+ response = Response(
+ {"detail": "Logged out"},
+ status=status.HTTP_200_OK,
+ )
+
+ response.delete_cookie(
+ key="refreshToken",
+ path=settings.JWT_AUTH_COOKIE.get("path", "/"),
+ domain=settings.JWT_AUTH_COOKIE.get("domain"),
+ samesite=settings.JWT_AUTH_COOKIE.get("samesite"),
+ )
+ response.delete_cookie(
+ key="accessToken",
+ path=settings.JWT_AUTH_COOKIE.get("path", "/"),
+ domain=settings.JWT_AUTH_COOKIE.get("domain"),
+ samesite=settings.JWT_AUTH_COOKIE.get("samesite"),
+ )
+
+ return response
diff --git a/ESSArch_Core/cli/main.py b/ESSArch_Core/cli/main.py
index 86a9c89c1..22c5f6706 100644
--- a/ESSArch_Core/cli/main.py
+++ b/ESSArch_Core/cli/main.py
@@ -45,6 +45,31 @@ def _loaddata(*fixture_labels):
def migrate(plan):
click.secho('Applying database migrations:', fg='green')
+ from django.db import connection
+
+ # MSSQL workaround for simplejwt blacklist migration
+ if connection.vendor == 'microsoft' and not plan:
+ try:
+ dj_call_command(
+ 'migrate',
+ 'token_blacklist',
+ '0007',
+ interactive=False,
+ verbosity=1,
+ )
+
+ dj_call_command(
+ 'migrate',
+ 'token_blacklist',
+ '0008',
+ fake=True,
+ interactive=False,
+ verbosity=1,
+ )
+ except Exception:
+ # Ignore if already migrated/faked
+ pass
+
dj_call_command(
'migrate',
interactive=False,
diff --git a/ESSArch_Core/cli/tests/test_main.py b/ESSArch_Core/cli/tests/test_main.py
index 032ae9a25..31d79cc44 100644
--- a/ESSArch_Core/cli/tests/test_main.py
+++ b/ESSArch_Core/cli/tests/test_main.py
@@ -20,7 +20,7 @@ def test_migrate(self):
with mock.patch('ESSArch_Core.cli.main.dj_call_command') as cmd:
result = runner.invoke(migrate)
- cmd.assert_called_once_with('migrate', interactive=False, verbosity=1, plan=False)
+ cmd.assert_any_call('migrate', interactive=False, verbosity=1, plan=False)
self.assertEqual(result.exit_code, 0)
def test_devserver(self):
diff --git a/ESSArch_Core/config/settings.py b/ESSArch_Core/config/settings.py
index d7214e851..e7c6a5419 100644
--- a/ESSArch_Core/config/settings.py
+++ b/ESSArch_Core/config/settings.py
@@ -1,4 +1,5 @@
import os
+import sys
import tarfile
from copy import deepcopy
from datetime import timedelta
@@ -46,6 +47,18 @@
# Set test runner
TEST_RUNNER = "ESSArch_Core.testing.runner.ESSArchTestRunner"
+IS_TESTING = len(sys.argv) > 1 and sys.argv[1] == "test"
+
+env.DB_SCHEMES['mssql'] = 'mssql'
+
+try:
+ from local_essarch_settings import DATABASE_URL
+ default_db_config = env.db_url_config(DATABASE_URL)
+except ImportError:
+ default_db_config = env.db_url('ESSARCH_DATABASE_URL', default=env.str(
+ 'DATABASE_URL_ESSARCH', default='sqlite:///db.sqlite'))
+
+IS_MSSQL = default_db_config.get('ENGINE') == 'mssql'
# Exclude file formats keys from content indexing. Example: ['fmt/569',]
EXCLUDE_FILE_FORMAT_FROM_INDEXING_CONTENT = env.list('ESSARCH_EXCLUDE_FILE_FORMAT_FROM_INDEXING_CONTENT', default=[])
@@ -64,6 +77,7 @@
'PAGE_SIZE': 10,
'DEFAULT_AUTHENTICATION_CLASSES': (
'knox.auth.TokenAuthentication',
+ "ESSArch_Core.auth.jwt_auth.CookieJWTAuthentication",
'ESSArch_Core.auth.SessionAuthentication',
'rest_framework.authentication.BasicAuthentication',
),
@@ -158,8 +172,37 @@
'ESSArch_Core.WorkflowEngine',
'ESSArch_Core.workflow',
])
+if not IS_MSSQL and not IS_TESTING:
+ INSTALLED_APPS.append('rest_framework_simplejwt.token_blacklist')
+
INSTALLED_APPS.extend(env.list('ESSARCH_INSTALLED_APPS_EXTRA', default=[]))
+AUTHENTICATION_BACKENDS = env.list('ESSARCH_AUTHENTICATION_BACKENDS', default=[
+ 'django.contrib.auth.backends.ModelBackend',
+ 'ESSArch_Core.auth.backends.GroupRoleBackend',
+ 'guardian.backends.ObjectPermissionBackend',
+])
+AUTHENTICATION_BACKENDS.extend(env.list('ESSARCH_AUTHENTICATION_BACKENDS_EXTRA', default=[]))
+
+MIDDLEWARE = env.list('ESSARCH_MIDDLEWARE', default=[
+ 'django.middleware.security.SecurityMiddleware',
+ 'django.contrib.sessions.middleware.SessionMiddleware',
+ 'django.middleware.locale.LocaleMiddleware',
+ 'corsheaders.middleware.CorsMiddleware',
+ 'django.middleware.common.CommonMiddleware',
+ 'django.middleware.csrf.CsrfViewMiddleware',
+ 'django.contrib.auth.middleware.AuthenticationMiddleware',
+ 'django.contrib.messages.middleware.MessageMiddleware',
+ 'django.middleware.clickjacking.XFrameOptionsMiddleware',
+ 'allauth.account.middleware.AccountMiddleware',
+])
+MIDDLEWARE.extend(env.list('ESSARCH_MIDDLEWARE_EXTRA', default=[]))
+
+# Database
+DATABASES = {'default': default_db_config}
+
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+
try:
import test_without_migrations # noqa
except ImportError:
@@ -167,12 +210,26 @@
else:
INSTALLED_APPS.append('test_without_migrations')
-AUTHENTICATION_BACKENDS = env.list('ESSARCH_AUTHENTICATION_BACKENDS', default=[
- 'django.contrib.auth.backends.ModelBackend',
- 'ESSArch_Core.auth.backends.GroupRoleBackend',
- 'guardian.backends.ObjectPermissionBackend',
-])
-AUTHENTICATION_BACKENDS.extend(env.list('ESSARCH_AUTHENTICATION_BACKENDS_EXTRA', default=[]))
+JWT_AUTH_COOKIE = {
+ "httponly": True,
+ "secure": True,
+ "samesite": "Lax",
+ "path": "/",
+ # "domain": ".dev.essarch.org",
+}
+
+SIMPLE_JWT = {
+ "ACCESS_TOKEN_LIFETIME": timedelta(minutes=5),
+ "REFRESH_TOKEN_LIFETIME": timedelta(days=1),
+ "AUTH_HEADER_TYPES": ("Bearer",),
+ "TOKEN_OBTAIN_SERIALIZER": "ESSArch_Core.auth.serializers.ESSArchTokenSerializer",
+ "SIGNING_KEY": "your-shared-secret",
+ "ROTATE_REFRESH_TOKENS": True,
+ "BLACKLIST_AFTER_ROTATION": True,
+}
+
+if IS_MSSQL and IS_TESTING:
+ SIMPLE_JWT["BLACKLIST_AFTER_ROTATION"] = False
GROUPS_MANAGER = {
'AUTH_MODELS_SYNC': True,
@@ -201,20 +258,6 @@
SITE_ID = 1
-MIDDLEWARE = env.list('ESSARCH_MIDDLEWARE', default=[
- 'django.middleware.security.SecurityMiddleware',
- 'django.contrib.sessions.middleware.SessionMiddleware',
- 'django.middleware.locale.LocaleMiddleware',
- 'corsheaders.middleware.CorsMiddleware',
- 'django.middleware.common.CommonMiddleware',
- 'django.middleware.csrf.CsrfViewMiddleware',
- 'django.contrib.auth.middleware.AuthenticationMiddleware',
- 'django.contrib.messages.middleware.MessageMiddleware',
- 'django.middleware.clickjacking.XFrameOptionsMiddleware',
- 'allauth.account.middleware.AccountMiddleware',
-])
-MIDDLEWARE.extend(env.list('ESSARCH_MIDDLEWARE_EXTRA', default=[]))
-
ROOT_URLCONF = 'ESSArch_Core.config.urls'
TEMPLATES = [
@@ -236,16 +279,6 @@
WSGI_APPLICATION = 'ESSArch_Core.config.wsgi.application'
-# Database
-env.DB_SCHEMES['mssql'] = 'mssql'
-try:
- from local_essarch_settings import DATABASE_URL
- DATABASES = {'default': env.db_url_config(DATABASE_URL)}
-except ImportError:
- DATABASES = {'default': env.db_url('ESSARCH_DATABASE_URL', default=env.str(
- 'DATABASE_URL_ESSARCH', default='sqlite:///db.sqlite'))}
-DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
-
# Cache
REDIS_CLIENT_CLASS = env.str('ESSARCH_REDIS_CLIENT_CLASS', 'redis.client.StrictRedis')
DJANGO_REDIS_CONNECTION_FACTORY = env.str('ESSARCH_DJANGO_REDIS_CONNECTION_FACTORY',
diff --git a/ESSArch_Core/frontend/static/frontend/scripts/controllers/StorageMigrationModalInstanceCtrl.js b/ESSArch_Core/frontend/static/frontend/scripts/controllers/StorageMigrationModalInstanceCtrl.js
index 18db458d8..3955fc46b 100644
--- a/ESSArch_Core/frontend/static/frontend/scripts/controllers/StorageMigrationModalInstanceCtrl.js
+++ b/ESSArch_Core/frontend/static/frontend/scripts/controllers/StorageMigrationModalInstanceCtrl.js
@@ -19,34 +19,61 @@ export default class StorageMigrationModalInstanceCtrl {
let methods = [];
let export_path = '';
$ctrl.migration.export_path = '';
+ let tempModel = {}; // Formly model for checkbox visibility
+
$ctrl.$onInit = function () {
if ($ctrl.data.ips == null && $ctrl.data.ip != null) {
$ctrl.data.ips = [$ctrl.data.ip];
}
+
$ctrl.pageLoading = true;
+
+ // Fetch paths
$http.get(appConfig.djangoUrl + 'paths/', {params: {pager: 'none'}}).then((response) => {
- $ctrl.pageLoading = false;
let temp = '';
response.data.forEach((x) => {
- if (x.entity === 'temp') {
- temp = x.value;
- } else if (x.entity === 'export') {
- export_path = x.value;
- }
+ if (x.entity === 'temp') temp = x.value;
+ else if (x.entity === 'export') export_path = x.value;
});
$ctrl.migration.temp_path = temp;
export_path = export_path || temp;
$ctrl.migration.policy = $ctrl.data.policy;
});
- $ctrl.migration.storage_methods = [];
+ // Fetch storage methods
let params = {policy: $ctrl.data.policy, page: 1, pager: 'none', has_enabled_target: true};
$http.get(appConfig.djangoUrl + 'storage-methods/', {params}).then((response) => {
methods = parseMethods(response.data);
- $ctrl.migration.storage_methods = methods.map((x) => {
- return x.id;
+ $ctrl.migration.storage_methods = methods.map((x) => x.id);
+ });
+
+ // -----------------------------
+ // Fetch default checkbox parameters
+ // -----------------------------
+ $http.get(appConfig.djangoUrl + 'parameters/', {params: {pager: 'none'}}).then((response) => {
+ let migrate_all_default = true; // fallback default
+ let export_copy_default = false; // fallback default
+
+ response.data.forEach((x) => {
+ if (x.entity === 'migrate_all_default') migrate_all_default = x.value === 'true';
+ if (x.entity === 'export_copy_default') export_copy_default = x.value === 'true';
});
+
+ // Apply defaults to tempModel
+ tempModel.migrate_all = migrate_all_default;
+ tempModel.export_copy = export_copy_default;
+
+ // Preselect storage methods if migrate_all_default
+ if (migrate_all_default && methods.length > 0) {
+ $ctrl.migration.storage_methods = methods.map((x) => x.id);
+ }
+
+ // Set export path if export_copy_default
+ if (export_copy_default && $ctrl.migration.export_path.length === 0) {
+ $ctrl.migration.export_path = export_path;
+ }
});
+
EditMode.enable();
};
@@ -55,19 +82,17 @@ export default class StorageMigrationModalInstanceCtrl {
const targetTranslation = $translate.instant('STORAGE_TARGET');
return methods.map((x) => {
- let temp = x.storage_method_target_relations.filter((relation) => {
- return relation.status === 1;
- });
+ let temp = x.storage_method_target_relations.filter((relation) => relation.status === 1);
let enabledTarget = {name: null, id: null};
- if (temp.length > 0) {
- enabledTarget = temp[0];
- }
+ if (temp.length > 0) enabledTarget = temp[0];
+
const methodWithTarget = `
-
- ${methodTranslation}: ${x.name}
- ${targetTranslation}: ${enabledTarget.name}
-
+
+ ${methodTranslation}: ${x.name}
+ ${targetTranslation}: ${enabledTarget.name}
+
`;
+
return {
methodWithTarget,
id: enabledTarget.storage_method,
@@ -75,8 +100,6 @@ export default class StorageMigrationModalInstanceCtrl {
});
};
- let tempModel = {};
-
$ctrl.fields = [
{
type: 'input',
@@ -88,18 +111,15 @@ export default class StorageMigrationModalInstanceCtrl {
{
type: 'checkbox',
key: 'migrate_all',
+ model: tempModel,
templateOptions: {
label: $translate.instant('MIGRATE_ALL_METHODS'),
},
- defaultValue: true,
- model: tempModel,
expressionProperties: {
'templateOptions.onChange': function ($viewValue, $modelValue, scope) {
- if ($modelValue === true && $ctrl.migration.storage_methods.length === 0) {
- $ctrl.migration.storage_methods = methods.map((x) => {
- return x.id;
- });
- } else if ($modelValue === false && $ctrl.migration.storage_methods.length !== 0) {
+ if ($modelValue && $ctrl.migration.storage_methods.length === 0) {
+ $ctrl.migration.storage_methods = methods.map((x) => x.id);
+ } else if (!$modelValue && $ctrl.migration.storage_methods.length !== 0) {
$ctrl.migration.storage_methods = [];
}
},
@@ -111,30 +131,27 @@ export default class StorageMigrationModalInstanceCtrl {
templateOptions: {
label: $translate.instant('STORAGE_METHODS'),
labelProp: 'methodWithTarget',
- multiple: true,
valueProp: 'id',
+ multiple: true,
optionsFunction: function () {
return methods;
},
appendToBody: false,
},
- hideExpression: ($viewValue, $modelValue, scope) => {
- return tempModel.migrate_all === false;
- },
+ hideExpression: () => tempModel.migrate_all === false,
},
{
type: 'checkbox',
key: 'export_copy',
+ model: tempModel,
templateOptions: {
label: $translate.instant('EXPORT_COPY'),
},
- defaultValue: false,
- model: tempModel,
expressionProperties: {
'templateOptions.onChange': function ($viewValue, $modelValue, scope) {
- if ($modelValue === true && $ctrl.migration.export_path.length === 0) {
+ if ($modelValue && $ctrl.migration.export_path.length === 0) {
$ctrl.migration.export_path = export_path;
- } else if ($modelValue === false && $ctrl.migration.export_path.length !== 0) {
+ } else if (!$modelValue && $ctrl.migration.export_path.length !== 0) {
$ctrl.migration.export_path = '';
}
},
@@ -146,9 +163,7 @@ export default class StorageMigrationModalInstanceCtrl {
templateOptions: {
label: $translate.instant('EXPORTPATH'),
},
- hideExpression: ($viewValue, $modelValue, scope) => {
- return tempModel.export_copy === false;
- },
+ hideExpression: () => tempModel.export_copy === false,
},
{
type: 'input',
@@ -164,26 +179,24 @@ export default class StorageMigrationModalInstanceCtrl {
$ctrl.form.$setSubmitted();
return;
}
+
let req_data = {};
if ($ctrl.data.ips) {
req_data = angular.extend(
{
- information_packages: $ctrl.data.ips.map((x) => {
- return x.id;
- }),
+ information_packages: $ctrl.data.ips.map((x) => x.id),
},
$ctrl.migration
);
} else if ($ctrl.data.mediums) {
req_data = angular.extend(
{
- storage_mediums: $ctrl.data.mediums.map((x) => {
- return x.id;
- }),
+ storage_mediums: $ctrl.data.mediums.map((x) => x.id),
},
$ctrl.migration
);
}
+
$ctrl.migrating = true;
return $http({
method: 'POST',
diff --git a/ESSArch_Core/ip/models.py b/ESSArch_Core/ip/models.py
index 470ef2244..51db0aa9c 100644
--- a/ESSArch_Core/ip/models.py
+++ b/ESSArch_Core/ip/models.py
@@ -344,7 +344,7 @@ def filter_objects(status):
storage_method__storage_policies=OuterRef('submission_agreement__policy'),
)
- return self.filter(
+ qs = self.filter(
Q(
Q(Exists(method_target_rel_with_migrate_target_with_ip)) &
Q(Exists(method_target_rel_with_enabled_target_without_ip)) |
@@ -353,6 +353,11 @@ def filter_objects(status):
archived=True,
).exclude(storage=None)
+ if not include_inactive_ips:
+ qs = qs.filter(active=True)
+
+ return qs
+
class InformationPackageManager(OrganizationManager):
def get_queryset(self):
diff --git a/ESSArch_Core/ip/views.py b/ESSArch_Core/ip/views.py
index ee5ae2f6a..0c8378bd4 100644
--- a/ESSArch_Core/ip/views.py
+++ b/ESSArch_Core/ip/views.py
@@ -2722,8 +2722,13 @@ def _prepare(self, request, pk):
label=parsed.get('label'),
package_mets_path=xmlfile,
)
- user_perms = perms.pop('owner', [])
+
member = Member.objects.get(django_user=request.user)
+ user_perms = perms.pop('owner', [])
+
+ organization.assign_object(ip, custom_permissions=perms)
+ organization.add_object(ip)
+
for perm in user_perms:
perm_name = get_permission_name(perm, ip)
assign_perm(perm_name, member.django_user, ip)
diff --git a/ESSArch_Core/storage/backends/tape.py b/ESSArch_Core/storage/backends/tape.py
index cb3a49722..7c08439ad 100644
--- a/ESSArch_Core/storage/backends/tape.py
+++ b/ESSArch_Core/storage/backends/tape.py
@@ -44,35 +44,37 @@ def lock_or_wait_for_drive(storage_medium, io_lock_key, wait_timeout=10 * 60):
else:
io_lock_key = pickle.dumps(io_lock_key)
- if not drive.is_locked():
- if cache.add(drive.get_lock_key(), io_lock_key):
- drive.last_change = timezone.now()
- drive.save(update_fields=['last_change'])
- logger.debug('Storage medium {} ({}) is now locked for request {}'.format(
- storage_medium.medium_id, str(storage_medium.pk), pickle.loads(io_lock_key)))
- else:
- raise ValueError('Failed to set drivelock for Storage medium {} ({}) with request {}'.format(
- storage_medium.medium_id, str(storage_medium.pk), pickle.loads(io_lock_key)))
- else:
- timeout_at = time.monotonic() + wait_timeout
- while drive.is_locked():
- drive_lock_key = pickle.loads(cache.get(drive.get_lock_key()))
- logger.debug('Storage medium {} ({}) is already locked with {} and not requested {}'.format(
- storage_medium.medium_id, str(storage_medium.pk),
- drive_lock_key, pickle.loads(io_lock_key)))
- if time.monotonic() > timeout_at:
- raise ValueError("Timeout waiting for drivelock for storage medium {} ({}) with \
-request {}".format(storage_medium.medium_id, str(storage_medium.pk), pickle.loads(io_lock_key)))
- time.sleep(1)
+ timeout_at = time.monotonic() + wait_timeout
+ while True:
+ # Try to acquire lock immediately
if cache.add(drive.get_lock_key(), io_lock_key):
drive.last_change = timezone.now()
drive.save(update_fields=['last_change'])
logger.debug('Storage medium {} ({}) is now locked for request {}'.format(
storage_medium.medium_id, str(storage_medium.pk), pickle.loads(io_lock_key)))
+ return
+
+ # Someone else holds the lock
+ raw_lock = cache.get(drive.get_lock_key())
+
+ if raw_lock is not None:
+ try:
+ drive_lock_key = pickle.loads(raw_lock)
+ except Exception:
+ drive_lock_key = ''
else:
- raise ValueError('Failed to set drivelock for Storage medium {} ({}) with request {}'.format(
- storage_medium.medium_id, str(storage_medium.pk), pickle.loads(io_lock_key)))
+ drive_lock_key = ''
+
+ logger.debug('Storage medium {} ({}) is already locked with {} and not requested {}'.format(
+ storage_medium.medium_id, str(storage_medium.pk),
+ drive_lock_key, pickle.loads(io_lock_key)))
+
+ if time.monotonic() > timeout_at:
+ raise ValueError(
+ "Timeout waiting for drivelock for storage medium {} ({}) with request {}".format(
+ storage_medium.medium_id, str(storage_medium.pk), pickle.loads(io_lock_key)))
+ time.sleep(1)
@staticmethod
def wait_for_media_transit(storage_medium, wait_timeout=10 * 60):
diff --git a/ESSArch_Core/workflow/tasks.py b/ESSArch_Core/workflow/tasks.py
index 523865d30..575d53fb6 100644
--- a/ESSArch_Core/workflow/tasks.py
+++ b/ESSArch_Core/workflow/tasks.py
@@ -86,15 +86,16 @@ def _get_workarea_path():
return os.path.join(workarea_user, ip.object_identifier_value)
ip = InformationPackage.objects.get(pk=self.ip)
+ user = User.objects.get(pk=self.responsible)
objpath = ip.object_path
workarea_path = _get_workarea_path()
shutil.copytree(objpath, workarea_path)
ip.object_path = Path(workarea_path).as_posix()
ip.save()
- Workarea.objects.create(ip=ip, user_id=self.responsible, type=Workarea.INGEST, read_only=False)
+ Workarea.objects.create(ip=ip, user=user, type=Workarea.INGEST, read_only=False)
- self.create_success_event("Received IP")
+ self.create_success_event("Received IP to workspace")
@app.task(bind=True, event_type=20100)
diff --git a/requirements/base.txt b/requirements/base.txt
index 5be950678..36dcaa501 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -1,15 +1,15 @@
asgiref==3.9.1
-boto3==1.42.89
+boto3==1.43.14
celery[tblib]==5.6.3
cffi==2.0.0
channels==4.3.2
channels-redis==4.3.0
chardet==5.2.0
-click==8.3.2
+click==8.4.1
cryptography==45.0.7
daphne==4.2.1
dj-rest-auth[with-social]==7.0.1
-django==5.2.13
+django==5.2.14
django-allauth==65.11.1
django-cors-headers==4.9.0
django-countries-plus==2.2.0
@@ -28,6 +28,7 @@ djangorestframework==3.17.1
django-json-widget==1.1.1
django-proxy==1.2.2
django-rest-knox==5.0.2
+djangorestframework-simplejwt[crypto]==5.5.1
distro==1.9.0
docker==7.1.0
drf-dynamic-fields==0.4.0
@@ -39,7 +40,7 @@ elasticsearch-dsl==7.4.1
gevent==24.11.1 ; platform_system=='Windows'
glob2==0.7
jsonfield==3.2.0
-lxml==6.0.4
+lxml==6.1.1
msoffcrypto-tool==5.4.2
natsort==8.4.0
opf-fido==1.6.1
@@ -47,8 +48,8 @@ pyfakefs==6.2.0
python-dateutil==2.8.2
pywin32==311 ; platform_system=='Windows'
redis==7.4.0
-regex==2026.4.4
-requests==2.33.1
+regex==2026.5.9
+requests==2.34.2
requests-toolbelt==1.0.0
setuptools==81.0.0
tenacity==9.1.4
diff --git a/requirements/optional.txt b/requirements/optional.txt
index 1dcb9bda7..0d841dc6c 100644
--- a/requirements/optional.txt
+++ b/requirements/optional.txt
@@ -9,5 +9,5 @@ wfastcgi==3.0.0
mod-wsgi==5.0.0
mssql-django==1.6
mysqlclient==2.2.8
-psycopg2==2.9.11
+psycopg2==2.9.12
python-logstash-async==3.0.0
diff --git a/requirements/tests.txt b/requirements/tests.txt
index e526af07e..6e83cb9e6 100644
--- a/requirements/tests.txt
+++ b/requirements/tests.txt
@@ -1,3 +1,3 @@
-coverage==7.13.5
+coverage==7.14.0
django-test-without-migrations==0.6
selenium==4.25.0