Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

services: updated file schema #1693

Closed
wants to merge 3 commits into from
Closed

services: updated file schema #1693

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
58e1a25
services: updated file schema
alejandromumo Mar 15, 2024
fa82e42
schema: added width and height to file metadata
alejandromumo Apr 24, 2024
d7ec85b
schema: added storage class and uri
alejandromumo Apr 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions invenio_rdm_records/records/mappings/os-v1/rdmrecords/drafts/draft-v6.0.0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1292,6 +1292,14 @@
},
"file_id": {
"enabled": false
},
"access": {
"type": "object",
"properties": {
"hidden": {
"type": "boolean"
}
}
}
}
}
Expand Down Expand Up @@ -1351,6 +1359,14 @@
},
"file_id": {
"enabled": false
},
"access": {
"type": "object",
"properties": {
"hidden": {
"type": "boolean"
}
}
}
}
}
Expand Down
16 changes: 16 additions & 0 deletions invenio_rdm_records/records/mappings/os-v1/rdmrecords/records/record-v6.0.0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1243,6 +1243,14 @@
},
"file_id": {
"enabled": false
},
"access": {
"type": "object",
"properties": {
"hidden": {
"type": "boolean"
}
}
}
}
}
Expand Down Expand Up @@ -1302,6 +1310,14 @@
},
"file_id": {
"enabled": false
},
"access": {
"type": "object",
"properties": {
"hidden": {
"type": "boolean"
}
}
}
}
}
Expand Down
16 changes: 16 additions & 0 deletions invenio_rdm_records/records/mappings/os-v2/rdmrecords/drafts/draft-v6.0.0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1292,6 +1292,14 @@
},
"file_id": {
"enabled": false
},
"access": {
"type": "object",
"properties": {
"hidden": {
"type": "boolean"
}
}
}
}
}
Expand Down Expand Up @@ -1351,6 +1359,14 @@
},
"file_id": {
"enabled": false
},
"access": {
"type": "object",
"properties": {
"hidden": {
"type": "boolean"
}
}
}
}
}
Expand Down
16 changes: 16 additions & 0 deletions invenio_rdm_records/records/mappings/os-v2/rdmrecords/records/record-v6.0.0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1243,6 +1243,14 @@
},
"file_id": {
"enabled": false
},
"access": {
"type": "object",
"properties": {
"hidden": {
"type": "boolean"
}
}
}
}
}
Expand Down Expand Up @@ -1302,6 +1310,14 @@
},
"file_id": {
"enabled": false
},
"access": {
"type": "object",
"properties": {
"hidden": {
"type": "boolean"
}
}
}
}
}
Expand Down
16 changes: 16 additions & 0 deletions invenio_rdm_records/records/mappings/v7/rdmrecords/drafts/draft-v6.0.0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1292,6 +1292,14 @@
},
"file_id": {
"enabled": false
},
"access": {
"type": "object",
"properties": {
"hidden": {
"type": "boolean"
}
}
}
}
}
Expand Down Expand Up @@ -1351,6 +1359,14 @@
},
"file_id": {
"enabled": false
},
"access": {
"type": "object",
"properties": {
"hidden": {
"type": "boolean"
}
}
}
}
}
Expand Down
16 changes: 16 additions & 0 deletions invenio_rdm_records/records/mappings/v7/rdmrecords/records/record-v6.0.0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1243,6 +1243,14 @@
},
"file_id": {
"enabled": false
},
"access": {
"type": "object",
"properties": {
"hidden": {
"type": "boolean"
}
}
}
}
}
Expand Down Expand Up @@ -1302,6 +1310,14 @@
},
"file_id": {
"enabled": false
},
"access": {
"type": "object",
"properties": {
"hidden": {
"type": "boolean"
}
}
}
}
}
Expand Down
9 changes: 9 additions & 0 deletions invenio_rdm_records/services/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@
from .result_items import GrantItem, GrantList, SecretLinkItem, SecretLinkList
from .schemas import RDMParentSchema, RDMRecordSchema
from .schemas.community_records import CommunityRecordsSchema
from .schemas.files import FileSchema
from .schemas.parent.access import AccessSettingsSchema
from .schemas.parent.access import Grant as GrantSchema
from .schemas.parent.access import Grants as GrantsSchema
Expand Down Expand Up @@ -263,6 +264,8 @@ class RDMFileRecordServiceConfig(FileServiceConfig, ConfiguratorMixin):
),
}

file_schema = FileSchema


class RDMRecordServiceConfig(RecordServiceConfig, ConfiguratorMixin):
"""RDM record draft service config."""
Expand Down Expand Up @@ -575,6 +578,8 @@ class RDMMediaFileRecordServiceConfig(FileServiceConfig, ConfiguratorMixin):
"content": FileLink("{+api}/records/{id}/media-files/{key}/content"),
}

file_schema = FileSchema


class RDMFileDraftServiceConfig(FileServiceConfig, ConfiguratorMixin):
"""Configuration for draft files."""
Expand Down Expand Up @@ -617,6 +622,8 @@ class RDMFileDraftServiceConfig(FileServiceConfig, ConfiguratorMixin):
),
}

file_schema = FileSchema


class RDMMediaFileDraftServiceConfig(FileServiceConfig, ConfiguratorMixin):
"""Configuration for draft media files."""
Expand Down Expand Up @@ -644,3 +651,5 @@ class RDMMediaFileDraftServiceConfig(FileServiceConfig, ConfiguratorMixin):
"content": FileLink("{+api}/records/{id}/draft/media-files/{key}/content"),
"commit": FileLink("{+api}/records/{id}/draft/media-files/{key}/commit"),
}

file_schema = FileSchema
25 changes: 23 additions & 2 deletions invenio_rdm_records/services/schemas/files.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,29 @@
from invenio_vocabularies.services.schema import (
VocabularyRelationSchema as VocabularySchema,
)
from marshmallow import Schema, fields, pre_load
from marshmallow import RAISE, Schema, fields, pre_load
from marshmallow_utils.fields import NestedAttribute, SanitizedUnicode
from marshmallow_utils.permissions import FieldPermissionsMixin


class FileMetadataSchema(Schema):
"""Schema for file metadata."""

class Meta:
"""Meta."""

unknown = RAISE

page = fields.Integer()
type = fields.String()
language = fields.String()
encoding = fields.String()
charset = fields.String()
previewer = fields.String()
width = fields.Integer()
height = fields.Integer()


class FileSchema(Schema):
"""File schema."""

Expand All @@ -25,10 +43,13 @@ class FileSchema(Schema):
ext = fields.String(attribute="file.ext")
size = fields.Integer(attribute="file.size")
mimetype = fields.String(attribute="file.mimetype")
storage_class = fields.String(attribute="file.storage_class")
uri = fields.String(attribute="file.uri")

# FileRecord fields
key = SanitizedUnicode()
metadata = fields.Dict()
metadata = fields.Nested(FileMetadataSchema)
access = fields.Dict(attribute="file.access")


class FilesSchema(Schema, FieldPermissionsMixin):
Expand Down
24 changes: 15 additions & 9 deletions tests/resources/test_iiif_image_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

from io import BytesIO

import dictdiffer
from PIL import Image
from werkzeug.utils import secure_filename

Expand Down Expand Up @@ -112,15 +113,20 @@ def test_iiif_info(
recid = publish_record_with_images(client, file_id, minimal_record, headers)
response = client.get(f"/iiif/record:{recid}:{file_id}/info.json")
assert response.status_code == 200
assert response.json == {
"@context": "http://iiif.io/api/image/2/context.json",
"profile": ["http://iiif.io/api/image/2/level2.json"],
"protocol": "http://iiif.io/api/image",
"@id": f"https://127.0.0.1:5000/api/iiif/record:{recid}:{file_id}",
"tiles": [{"width": 256, "scaleFactors": [1, 2, 4, 8, 16, 32, 64]}],
"width": 1280,
"height": 1024,
}
assert not list(
dictdiffer.diff(
response.json,
{
"@context": "http://iiif.io/api/image/2/context.json",
"profile": ["http://iiif.io/api/image/2/level2.json"],
"protocol": "http://iiif.io/api/image",
"@id": f"https://127.0.0.1:5000/api/iiif/record:{recid}:{file_id}",
"tiles": [{"width": 256, "scaleFactors": [1, 2, 4, 8, 16, 32, 64]}],
"width": 1280,
"height": 1024,
},
)
)

## Testing with filename with a slash ##

Expand Down
78 changes: 78 additions & 0 deletions tests/services/files/test_metadata_files.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2024 CERN.
#
# Invenio-RDM-Records is free software; you can redistribute it and/or modify
# it under the terms of the MIT License; see LICENSE file for more details.
"""Test RDM records files (metadata)."""

from io import BytesIO

import pytest
from marshmallow.exceptions import ValidationError

from invenio_rdm_records.proxies import current_rdm_records_service


def test_valid_metadata_set(running_app, db, location, minimal_record, identity_simple):
"""Test setting file metadata."""
data = minimal_record.copy()
data["files"] = {"enabled": True}
service = current_rdm_records_service

file_service = service.files

# Create
draft = service.create(identity_simple, data)

# Initialize files and add valid metadata
metadata = {
"page": 1,
"encoding": "utf-8",
"language": "en",
"charset": "utf-8",
"type": "pdf",
}
service.draft_files.init_files(
identity_simple,
draft.id,
data=[{"key": "test.pdf", "metadata": metadata, "access": {"hidden": False}}],
)
service.draft_files.set_file_content(
identity_simple, draft.id, "test.pdf", BytesIO(b"test file")
)
service.draft_files.commit_file(identity_simple, draft.id, "test.pdf")

# Publish the record
record = service.publish(identity_simple, draft.id)

# Get file metadata
result = file_service.list_files(identity_simple, draft.id)
assert result.to_dict()["entries"][0]["metadata"] == metadata

Check failure on line 51 in tests/services/files/test_metadata_files.py

View workflow job for this annotation

GitHub Actions / Tests (3.9, postgresql14, opensearch2, 18.x) 

test_valid_metadata_set

AssertionError: assert {} == {'charset': '...page': 1, ...}
  Right contains 5 more items:
  {'charset': 'utf-8',
   'encoding': 'utf-8',
   'language': 'en',
   'page': 1,
   'type': 'pdf'}
  Full diff:
    {
  +  ,
  -  'charset': 'utf-8',
  -  'encoding': 'utf-8',
  -  'language': 'en',
  -  'page': 1,
  -  'type': 'pdf',
    }

Check failure on line 51 in tests/services/files/test_metadata_files.py

View workflow job for this annotation

GitHub Actions / Tests (3.9, postgresql14, opensearch2, 20.x) 

test_valid_metadata_set

AssertionError: assert {} == {'charset': '...page': 1, ...}
  Right contains 5 more items:
  {'charset': 'utf-8',
   'encoding': 'utf-8',
   'language': 'en',
   'page': 1,
   'type': 'pdf'}
  Full diff:
    {
  +  ,
  -  'charset': 'utf-8',
  -  'encoding': 'utf-8',
  -  'language': 'en',
  -  'page': 1,
  -  'type': 'pdf',
    }

Check failure on line 51 in tests/services/files/test_metadata_files.py

View workflow job for this annotation

GitHub Actions / Tests (3.12, postgresql14, opensearch2, 18.x) 

test_valid_metadata_set

AssertionError: assert {} == {'charset': '...page': 1, ...}
  Right contains 5 more items:
  {'charset': 'utf-8',
   'encoding': 'utf-8',
   'language': 'en',
   'page': 1,
   'type': 'pdf'}
  Full diff:
    {
  +  ,
  -  'charset': 'utf-8',
  -  'encoding': 'utf-8',
  -  'language': 'en',
  -  'page': 1,
  -  'type': 'pdf',
    }

Check failure on line 51 in tests/services/files/test_metadata_files.py

View workflow job for this annotation

GitHub Actions / Tests (3.12, postgresql14, opensearch2, 20.x) 

test_valid_metadata_set

AssertionError: assert {} == {'charset': '...page': 1, ...}
  Right contains 5 more items:
  {'charset': 'utf-8',
   'encoding': 'utf-8',
   'language': 'en',
   'page': 1,
   'type': 'pdf'}
  Full diff:
    {
  +  ,
  -  'charset': 'utf-8',
  -  'encoding': 'utf-8',
  -  'language': 'en',
  -  'page': 1,
  -  'type': 'pdf',
    }
assert result.to_dict()["entries"][0]["access"]["hidden"] is False # default value


def test_invalid_metadata_set(
running_app, db, location, minimal_record, identity_simple
):
"""Test setting file metadata with invalid data."""
data = minimal_record.copy()
data["files"] = {"enabled": True}
service = current_rdm_records_service

# Create
draft = service.create(identity_simple, data)

# Initialize files and add valid metadata
metadata = {
"page": 1,
"encoding": "utf-8",
"language": "en",
"charset": "utf-8",
"type": "pdf",
"invalid": "invalid",
}
with pytest.raises(ValidationError):
service.draft_files.init_files(
identity_simple, draft.id, data=[{"key": "test.pdf", "metadata": metadata}]
)
Loading