fix: make fields optional in Pydantic models for better flexibility

welearn_datastack/data/source_models/hal.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import List, Optional
 
 from pydantic import BaseModel
 
@@ -7,12 +7,12 @@ class Doc(BaseModel):
     docid: str
     title_s: List[str]
     abstract_s: List[str]
-    authFullName_s: List[str]
-    language_s: List[str]
+    authFullName_s: Optional[List[str]]
+    language_s: Optional[List[str]] = None
     uri_s: str
-    docType_s: str
-    producedDate_tdate: str
-    publicationDate_tdate: str
+    docType_s: Optional[str]
+    producedDate_tdate: Optional[str]
+    publicationDate_tdate: Optional[str] = None
     halId_s: str
 
 
@@ -25,4 +25,4 @@ class Response(BaseModel):
 
 class HALModel(BaseModel):
     response: Response
-    nextCursorMark: str
+    nextCursorMark: Optional[str] = None

welearn_datastack/data/source_models/oapen.py

@@ -6,53 +6,53 @@
 
 
 class CheckSum(BaseModel):
-    value: str
-    checkSumAlgorithm: str
+    value: Optional[str] = None
+    checkSumAlgorithm: Optional[str] = None
 
 
 class Bitstream(BaseModel):
-    uuid: str
-    name: str
-    handle: Any
-    type: str
-    expand: List[str]
-    bundleName: str
-    description: Optional[str]
-    format: str
-    mimeType: str
-    sizeBytes: int
-    parentObject: Any
-    retrieveLink: str
-    checkSum: CheckSum
-    sequenceId: int
-    code: str
-    policies: Any
-    link: str
-    metadata: List[Metadatum]
+    uuid: Optional[str] = None
+    name: Optional[str] = None
+    handle: Optional[Any] = None
+    type: Optional[str] = None
+    expand: Optional[List[str]] = None
+    bundleName: Optional[str] = None
+    description: Optional[str] = None
+    format: Optional[str] = None
+    mimeType: Optional[str] = None
+    sizeBytes: Optional[int] = None
+    parentObject: Optional[Any] = None
+    retrieveLink: Optional[str] = None
+    checkSum: Optional[CheckSum] = None
+    sequenceId: Optional[int] = None
+    code: Optional[str] = None
+    policies: Optional[Any] = None
+    link: Optional[str] = None
+    metadata: Optional[List[Metadatum]] = None
 
 
 class Metadatum(BaseModel):
-    key: str
-    value: str
-    language: Optional[str]
-    schema_: str
-    element: str
-    qualifier: Optional[str]
+    key: Optional[str] = None
+    value: Optional[str] = None
+    language: Optional[str] = None
+    schema_: Optional[str] = None
+    element: Optional[str] = None
+    qualifier: Optional[str] = None
     code: Optional[str] = None
 
 
 class OapenModel(BaseModel):
-    uuid: str
+    uuid: Optional[str] = None
     name: str
     handle: str
-    type: str
-    expand: List[str]
-    lastModified: str
-    parentCollection: Any
-    parentCollectionList: Any
-    parentCommunityList: Any
+    type: Optional[str] = None
+    expand: Optional[List[str]] = None
+    lastModified: Optional[str] = None
+    parentCollection: Optional[Any] = None
+    parentCollectionList: Optional[Any] = None
+    parentCommunityList: Optional[Any] = None
     bitstreams: List[Bitstream]
-    archived: str
-    withdrawn: str
-    link: str
+    archived: Optional[str] = None
+    withdrawn: Optional[str] = None
+    link: Optional[str] = None
     metadata: List[Metadatum]

welearn_datastack/data/source_models/ted.py

@@ -1,3 +1,5 @@
+from typing import List, Optional
+
 from pydantic import BaseModel
 
 
@@ -21,17 +23,17 @@ class Cue(BaseModel):
 
 
 class Paragraph(BaseModel):
-    cues: list[Cue]
+    cues: Optional[List[Cue]] = None
 
 
 class Translation(BaseModel):
-    paragraphs: list[Paragraph]
+    paragraphs: Optional[List[Paragraph]] = None
 
 
 class TEDData(BaseModel):
-    video: Video
-    translation: Translation
+    video: Optional[Video] = None
+    translation: Optional[Translation] = None
 
 
 class TEDModel(BaseModel):
-    data: TEDData
+    data: Optional[TEDData] = None