fixed migration to psql

isatools/database/__init__.py

@@ -1,5 +1,5 @@
 """
-The isatools.database package contains the SQLAlchemy models for the ISA tools library.
+The isatools database package contains the SQLAlchemy models for the ISA tools library.
 It works by dynamically adding methods to the models defined in the isatools.model package.
 
 The database model is highly sensitive to identifiers. We suggest serializing all your ISA-tab files to json
@@ -10,4 +10,8 @@
 """
 
 from isatools.database.utils import app, db
-from isatools.database.models import *
+from isatools.database.models import (
+    Comment, Publication, Investigation, Study, OntologyAnnotation, OntologySource,
+    Parameter, Person, Process, Protocol, Source, Characteristic, Factor, Sample,
+    FactorValue, Material, ParameterValue, Assay, Datafile as DataFile
+)

isatools/database/models/__init__.py

@@ -54,7 +54,7 @@
     AssayModel as Assay, Assay as AssayTable, make_assay_methods
 )
 from isatools.database.models.datafile import (
-    DataFileModel as DataFile, DataFile as DataFileTable, make_datafile_methods
+    DataFileModel as Datafile, Datafile as DatafileTable, make_datafile_methods
 )
 
 

isatools/database/models/assay.py

@@ -29,10 +29,10 @@ class Assay(Base):
     studies: relationship = relationship('Study', secondary=study_assays, back_populates='assays')
 
     # Relationship many-to-one
-    measurement_type_id: int = Column(Integer, ForeignKey('ontology_annotation.ontology_annotation_id'))
+    measurement_type_id: str = Column(String, ForeignKey('ontology_annotation.ontology_annotation_id'))
     measurement_type: relationship = relationship(
         'OntologyAnnotation', backref='measurement_type', foreign_keys=[measurement_type_id])
-    technology_type_id: int = Column(Integer, ForeignKey('ontology_annotation.ontology_annotation_id'))
+    technology_type_id: str = Column(String, ForeignKey('ontology_annotation.ontology_annotation_id'))
     technology_type: relationship = relationship(
         'OntologyAnnotation', backref='technology_type', foreign_keys=[technology_type_id])
 
@@ -44,7 +44,7 @@ class Assay(Base):
         'OntologyAnnotation', secondary=assay_characteristic_categories, back_populates='assays_characteristics')
     samples: relationship = relationship('Sample', secondary=assay_samples, back_populates='assays')
     materials: relationship = relationship('Material', secondary=assay_materials, back_populates='assays')
-    datafiles: relationship = relationship('DataFile', secondary=assay_data_files, back_populates='assays')
+    datafiles: relationship = relationship('Datafile', secondary=assay_data_files, back_populates='assays')
 
     # Relationships: one-to-many
     comments: relationship = relationship('Comment', back_populates='assay')

isatools/database/models/characteristic.py

@@ -1,7 +1,7 @@
 from sqlalchemy import Column, Integer, ForeignKey, Float, String
 from sqlalchemy.orm import relationship, Session
 
-from isatools.model import Characteristic as CharacteristicModel
+from isatools.model import Characteristic as CharacteristicModel, OntologyAnnotation as OntologyAnnotationModel
 from isatools.database.models.relationships import (
     source_characteristics,
     sample_characteristics,
@@ -31,18 +31,18 @@ class Characteristic(Base):
         'Material', secondary=materials_characteristics, back_populates='characteristics')
 
     # Relationships many-to-one
-    value_id: int = Column(Integer, ForeignKey(
+    value_id: str = Column(String, ForeignKey(
         'ontology_annotation.ontology_annotation_id'), comment='Value of the characteristic as an OntologyAnnotation')
     value_oa: relationship = relationship(
         'OntologyAnnotation', backref='characteristics_value', foreign_keys=[value_id])
 
-    unit_id: int = Column(
-        Integer, ForeignKey('ontology_annotation.ontology_annotation_id'),
+    unit_id: str = Column(
+        String, ForeignKey('ontology_annotation.ontology_annotation_id'),
         comment='Characteristic unit as an ontology annotation')
     unit_oa: relationship = relationship('OntologyAnnotation', backref='characteristics_unit', foreign_keys=[unit_id])
 
-    category_id: int = Column(
-        Integer, ForeignKey('ontology_annotation.ontology_annotation_id'),
+    category_id: str = Column(
+        String, ForeignKey('ontology_annotation.ontology_annotation_id'),
         comment='Characteristic category as an ontology annotation')
     category_oa: relationship = relationship(
         'OntologyAnnotation', backref='characteristics_category', foreign_keys=[category_id])
@@ -91,8 +91,11 @@ def to_sql(self, session: Session) -> Characteristic:
             if isinstance(self.value, int):
                 value = float(self.value)
             characteristic["value_int"] = value
-        else:
+        elif not isinstance(self.value, str):
             characteristic["value_oa"] = self.value.to_sql(session)
+        else:
+            value = OntologyAnnotationModel(term=self.value)
+            characteristic["value_oa"] = value.to_sql(session)
 
         if isinstance(self.unit, str):
             characteristic["unit_str"] = self.unit

isatools/database/models/comment.py

@@ -23,31 +23,31 @@ class Comment(Base):
     assay: relationship = relationship('Assay', back_populates='comments')
     characteristic_id: int = Column(Integer, ForeignKey('characteristic.characteristic_id'))
     characteristic: relationship = relationship('Characteristic', back_populates='comments')
-    datafile_id: int = Column(Integer, ForeignKey('datafile.datafile_id'))
-    datafile: relationship = relationship('DataFile', back_populates='comments')
+    datafile_id: str = Column(String, ForeignKey('datafile.datafile_id'))
+    datafile: relationship = relationship('Datafile', back_populates='comments')
     factor_value_id: int = Column(Integer, ForeignKey('factor_value.factor_value_id'))
     factor_value: relationship = relationship('FactorValue', back_populates='comments')
     investigation_id: int = Column(Integer, ForeignKey('investigation.investigation_id'))
     investigation: relationship = relationship('Investigation', back_populates='comments')
-    material_id: int = Column(Integer, ForeignKey('material.material_id'))
+    material_id: str = Column(String, ForeignKey('material.material_id'))
     material: relationship = relationship('Material', back_populates='comments')
-    ontology_source_id: int = Column(Integer, ForeignKey('ontology_source.ontology_source_id'))
+    ontology_source_id: str = Column(String, ForeignKey('ontology_source.ontology_source_id'))
     ontology_source: relationship = relationship('OntologySource', back_populates='comments')
-    ontology_annotation_id: int = Column(Integer, ForeignKey('ontology_annotation.ontology_annotation_id'))
+    ontology_annotation_id: str = Column(String, ForeignKey('ontology_annotation.ontology_annotation_id'))
     ontology_annotation: relationship = relationship('OntologyAnnotation', back_populates='comments')
     person_id: int = Column(Integer, ForeignKey('person.person_id'))
     person: relationship = relationship('Person', back_populates='comments')
-    process_id: int = Column(Integer, ForeignKey('process.process_id'))
+    process_id: str = Column(String, ForeignKey('process.process_id'))
     process: relationship = relationship('Process', back_populates='comments')
-    protocol_id: int = Column(Integer, ForeignKey('protocol.protocol_id'))
+    protocol_id: str = Column(String, ForeignKey('protocol.protocol_id'))
     protocol: relationship = relationship('Protocol', back_populates='comments')
-    publication_id: int = Column(Integer, ForeignKey('publication.publication_id'))
+    publication_id: str = Column(String, ForeignKey('publication.publication_id'))
     publication: relationship = relationship('Publication', back_populates='comments')
-    sample_id: int = Column(Integer, ForeignKey('sample.sample_id'))
+    sample_id: str = Column(String, ForeignKey('sample.sample_id'))
     sample: relationship = relationship('Sample', back_populates='comments')
-    source_id: int = Column(Integer, ForeignKey('source.source_id'))
+    source_id: str = Column(String, ForeignKey('source.source_id'))
     source: relationship = relationship('Source', back_populates='comments')
-    study_factor_id: int = Column(Integer, ForeignKey('factor.factor_id'))
+    study_factor_id: str = Column(String, ForeignKey('factor.factor_id'))
     study_factor: relationship = relationship('StudyFactor', back_populates='comments')
     study_id: int = Column(Integer, ForeignKey('study.study_id'))
     study: relationship = relationship('Study', back_populates='comments')

isatools/database/models/constraints.py

@@ -51,5 +51,5 @@ def build_material_constraints() -> CheckConstraint:
     :return: A CheckConstraint object used to validate the Material table.
     """
     statement = '''NOT (material_type IS NOT NULL 
-                AND material_type != "Extract Name" AND material_type != "Labeled Extract Name")'''
+                AND material_type != 'Extract Name' AND material_type != 'Labeled Extract Name')'''
     return CheckConstraint(statement, name='material_type_must_be_extract_name_or_labeled_extract_name')

isatools/database/models/datafile.py

@@ -7,11 +7,11 @@
 from isatools.database.models.utils import make_get_table_method
 
 
-class DataFile(InputOutput):
+class Datafile(InputOutput):
     """ The SQLAlchemy model for the Material table """
 
     __tablename__: str = 'datafile'
-    __mapper_args__: dict = {"polymorphic_identity": "datafile", "concrete": True}
+    __mapper_args__: dict = {"polymorphic_identity": "Datafile", "concrete": True}
 
     # Base fields
     datafile_id: str = Column(String, primary_key=True)
@@ -27,22 +27,22 @@ class DataFile(InputOutput):
     def to_json(self):
         return {
             '@id': self.datafile_id,
-            'filename': self.filename,
+            'name': self.filename,
             'type': self.label,
             'comments': [comment.to_json() for comment in self.comments]
         }
 
 
 def make_datafile_methods():
-    def to_sql(self, session: Session) -> DataFile:
-        datafile = session.query(DataFile).get(self.id)
+    def to_sql(self, session: Session) -> Datafile:
+        datafile = session.query(Datafile).get(self.id)
         if datafile:
             return datafile
-        return DataFile(
+        return Datafile(
             datafile_id=self.id,
             filename=self.filename,
             label=self.label,
             comments=[comment.to_sql() for comment in self.comments]
         )
     setattr(DataFileModel, 'to_sql', to_sql)
-    setattr(DataFileModel, 'get_table', make_get_table_method(DataFile))
+    setattr(DataFileModel, 'get_table', make_get_table_method(Datafile))

isatools/database/models/factor_value.py

@@ -49,7 +49,7 @@ def to_json(self) -> dict:
         elif self.value_str:
             value = self.value_str
         else:
-            value = self.value_oa.to_json()
+            value = self.value_oa.to_json() if self.value_oa else None
         if self.factor_name:
             category = {"@id": self.factor_name.factor_id}
         if self.factor_unit:
@@ -75,8 +75,8 @@ def to_sql(self, session: Session) -> FactorValue:
             'comments': [comment.to_sql(session) for comment in self.comments]
         }
         value = self.value if self.value else ''
-        if isinstance(value, int):
-            factor_value['value_int'] = value
+        if isinstance(value, int) or isinstance(value, float):
+            factor_value['value_int'] = float(value)
         elif isinstance(value, str):
             factor_value['value_str'] = value
         elif isinstance(value, OntologyAnnotationModel):

isatools/database/models/ontology_annotation.py

@@ -36,7 +36,7 @@ class OntologyAnnotation(Base):
         'Assay', secondary=assay_characteristic_categories, back_populates='characteristic_categories')
 
     # Relationships many-to-one
-    term_source_id: int = Column(Integer, ForeignKey('ontology_source.ontology_source_id'))
+    term_source_id: str = Column(String, ForeignKey('ontology_source.ontology_source_id'))
     term_source: relationship = relationship('OntologySource', backref='ontology_annotations')
 
     # References: one-to-many
@@ -73,11 +73,12 @@ def to_sql(self, session):
         oa = session.query(OntologyAnnotation).get(self.id)
         if oa:
             return oa
+        term_source_id = self.term_source.to_sql(session) if self.term_source else None
         oa = OntologyAnnotation(
             ontology_annotation_id=self.id,
             annotation_value=self.term,
             term_accession=self.term_accession,
-            term_source_id=self.term_source.name if self.term_source else None,
+            term_source_id=term_source_id.ontology_source_id if term_source_id else None,
             comments=[comment.to_sql() for comment in self.comments]
         )
         session.add(oa)

isatools/database/models/ontology_source.py

@@ -57,12 +57,15 @@ def to_sql(self, session) -> OntologySource:
         ontology_source = session.query(OntologySource).get(self.name)
         if ontology_source:
             return ontology_source
-        return OntologySource(
+        os = OntologySource(
             ontology_source_id=self.name,
             name=self.name,
             file=self.file,
             version=self.version,
             description=self.description,
         )
+        session.add(os)
+        session.commit()
+        return os
     setattr(OntologySourceModel, 'to_sql', to_sql)
     setattr(OntologySourceModel, 'get_table', make_get_table_method(OntologySource))

isatools/database/models/parameter_value.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Column, Integer, ForeignKey
+from sqlalchemy import Column, Integer, ForeignKey, String
 from sqlalchemy.orm import relationship, Session
 
 from isatools.model import ParameterValue as ParameterValueModel
@@ -23,13 +23,13 @@ class ParameterValue(Base):
     )
 
     # Relationships many-to-one
-    value_id: int = Column(Integer, ForeignKey('ontology_annotation.ontology_annotation_id'))
+    value_id: str = Column(String, ForeignKey('ontology_annotation.ontology_annotation_id'))
     value_oa: relationship = relationship(
         'OntologyAnnotation', backref='parameter_values', foreign_keys=[value_id])
-    unit_id: int = Column(Integer, ForeignKey('ontology_annotation.ontology_annotation_id'))
+    unit_id: str = Column(String, ForeignKey('ontology_annotation.ontology_annotation_id'))
     unit: relationship = relationship(
         'OntologyAnnotation', backref='parameter_values_unit', foreign_keys=[unit_id])
-    category_id: int = Column(Integer, ForeignKey('parameter.parameter_id'))
+    category_id: str = Column(String, ForeignKey('parameter.parameter_id'))
     category: relationship = relationship('Parameter', backref='parameter_values')
 
     def to_json(self) -> dict:

isatools/database/models/process.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 
-from sqlalchemy import Column, Integer, String, ForeignKey, Date
+from sqlalchemy import Column, Integer, String, ForeignKey, Date, update
 from sqlalchemy.orm import relationship, Session
 
 from isatools.model import Process as ProcessModel
@@ -21,8 +21,8 @@ class Process(Base):
     date: datetime = Column(Date)
 
     # Relationships self-referential
-    previous_process_id: int = Column(Integer, ForeignKey('process.process_id'))
-    next_process_id: int = Column(Integer, ForeignKey('process.process_id'))
+    previous_process_id: str = Column(String, ForeignKey('process.process_id'))
+    next_process_id: str = Column(String, ForeignKey('process.process_id'))
 
     # Relationships back reference
     study_id: int = Column(Integer, ForeignKey('study.study_id'))
@@ -31,7 +31,7 @@ class Process(Base):
     assay: relationship = relationship('Assay', back_populates='process_sequence')
 
     # Relationships: many-to-one
-    protocol_id: int = Column(Integer, ForeignKey('protocol.protocol_id'))
+    protocol_id: str = Column(String, ForeignKey('protocol.protocol_id'))
     protocol: relationship = relationship('Protocol', backref='processes')
 
     # Relationships: many-to-many
@@ -52,9 +52,9 @@ def to_json(self) -> dict:
             '@id': self.process_id,
             'name': self.name,
             'performer': self.performer,
-            'date': str(self.date),
-            'input': [{"@id": data_input.io_id} for data_input in self.inputs],
-            'output': [{"@id": output.io_id} for output in self.outputs],
+            'date': str(self.date) if self.date else '',
+            'inputs': [{"@id": data_input.io_id} for data_input in self.inputs],
+            'outputs': [{"@id": data_output.io_id} for data_output in self.outputs],
             'parameterValues': [pv.to_json() for pv in self.parameter_values],
             'previous_process': {"@id": self.previous_process_id} if self.previous_process_id else None,
             'next_process': {"@id": self.next_process_id} if self.next_process_id else None,
@@ -77,40 +77,43 @@ def to_sql(self, session: Session) -> Process:
 
         :return: The SQLAlchemy object ready to be committed to the database session.
         """
-
         process = session.query(Process).get(self.id)
         if process:
             return process
 
         inputs = []
         for data_input in self.inputs:
-            input_ = session.query(InputOutput).filter(InputOutput.io_id == data_input.id).first()
-            if input_:
-                inputs.append(input_)
-            else:
-                inputs.append(InputOutput(io_id=data_input.id, io_type='input'))
+            inputs.append(InputOutput(io_id=data_input.id, io_type='input'))
 
         outputs = []
         for data_output in self.outputs:
-            output_ = session.query(InputOutput).filter(InputOutput.io_id == data_output.id).first()
-            if output_:
-                outputs.append(output_)
-            else:
-                outputs.append(InputOutput(io_id=data_output.id, io_type='output'))
+            outputs.append(InputOutput(io_id=data_output.id, io_type='output'))
 
         return Process(
             process_id=self.id,
             name=self.name,
             performer=self.performer,
             date=datetime.strptime(self.date) if self.date else None,
             comments=[comment.to_sql() for comment in self.comments],
-            previous_process_id=self.prev_process.id if self.prev_process else None,
-            next_process_id=self.next_process.id if self.next_process else None,
             protocol_id=self.executes_protocol.id,
             inputs=inputs,
             outputs=outputs,
             parameter_values=[parameter_value.to_sql(session) for parameter_value in self.parameter_values]
         )
 
+    def update_plink(self, session: Session):
+        """ Update the previous and next process links for the process.
+
+        :param self: The Process object. Will be injected automatically.
+        :param session: The SQLAlchemy session to use.
+        """
+        statement = update(Process).where(Process.process_id == self.id).values(
+            previous_process_id=self.prev_process.id if self.prev_process else None,
+            next_process_id=self.next_process.id if self.next_process else None
+        )
+        session.execute(statement)
+        session.commit()
+
     setattr(ProcessModel, 'to_sql', to_sql)
+    setattr(ProcessModel, 'update_plink', update_plink)
     setattr(ProcessModel, 'get_table', make_get_table_method(Process))

isatools/database/models/protocol.py

@@ -30,7 +30,7 @@ class Protocol(Base):
         'Parameter', secondary=protocol_parameters, back_populates='protocols')
 
     # Relationships many-to-one
-    protocol_type_id: int = Column(Integer, ForeignKey('ontology_annotation.ontology_annotation_id'))
+    protocol_type_id: str = Column(String, ForeignKey('ontology_annotation.ontology_annotation_id'))
     protocol_type: relationship = relationship('OntologyAnnotation', backref='protocols')
 
     def to_json(self) -> dict: