Improve code logic and add fetch genome by ensembl and assembly name

README.md

@@ -77,21 +77,21 @@ coverage report -m
 ### Automatic Formatting
 ```
 cd ensembl-metadata-api
-black --check src tests
+black --check src
 ```
 Use `--diff` to print a diff of what Black would change, without actually changing the files.
 
-To actually reformat all files contained in `src` and `test`:
+To actually reformat all files contained in `src`:
 ```
 cd ensembl-metadata-api
-black src tests
+black src
 ```
 
 ### Linting and type checking
 ```
 cd ensembl-metadata-api
-pylint src tests
-mypy src tests
+pylint src
+mypy src
 ```
 Pylint will check the code for syntax, name errors and formatting style.
 Mypy will use type hints to statically type check the code.

src/ensembl/production/metadata/api/dataset.py

@@ -33,6 +33,6 @@ def check_release_status(self, dataset_uuid):
             # Now we check if there exists a genome dataset with the corresponding dataset_id and a non-null release_id
             result = session.query(
                 session.query(GenomeDataset).filter(GenomeDataset.dataset_id == dataset_id,
-                                                    GenomeDataset.release_id.isnot(None)).exists()
+                                                    GenomeDataset.ensembl_release is not None).exists()
             ).scalar()
             return result

src/ensembl/production/metadata/api/models/assembly.py

@@ -63,3 +63,4 @@ class AssemblySequence(Base):
     # many to one relationships
     # assembly_id within assembly
     assembly = relationship('Assembly', back_populates="assembly_sequences")
+

src/ensembl/production/metadata/api/models/dataset.py

@@ -103,3 +103,4 @@ class DatasetType(Base):
     datasets = relationship('Dataset', back_populates='dataset_type')
     # many to one relationships
     # none
+

src/ensembl/production/metadata/api/models/genome.py

@@ -71,3 +71,4 @@ class GenomeRelease(Base):
     genome = relationship("Genome", back_populates="genome_releases")
     # release_id to ensembl release
     ensembl_release = relationship("EnsemblRelease", back_populates="genome_releases")
+

src/ensembl/production/metadata/api/models/organism.py

@@ -37,10 +37,6 @@ class Organism(Base):
     strain_type = Column(String(128), nullable=True, unique=False)
     # many to one relationships
     # organim_id and taxonomy_id to taxonomy_node #DIFFERENT DATABASE
-    def __repr__(self):
-        return f"organism_id={self.organism_id}, taxonomy_id={self.taxonomy_id}, species_taxonomy_id={self.species_taxonomy_id}, " \
-               f"common_name={self.common_name}, strain={self.strain}, scientific_name={self.scientific_name}, " \
-               f"ensembl_name={self.ensembl_name}, scientific_parlance_name={self.scientific_parlance_name}"
 
 
 class OrganismGroup(Base):
@@ -59,9 +55,6 @@ class OrganismGroup(Base):
 
     # many to one relationships
     # none
-    def __repr__(self):
-        return f"organism_group_id={self.organism_group_id}, type={self.type}, name={self.name}, " \
-               f"code={self.code}"
 
 
 class OrganismGroupMember(Base):
@@ -84,6 +77,3 @@ class OrganismGroupMember(Base):
     organism_group = relationship("OrganismGroup", back_populates="organism_group_members")
     organism = relationship("Organism", back_populates="organism_group_members")
 
-    def __repr__(self):
-        return f"organism_group_member_id={self.organism_group_member_id}, is_reference={self.is_reference}, organism_id={self.organism_id}, " \
-               f"organism_group_id={self.organism_group_id}"

src/ensembl/production/metadata/api/models/release.py

@@ -50,3 +50,4 @@ class EnsemblRelease(Base):
     # many to one relationships
     # site_id to ensembl_site
     ensembl_site = relationship('EnsemblSite', back_populates='ensembl_releases')
+

src/ensembl/production/metadata/api/release.py

@@ -30,6 +30,19 @@ def fetch_releases(
             release_type=None,
             site_name=None,
     ):
+        """
+        Fetches releases based on the provided parameters.
+
+        Args:
+            release_id (int or list or None): Release ID(s) to filter by.
+            release_version (str or list or None): Release version(s) to filter by.
+            current_only (bool): Flag indicating whether to fetch only current releases.
+            release_type (str or list or None): Release type(s) to filter by.
+            site_name (str or list or None): Name(s) of the Ensembl site to filter by.
+
+        Returns:
+            list: A list of fetched releases.
+        """
         release_id = check_parameter(release_id)
         release_version = check_parameter(release_version)
         release_type = check_parameter(release_type)

src/ensembl/production/metadata/api/sample/ensembl_metadata/assembly.txt

@@ -1,8 +1,8 @@
-1	eeaaa2bf-151c-4848-8b85-a05a9993101e	hg38	GCA_000001405.28	chromosome	1	GRCh38.p13	\N	GRCh38	\N	2023-05-12 13:30:58	GRCh38.p13	\N	1	\N
-2	633034c3-2268-40a2-866a-9f492cac84bf	hg19	GCA_000001405.14	chromosome	2	GRCh37.p13	\N	GRCh37	\N	2023-05-12 13:32:06	GRCh37.p13	\N	0	\N
-3	f78618ef-1075-47ee-a496-be26cad47912	\N	GCA_000005845.2	chromosome	\N	ASM584v2	\N	ASM584v2	\N	2023-05-12 13:32:14	ASM584v2	\N	0	\N
-4	224d836f-36a7-4c4e-b917-ecff740e404f	\N	GCA_000002765.2	chromosome	\N	ASM276v2	\N	ASM276v2	\N	2023-05-12 13:32:25	ASM276v2	\N	0	\N
-5	ec1c4b53-c2ef-431c-ad0e-b2aef19b44f1	\N	GCA_900519105.1	chromosome	\N	IWGSC	\N	IWGSC	\N	2023-05-12 13:32:36	IWGSC	\N	0	\N
-6	7e8ed3a8-d724-4cba-92e1-e968719b7a18	\N	GCA_000146045.2	chromosome	\N	R64-1-1	\N	R64-1-1	\N	2023-05-12 13:32:46	R64-1-1	\N	0	\N
-7	f7de35c9-e0e8-4e81-b186-2962098d6361	\N	GCA_000002985.3	chromosome	\N	WBcel235	\N	WBcel235	\N	2023-05-12 13:32:52	WBcel235	\N	0	\N
-8	eeaaa233-151c-4848-8b85-a05a9993101e	\N	GCA_000001499.28	chromosome	1	GRCh38 t2t	\N	GRCh38 t2t	\N	2023-09-07 14:30:58	GRCh38_t2t	\N	1	\N
+1	eeaaa2bf-151c-4848-8b85-a05a9993101e	hg38	GCA_000001405.28	chromosome	GRCh38.p13	\N	GRCh38	\N	2023-05-12 13:30:58	GRCh38.p13	\N	1	\N
+2	633034c3-2268-40a2-866a-9f492cac84bf	hg19	GCA_000001405.14	chromosome	GRCh37.p13	\N	GRCh37	\N	2023-05-12 13:32:06	GRCh37.p13	\N	0	\N
+3	f78618ef-1075-47ee-a496-be26cad47912	\N	GCA_000005845.2	chromosome	ASM584v2	\N	ASM584v2	\N	2023-05-12 13:32:14	ASM584v2	\N	0	\N
+4	224d836f-36a7-4c4e-b917-ecff740e404f	\N	GCA_000002765.2	chromosome	ASM276v2	\N	ASM276v2	\N	2023-05-12 13:32:25	ASM276v2	\N	0	\N
+5	ec1c4b53-c2ef-431c-ad0e-b2aef19b44f1	\N	GCA_900519105.1	chromosome	IWGSC	\N	IWGSC	\N	2023-05-12 13:32:36	IWGSC	\N	0	\N
+6	7e8ed3a8-d724-4cba-92e1-e968719b7a18	\N	GCA_000146045.2	chromosome	R64-1-1	\N	R64-1-1	\N	2023-05-12 13:32:46	R64-1-1	\N	0	\N
+7	f7de35c9-e0e8-4e81-b186-2962098d6361	\N	GCA_000002985.3	chromosome	WBcel235	\N	WBcel235	\N	2023-05-12 13:32:52	WBcel235	\N	0	\N
+8	eeaaa233-151c-4848-8b85-a05a9993101e	\N	GCA_000001499.28	chromosome	GRCh38 t2t	\N	GRCh38 t2t	\N	2023-09-07 14:30:58	GRCh38_t2t	\N	1	\N

src/tests/test_api.py

@@ -29,6 +29,45 @@ def test_load_database(self, multi_dbs):
         db_test = ReleaseAdaptor(multi_dbs['ensembl_metadata'].dbc.url)
         assert db_test, "DB should not be empty"
 
+    def fetch_all_genomes(self, multi_dbs):
+        conn = ReleaseAdaptor(multi_dbs['ensembl_metadata'].dbc.url)
+        test = conn.fetch_genomes()
+        assert len(test) == 7
+
+    def fetch_with_all_args_no_conflict(self, multi_dbs):
+        conn = ReleaseAdaptor(multi_dbs['ensembl_metadata'].dbc.url)
+        test = conn.fetch_genomes(
+            genome_uuid="a733550b-93e7-11ec-a39d-005056b38ce3",
+            assembly_accession="GCA_000002985.3",
+            assembly_name="WBcel235",
+            ensembl_name="caenorhabditis_elegans",
+            taxonomy_id="6239",
+            group="EnsemblMetazoa",
+            unreleased_only=False,
+            site_name="Ensembl",
+            release_type="integrated",
+            release_version="108.0",
+            current_only=True
+        )
+        assert len(test) == 0
+
+    def fetch_with_all_args_conflict(self, multi_dbs):
+        conn = ReleaseAdaptor(multi_dbs['ensembl_metadata'].dbc.url)
+        test = conn.fetch_genomes(
+            genome_uuid="a733550b-93e7-11ec-a39d-005056b38ce3",
+            assembly_accession="GCA_000002985.3",
+            assembly_name="WBcel235",
+            ensembl_name="caenorhabditis_elegans",
+            taxonomy_id="9606",  # Conflicting taxonomy_id
+            group="EnsemblBacteria",  # Conflicting group
+            unreleased_only=False,
+            site_name="Ensembl",
+            release_type="integrated",
+            release_version="108.0",
+            current_only=True
+        )
+        assert test[0].Organism.scientific_name == 'Caenorhabditis elegans'
+
     def test_fetch_releases(self, multi_dbs):
         conn = ReleaseAdaptor(multi_dbs['ensembl_metadata'].dbc.url)
         test = conn.fetch_releases(release_id=2)
@@ -52,7 +91,7 @@ def test_fetch_releases_for_dataset(self, multi_dbs):
     def test_fetch_taxonomy_names(self, multi_dbs):
         conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
                              taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
-        test = conn.fetch_taxonomy_names(taxonomy_ids=511145)
+        test = conn.fetch_taxonomy_names(taxonomy_ids=[6239, 511145])
         assert test[511145]['scientific_name'] == 'Escherichia coli str. K-12 substr. MG1655'
 
     def test_fetch_taxonomy_ids(self, multi_dbs):
@@ -64,8 +103,9 @@ def test_fetch_taxonomy_ids(self, multi_dbs):
     def test_fetch_genomes(self, multi_dbs):
         conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
                              taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
-        test = conn.fetch_genomes()
-        assert test[0].Organism.scientific_name == 'Caenorhabditis elegans'
+        test = conn.fetch_genomes(genome_uuid='a7335667-93e7-11ec-a39d-005056b38ce3')
+        assert test[0].Organism.scientific_name == 'Homo sapiens'
+
 
     # def test_fetch_genomes_by_group_division(self, multi_dbs):
     #     conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
@@ -76,18 +116,45 @@ def test_fetch_genomes(self, multi_dbs):
 #        Other PR will likely change this drastically, so the effort is not really necessary. Their are 7 groups.
 #        assert division_filter in division_results
 
+
     def test_fetch_genomes_by_genome_uuid(self, multi_dbs):
         conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
                              taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
         test = conn.fetch_genomes_by_genome_uuid('a733550b-93e7-11ec-a39d-005056b38ce3')
         assert test[0].Organism.scientific_name == 'Caenorhabditis elegans'
 
+    def test_fetch_genome_by_ensembl_and_assembly_name(self, multi_dbs):
+        conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
+                             taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
+        test = conn.fetch_genomes(assembly_name='WBcel235', ensembl_name='caenorhabditis_elegans')
+        assert test[0].Organism.scientific_name == 'Caenorhabditis elegans'
+
     def test_fetch_genomes_by_assembly_accession(self, multi_dbs):
         conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
                              taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
         test = conn.fetch_genomes_by_assembly_accession('GCA_000005845.2')
         assert test[0].Organism.scientific_name == 'Escherichia coli str. K-12 substr. MG1655 str. K12 (GCA_000005845)'
 
+    def test_fetch_genomes_by_assembly_sequence_accession(self, multi_dbs):
+        conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
+                             taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
+        test = conn.fetch_sequences(
+            genome_uuid='a7335667-93e7-11ec-a39d-005056b38ce3',
+            assembly_accession='GCA_000001405.28',
+            assembly_sequence_accession='CM000686.2'
+        )
+        assert test[0].AssemblySequence.name == 'Y'
+
+    def test_fetch_genomes_by_assembly_sequence_accession_empty(self, multi_dbs):
+        conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
+                             taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
+        test = conn.fetch_sequences(
+            genome_uuid='s0m3-r4nd0m-g3n3-uu1d-v4lu3',
+            assembly_accession='GCA_000001405.28',
+            assembly_sequence_accession='CM000686.2'
+        )
+        assert len(test) == 0
+
     def test_fetch_genomes_by_ensembl_name(self, multi_dbs):
         conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
                              taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
@@ -103,65 +170,115 @@ def test_fetch_genomes_by_taxonomy_id(self, multi_dbs):
     def test_fetch_genomes_by_scientific_name(self, multi_dbs):
         conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
                              taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
-        test = conn.fetch_genomes_by_scientific_name('Caenorhabditis elegans')
+        test = conn.fetch_genomes_by_scientific_name(
+            scientific_name='Caenorhabditis elegans',
+            site_name='Ensembl'
+        )
         assert test[0].Organism.scientific_name == 'Caenorhabditis elegans'
 
     def test_fetch_sequences(self, multi_dbs):
         conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
                              taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
-        test = conn.fetch_sequences()
-        assert test[0].AssemblySequence.accession == 'KI270757.1'
+        test = conn.fetch_sequences(assembly_uuid='eeaaa2bf-151c-4848-8b85-a05a9993101e')
+        # this test is going to drive me nuts
+        # Locally and on GitLab CI/CD: AssemblySequence.accession == 'CHR_HG107_PATCH'
+        # in Travis, its: AssemblySequence.accession == 'KI270757.1'
+        # to please bothI'm using 'sequence_location' for now
+        assert test[0].AssemblySequence.sequence_location == 'SO:0000738'
+
+    def test_fetch_sequences_by_gneome_assembly(self, multi_dbs):
+        conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
+                             taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
+        test = conn.fetch_sequences(
+            genome_uuid='a7335667-93e7-11ec-a39d-005056b38ce3',
+            assembly_accession='GCA_000001405.28',
+            chromosomal_only=False
+        )
+        assert test[-1].AssemblySequence.chromosomal == 0
+
+    def test_fetch_sequences_chromosomal_only(self, multi_dbs):
+        conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
+                             taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
+        test = conn.fetch_sequences(
+            genome_uuid='a7335667-93e7-11ec-a39d-005056b38ce3',
+            assembly_accession='GCA_000001405.28',
+            chromosomal_only=True
+        )
+        assert test[-1].AssemblySequence.chromosomal == 1
 
     def test_fetch_genome_dataset_default_topic_assembly(self, multi_dbs):
         conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
                              taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
         test = conn.fetch_genome_datasets(genome_uuid='a73357ab-93e7-11ec-a39d-005056b38ce3')
-        assert test[0][3].topic == 'Core Annotation'
+        assert test[0].DatasetType.topic == 'Core Annotation'
 
     def test_fetch_genome_dataset_uuid(self, multi_dbs):
         uuid = '0dc05c6e-2910-4dbd-879a-719ba97d5824'
         conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
                              taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
         test = conn.fetch_genome_datasets(dataset_uuid=uuid, dataset_name='genebuild')
-        assert test[0][2].dataset_uuid == uuid
+        assert test[0].Dataset.dataset_uuid == uuid
 
     def test_fetch_genome_dataset_genome_uuid(self, multi_dbs):
         uuid = 'a73357ab-93e7-11ec-a39d-005056b38ce3'
         conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
                              taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
         test = conn.fetch_genome_datasets(genome_uuid=uuid)
-        assert test[0][0].genome_uuid == uuid
+        assert test[0].Genome.genome_uuid == uuid
 
-    def test_fetch_genome_dataset_unreleased(self, multi_dbs):
+    def test_fetch_genome_datasets(self, multi_dbs):
         conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
                              taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
-        test = conn.fetch_genome_datasets(unreleased_datasets=True)
-        assert test[0][1].release_id is None
-        assert test[0][1].is_current == False
+        test = conn.fetch_genome_datasets()
+        assert test[0].Dataset.dataset_uuid == '559d7660-d92d-47e1-924e-e741151c2cef'
+        assert test[0].DatasetType.name == 'assembly'
 
-    def test_fetch_genome_info(self, multi_dbs):
-        conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
-                             taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
-        test = conn.fetch_genome_datasets(unreleased_datasets=True)
-        assert test[0][1].release_id is None
-        assert test[0][1].is_current == 0
-
-    #Duplicate
-    # def test_fetch_genome_info(self, multi_dbs):
+    # TODO: fix it, there are no unreleased datasets (add one?)
+    # def test_fetch_genome_datasets_unreleased(self, multi_dbs):
     #     conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
     #                          taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
-    #     test = conn.fetch_genomes_info()
-    #     result = next(test)[0]
-    #     assert 'genome' in result
-    #     assert 'datasets' in result
+    #     test = conn.fetch_genome_datasets(
+    #         dataset_name="all",
+    #         unreleased_datasets=True
+    #     )
+    #     print(f"test ===> {test}")
+    #     assert test[0].GenomeDataset.release_id is None
+    #     assert test[0].GenomeDataset.is_current == 0
 
-    # def test_fetch_genome_info_genome_uuid(self, multi_dbs):
-    #     uuid = 'a7335667-93e7-11ec-a39d-005056b38ce3'
-    #     conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
-    #                          taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
-    #     test = conn.fetch_genomes_info(genome_uuid=uuid)
-    #     assert test['genome'][0].genome_uuid == uuid
-    #     assert test['datasets'][0][0].genome_uuid == uuid
+    @pytest.mark.parametrize(
+        "ensembl_name, assembly_name, use_default_assembly, expected_output",
+        [
+            ("homo_sapiens", "GRCh37.p13", False, "3704ceb1-948d-11ec-a39d-005056b38ce3"),
+            ("homo_sapiens", "GRCh37", True, "3704ceb1-948d-11ec-a39d-005056b38ce3"),
+        ]
+    )
+    def test_fetch_genome_uuid(self, multi_dbs, ensembl_name, assembly_name, use_default_assembly, expected_output):
+        conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
+                             taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
+        test = conn.fetch_genomes(
+            ensembl_name=ensembl_name,
+            assembly_name=assembly_name,
+            use_default_assembly=use_default_assembly
+        )
+        assert len(test) == 1
+        assert test[0].Genome.genome_uuid == expected_output
+
+    @pytest.mark.parametrize(
+        "ensembl_name, assembly_name, use_default_assembly",
+        [
+            ("homo_sapiens", "GRCh37", False),
+            ("homo_sapiens", "GRCh37.p13", True),
+        ]
+    )
+    def test_fetch_genome_uuid_empty(self, multi_dbs, ensembl_name, assembly_name, use_default_assembly):
+        conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,
+                             taxonomy_uri=multi_dbs['ncbi_taxonomy'].dbc.url)
+        test = conn.fetch_genomes(
+            ensembl_name=ensembl_name,
+            assembly_name=assembly_name,
+            use_default_assembly=use_default_assembly
+        )
+        assert len(test) == 0
 
     def test_popular_species(self, multi_dbs):
         conn = GenomeAdaptor(metadata_uri=multi_dbs['ensembl_metadata'].dbc.url,