Added server reflection to services + quick tests

.travis.yml

@@ -6,11 +6,8 @@ python:
 services:
   - mysql
 before_script:
-  - pip install -r requirements.txt
   - pip install -r requirements-test.txt
   - pip install .
-  - export PYTHONPATH=$PYTHONPATH:$PWD/src
-
 script:
-  - echo "DB_HOST ENV $DB_HOST"
-  - coverage run -m pytest --server mysql://travis@127.0.0.1:3306
+  - echo "DB_HOST $METADATA_URI $TAXONOMY_URI"
+  - coverage run -m pytest -c pyproject.toml --server mysql://travis@127.0.0.1:3306

conftest.py

@@ -9,41 +9,5 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-import os
-from pathlib import Path
-
-from _pytest.config import Config
-import pytest
-import sqlalchemy as db
-
-from ensembl.production.metadata.grpc.adaptors.genome import GenomeAdaptor
-from ensembl.production.metadata.grpc.adaptors.release import ReleaseAdaptor
-
+"""Global pytest configuration for Ensembl Metadata GRPC tests."""
 pytest_plugins = ("ensembl.plugins.pytest_unittest",)
-
-
-def pytest_configure(config: Config) -> None:
-    pytest.dbs_dir = Path(__file__).parent / 'src' / 'ensembl' / 'production' / 'metadata' / 'api' / 'sample'
-
-@pytest.fixture(scope="class")
-def engine(multi_dbs):
-    os.environ["METADATA_URI"] = multi_dbs["ensembl_metadata"].dbc.url
-    os.environ["TAXONOMY_URI"] = multi_dbs["ncbi_taxonomy"].dbc.url
-    yield db.create_engine(multi_dbs["ensembl_metadata"].dbc.url)
-
-
-@pytest.fixture(scope="class")
-def genome_db_conn(multi_dbs):
-    genome_conn = GenomeAdaptor(
-        metadata_uri=multi_dbs["ensembl_metadata"].dbc.url,
-        taxonomy_uri=multi_dbs["ncbi_taxonomy"].dbc.url
-    )
-    yield genome_conn
-
-
-@pytest.fixture(scope="class")
-def release_db_conn(multi_dbs):
-    release_conn = ReleaseAdaptor(
-        metadata_uri=multi_dbs["ensembl_metadata"].dbc.url
-    )
-    yield release_conn

requirements-test.txt

@@ -3,3 +3,4 @@ pytest
 pylint
 mypy
 coverage[toml]
+pytest-grpc

requirements.in

@@ -1,5 +1,7 @@
 ensembl-py@git+https://github.com/Ensembl/ensembl-py.git@1.2.2
 grpcio
 grpcio-tools
+grpcio-reflection
 sqlalchemy
 types-pymysql
+yagrc

requirements.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile requirements.in
 #
-certifi==2023.11.17
+certifi==2024.2.2
     # via requests
 charset-normalizer==3.3.2
     # via requests
@@ -16,11 +16,17 @@ exceptiongroup==1.2.0
     # via pytest
 greenlet==3.0.3
     # via sqlalchemy
-grpcio==1.60.0
+grpcio==1.62.0
     # via
     #   -r requirements.in
+    #   grpcio-reflection
     #   grpcio-tools
-grpcio-tools==1.60.0
+    #   yagrc
+grpcio-reflection==1.62.0
+    # via
+    #   -r requirements.in
+    #   yagrc
+grpcio-tools==1.62.0
     # via -r requirements.in
 idna==3.6
     # via requests
@@ -30,11 +36,14 @@ mysqlclient==2.1.1
     # via ensembl-py
 packaging==23.2
     # via pytest
-pluggy==1.3.0
+pluggy==1.4.0
     # via pytest
-protobuf==4.25.2
-    # via grpcio-tools
-pytest==7.4.4
+protobuf==4.25.3
+    # via
+    #   grpcio-reflection
+    #   grpcio-tools
+    #   yagrc
+pytest==8.0.1
     # via
     #   ensembl-py
     #   pytest-dependency
@@ -57,8 +66,10 @@ tomli==2.0.1
     # via pytest
 types-pymysql==1.1.0.1
     # via -r requirements.in
-urllib3==2.1.0
+urllib3==2.2.1
     # via requests
+yagrc==1.1.2
+    # via -r requirements.in
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

src/ensembl/production/metadata/api/factory.py

@@ -40,7 +40,7 @@ def meta_factory(db_uri, metadata_uri, taxonomy_uri,force=False):
     # Dealing with other versionned databases like mart, ontology,...
     elif re.match(r'^\w+_?\d*_\d+$', db_url.database):
         raise Exception("other not implemented yet")
-    elif re.match(r'^ensembl_accounts|ensembl_archive|ensembl_autocomplete|ensembl_metadata|ensembl_production|'
+    elif re.match(r'^ensembl_accounts|ensembl_archive|ensembl_autocomplete|ensembl_genome_metadata|ensembl_production|'
                   r'ensembl_stable_ids|ncbi_taxonomy|ontology|website',
             db_url.database):
         raise Exception("other not implemented yet")

src/ensembl/production/metadata/api/sample/ensembl_metadata/table.sql → src/ensembl/production/metadata/api/sample/ensembl_genome_metadata/table.sql

@@ -167,7 +167,7 @@ CREATE TABLE genome
     assembly_id     int                            not null,
     organism_id     int                            not null,
     is_best         tinyint(1)   default 0         not null,
-    production_name varchar(255) default 'default' not null,
+    production_name varchar(255) not null,
     constraint genome_genome_uuid_6b62d0ad_uniq
         unique (genome_uuid),
     constraint genome_assembly_id_0a748388_fk_assembly_assembly_id

src/ensembl/production/metadata/grpc/adaptors/base.py

@@ -10,7 +10,7 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 from ensembl.database import DBConnection
-from ensembl.production.metadata.grpc.config import MetadataConfig as config
+from ensembl.production.metadata.grpc.config import MetadataConfig
 
 
 ##Todo: Add in OrganismAdapator. Subfunction fetches all organism in popular group. and # of genomes from distinct assemblies.
@@ -20,7 +20,9 @@
 
 class BaseAdaptor:
     def __init__(self, metadata_uri):
-        self.metadata_db = DBConnection(metadata_uri, pool_size=config.pool_size, pool_recycle=config.pool_recycle)
+        self.config = MetadataConfig()
+        self.metadata_db = DBConnection(metadata_uri, pool_size=self.config.pool_size,
+                                        pool_recycle=self.config.pool_recycle)
 
 
 def check_parameter(param):

src/ensembl/production/metadata/grpc/adaptors/genome.py

@@ -28,7 +28,8 @@
 class GenomeAdaptor(BaseAdaptor):
     def __init__(self, metadata_uri: str, taxonomy_uri: str):
         super().__init__(metadata_uri)
-        self.taxonomy_db = DBConnection(taxonomy_uri, pool_size=MetadataConfig.pool_size, pool_recycle=MetadataConfig.pool_recycle)
+        self.taxonomy_db = DBConnection(taxonomy_uri, pool_size=self.config.pool_size,
+                                        pool_recycle=self.config.pool_recycle)
 
     def fetch_taxonomy_names(self, taxonomy_ids, synonyms=None):
 

src/ensembl/production/metadata/grpc/config.py

@@ -48,10 +48,14 @@ def parse_boolean_var(var):
 
 
 class MetadataConfig:
-    metadata_uri = os.environ.get("METADATA_URI", f"mysql://ensembl@localhost:3306/ensembl_genome_metadata")
-    taxon_uri = os.environ.get("TAXONOMY_URI", f"mysql://ensembl@localhost:3306/ncbi_taxonomy")
-    pool_size = os.environ.get("POOL_SIZE", 20)
-    max_overflow = os.environ.get("MAX_OVERFLOW", 0)
-    pool_recycle = os.environ.get("POOL_RECYCLE", 50)
-    allow_unreleased = parse_boolean_var(os.environ.get("ALLOW_UNRELEASED", False))
-    debug_mode = parse_boolean_var(os.environ.get("DEBUG", False))
+
+    def __init__(self):
+        super().__init__()
+        self.metadata_uri = os.environ.get("METADATA_URI",
+                                           f"mysql://ensembl@localhost:3306/ensembl_genome_metadata")
+        self.taxon_uri = os.environ.get("TAXONOMY_URI", f"mysql://ensembl@localhost:3306/marco_ncbi_taxonomy")
+        self.pool_size = os.environ.get("POOL_SIZE", 20)
+        self.max_overflow = os.environ.get("MAX_OVERFLOW", 0)
+        self.pool_recycle = os.environ.get("POOL_RECYCLE", 50)
+        self.allow_unreleased = parse_boolean_var(os.environ.get("ALLOW_UNRELEASED", False))
+        self.debug_mode = parse_boolean_var(os.environ.get("DEBUG", False))

src/ensembl/production/metadata/grpc/service.py

@@ -13,30 +13,39 @@
 import grpc
 import logging
 
-from ensembl.production.metadata.grpc.config import MetadataConfig as cfg
-from ensembl.production.metadata.grpc import ensembl_metadata_pb2_grpc
+from grpc_reflection.v1alpha import reflection
+
+from ensembl.production.metadata.grpc.config import MetadataConfig
+from ensembl.production.metadata.grpc import ensembl_metadata_pb2_grpc, ensembl_metadata_pb2
 from ensembl.production.metadata.grpc.servicer import EnsemblMetadataServicer
 
 logger = logging.getLogger(__name__)
 
-# Determine the logging level based on the value of cfg.debug_mode
-log_level = logging.DEBUG if cfg.debug_mode else logging.WARNING
-
-logging.basicConfig(
-    level=log_level,
-    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-)
-
 
 def serve():
+    cfg = MetadataConfig()
+    log_level = logging.DEBUG if cfg.debug_mode else logging.WARNING
+
+    logging.basicConfig(
+        level=log_level,
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    )
     server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
     ensembl_metadata_pb2_grpc.add_EnsemblMetadataServicer_to_server(
         EnsemblMetadataServicer(), server
     )
+    SERVICE_NAMES = (
+        ensembl_metadata_pb2.DESCRIPTOR.services_by_name['EnsemblMetadata'].full_name,
+        reflection.SERVICE_NAME
+    )
+    reflection.enable_server_reflection(SERVICE_NAMES, server)
     server.add_insecure_port("[::]:50051")
     server.start()
     try:
+        logger.info(f"Starting GRPC Server from {cfg.metadata_uri}")
+        logger.info(f"DEBUG: {cfg.debug_mode}")
         server.wait_for_termination()
+        yield server
     except KeyboardInterrupt:
         logger.info("KeyboardInterrupt caught, stopping the server...")
         server.stop(grace=0)  # Immediately stop the server
@@ -45,5 +54,4 @@ def serve():
 
 if __name__ == "__main__":
     logger.info("gRPC server starting on port 50051...")
-    logger.info(f"DEBUG: {cfg.debug_mode}")
     serve()

src/ensembl/production/metadata/grpc/servicer.py

@@ -21,6 +21,7 @@
 class EnsemblMetadataServicer(ensembl_metadata_pb2_grpc.EnsemblMetadataServicer):
     def __init__(self):
         self.db = utils.connect_to_db()
+        super().__init__()
 
     def GetSpeciesInformation(self, request, context):
         logger.debug(f"Received RPC for GetSpeciesInformation with request: {request}")