tries fixing CI

Author: WolfgangFahl

lodstorage/query.py

@@ -17,12 +17,14 @@
 from lodstorage.mwTable import MediaWikiTable
 from lodstorage.params import Param, Params
 from lodstorage.prefix_config import PrefixConfigs
+from lodstorage.prefixes import Prefixes
 from pygments import highlight
 from pygments.formatters.html import HtmlFormatter
 from pygments.formatters.latex import LatexFormatter
 from pygments.lexers import get_lexer_by_name
 from tabulate import tabulate
 import yaml
+
 from lodstorage.yaml_path import YamlPath
 
 
@@ -424,11 +426,29 @@ def getLink(self, url, title, tablefmt):
             markup = r"\href{%s}{%s}" % (url, title)
         return markup
 
-    def add_endpoint_prefixes(self, endpoint: "Endpoint", prefix_configs: PrefixConfigs):
+
+    def add_endpoint_prefixes(self, endpoint: "Endpoint", prefix_configs: PrefixConfigs) -> None:
+        """
+        Add endpoint-specific PREFIX declarations to this query (via prefix_sets or legacy prefixes).
+
+        Merges (deduplicates by prefix name) endpoint prefixes into self.query using Prefixes.merge_prefixes().
+        Updates self.prefixes to full unique PREFIX lines list. Safe/idempotent (no-op if prefixes_str empty).
+
+        Args:
+            endpoint (Endpoint): Endpoint config with prefix_sets or legacy prefixes.
+            prefix_configs (PrefixConfigs): Loaded prefix configurations resolver.
+        """
         prefixes_str = endpoint.get_prefixes(prefix_configs)
-        if prefixes_str:
-            self.prefixes = prefixes_str.splitlines()
-            self.query = f"{prefixes_str}\n{self.query}"
+        if not prefixes_str.strip():
+            return
+
+        # Merge: Prepend ONLY missing prefixes (no dups like 'rdfs')
+        self.query = Prefixes.merge_prefixes(self.query, prefixes_str)
+
+        # Update self.prefixes: Full unique lines from merged query
+        prefix_dict = Prefixes.extract_prefixes(self.query)
+        self.prefixes = [Prefixes.prefix_line(prefix_dict, prefix) for prefix in sorted(prefix_dict)]
+
 
     def prefixToLink(self, lod: list, prefix: str, tablefmt: str):
         """

lodstorage/querymain.py

@@ -4,31 +4,33 @@
 @author: wf
 """
 
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
 import logging
+import os
+import re
+import sys
+import traceback
 import urllib.parse
 
 from lodstorage.mysql import MySqlQuery
+from lodstorage.prefix_config import PrefixConfigs
+from lodstorage.query import Endpoint, EndpointManager, Format, ValueFormatter
+from lodstorage.query_cmd import QueryCmd
 from lodstorage.rate_limiter import RateLimiter
+from lodstorage.sparql import SPARQL
+from lodstorage.sql import SQLDB
 from lodstorage.version import Version  # Use sqlq.py module for MySQL endpoints
+import requests
+
 
 __version__ = Version.version
 __date__ = Version.date
 __updated__ = Version.updated
 
 DEBUG = 0
 
-import os
-import re
-import sys
-import traceback
-from argparse import ArgumentParser, RawDescriptionHelpFormatter
 
-import requests
 
-from lodstorage.query import Endpoint, EndpointManager, Format, ValueFormatter
-from lodstorage.query_cmd import QueryCmd
-from lodstorage.sparql import SPARQL
-from lodstorage.sql import SQLDB
 
 
 class QueryMain(QueryCmd):
@@ -78,9 +80,7 @@ def handle_args(self) -> bool:
             if args.language == "sparql":
                 sparql = SPARQL.fromEndpointConf(endpointConf)
                 if args.prefixes and endpointConf is not None:
-                    prefixes_list = endpointConf.prefixes.split("\n")
-                    self.query.prefixes = prefixes_list
-                    self.queryCode = f"{endpointConf.prefixes}\n{self.queryCode}"
+                    self.query.add_endpoint_prefixes(endpointConf, PrefixConfigs.get_instance())
                 if args.raw:
                     qres = self.rawQuery(
                         endpointConf,

tests/test_queries.py

@@ -4,16 +4,15 @@
 @author: wf
 """
 
+from argparse import Namespace
+from contextlib import redirect_stdout
 import copy
 import io
 import json
 import os
 import traceback
-from argparse import Namespace
-from contextlib import redirect_stdout
 
-import tests.test_sqlite3
-from tests.action_stats import ActionStats
+from lodstorage.prefix_config import PrefixConfigs
 from lodstorage.prefixes import Prefixes
 from lodstorage.query import (
     EndpointManager,
@@ -27,7 +26,11 @@
 from lodstorage.querymain import QueryMain
 from lodstorage.querymain import main as queryMain
 from lodstorage.sparql import SPARQL
+
+from tests.action_stats import ActionStats
 from tests.endpoint_test import EndpointTest
+import tests.test_sqlite3
+
 
 class TestQueries(EndpointTest):
     """
@@ -171,6 +174,64 @@ def testQueryCommandLine(self):
                 print(f"{resultFormat}:{result}")
             self.assertTrue(expected in result, f"{expected}({resultFormat})")
 
+    def test_add_endpoint_prefixes(self):
+        """
+        Test Query.add_endpoint_prefixes() merges endpoint prefixes without duplicates.
+        https://github.com/WolfgangFahl/pyLoDStorage/issues/140 (related prefix handling)
+        """
+        debug = self.debug
+        #debug=True
+        endpoints = EndpointManager.getEndpoints(lang="sparql")
+        wikidata_ep = endpoints["wikidata"]
+        prefix_configs = PrefixConfigs.get_instance()
+
+        qm = QueryManager(lang="sparql", debug=False)
+        cities_query = qm.queriesByName["cities"]  # Has rdfs, wd, wdt
+
+        original_query = cities_query.query
+        original_prefixes = Prefixes.extract_prefixes(original_query)
+        self.assertEqual(3, len(original_prefixes))  # rdfs, wd, wdt
+
+        # Add endpoint prefixes (Wikidata full set → overlaps)
+        cities_query.add_endpoint_prefixes(wikidata_ep, prefix_configs)
+
+        # Debug: Always show merged state (before asserts → visible on fail)
+        merged_query = cities_query.query
+        prefix_dict = Prefixes.extract_prefixes(merged_query)
+        prefix_lines = [line for line in merged_query.splitlines() if line.strip().startswith('PREFIX')]
+        if debug:
+            print("Merged query prefixes:")
+            print('\n'.join(prefix_lines))
+            print(f"Unique prefixes: {len(prefix_dict)}")
+            print(f"self.prefixes len: {len(cities_query.prefixes)}")
+            print("self.prefixes sample:", cities_query.prefixes[:3])
+
+        # Verify: Unique prefixes (no dups like 'rdfs')
+        self.assertEqual(len(prefix_dict), len(prefix_lines))  # 1:1 → no dup lines
+
+        # Specific: 'rdfs', 'wd', 'wdt' appear exactly once
+        for prefix in ['rdfs', 'wd', 'wdt']:
+            count = sum(1 for line in prefix_lines if f'{prefix}: ' in line)
+            self.assertEqual(1, count)
+
+        # Full Wikidata set merged (many more)
+        self.assertGreater(len(prefix_dict), 10)
+
+        # Query body preserved
+        self.assertIn('SELECT ?city_id ?name', merged_query)
+
+        # self.prefixes: Unique PREFIX lines list (len + sorted names match)
+        self.assertEqual(len(prefix_lines), len(cities_query.prefixes))  # same count
+        def get_prefix_names(lines):
+            return sorted([line.split(':')[0].strip('PREFIX ').strip() for line in lines])
+        self.assertEqual(
+            get_prefix_names(prefix_lines),
+            get_prefix_names(cities_query.prefixes)
+        )  # same set (order-insensitive)
+        used_prefixes = {line.split(':')[0].strip() for line in cities_query.prefixes}
+        self.assertEqual(len(used_prefixes), len(cities_query.prefixes))  # no dups
+
+
     def testQueryEndpoints(self):
         """
         tests the sparql endpoint commandline endpoint selection