Add support for endpoints in GOCDB (#234)

* Add support for endpoints in GOCDB * Update changelog * Add missing library How this was not missing before? * Do not debug... * More test requirements * Add missing requirements * Update actions * Py 2.7 should be gone by now * Fix linting issues * lovely yaml * Do not use real sites names
EGI-Federation · Mar 20, 2023 · 5775ad4 · 5775ad4
1 parent f6f6a2e
commit 5775ad4
Show file tree

Hide file tree

Showing 9 changed files with 82 additions and 25 deletions.
diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml
@@ -21,7 +21,7 @@ jobs:
         python setup.py sdist
     - name: Get version
       id: version
-      run: echo "::set-output name=version::$(python setup.py --version)"
+      run: echo "version=$(python setup.py --version)" >> "$GITHUB_OUTPUT"
     - name: Upload output file
       uses: actions/upload-artifact@v2
       with:

diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml
@@ -35,8 +35,8 @@ jobs:
         version=$(grep "^## \[[0-9].*\]" CHANGELOG | head -1 | sed -e "s/^.*\[\(.*\)\]/\1/")
         changes=$(sed -e "/^## \[$version\]\$/,/^##/!d;//d;/^\$/d" CHANGELOG | \
                   sed -e "s/\%/%25/g" | sed -e "s/\r/%0D/" | sed -z -e "s/\n/%0A/g")
-        echo "::set-output name=changes::$changes"
-        echo "::set-output name=version::$version"
+        echo "changes=$changes" >> "$GITHUB_OUTPUT"
+        echo "version=$version" >> "$GITHUB_OUTPUT"
     - name: bump versions
       run: |
         {

diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [2.7, 3.7, 3.8]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v2

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -20,8 +20,8 @@ jobs:
         version=$(grep "^## \[[0-9].*\]" CHANGELOG | head -1 | sed -e "s/^.*\[\(.*\)\]/\1/")
         changes=$(sed -e "/^## \[$version\]\$/,/^##/!d;//d;/^\$/d" CHANGELOG | \
                   sed -e "s/\%/%25/g" | sed -e "s/\r/%0D/" | sed -z -e "s/\n/%0A/g")
-        echo "::set-output name=changes::$changes"
-        echo "::set-output name=version::$version"
+        echo "changes=$changes" >> "$GITHUB_OUTPUT"
+        echo "version=$version" >> "$GITHUB_OUTPUT"
     - name: Create Release
       id: create_release
       uses: actions/create-release@v1
@@ -52,7 +52,7 @@ jobs:
         python setup.py sdist
     - name: Get version
       id: version
-      run: echo "::set-output name=version::$(python setup.py --version)"
+      run: echo "version=$(python setup.py --version)" >> "$GITHUB_OUTPUT"
     - name: Upload output file
       uses: actions/upload-artifact@v2
       with:

diff --git a/CHANGELOG b/CHANGELOG
@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 - Add flavor name to templates
+- Support sites with URLs in the endpoints of GOCDB
 
 ## [0.12.2]
 - Support crendential refresh without client secret (#222) (Enol Fernández)

diff --git a/cloud_info_provider/providers/gocdb.py b/cloud_info_provider/providers/gocdb.py
@@ -35,16 +35,24 @@ def _find_url_in_result(svc_url, result):
 
     svc_url = urlparse(svc_url)
     for svc in result:
+        urls = []
         try:
-            url = urlparse(svc.find("URL").text)
+            urls.append(urlparse(svc.find("URL").text))
         except AttributeError:
-            # something is wrong at GOCDB, skip this one
-            continue
-        if _are_url_similar(svc_url, url):
-            return {
-                "gocdb_id": svc.attrib["PRIMARY_KEY"],
-                "site_name": svc.find("SITENAME").text,
-            }
+            # something is wrong at GOCDB, do not care
+            pass
+        for ep in svc.find("ENDPOINTS"):
+            try:
+                urls.append(urlparse(ep.find("URL").text))
+            except AttributeError:
+                # something is wrong at GOCDB, do not care
+                pass
+        for url in urls:
+            if _are_url_similar(svc_url, url):
+                return {
+                    "gocdb_id": svc.attrib["PRIMARY_KEY"],
+                    "site_name": svc.find("SITENAME").text,
+                }
     logger.warning("Unable to find URL %s in GOCDB!", svc_url)
     return {}
 

diff --git a/cloud_info_provider/tests/test_gocdb.py b/cloud_info_provider/tests/test_gocdb.py
@@ -4,21 +4,55 @@
 
 sample_goc_response = """<?xml version="1.0" encoding="UTF-8"?>
 <results>
-    <SERVICE_ENDPOINT PRIMARY_KEY="7513G0">
-        <PRIMARY_KEY>7513G0</PRIMARY_KEY>
-        <HOSTNAME>nova.cloud.ifca.es</HOSTNAME>
+    <SERVICE_ENDPOINT PRIMARY_KEY="1234G0">
+        <PRIMARY_KEY>1234G0</PRIMARY_KEY>
+        <HOSTNAME>simple.example.com</HOSTNAME>
         <SERVICE_TYPE>org.openstack.nova</SERVICE_TYPE>
         <CORE></CORE>
         <IN_PRODUCTION>Y</IN_PRODUCTION>
         <NODE_MONITORED>Y</NODE_MONITORED>
         <NOTIFICATIONS>Y</NOTIFICATIONS>
-        <SITENAME>IFCA-LCG2</SITENAME>
+        <SITENAME>FOO-BAR-SITE</SITENAME>
         <COUNTRY_NAME>Spain</COUNTRY_NAME>
         <COUNTRY_CODE>ES</COUNTRY_CODE>
         <ROC_NAME>NGI_IBERGRID</ROC_NAME>
-        <URL>https://keystone.ifca.es:5000/v2.0/?image=18d99a06-c3e5-4157-a0e3-37ec34bdfc24&amp;resource=m1.tiny</URL>
+        <URL>https://keystone.example.com:5000/v2.0/?image=18d99a06-c3e5-4157-a0e3-37ec34bdfc24&amp;resource=m1.tiny</URL>
         <ENDPOINTS/>
-  </SERVICE_ENDPOINT>'
+  </SERVICE_ENDPOINT>
+</results>"""
+
+sample_goc_ep_response = """<?xml version="1.0" encoding="UTF-8"?>
+<results>
+  <SERVICE_ENDPOINT PRIMARY_KEY="00000G0">
+    <PRIMARY_KEY>00000G0</PRIMARY_KEY>
+    <HOSTNAME>www.baz.example.com</HOSTNAME>
+    <GOCDB_PORTAL_URL>https://goc.egi.eu/portal/index.php?Page_Type=Service&amp;id=14541</GOCDB_PORTAL_URL>
+    <BETA>N</BETA>
+    <SERVICE_TYPE>org.openstack.nova</SERVICE_TYPE>
+    <CORE/>
+    <IN_PRODUCTION>Y</IN_PRODUCTION>
+    <NODE_MONITORED>Y</NODE_MONITORED>
+    <NOTIFICATIONS>Y</NOTIFICATIONS>
+    <SITENAME>BAR-FOO-SITE</SITENAME>
+    <COUNTRY_NAME>Ireland</COUNTRY_NAME>
+    <COUNTRY_CODE>IE</COUNTRY_CODE>
+    <ROC_NAME>NGI_IE</ROC_NAME>
+    <URL/>
+    <ENDPOINTS>
+      <ENDPOINT>
+        <ID>8282</ID>
+        <NAME>Name-it</NAME>
+        <EXTENSIONS/>
+        <URL>https://horizon.baz.example.com:5000/v3</URL>
+        <INTERFACENAME>org.openstack.nova</INTERFACENAME>
+        <ENDPOINT_MONITORED>Y</ENDPOINT_MONITORED>
+      </ENDPOINT>
+    </ENDPOINTS>
+    <SCOPES>
+      <SCOPE>EGI</SCOPE>
+    </SCOPES>
+    <EXTENSIONS/>
+  </SERVICE_ENDPOINT>
 </results>"""
 
 
@@ -88,10 +122,10 @@ def test_goc_found_same_path(self):
             r.status_code = 200
             r.text = sample_goc_response
             m_requests.return_value = r
-            expected = {"gocdb_id": "7513G0", "site_name": "IFCA-LCG2"}
+            expected = {"gocdb_id": "1234G0", "site_name": "FOO-BAR-SITE"}
             self.assertEqual(
                 expected,
-                gocdb.find_in_gocdb("https://keystone.ifca.es:5000/v2.0/", "bar"),
+                gocdb.find_in_gocdb("https://keystone.example.com:5000/v2.0/", "bar"),
             )
 
     def test_goc_found_similar_path(self):
@@ -100,8 +134,20 @@ def test_goc_found_similar_path(self):
             r.status_code = 200
             r.text = sample_goc_response
             m_requests.return_value = r
-            expected = {"gocdb_id": "7513G0", "site_name": "IFCA-LCG2"}
+            expected = {"gocdb_id": "1234G0", "site_name": "FOO-BAR-SITE"}
+            self.assertEqual(
+                expected,
+                gocdb.find_in_gocdb("https://keystone.example.com:5000/v2.0", "bar"),
+            )
+
+    def test_goc_multiple_endpoints(self):
+        with mock.patch("requests.get") as m_requests:
+            r = mock.MagicMock()
+            r.status_code = 200
+            r.text = sample_goc_ep_response
+            m_requests.return_value = r
+            expected = {"gocdb_id": "00000G0", "site_name": "BAR-FOO-SITE"}
             self.assertEqual(
                 expected,
-                gocdb.find_in_gocdb("https://keystone.ifca.es:5000/v2.0", "bar"),
+                gocdb.find_in_gocdb("https://horizon.baz.example.com:5000/v3", "bar"),
             )
diff --git a/test-requirements.txt b/test-requirements.txt
@@ -7,3 +7,4 @@ requests
 pyOpenSSL
 python-novaclient
 python-glanceclient
+testtools
diff --git a/tox.ini b/tox.ini
@@ -25,6 +25,7 @@ commands = {posargs}
 
 [testenv:cobertura]
 deps = pytest-cov
+       -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 commands = py.test --cov=cloud_info_provider --cov-report=xml --cov-report=term-missing cloud_info_provider/tests