Merge pull request #1546

python: adapt for new Python module versions
bareos · Sep 28, 2023 · 9ecffc4 · 9ecffc4
2 parents 50997a7 + 13c3546
commit 9ecffc4
Show file tree

Hide file tree

Showing 12 changed files with 122 additions and 188 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -101,6 +101,7 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https:
 - utils: add a thread-safe single-producer/single-consumer queue [PR #1504]
 - require TLS by default [PR #1529]
 - build: introduce fedora38 [PR #1563]
+- python: adapt for new Python module versions [PR #1546]
 
 ### Removed
 - remove no longer used pkglists [PR #1335]
@@ -257,6 +258,7 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https:
 [PR #1532]: https://github.com/bareos/bareos/pull/1532
 [PR #1542]: https://github.com/bareos/bareos/pull/1542
 [PR #1543]: https://github.com/bareos/bareos/pull/1543
+[PR #1546]: https://github.com/bareos/bareos/pull/1546
 [PR #1550]: https://github.com/bareos/bareos/pull/1550
 [PR #1556]: https://github.com/bareos/bareos/pull/1556
 [PR #1563]: https://github.com/bareos/bareos/pull/1563

diff --git a/python-bareos/bareos/util/__init__.py b/python-bareos/bareos/util/__init__.py
@@ -1,6 +1,6 @@
 #   BAREOS - Backup Archiving REcovery Open Sourced
 #
-#   Copyright (C) 2015-2021 Bareos GmbH & Co. KG
+#   Copyright (C) 2015-2023 Bareos GmbH & Co. KG
 #
 #   This program is Free Software; you can redistribute it and/or
 #   modify it under the terms of version three of the GNU Affero General Public
@@ -24,5 +24,6 @@
 from bareos.util.bareosbase64 import BareosBase64
 from bareos.util.password import Password
 from bareos.util.path import Path
+from bareos.util.version import Version
 
-__all__ = ["BareosBase64", "Password", "Path"]
+__all__ = ["BareosBase64", "Password", "Path", "Version"]
diff --git a/python-bareos/bareos/util/version.py b/python-bareos/bareos/util/version.py
@@ -0,0 +1,35 @@
+#   BAREOS - Backup Archiving REcovery Open Sourced
+#
+#   Copyright (C) 2023-2023 Bareos GmbH & Co. KG
+#
+#   This program is Free Software; you can redistribute it and/or
+#   modify it under the terms of version three of the GNU Affero General Public
+#   License as published by the Free Software Foundation and included
+#   in the file LICENSE.
+#
+#   This program is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+#   Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+#   02110-1301, USA.
+
+import re
+
+
+class Version(object):
+    def __init__(self, version):
+        self.version = version
+
+    def __str__(self):
+        return self.version
+
+    def as_python_version(self):
+        """
+        Translate a Bareos version number
+        into a version compatible with PEP 440.
+        """
+        return re.sub(r"~pre([0-9]+\.)", r".dev+\1", self.version)
diff --git a/python-bareos/setup.py b/python-bareos/setup.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 #   BAREOS - Backup Archiving REcovery Open Sourced
 #
-#   Copyright (C) 2019-2020 Bareos GmbH & Co. KG
+#   Copyright (C) 2019-2023 Bareos GmbH & Co. KG
 #
 #   This program is Free Software; you can redistribute it and/or
 #   modify it under the terms of version three of the GNU Affero General Public
@@ -33,7 +33,7 @@ def get_version():
             # and adapt it according to
             # https://www.python.org/dev/peps/pep-0440/.
             fullversion = version_file.read().strip()
-            __version__ = re.compile(r"~pre([0-9]+).*").sub(r".dev\1", fullversion)
+            __version__ = re.sub(r"~pre([0-9]+\.)", r".dev+\1", fullversion)
     except IOError:
         # Fallback version.
         # First protocol implemented

diff --git a/restapi/bareos_restapi/__init__.py b/restapi/bareos_restapi/__init__.py
@@ -4,7 +4,7 @@
 #
 # BAREOS - Backup Archiving REcovery Open Sourced
 #
-# Copyright (C) 2020-2021 Bareos GmbH & Co. KG
+# Copyright (C) 2020-2023 Bareos GmbH & Co. KG
 #
 # This program is Free Software; you can redistribute it and/or
 # modify it under the terms of version three of the GNU Affero General Public
@@ -37,6 +37,7 @@
 import yaml
 
 import bareos.bsock
+import bareos.util
 from bareos_restapi.models import *
 
 # Read config from api.ini
@@ -199,23 +200,25 @@ def versionCheck(
     else:
         result = read_director_version(response=response, current_user=current_user)
         if "version" in result:
-            myVersion = result["version"]
+            myVersion = bareos.util.Version(result["version"]).as_python_version()
             current_user.directorVersion = myVersion
         else:
             raise HTTPException(
                 status_code=500,
                 detail="Could not read version from director. Need at least version %s"
                 % (minVersion),
             )
-    # print (myVersion)
-    if not (version.parse(myVersion) >= version.parse(minVersion)):
-        raise HTTPException(
-            status_code=501,
-            detail="Not implemented in Bareos %s. Need at least version %s"
-            % (myVersion, minVersion),
-        )
-    else:
-        return True
+    try:
+        if not (version.parse(myVersion) >= version.parse(minVersion)):
+            raise HTTPException(
+                status_code=501,
+                detail="Not implemented in Bareos %s. Need at least version %s"
+                % (myVersion, minVersion),
+            )
+        else:
+            return True
+    except version.InvalidVersion as exception:
+        raise HTTPException(status_code=500, detail=str(exception))
 
 
 def configure_add_standard_component(
@@ -321,7 +324,7 @@ def show_configuration_items(
     versionCheck(
         response=response,
         current_user=current_user,
-        minVersion="20.0.0~pre996.de46d0b15",
+        minVersion="20.0.1",
     )
     # Sometimes config type identificator differs from key returned by director, we need to map
     itemKey = itemType

diff --git a/restapi/bareos_restapi/models.py b/restapi/bareos_restapi/models.py
@@ -1,6 +1,6 @@
 #   BAREOS - Backup Archiving REcovery Open Sourced
 #
-#   Copyright (C) 2020-2021 Bareos GmbH & Co. KG
+#   Copyright (C) 2020-2023 Bareos GmbH & Co. KG
 #
 #   This program is Free Software; you can redistribute it and/or
 #   modify it under the terms of version three of the GNU Affero General Public
@@ -21,6 +21,7 @@
 from enum import Enum
 import pathlib
 from typing import Optional, List
+from typing_extensions import Annotated
 from fastapi import Depends, FastAPI, HTTPException, status, Response, Path, Body, Query
 
 
@@ -68,12 +69,6 @@ def __bool__(self):
         return self.name == "yes"
 
 
-class bareosTime(str):
-    # TODO: define this, stuff like "20 days" or "1 months"
-    def __str__(self):
-        return self.name
-
-
 class bareosReplaceOption(str, Enum):
     """
     Replace option used by restore command
@@ -88,22 +83,12 @@ def __str__(self):
         return self.name
 
 
-class bareosSpeed(str):
-    # TODO: define this
-    def __str__(self):
-        return self.name
-
-
-class bareosSize(str):
-    # TODO: define this. Samples: 300, 10G
-    def __str__(self):
-        return self.name
-
-
-class bareosACL(str):
-    # TODO: define this.
-    def __str__(self):
-        return self.name
+# TODO: define this. Samples: "20 days" or "1 months"
+bareosTime = Annotated[str, Field()]
+bareosSpeed = Annotated[str, Field()]
+# TODO: define this. Samples: 300, 10G
+bareosSize = Annotated[str, Field()]
+bareosACL = Annotated[str, Field()]
 
 
 class jobStatus(str, Enum):

diff --git a/restapi/setup.py b/restapi/setup.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 #   BAREOS - Backup Archiving REcovery Open Sourced
 #
-#   Copyright (C) 2020-2021 Bareos GmbH & Co. KG
+#   Copyright (C) 2020-2023 Bareos GmbH & Co. KG
 #
 #   This program is Free Software; you can redistribute it and/or
 #   modify it under the terms of version three of the GNU Affero General Public
@@ -36,7 +36,7 @@ def get_version():
             # and adapt it according to
             # https://www.python.org/dev/peps/pep-0440/.
             fullversion = version_file.read().strip()
-            __version__ = re.compile(r"~pre([0-9]+).*").sub(r".dev\1", fullversion)
+            __version__ = re.sub(r"~pre([0-9]+\.)", r".dev+\1", fullversion)
     except IOError:
         # Fallback version.
         # First protocol implemented

diff --git a/systemtests/tests/restapi/api/curl-auth.sh b/systemtests/tests/restapi/api/curl-auth.sh
@@ -3,7 +3,7 @@
 #
 # BAREOS - Backup Archiving REcovery Open Sourced
 #
-# Copyright (C) 2021-2021 Bareos GmbH & Co. KG
+# Copyright (C) 2021-2023 Bareos GmbH & Co. KG
 #
 # This program is Free Software; you can redistribute it and/or
 # modify it under the terms of version three of the GNU Affero General Public
@@ -23,7 +23,7 @@
 
 . $(dirname "$BASH_SOURCE")/../environment-local
 
-USERNAME=${1:-admin-tls}
+USERNAME=${1:-admin-notls}
 PASSWORD=${2:-secret}
 
-curl --silent -X POST "${REST_API_URL}/token" -H  "accept: application/json" -H  "Content-Type: application/x-www-form-urlencoded" -d "username=admin-tls&password=secret" | grep access_token | cut -d '"' -f 4
+curl --silent -X POST "${REST_API_URL}/token" -H  "accept: application/json" -H  "Content-Type: application/x-www-form-urlencoded" -d "username=${USERNAME}&password=${PASSWORD}" | grep access_token | cut -d '"' -f 4
diff --git a/systemtests/tests/restapi/api/restapi.sh b/systemtests/tests/restapi/api/restapi.sh
@@ -3,7 +3,7 @@
 #
 # BAREOS - Backup Archiving REcovery Open Sourced
 #
-# Copyright (C) 2021-2021 Bareos GmbH & Co. KG
+# Copyright (C) 2021-2023 Bareos GmbH & Co. KG
 #
 # This program is Free Software; you can redistribute it and/or
 # modify it under the terms of version three of the GNU Affero General Public
@@ -34,12 +34,14 @@ else
     export PYTHONPATH=${CMAKE_SOURCE_DIR}/restapi/
 fi
 
+LSOF_CMD="lsof -n -iTCP:$REST_API_PORT -sTCP:LISTEN"
+
 start()
 {
     printf "Starting bareos-restapi: "
-
-    if lsof -i:$REST_API_PORT >/dev/null; then
-        printf " FAILED (port $REST_API_PORT already in use)\n"
+    if LSOF=$(${LSOF_CMD}); then
+        printf " FAILED: port $REST_API_PORT already in use\n"
+        printf " %s\n" "${LSOF}"
         exit 1
     fi
 
@@ -84,16 +86,16 @@ status()
   printf "bareos-restapi: "
   if ! lsof -i:$REST_API_PORT >/dev/null; then
     printf "not running\n"
-    exit 1
+    return 1
   fi
-  PORTPID=$(lsof -t -i:$REST_API_PORT)
+  PORTPID=$(${LSOF_CMD} -t)
   PID=$(cat api.pid)
   if [ "$PORTPID" != "$PID" ]; then
     printf "running with unexpected PID (expected PID=$PID, running PID=$PORTPID)\n"
-    exit 1
+    return 1
   fi
   printf "running (PORT=${REST_API_PORT}, PID=$PID)\n"
-  exit 0
+  return 0
 }
 
 case "$1" in
@@ -111,8 +113,19 @@ case "$1" in
       start
       ;;
 
+  forcestart)
+      status || true
+      stop >/dev/null 2>&1 || true
+      if PID=$(${LSOF_CMD} -t); then
+        kill $PID
+        sleep 1
+      fi
+      start
+      ;;
+
    status)
       status
+      exit $?
       ;;
 
    *)

diff --git a/systemtests/tests/restapi/etc/bareos/bareos-dir.d/console/admin-notls.conf b/systemtests/tests/restapi/etc/bareos/bareos-dir.d/console/admin-notls.conf
@@ -0,0 +1,7 @@
+Console {
+   Name = admin-notls
+   Password = secret
+   TLS Enable = no
+
+   Profile = all
+}
diff --git a/systemtests/tests/restapi/testrunner b/systemtests/tests/restapi/testrunner
@@ -1,8 +1,9 @@
 #!/bin/bash
+
 set -e
 set -o pipefail
 set -u
-#
+
 TestName="$(basename "$(pwd)")"
 export TestName
 
@@ -11,10 +12,19 @@ export TestName
 #shellcheck source=environment-local.in
 . ./environment-local
 
-requirements="${CMAKE_SOURCE_DIR}/restapi/requirements.txt"
+# convert requirements.txt into Python list,
+# but exclude comments and Bareos packages
+# (as Bareos packages are handled from the local repo)
+requirements=$(
+  printf "%s" "[ "
+  grep -v "^#\|bareos" ${CMAKE_SOURCE_DIR}/restapi/requirements.txt | while read pkg; do
+    printf "'%s', " "$pkg"
+  done
+  printf "%s" "]"
+)
 
 "${PYTHON_EXECUTABLE}" -c \
-  "import pkg_resources;pkg_resources.require(open('$requirements',mode='r'))" \
+  "import pkg_resources;pkg_resources.require($requirements)" \
   || exit 77
 
 #shellcheck source=../scripts/functions
@@ -77,6 +87,7 @@ done
         printf "NOT FOUND\n" >> ${tmp}/../log/curl.log
         printf -- "--------------------------------------\n\n" >> ${tmp}/../log/curl.log
         print_debug "ERROR getting endpoint $2 with method $1 in $maxTry tries, expected output: \"$3\""
+        cat ${tmp}/curl.out
         exit 1
     fi
 }
@@ -87,7 +98,7 @@ done
 mkdir -p etc/bareos/bareos-dir.d/schedule
 mkdir -p etc/bareos/bareos-dir.d/user
 
-api/restapi.sh start
+api/restapi.sh forcestart
 
 TOKEN=$(api/curl-auth.sh)
 endpoint_check GET "configuration/clients/bareos-fd" bareos-fd "" 1
@@ -154,6 +165,15 @@ endpoint_check PUT "control/directors/reload" "success" "" 1
 
 api/restapi.sh stop
 
+# wait for Director commands to finish
+cat <<END_OF_DATA >"$tmp/bconcmds"
+@$out /dev/null
+status director
+wait
+status director
+END_OF_DATA
+run_bconsole
+
 #check_for_zombie_jobs storage=File client=bareos-fd
 
 stop_bareos