Merge pull request #1522

plugins: switch python-ldap plugin to  python3

CHANGELOG.md

@@ -87,6 +87,7 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https:
 - dird: add prev and new jobid variables [PR #1499]
 - improve default configuration [PR #1508]
 - stored: add AccessMode SD->Device directive to reserve devices exclusively for reading or writing [PR #1464]
+- plugins: switch python-ldap plugin to  python3 [PR #1522]
 
 ### Removed
 - remove no longer used pkglists [PR #1335]
@@ -224,4 +225,5 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https:
 [PR #1508]: https://github.com/bareos/bareos/pull/1508
 [PR #1511]: https://github.com/bareos/bareos/pull/1511
 [PR #1512]: https://github.com/bareos/bareos/pull/1512
+[PR #1522]: https://github.com/bareos/bareos/pull/1522
 [unreleased]: https://github.com/bareos/bareos/tree/master

core/src/plugins/filed/python/ldap/BareosFdPluginLDAP.py

@@ -28,7 +28,6 @@
 import bareosfd
 
 import BareosFdPluginBaseclass
-from StringIO import StringIO
 from stat import S_IFREG, S_IFDIR, S_IRWXU
 import io
 import ldif
@@ -41,23 +40,11 @@
 
 
 def _safe_encode(data):
-    if isinstance(data, unicode):
+    if isinstance(data, str):
         return data.encode("utf-8")
     return data
 
 
-class BytesLDIFRecordList(ldif.LDIFRecordList):
-    """Simple encoding wrapper for LDIFRecordList that converts keys to UTF-8"""
-
-    def _next_key_and_value(self):
-        # we do not descend from object, so we cannot use super()
-        k, v = ldif.LDIFRecordList._next_key_and_value(self)
-        try:
-            return _safe_encode(k), v
-        except:
-            return k, v
-
-
 class BareosFdPluginLDAP(BareosFdPluginBaseclass.BareosFdPluginBaseclass):
     """
     LDAP related backup and restore stuff
@@ -134,14 +121,14 @@ def create_file(self, restorepkt):
         bareosfd.DebugMessage(
             100, "BareosFdPluginLDAP:create_file() called with %s\n" % (restorepkt)
         )
-        if restorepkt.type == bareosfd.bFileType["FT_DIREND"]:
-            restorepkt.create_status = bareosfd.bCFs["CF_SKIP"]
-        elif restorepkt.type == bareosfd.bFileType["FT_REG"]:
+        if restorepkt.type == bareosfd.FT_DIREND:
+            restorepkt.create_status = bareosfd.CF_SKIP
+        elif restorepkt.type == bareosfd.FT_REG:
             self.ldap.set_new_dn(restorepkt.ofname)
-            restorepkt.create_status = bareosfd.bCFs["CF_EXTRACT"]
+            restorepkt.create_status = bareosfd.CF_EXTRACT
         else:
             bareosfd.JobMessage(
-                bareosfd.bJobMessageType["M_FATAL"],
+                bareosfd.M_FATAL,
                 "Request to restore illegal filetype %s\n" % (restorepkt.type),
             )
             return bareosfd.bRC_Error
@@ -153,24 +140,24 @@ def plugin_io(self, IOP):
             100, "BareosFdPluginLDAP:plugin_io() called with function %s\n" % (IOP.func)
         )
 
-        if IOP.func == bareosfd.bIOPS["IO_OPEN"]:
+        if IOP.func == bareosfd.IO_OPEN:
             self.last_op = IOP.func
             return bareosfd.bRC_OK
 
-        elif IOP.func == bareosfd.bIOPS["IO_CLOSE"]:
-            if self.last_op == bareosfd.bIOPS["IO_OPEN"]:
+        elif IOP.func == bareosfd.IO_CLOSE:
+            if self.last_op == bareosfd.IO_OPEN:
                 bareosfd.JobMessage(
-                    bareosfd.bJobMessageType["M_WARNING"],
+                    bareosfd.M_WARNING,
                     "Missing data for DN %s\n" % self.ldap.dn,
                 )
             self.last_op = IOP.func
             return bareosfd.bRC_OK
 
-        elif IOP.func == bareosfd.bIOPS["IO_SEEK"]:
+        elif IOP.func == bareosfd.IO_SEEK:
             self.last_op = IOP.func
             return bareosfd.bRC_OK
 
-        elif IOP.func == bareosfd.bIOPS["IO_READ"]:
+        elif IOP.func == bareosfd.IO_READ:
             if not self.data_stream:
                 self.data_stream = io.BytesIO(_safe_encode(self.ldap.ldif))
             IOP.buf = bytearray(IOP.count)
@@ -184,7 +171,7 @@ def plugin_io(self, IOP):
             self.last_op = IOP.func
             return bareosfd.bRC_OK
 
-        elif IOP.func == bareosfd.bIOPS["IO_WRITE"]:
+        elif IOP.func == bareosfd.IO_WRITE:
             if not self.data_stream:
                 self.data_stream = io.BytesIO()
 
@@ -238,15 +225,9 @@ def connect_and_bind(self, options, bulk=False):
         Bind to LDAP URI using the given authentication tokens
         """
         if bulk:
-            try:
-                self.ld = BulkLDAP(options["uri"], bytes_mode=True)
-            except TypeError:
-                self.ld = BulkLDAP(options["uri"])
+            self.ld = BulkLDAP(options["uri"])
         else:
-            try:
-                self.ld = ldap.initialize(options["uri"], bytes_mode=True)
-            except TypeError:
-                self.ld = ldap.initialize(options["uri"])
+            self.ld = ldap.initialize(options["uri"])
 
         try:
             self.ld.protocol_version = ldap.VERSION3
@@ -256,7 +237,7 @@ def connect_and_bind(self, options, bulk=False):
                 self.ld.simple_bind_s("", "")
         except ldap.INVALID_CREDENTIALS:
             bareosfd.JobMessage(
-                bareosfd.bJobMessageType["M_FATAL"],
+                bareosfd.M_FATAL,
                 "Failed to bind to LDAP uri due to invalid credentials %s\n"
                 % (options["uri"]),
             )
@@ -265,13 +246,13 @@ def connect_and_bind(self, options, bulk=False):
         except ldap.LDAPError as e:
             if type(e.message) == dict and "desc" in e.message:
                 bareosfd.JobMessage(
-                    bareosfd.bJobMessageType["M_FATAL"],
+                    bareosfd.M_FATAL,
                     "Failed to bind to LDAP uri %s: %s %s\n"
                     % (options["uri"], e.message["desc"], e.message["info"]),
                 )
             else:
                 bareosfd.JobMessage(
-                    bareosfd.bJobMessageType["M_FATAL"],
+                    bareosfd.M_FATAL,
                     "Failed to bind to LDAP uri %s: %s\n" % (options["uri"], e),
                 )
 
@@ -313,13 +294,13 @@ def prepare_backup(self, options):
         except ldap.LDAPError as e:
             if type(e.message) == dict and "desc" in e.message:
                 bareosfd.JobMessage(
-                    bareosfd.bJobMessageType["M_FATAL"],
+                    bareosfd.M_FATAL,
                     "Failed to execute LDAP search on LDAP uri %s: %s\n"
                     % (options["uri"], e.message["desc"]),
                 )
             else:
                 bareosfd.JobMessage(
-                    bareosfd.bJobMessageType["M_FATAL"],
+                    bareosfd.M_FATAL,
                     "Failed to execute LDAP search on LDAP uri %s: %s\n"
                     % (options["uri"], e),
                 )
@@ -358,18 +339,15 @@ def get_next_file_to_backup(self, savepkt):
             # Remove some attributes from entry before creating the LDIF.
             ignore_attribute = ["createTimestamp", "modifyTimestamp"]
 
-            keys = self.entry.keys()
+            keys = list(self.entry.keys())
             for value in keys:
                 if value in ignore_attribute:
                     del self.entry[value]
 
             # Dump the content of the LDAP entry as LDIF text
-            ldif_dump = StringIO()
+            ldif_dump = io.StringIO()
             ldif_out = ldif.LDIFWriter(ldif_dump)
-            try:
-                ldif_out.unparse(self.dn, self.entry)
-            except UnicodeDecodeError:
-                ldif_out.unparse(self.dn.decode("utf-8"), self.entry)
+            ldif_out.unparse(self.dn, self.entry)
 
             self.ldif = ldif_dump.getvalue()
             self.ldif_len = len(self.ldif)
@@ -384,7 +362,7 @@ def get_next_file_to_backup(self, savepkt):
                 statp.st_mtime = self.unix_modify_time
 
             savepkt.statp = statp
-            savepkt.type = bareosfd.bFileType["FT_REG"]
+            savepkt.type = bareosfd.FT_REG
             savepkt.fname = self.file_to_backup + "/data.ldif"
             # Read the content of a file
             savepkt.no_read = False
@@ -398,7 +376,7 @@ def get_next_file_to_backup(self, savepkt):
                 # Try to get the first result set from the query,
                 # if there is nothing return an error.
                 try:
-                    res_type, res_data, res_msgid, res_controls = self.resultset.next()
+                    res_type, res_data, res_msgid, res_controls = next(self.resultset)
                     self.ldap_entries = collections.deque(res_data)
                 except ldap.NO_SUCH_OBJECT:
                     return bareosfd.bRC_Error
@@ -414,15 +392,19 @@ def get_next_file_to_backup(self, savepkt):
                     # convert it to an UNIX timestamp
                     self.unix_create_time = None
                     try:
-                        createTimestamp = self.entry["createTimestamp"][0]
+                        createTimestamp = self.entry["createTimestamp"][0].decode(
+                            "ascii"
+                        )
                     except KeyError:
                         pass
                     else:
                         self.unix_create_time = self.to_unix_timestamp(createTimestamp)
 
                     self.unix_modify_time = None
                     try:
-                        modifyTimestamp = self.entry["modifyTimestamp"][0]
+                        modifyTimestamp = self.entry["modifyTimestamp"][0].decode(
+                            "ascii"
+                        )
                     except KeyError:
                         pass
                     else:
@@ -442,7 +424,8 @@ def get_next_file_to_backup(self, savepkt):
                         statp.st_mtime = self.unix_modify_time
 
                     savepkt.statp = statp
-                    savepkt.type = bareosfd.bFileType["FT_DIREND"]
+
+                    savepkt.type = bareosfd.FT_DIREND
                     savepkt.fname = self.file_to_backup
                     # A directory has a link field which contains
                     # the fname + a trailing '/'
@@ -452,7 +435,7 @@ def get_next_file_to_backup(self, savepkt):
 
                     if "/" in self.dn:
                         bareosfd.JobMessage(
-                            bareosfd.bJobMessageType["M_ERROR"],
+                            bareosfd.M_ERROR,
                             "Slashes (/) in DN not supported. Skipping %s" % self.dn,
                         )
                         # set to none, so the object will not be picket up
@@ -469,9 +452,7 @@ def set_new_dn(self, fname):
         3. filter out anything without an equals sign("=")
         4. join components into comma-separated string
         """
-        self.dn = _safe_encode(
-            ",".join(filter(lambda x: "=" in x, reversed(fname.split("/"))))
-        )
+        self.dn = ",".join(filter(lambda x: "=" in x, reversed(fname.split("/"))))
 
     def has_next_file(self):
         # See if we are currently handling the LDIF file or
@@ -484,7 +465,7 @@ def has_next_file(self):
             if self.resultset:
                 try:
                     # Get the next result set
-                    res_type, res_data, res_msgid, res_controls = self.resultset.next()
+                    res_type, res_data, res_msgid, res_controls = next(self.resultset)
                     self.ldap_entries = collections.deque(res_data)
 
                     # We expect something to be in the result set but better check
@@ -502,16 +483,15 @@ def restore_entry(self):
         # Restore the entry
         if self.ldif:
             # Parse the LDIF back to an attribute list
-            ldif_dump = StringIO(str(self.ldif))
-            ldif_parser = BytesLDIFRecordList(ldif_dump, max_entries=1)
+            ldif_dump = io.StringIO(self.ldif)
+            ldif_parser = ldif.LDIFRecordList(ldif_dump, max_entries=1)
             ldif_parser.parse()
             dn, entry = ldif_parser.all_records[0]
-            dn = _safe_encode(dn)
             ldif_dump.close()
 
             if self.dn != dn:
                 bareosfd.JobMessage(
-                    bareosfd.bJobMessageType["M_INFO"],
+                    bareosfd.M_INFO,
                     "Restoring original DN %s as %s\n" % (dn, self.dn),
                 )
 
@@ -529,20 +509,20 @@ def restore_entry(self):
                         except ldap.LDAPError as e:
                             if type(e.message) == dict and "desc" in e.message:
                                 bareosfd.JobMessage(
-                                    bareosfd.bJobMessageType["M_ERROR"],
+                                    bareosfd.M_ERROR,
                                     "Failed to restore LDAP DN %s: %s\n"
                                     % (self.dn, e.message["desc"]),
                                 )
                             else:
                                 bareosfd.JobMessage(
-                                    bareosfd.bJobMessageType["M_ERROR"],
+                                    bareosfd.M_ERROR,
                                     "Failed to restore LDAP DN %s: %s\n" % (self.dn, e),
                                 )
                             self.ldif = None
                             return bareosfd.bRC_Error
                 else:
                     bareosfd.JobMessage(
-                        bareosfd.bJobMessageType["M_ERROR"],
+                        bareosfd.M_ERROR,
                         "Failed to restore LDAP DN %s no writable binding to LDAP exists\n"
                         % (self.dn),
                     )

systemtests/tests/CMakeLists.txt

@@ -58,7 +58,6 @@ add_subdirectory(passive)
 add_subdirectory(pruning)
 add_subdirectory(py2plug-dir)
 add_subdirectory(py2plug-fd-contrib-bareos_tasks_mysql)
-add_subdirectory(py2plug-fd-ldap)
 add_subdirectory(py2plug-fd-libcloud)
 add_subdirectory(py2plug-fd-local-fileset-restoreobject)
 add_subdirectory(py2plug-fd-mariabackup)
@@ -67,6 +66,7 @@ add_subdirectory(py2plug-fd-percona-xtrabackup)
 add_subdirectory(py2plug-fd-vmware)
 add_subdirectory(py2plug-sd)
 add_subdirectory(py2plug-fd-local-fileset-basic)
+add_subdirectory(py3plug-fd-ldap)
 add_subdirectory(py3plug-fd-local-fileset-basic)
 add_subdirectory(py3plug-fd-basic)
 add_subdirectory(py3plug-fd-contrib-mysql_dump)

systemtests/tests/py2plug-fd-ldap/CMakeLists.txt → systemtests/tests/py3plug-fd-ldap/CMakeLists.txt

@@ -1,6 +1,6 @@
 #   BAREOS® - Backup Archiving REcovery Open Sourced
 #
-#   Copyright (C) 2021-2021 Bareos GmbH & Co. KG
+#   Copyright (C) 2021-2023 Bareos GmbH & Co. KG
 #
 #   This program is Free Software; you can redistribute it and/or
 #   modify it under the terms of version three of the GNU Affero General Public
@@ -35,13 +35,12 @@ set(SYSTEMTEST_LDAP_PASSWORD
     "admin"
     CACHE STRING "LDAP bind password for plugin test"
 )
-check_pymodule_available(2 ldap)
-if(TARGET python-fd
-   AND PYMODULE_2_LDAP_FOUND
-   AND Python2_EXECUTABLE
-   AND SYSTEMTEST_LDAP_ADDRESS
-)
-  create_systemtest(${SYSTEMTEST_PREFIX} ${BASENAME})
-else()
-  create_systemtest(${SYSTEMTEST_PREFIX} ${BASENAME} DISABLED)
+
+if(TARGET python3-fd)
+  check_pymodule_available(3 ldap)
+  if(PYMODULE_3_LDAP_FOUND AND SYSTEMTEST_LDAP_ADDRESS)
+    create_systemtest(${SYSTEMTEST_PREFIX} ${BASENAME})
+  else()
+    create_systemtest(${SYSTEMTEST_PREFIX} ${BASENAME} DISABLED)
+  endif()
 endif()