Skip to content

Commit

Permalink
Merge pull request #572 from bareos/dev/pstorz/master/python3-modules…
Browse files Browse the repository at this point in the history
…-rebase

add python3 support for fd, sd and dir plugins
  • Loading branch information
pstorz committed Aug 31, 2020
2 parents 6bf4ca2 + c63130c commit bfd7053
Show file tree
Hide file tree
Showing 525 changed files with 13,944 additions and 10,721 deletions.
9 changes: 9 additions & 0 deletions .gitignore
Expand Up @@ -6,6 +6,12 @@
*.lo
*.o

# Python compiled files
*.pyc

# vim swap file
*.swp

# Compiled Dynamic libraries
*.so
*.dylib
Expand Down Expand Up @@ -77,3 +83,6 @@ compile_commands.json

# JetBrains Pycharm
.idea

# ctest testing subdirs
*/Testing/*
1 change: 1 addition & 0 deletions AUTHORS
Expand Up @@ -13,6 +13,7 @@ Alessandro Rigopoulos
Alexander Bergolth
Alexander Kushnirenko
Alexandre Baron
Alexandre Bruyelles
Alexandre Simon
Allan Black
Alvaro Flaño
Expand Down
8 changes: 4 additions & 4 deletions CMakeLists.txt
Expand Up @@ -24,11 +24,11 @@ project(bareos NONE)
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/core/cmake")

add_custom_target(uninstall
"${CMAKE_COMMAND}" -P "${CMAKE_CURRENT_SOURCE_DIR}/cmake/uninstall.cmake"
add_custom_target(
uninstall "${CMAKE_COMMAND}" -P
"${CMAKE_CURRENT_SOURCE_DIR}/cmake/uninstall.cmake"
)


set(default_build_type "Debug")

find_package(Git)
Expand Down Expand Up @@ -74,7 +74,7 @@ endif()
if(BUILD_BAREOS_BINARIES)
add_subdirectory(core)
add_subdirectory(webui)
if(NOT client-only AND NOT CMAKE_SYSTEM_NAME STREQUAL "Windows")
if(NOT client-only AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows")
add_subdirectory(systemtests)
endif()
endif()
Expand Down
81 changes: 64 additions & 17 deletions core/CMakeLists.txt
Expand Up @@ -161,6 +161,7 @@ include(CTest)
if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(OPENSSL_ROOT_DIR /usr/local/opt/openssl)
set(Readline_ROOT_DIR /usr/local/opt/readline)
set(Intl_ROOT_DIR /usr/local/opt/gettext)
endif()

if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
Expand Down Expand Up @@ -299,21 +300,21 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")

set(WINDOWS_LIBRARIES ws2_32)

set(PYTHON_LIBRARIES
${PROJECT_SOURCE_DIR}/src/win32/plugins/python/lib/${WINDOWS_BITS}/python27.dll
)
set(PYTHON_INCLUDE_DIRS
${PROJECT_SOURCE_DIR}/src/win32/plugins/python/include
)
set(HAVE_PYTHON 1)
set(Python2_LIBRARIES python27.dll)
set(Python3_LIBRARIES python38.dll)

set(PostgreSQL_LIBRARY
${PROJECT_SOURCE_DIR}/src/win32/cats/pgsql/lib/${WINDOWS_BITS}/libpq.dll
)
set(PostgreSQL_INCLUDE_DIR ${PROJECT_SOURCE_DIR}/src/win32/cats/pgsql/include)
set(PostgreSQL_TYPE_INCLUDE_DIR
${PROJECT_SOURCE_DIR}/src/win32/plugins/python/include
)
if(${WINDOWS_BITS} MATCHES "32")
set(cross-prefix i686-w64-mingw32)
else()
set(cross-prefix x86_64-w64-mingw32)
endif()

set(Python2_INCLUDE_DIRS /usr/${cross-prefix}/include/python2/)
set(Python3_INCLUDE_DIRS /usr/${cross-prefix}/include/python3/)

set(PostgreSQL_LIBRARY libpq.dll)
set(PostgreSQL_INCLUDE_DIR "")
set(PostgreSQL_TYPE_INCLUDE_DIR "")

set(dynamic-storage-backends OFF)
set(HAVE_DYNAMIC_SD_BACKENDS 0)
Expand Down Expand Up @@ -407,9 +408,9 @@ include(BareosDetermineHaveLowLevelScsiInterface)

include(acltypes)

include_directories(SYSTEM ${PYTHON_INCLUDE_DIRS})
# include_directories(SYSTEM ${Python_INCLUDE_DIRS})

include_directories(SYSTEM ${PYTHON_INCLUDE_PATH})
# include_directories(SYSTEM ${Python_INCLUDE_PATH})

include_directories(${PROJECT_SOURCE_DIR}/src)

Expand Down Expand Up @@ -674,6 +675,46 @@ if(${lmdb})
endif()

# info what the config files need to be installed PLUGINS ############
set(PLUGINS python-ldap)

list(APPEND PLUGINS python-ovirt)

if(VIXDISKLIB_FOUND)
list(APPEND PLUGINS python-vmware)
endif()

if(${HAVE_CEPHFS})
list(APPEND PLUGINS cephfs)
endif()
if(${HAVE_CEPH_RADOS})
list(APPEND PLUGINS rados)
endif()

if(${HAVE_GLUSTERFS})
list(APPEND PLUGINS gfapi)
endif()

set(PLUGINS
${PLUGINS}
PARENT_SCOPE
)

# BACKENDS ####
if(build_client_only)
set(BACKENDS "")
else()
set(BACKENDS unix_tape_device.d)
list(APPEND BACKENDS unix_fifo_device.d)
if(${HAVE_CEPHFS})
list(APPEND BACKENDS rados_device.d)
endif()
if(${HAVE_GLUSTERFS})
list(APPEND BACKENDS gfapi_device.d)
endif()
if(${HAVE_DROPLET})
list(APPEND BACKENDS droplet_device.d)
endif()
endif()

set(support_systemd "")
set(batch_insert_db_backends "")
Expand Down Expand Up @@ -837,10 +878,14 @@ message(
" CEPHFS support: ${CEPHFS_FOUND} ${CEPHFS_LIBRARIES} ${CEPHFS_INCLUDE_DIRS} "
)
message(
" Python support: ${PYTHONLIBS_FOUND} ${PYTHONLIBS_VERSION_STRING} ${PYTHON_INCLUDE_PATH}"
" Python2 support: ${Python2_FOUND} ${Python2_VERSION} ${Python2_INCLUDE_DIRS} ${Python2_EXECUTABLE}"
)
message(
" Python3 support: ${Python3_FOUND} ${Python3_VERSION} ${Python3_INCLUDE_DIRS} ${Python3_EXECUTABLE}"
)
message(" systemd support: ${WITH_SYSTEMD} ${SYSTEMD_UNITDIR}")
message(" Batch insert enabled: ${USE_BATCH_FILE_INSERT}")
message(" PostgreSQL Version: ${PostgreSQL_VERSION_STRING} ")
if(GTEST_FOUND)
message(
" gtest support: ${GTEST_FOUND} ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} "
Expand Down Expand Up @@ -880,6 +925,8 @@ message(" AWK: ${AWK}")
message(" GAWK: ${GAWK}")
message(" RPCGEN: ${RPCGEN}")
message(" MTX: ${MTX}")
message(" MINIO: ${MINIO}")
message(" S3CMD: ${S3CMD}")
message(" XTRABACKUP: ${XTRABACKUP}")
message(" DEVELOPER: ${developer}")
message(" LocalBuildDefinitionsFile: ${BareosLocalBuildDefinitionsFile}")
Expand Down
81 changes: 76 additions & 5 deletions core/cmake/BareosFindAllLibraries.cmake
Expand Up @@ -23,15 +23,86 @@ if(${SYSTEMD_FOUND})
endif()

if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows")
# make sure we get python 2 not 3
set(Python_ADDITIONAL_VERSIONS 2.5 2.6 2.7 2.8 2.9)
find_package(PythonInterp)
include(FindPythonLibs)
if(${CMAKE_VERSION} VERSION_LESS "3.12.0")
message(STATUS "CMake too old for FindPython2/3, using FindPythonInterp")
set(Python2_FOUND FALSE)
set(Python3_FOUND FALSE)
set( Python_ADDITIONAL_VERSIONS 2.6 2.7)
find_package(PythonInterp)
find_package(PythonLibs)
message(STATUS "Found PYTHON_VERSION_MAJOR" ${PYTHON_VERSION_MAJOR})
if (PYTHON_VERSION_MAJOR EQUAL 2)
set(Python2_FOUND ${PYTHONLIBS_FOUND})
set(Python2_LIBRARIES ${PYTHON_LIBRARIES})
set(Python2_INCLUDE_DIRS ${PYTHON_INCLUDE_DIRS})
set(Python2_EXECUTABLE ${PYTHON_EXECUTABLE})

elseif (PYTHON_VERSION_MAJOR EQUAL 3)
set(Python3_FOUND ${PYTHONLIBS_FOUND})
set(Python3_LIBRARIES ${PYTHON_LIBRARIES})
set(Python3_INCLUDE_DIRS ${PYTHON_INCLUDE_DIRS})
set(Python3_EXECUTABLE ${PYTHON_EXECUTABLE})
endif()

set( Python_ADDITIONAL_VERSIONS 3.6 3.7 3.8 3.9)
find_package(PythonInterp)
find_package(PythonLibs)
message(STATUS "Found PYTHON_VERSION_MAJOR" ${PYTHON_VERSION_MAJOR})

if (PYTHON_VERSION_MAJOR EQUAL 2)
set(Python2_FOUND ${PYTHONLIBS_FOUND})
set(Python2_LIBRARIES ${PYTHON_LIBRARIES})
set(Python2_INCLUDE_DIRS ${PYTHON_INCLUDE_DIRS})
set(Python2_EXECUTABLE ${PYTHON_EXECUTABLE})
elseif (PYTHON_VERSION_MAJOR EQUAL 3)
set(Python3_FOUND ${PYTHONLIBS_FOUND})
set(Python3_LIBRARIES ${PYTHON_LIBRARIES})
set(Python3_INCLUDE_DIRS ${PYTHON_INCLUDE_DIRS})
set(Python3_EXECUTABLE ${PYTHON_EXECUTABLE})
endif()

if(${PYTHONLIBS_FOUND})
else()
find_package (Python2 COMPONENTS Interpreter Development)
find_package (Python3 COMPONENTS Interpreter Development)
endif()

if(${Python2_FOUND} OR ${Python3_FOUND})
set(HAVE_PYTHON 1)
endif()

if(${Python2_FOUND})
set(PYTHON_EXECUTABLE ${Python2_EXECUTABLE} PARENT_SCOPE)
set(Python2_EXECUTABLE ${Python2_EXECUTABLE} PARENT_SCOPE)
execute_process(
COMMAND ${Python2_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/get_python_compile_settings.py
OUTPUT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/py2settings.cmake
)
include(${CMAKE_CURRENT_SOURCE_DIR}/py2settings.cmake)
endif()

if(${Python3_FOUND})
set(PYTHON_EXECUTABLE ${Python3_EXECUTABLE} PARENT_SCOPE)
set(Python3_EXECUTABLE ${Python3_EXECUTABLE} PARENT_SCOPE)
execute_process(
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/get_python_compile_settings.py
OUTPUT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/py3settings.cmake
)
include(${CMAKE_CURRENT_SOURCE_DIR}/py3settings.cmake)
endif()

else() # windows
set(HAVE_PYTHON 1)
set(Python2_FOUND 1)
set(Python2_EXT_SUFFIX ".dll")

set(Python3_FOUND 1)
set(Python3_EXT_SUFFIX ".pyd")
endif()




if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows")
include(FindPostgreSQL)
endif()

Expand Down
2 changes: 2 additions & 0 deletions core/cmake/BareosFindPrograms.cmake
Expand Up @@ -39,4 +39,6 @@ find_program(GDB gdb)
find_program(DBX dbx)
find_program(MDB mdb)
find_program(XTRABACKUP xtrabackup)
find_program(S3CMD s3cmd)
find_program(MINIO minio)

12 changes: 11 additions & 1 deletion core/cmake/BareosInstallConfigFiles.cmake
Expand Up @@ -159,8 +159,18 @@ macro(
# install configs from fd plugins
foreach(PLUGIN ${PLUGINS})
message(STATUS "install config files for PLUGIN ${PLUGIN}")

# if plugin has -, conf.d directory is:
# python-ovirt -> python/ovirt/python-ovirt-conf.d
# else it is :
# cephfs -> cephfs/cephfs-conf.d

string(REPLACE "-" "/" PLUGINPATH "${PLUGIN}")
string(APPEND PLUGINPATH "/" ${PLUGIN} "-conf.d")
message(STATUS "PLUGINPATH for PLUGIN ${PLUGIN} is ${PLUGINPATH}")

file(GLOB resourcedirs
"${SRC_DIR}/src/plugins/filed/${PLUGIN}/${CONFIGBASEDIRECTORY}/*")
"${SRC_DIR}/src/plugins/filed/${PLUGINPATH}/${CONFIGBASEDIRECTORY}/*")
foreach(resdir ${resourcedirs})
file(GLOB configfiles "${resdir}/*.conf*")
get_filename_component(resname ${resdir} NAME)
Expand Down
57 changes: 57 additions & 0 deletions core/cmake/get_python_compile_settings.py
@@ -0,0 +1,57 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-

# BAREOS(R) - Backup Archiving REcovery Open Sourced
#
# Copyright (C) 2020-2020 Bareos GmbH & Co. KG
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of version three of the GNU Affero General Public
# License as published by the Free Software Foundation, which is
# listed in the file LICENSE.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.

# This program extracts the build configuration from the python interpreter.
# The output is consumed by cmake where appropriate variables are set.
# This is required to build python modules without setuptools.

import sys
try:
import sysconfig
except:
import distutils.sysconfig as sysconfig

for var in ("CC", "BLDSHARED"):
value = sysconfig.get_config_var(var)
print ("message(STATUS \"Python{0}_{1}\ is\ {2}\")" . format(sys.version_info[0], var, value))
print ("set(Python{0}_{1} \"{2}\")" . format(sys.version_info[0], var, value))

# if nothing comes after the compile command, the flags are empty
try:
value = value.split(' ',1)[1]
except:
value = ''
# as these vars contain the compiler itself, we remove the first word and return it as _FLAGS
print ("message(STATUS \"Python{0}_{1}_FLAGS\ is\ {2}\")" . format(sys.version_info[0], var, value))
print ("set(Python{0}_{1}_FLAGS \"{2}\")" . format(sys.version_info[0], var, value))

for var in ("CFLAGS","CCSHARED","INCLUDEPY","LDFLAGS"):
value = sysconfig.get_config_var(var)
print ("message(STATUS \"Python{0}_{1}\ is\ {2}\")" . format(sys.version_info[0], var, value))
print ("set(Python{0}_{1} \"{2}\")" . format(sys.version_info[0], var, value))

for var in ("EXT_SUFFIX",):
value = sysconfig.get_config_var(var)
if value is None:
value = ''
print ("message(STATUS \"Python{0}_{1}\ is\ {2}\")" . format(sys.version_info[0], var, value))
print ("set(Python{0}_{1} \"{2}\")" . format(sys.version_info[0], var, value))
Expand Up @@ -204,9 +204,6 @@ lib/bareos/plugins/bareos-fd-mock-test.py
lib/bareos/plugins/bareos-fd.py.template
lib/bareos/plugins/bareos-sd-class-plugin.py
lib/bareos/plugins/bareos-sd.py.template
lib/bareos/plugins/bareos_dir_consts.py
lib/bareos/plugins/bareos_fd_consts.py
lib/bareos/plugins/bareos_sd_consts.py
lib/bareos/plugins/bpipe-fd.so
lib/bareos/plugins/python-dir.so
lib/bareos/plugins/python-fd.so
Expand Down
@@ -1,7 +1,6 @@

lib/bareos/plugins/python-dir.so
lib/bareos/plugins/bareos-dir.py.template
lib/bareos/plugins/bareos_dir_consts.py
lib/bareos/plugins/BareosDirPluginBaseclass.py
lib/bareos/plugins/bareos-dir-class-plugin.py
lib/bareos/plugins/BareosDirWrapper.py
Expand Up @@ -6,4 +6,3 @@ lib/bareos/plugins/bareos-fd-mock-test.py
lib/bareos/plugins/BareosFdPluginBaseclass.py
lib/bareos/plugins/BareosFdPluginLocalFileset.py
lib/bareos/plugins/BareosFdWrapper.py
lib/bareos/plugins/bareos_fd_consts.py
@@ -1,7 +1,6 @@

lib/bareos/plugins/python-sd.so
lib/bareos/plugins/bareos-sd.py.template
lib/bareos/plugins/bareos_sd_consts.py
lib/bareos/plugins/BareosSdPluginBaseclass.py
lib/bareos/plugins/BareosSdWrapper.py
lib/bareos/plugins/bareos-sd-class-plugin.py
Expand Up @@ -204,9 +204,6 @@ lib/bareos/plugins/bareos-fd-mock-test.py
lib/bareos/plugins/bareos-fd.py.template
lib/bareos/plugins/bareos-sd-class-plugin.py
lib/bareos/plugins/bareos-sd.py.template
lib/bareos/plugins/bareos_dir_consts.py
lib/bareos/plugins/bareos_fd_consts.py
lib/bareos/plugins/bareos_sd_consts.py
lib/bareos/plugins/bpipe-fd.so
lib/bareos/plugins/python-dir.so
lib/bareos/plugins/python-fd.so
Expand Down

0 comments on commit bfd7053

Please sign in to comment.