Skip to content

Commit

Permalink
[sipify] use configuration file for SIP sources (*.sip.in)
Browse files Browse the repository at this point in the history 
to allow preprocessing the files to handle version specific features such as template based classes documentation in 4.19.7+
  • Loading branch information
@3nids
3nids committed Jan 17, 2018
1 parent 5327978 commit 1d4e6a8
Show file tree
Hide file tree
Showing 5 changed files with 50 additions and 43 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -704,7 +704,7 @@ IF (WITH_CORE AND WITH_BINDINGS)
INCLUDE(PyQtMacros)
INCLUDE(SIPMacros)

SET(SIP_INCLUDES ${PYQT_SIP_DIR} ${CMAKE_SOURCE_DIR}/python ${CMAKE_SOURCE_DIR}/python/core)
SET(SIP_INCLUDES ${PYQT_SIP_DIR} ${CMAKE_SOURCE_DIR}/python)
SET(SIP_CONCAT_PARTS 4)

IF (NOT BINDINGS_GLOBAL_INSTALL)
Expand Down
22 changes: 14 additions & 8 deletions cmake/SIPMacros.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ SET(SIP_DISABLE_FEATURES)
SET(SIP_EXTRA_OPTIONS)
SET(SIP_EXTRA_OBJECTS)

MACRO(GENERATE_SIP_PYTHON_MODULE_CODE MODULE_NAME MODULE_SIP CPP_FILES)
MACRO(GENERATE_SIP_PYTHON_MODULE_CODE MODULE_NAME MODULE_SIP SIP_FILES CPP_FILES)
STRING(REPLACE "." "/" _x ${MODULE_NAME})
GET_FILENAME_COMPONENT(_parent_module_path ${_x} PATH)
GET_FILENAME_COMPONENT(_child_module_name ${_x} NAME)
Expand All @@ -52,13 +52,19 @@ MACRO(GENERATE_SIP_PYTHON_MODULE_CODE MODULE_NAME MODULE_SIP CPP_FILES)

FILE(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${_module_path}) # Output goes in this dir.

# If this is not need anymore (using input configuration file for SIP modules)
# Then SIP could build against the file in the source rather than in CMake current directory
# and thus remove the 2 extras includes:
# - hereafter in the custom command: -I ${CMAKE_CURRENT_SOURCE_DIR}/${_module_path}
# - in top CMakeLists.txt in SIP_INCLUDES declaraiton the core part
# If this is not need anymore (using input configuration file for SIP files)
# SIP could be run in the source rather than in binary directory
SET(_configured_module_sip ${CMAKE_CURRENT_BINARY_DIR}/${_module_path}/${_module_path}.sip)
CONFIGURE_FILE(${_abs_module_sip}.in ${_configured_module_sip})
FOREACH (_sip_file ${SIP_FILES})
GET_FILENAME_COMPONENT(_sip_file_path ${_sip_file} PATH)
GET_FILENAME_COMPONENT(_sip_file_name_we ${_sip_file} NAME_WE)
FILE(RELATIVE_PATH _sip_file_relpath ${CMAKE_CURRENT_SOURCE_DIR} "${_sip_file_path}/${_sip_file_name_we}")
SET(_out_sip_file "${CMAKE_CURRENT_BINARY_DIR}/${_sip_file_relpath}.sip")
CONFIGURE_FILE(${_sip_file} ${_out_sip_file})
MESSAGE(${_sip_file})
MESSAGE(${_out_sip_file})
ENDFOREACH (_sip_file)


SET(_sip_includes)
FOREACH (_inc ${SIP_INCLUDES})
Expand Down Expand Up @@ -107,7 +113,7 @@ MACRO(GENERATE_SIP_PYTHON_MODULE_CODE MODULE_NAME MODULE_SIP CPP_FILES)
ADD_DEFINITIONS( /bigobj )
ENDIF(MSVC)

SET(SIPCMD ${SIP_BINARY_PATH} ${_sip_tags} -w -e ${_sip_x} ${SIP_EXTRA_OPTIONS} -j ${SIP_CONCAT_PARTS} -c ${CMAKE_CURRENT_BINARY_DIR}/${_module_path} -I ${CMAKE_CURRENT_SOURCE_DIR}/${_module_path} ${_sip_includes} ${_configured_module_sip})
SET(SIPCMD ${SIP_BINARY_PATH} ${_sip_tags} -w -e ${_sip_x} ${SIP_EXTRA_OPTIONS} -j ${SIP_CONCAT_PARTS} -c ${CMAKE_CURRENT_BINARY_DIR}/${_module_path} -I ${CMAKE_BINARY_SOURCE_DIR}/${_module_path} ${_sip_includes} ${_configured_module_sip})

This comment has been minimized.

Sign in to view

Copy link
@nirvn

nirvn Jan 22, 2018

Contributor

@3nids , it seems reverting that specific line un-broke my build locally.
ADD_CUSTOM_COMMAND(
OUTPUT ${_sip_output_files}
COMMAND ${CMAKE_COMMAND} -E echo ${message}
Expand Down
29 changes: 13 additions & 16 deletions python/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,16 +171,20 @@ IF(NOT QT_MOBILITY_LOCATION_FOUND)
SET(SIP_DISABLE_FEATURES ${SIP_DISABLE_FEATURES} MOBILITY_LOCATION)
ENDIF(NOT QT_MOBILITY_LOCATION_FOUND)

# SIP 4.19.7+ can prepend auto-generated Python signature to existing Docstrings
# SIP 4.19.7+ can:
# * prepend auto-generated Python signature to existing Docstrings
# * document template based classes
SET(DOCSTRINGSTEMPLATE "//")
IF(${SIP_VERSION_STR} VERSION_GREATER 4.19.6)
SET(DEFAULTDOCSTRINGSIGNATURE "%DefaultDocstringSignature \"prepended\"")
SET(DOCSTRINGSTEMPLATE "")
ENDIF(${SIP_VERSION_STR} VERSION_GREATER 4.19.6)

# core module
FILE(GLOB_RECURSE sip_files_core core/*.sip)
FILE(GLOB_RECURSE sip_files_core core/*.sip core/*.sip.in)
SET(SIP_EXTRA_FILES_DEPEND ${sip_files_core})
SET(SIP_EXTRA_OPTIONS ${PYQT_SIP_FLAGS} -o -a ${CMAKE_BINARY_DIR}/python/qgis.core.api)
GENERATE_SIP_PYTHON_MODULE_CODE(qgis._core core/core.sip cpp_files)
GENERATE_SIP_PYTHON_MODULE_CODE(qgis._core core/core.sip "${sip_files_core}" cpp_files)
BUILD_SIP_PYTHON_MODULE(qgis._core core/core.sip ${cpp_files} "" qgis_core)
SET(SIP_CORE_CPP_FILES ${cpp_files})

Expand All @@ -195,7 +199,7 @@ SET(PY_MODULES core analysis)
IF (WITH_GUI)
SET(PY_MODULES ${PY_MODULES} gui)

FILE(GLOB_RECURSE sip_files_gui gui/*.sip)
FILE(GLOB_RECURSE sip_files_gui gui/*.sip gui/*.sip.in)
SET(SIP_EXTRA_FILES_DEPEND ${sip_files_core} ${sip_files_gui})
SET(SIP_EXTRA_OPTIONS ${PYQT_SIP_FLAGS} -o -a ${CMAKE_BINARY_DIR}/python/qgis.gui.api)

Expand All @@ -206,7 +210,7 @@ IF (WITH_GUI)
SET(SIP_DISABLE_FEATURES ${SIP_DISABLE_FEATURES} HAVE_QSCI_SIP)
ENDIF(QSCI_SIP_DIR)

GENERATE_SIP_PYTHON_MODULE_CODE(qgis._gui gui/gui.sip cpp_files)
GENERATE_SIP_PYTHON_MODULE_CODE(qgis._gui gui/gui.sip "${sip_files_gui}" cpp_files)
BUILD_SIP_PYTHON_MODULE(qgis._gui gui/gui.sip ${cpp_files} "" qgis_core qgis_gui)
ENDIF (WITH_GUI)

Expand All @@ -219,10 +223,10 @@ IF (WITH_SERVER AND WITH_SERVER_PLUGINS)

SET(PY_MODULES ${PY_MODULES} server)

FILE(GLOB_RECURSE sip_files_server server/*.sip)
FILE(GLOB_RECURSE sip_files_server server/*.sip server/*.sip.in)
SET(SIP_EXTRA_FILES_DEPEND ${sip_files_core} ${sip_files_server})
SET(SIP_EXTRA_OPTIONS ${PYQT_SIP_FLAGS} -o -a ${CMAKE_BINARY_DIR}/python/qgis.server.api)
GENERATE_SIP_PYTHON_MODULE_CODE(qgis._server server/server.sip cpp_files)
GENERATE_SIP_PYTHON_MODULE_CODE(qgis._server server/server.sip "${sip_files_server}" cpp_files)
BUILD_SIP_PYTHON_MODULE(qgis._server server/server.sip ${cpp_files} "" qgis_core qgis_server)
ENDIF (WITH_SERVER AND WITH_SERVER_PLUGINS)

Expand All @@ -243,17 +247,10 @@ INCLUDE_DIRECTORIES(BEFORE
)

# analysis module
FILE(GLOB sip_files_analysis
analysis/*.sip
analysis/raster/*.sip
analysis/vector/*.sip
analysis/network/*.sip
analysis/interpolation/*.sip
analysis/openstreetmap/*.sip
)
FILE(GLOB_RECURSE sip_files_analysis analysis/*.sip analysis/*.sip.in)
SET(SIP_EXTRA_FILES_DEPEND ${sip_files_core} ${sip_files_analysis})
SET(SIP_EXTRA_OPTIONS ${PYQT_SIP_FLAGS} -o -a ${CMAKE_BINARY_DIR}/python/qgis.analysis.api)
GENERATE_SIP_PYTHON_MODULE_CODE(qgis._analysis analysis/analysis.sip cpp_files)
GENERATE_SIP_PYTHON_MODULE_CODE(qgis._analysis analysis/analysis.sip "${sip_files_analysis}" cpp_files)
BUILD_SIP_PYTHON_MODULE(qgis._analysis analysis/analysis.sip ${cpp_files} "" qgis_core qgis_analysis)

SET(QGIS_PYTHON_DIR ${PYTHON_SITE_PACKAGES_DIR}/qgis)
Expand Down
18 changes: 11 additions & 7 deletions scripts/sipify.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@

# read arguments
my $debug = 0;
my $SUPPORT_TEMPLATE_DOCSTRING = 0;
die("usage: $0 [-debug] [-template-doc] headerfile\n") unless GetOptions ("debug" => \$debug, "template-doc" => \$SUPPORT_TEMPLATE_DOCSTRING) && @ARGV == 1;
#my $SUPPORT_TEMPLATE_DOCSTRING = 0;
#die("usage: $0 [-debug] [-template-doc] headerfile\n") unless GetOptions ("debug" => \$debug, "template-doc" => \$SUPPORT_TEMPLATE_DOCSTRING) && @ARGV == 1;
die("usage: $0 [-debug] headerfile\n") unless GetOptions ("debug" => \$debug) && @ARGV == 1;
my $headerfile = $ARGV[0];

# read file
Expand Down Expand Up @@ -967,9 +968,10 @@ sub detect_non_method_member{
$LINE =~ s/^(\s*struct )\w+_EXPORT (.+)$/$1$2/;

# Skip comments
if ( $SUPPORT_TEMPLATE_DOCSTRING == 1 &&
$LINE =~ m/^\s*typedef\s+\w+\s*<\s*\w+\s*>\s+\w+\s+.*SIP_DOC_TEMPLATE/ ) {
$COMMENT_TEMPLATE_DOCSTRING = 0;
if ( $LINE =~ m/^\s*typedef\s+\w+\s*<\s*\w+\s*>\s+\w+\s+.*SIP_DOC_TEMPLATE/ ) {
# support Docstring for template based classes in SIP 4.19.7+
$COMMENT_TEMPLATE_DOCSTRING = 1;
}
elsif ( $LINE =~ m/\/\// ||
$LINE =~ m/^\s*typedef / ||
Expand Down Expand Up @@ -1049,22 +1051,24 @@ sub detect_non_method_member{
else {
dbg_info('writing comment');
if ( $COMMENT !~ m/^\s*$/ ){
write_output("CM1", "%Docstring\n");
my $doc_prepend = "";
$doc_prepend = "\@TEMPLATE_DOCSTRING\@" if $COMMENT_TEMPLATE_DOCSTRING == 1;
write_output("CM1", "$doc_prepend%Docstring\n");
my @comment_lines = split /\n/, $COMMENT;
foreach my $comment_line (@comment_lines) {
# if ( $RETURN_TYPE ne '' && $comment_line =~ m/^\s*\.\. \w/ ){
# # return type must be added before any other paragraph-level markup
# write_output("CM5", ":rtype: $RETURN_TYPE\n\n");
# $RETURN_TYPE = '';
# }
write_output("CM2", "$comment_line\n");
write_output("CM2", "$doc_prepend$comment_line\n");
# if ( $RETURN_TYPE ne '' && $comment_line =~ m/:return:/ ){
# # return type must be added before any other paragraph-level markup
# write_output("CM5", ":rtype: $RETURN_TYPE\n\n");
# $RETURN_TYPE = '';
# }
}
write_output("CM4", "%End\n");
write_output("CM4", "$doc_prepend%End\n");
}
# if ( $RETURN_TYPE ne '' ){
# write_output("CM3", "\n:rtype: $RETURN_TYPE\n");
Expand Down
22 changes: 11 additions & 11 deletions scripts/sipify_all.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,15 @@
###########################################################################
set -e

TEMPLATE_DOC=""
while :; do
case $1 in
-t|--template-doc) TEMPLATE_DOC="-template-doc"
;;
*) break
esac
shift
done
# TEMPLATE_DOC=""
# while :; do
# case $1 in
# -t|--template-doc) TEMPLATE_DOC="-template-doc"
# ;;
# *) break
# esac
# shift
# done

DIR=$(git rev-parse --show-toplevel)

Expand All @@ -40,14 +40,14 @@ count=0
modules=(core gui analysis server)
for module in "${modules[@]}"; do
while read -r sipfile; do
echo "$sipfile"
echo "$sipfile.in"
header=$(${GP}sed -E 's/(.*)\.sip/src\/\1.h/' <<< $sipfile)
if [ ! -f $header ]; then
echo "*** Missing header: $header for sipfile $sipfile"
else
path=$(${GP}sed -r 's@/[^/]+$@@' <<< $sipfile)
mkdir -p python/$path
./scripts/sipify.pl $TEMPLATE_DOC $header > python/$sipfile
./scripts/sipify.pl $header > python/$sipfile.in
fi
count=$((count+1))
done < <( ${GP}sed -n -r "s/^%Include (.*\.sip)/${module}\/\1/p" python/${module}/${module}_auto.sip )
Expand Down

5 comments on commit 1d4e6a8

@tudorbarascu
Copy link
Member

@tudorbarascu tudorbarascu commented on 1d4e6a8 Jan 19, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@3nids This commit introduces the following build error on Debian Testing:

[1186/3652] Generating core/sip_corepart0.cpp, core/sip_corepart1.cpp, core/sip_corepart2.cpp, core/sip_corepart3.cpp
FAILED: python/core/sip_corepart0.cpp python/core/sip_corepart1.cpp python/core/sip_corepart2.cpp python/core/sip_corepart3.cpp 
cd /home/web/dev/cpp/qgis/build-master/python && /usr/bin/cmake -E echo && /usr/bin/cmake -E touch /home/web/dev/cpp/qgis/build-master/python/core/sip_corepart0.cpp /home/web/dev/cpp/qgis/build-master/python/core/sip_corepart1.cpp /home/web/dev/cpp/qgis/build-master/python/core/sip_corepart2.cpp /home/web/dev/cpp/qgis/build-master/python/core/sip_corepart3.cpp && /usr/bin/sip -w -e -x ANDROID -x ARM -x MOBILITY_LOCATION -t WS_X11 -t Qt_5_9_2 -o -a /home/web/dev/cpp/qgis/build-master/python/qgis.core.api -j 4 -c /home/web/dev/cpp/qgis/build-master/python/core -I /core -I /usr/share/sip/PyQt5 -I /home/web/dev/cpp/qgis/python /home/web/dev/cpp/qgis/build-master/python/core/core.sip

sip: Error in opening file /core/QtXml/QtXmlmod.sip

I did a git bisect to find it.

If it helps:

-- The C compiler identification is GNU 7.2.0
-- The CXX compiler identification is GNU 7.2.0
-- Check for working C compiler: /usr/bin/cc
-- Check for working C compiler: /usr/bin/cc -- works
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Detecting C compile features
-- Detecting C compile features - done
-- Check for working CXX compiler: /usr/bin/c++
-- Check for working CXX compiler: /usr/bin/c++ -- works
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- QGIS version: 2.99.0 Master (29900)
-- Qt WebKit support enabled
-- Found Qt version: 5.9.2
-- Found QScintilla2: /usr/lib/libqscintilla2_qt5.so (2.10.2)
-- Pedantic compiler settings enabled
-- Debug output enabled
-- Found PythonInterp: /usr/bin/python3 (found suitable version "3.6.4", minimum required is "3") 
-- Found Python executable: /usr/bin/python3
-- Found Python version: 3.6.4
-- Found Python library: /usr/lib/x86_64-linux-gnu/libpython3.6m.so
-- Found Python site-packages: /usr/lib/python3/dist-packages
-- Found PyQt5 version: 5.9.2
-- Found SIP version: 4.19.6

What's curious I get this on my server and not on my laptop although it's the same debian version with updated packages.
Any ideas? Thanks

@3nids
Copy link
Member Author

@3nids 3nids commented on 1d4e6a8 Jan 19, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

have you tried to make clean?

@tudorbarascu
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes.

  • I even git pulled the QGIS source again just to be sure everything is clean
  • I ran debsums to check for packages integrity
  • The build doesn't work from this commit on and I have no clue why.
    There's also https://issues.qgis.org/issues/16034 but for 2.18

@nirvn
Copy link
Contributor

@nirvn nirvn commented on 1d4e6a8 Jan 22, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@3nids , I see this issue on my system too, on a clean (i.e. new) build directory.

I'm on Ubuntu 17.10.

@3nids
Copy link
Member Author

@3nids 3nids commented on 1d4e6a8 Jan 22, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should be fixed in 9c5797e

Please sign in to comment.