[pyqgis-console] Add targets and python script for generating console's QScintilla .pap file, for auto-completion

- Run 'make;make qsci-pap-src; make install' to update the default .pap file in source tree
- Run 'make;make qsci-pap-master[; make install]' for local-only .pap that overrides the default .pap (when testing new bindings)
- Update console to override default .pap if master.pap is available
- Include new qgis.core.NULL attribute defined in <src>/python/__init__.py for QPyNullVariant comparisons
- Update 11-month-old default .pap

Author: dakcarto

CMakeLists.txt

@@ -86,7 +86,7 @@ IF (WITH_BINDINGS)
   SET (WITH_STAGED_PLUGINS TRUE CACHE BOOL "Stage-install core Python plugins to run from build directory? (utilities and console are always staged)")
   SET (WITH_PY_COMPILE FALSE CACHE BOOL "Determines whether Python modules in staged or installed locations are byte-compiled")
   # concatenate QScintilla2 API files
-  SET (WITH_QSCIAPI TRUE CACHE BOOL "Determines whether the QScintilla2 API files will be updated and concatenated")
+  SET (WITH_QSCIAPI TRUE CACHE BOOL "Whether to generate PyQGIS QScintilla2 API file. Run 'make qsci-pap-master or qsci-pap-src' in between QGIS build and install, to generate .pap file for console auto-completion.")
   # path to custom Python framework on Mac
   IF (APPLE)
     SET (PYTHON_CUSTOM_FRAMEWORK "" CACHE PATH "Path to custom Python.framework on Mac. (should not have to specify other Python options)")

cmake/QsciAPI.cmake

@@ -7,6 +7,9 @@
 
 SET(QGIS_PYTHON_API_FILE "${CMAKE_BINARY_DIR}/python/qsci_apis/PyQGIS.api")
 
+# create empty destination api file
+FILE(WRITE "${QGIS_PYTHON_API_FILE}" "")
+
 IF(EXISTS "${CMAKE_BINARY_DIR}/python/qgis.gui.api")
   FILE(READ "${CMAKE_BINARY_DIR}/python/qgis.gui.api" FILE_CONTENT)
   STRING(REGEX MATCHALL "gui\\.QgisInterface([^\n]+)" MATCHED_CONTENT "${FILE_CONTENT}")
@@ -16,6 +19,9 @@ IF(EXISTS "${CMAKE_BINARY_DIR}/python/qgis.gui.api")
   ENDFOREACH(matchedLine)
 ENDIF(EXISTS "${CMAKE_BINARY_DIR}/python/qgis.gui.api")
 
+# add qgis.core.NULL attribute defined in <src>/python/__init__.py for QPyNullVariant
+FILE(APPEND "${QGIS_PYTHON_API_FILE}" "qgis.core.NULL?7\n")
+
 FOREACH(apiFile qgis.core.api qgis.gui.api qgis.analysis.api qgis.networkanalysis.api)
   SET(api "${CMAKE_BINARY_DIR}/python/${apiFile}")
   IF(EXISTS "${api}")

python/CMakeLists.txt

@@ -127,17 +127,51 @@ SET(QGIS_PYTHON_DIR ${PYTHON_SITE_PACKAGES_DIR}/qgis)
 IF(WITH_QSCIAPI)
   # wait until after python module builds for api files to be available
   SET(QGIS_PYTHON_API_FILE "${CMAKE_BINARY_DIR}/python/qsci_apis/PyQGIS.api")
-  # create empty destination api file
-  FILE(WRITE "${QGIS_PYTHON_API_FILE}" "")
 
-  # run update/concatenate command after last python module is built (currently python_module_qgis_gui)
-  ADD_CUSTOM_COMMAND(TARGET python_module_qgis_gui
+  ADD_CUSTOM_TARGET(qsci-api ALL
+    DEPENDS python_module_qgis_gui python_module_qgis_core python_module_qgis_analysis python_module_qgis_networkanalysis)
+
+  # run update/concatenate command
+  ADD_CUSTOM_COMMAND(TARGET qsci-api
     POST_BUILD
     COMMAND "${CMAKE_COMMAND}" -P "${CMAKE_SOURCE_DIR}/cmake/QsciAPI.cmake"
     WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
     COMMENT "Generating pyqgis api file" VERBATIM)
 
   INSTALL(FILES ${QGIS_PYTHON_API_FILE} DESTINATION "${QGIS_DATA_DIR}/python/qsci_apis")
+
+  # create targets for generating console auto-completion *.pap binary file
+  # these take too long to build (> 1 minute) for targets to have ALL property
+  SET(APIS_SRC_DIR "${CMAKE_SOURCE_DIR}/python/qsci_apis")
+  SET(APIS_BIN_DIR "${CMAKE_BINARY_DIR}/python/qsci_apis")
+
+  # generate a local-only .pap file based upon current master branch,
+  #   which will override (but not replace) the default .pap from source
+  # run before 'make install' to have the local-only .pap file included in the install;
+  #   console will use it when running from build directory, regardless of whether it is installed
+  ADD_CUSTOM_TARGET(qsci-pap-master
+    DEPENDS qsci-api ${QGIS_PYTHON_API_FILE})
+
+  SET(PAP_NAME "pyqgis-master.pap")
+  ADD_CUSTOM_COMMAND(TARGET qsci-pap-master
+    POST_BUILD
+    COMMAND ${PYTHON_EXECUTABLE} "${APIS_SRC_DIR}/generate_console_pap.py" "${PAP_NAME}" "${APIS_SRC_DIR}" "${APIS_BIN_DIR}"
+    WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+    COMMENT "Generating local-only ${PAP_NAME} for console auto-completion (MAY TAKE > 1 MINUTE!)" VERBATIM)
+
+  INSTALL(FILES "${APIS_BIN_DIR}/pyqgis-master.pap" DESTINATION "${QGIS_DATA_DIR}/python/qsci_apis")
+
+  # generate a .pap file to be immediately installed in QGIS source tree (the default .pap)
+  ADD_CUSTOM_TARGET(qsci-pap-src
+    DEPENDS qsci-api ${QGIS_PYTHON_API_FILE})
+
+  SET(PAP_NAME "pyqgis.pap")
+  ADD_CUSTOM_COMMAND(TARGET qsci-pap-src
+    POST_BUILD
+    COMMAND ${PYTHON_EXECUTABLE} "${APIS_SRC_DIR}/generate_console_pap.py" "${PAP_NAME}" "${APIS_SRC_DIR}" "${APIS_SRC_DIR}"
+    WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+    COMMENT "Generating ${PAP_NAME} for console auto-completion (MAY TAKE > 1 MINUTE!)" VERBATIM)
+
 ENDIF(WITH_QSCIAPI)
 
 # Plugin utilities files to copy to staging or install

python/console/console_sci.py

@@ -192,7 +192,18 @@ def setLexers(self):
         chekBoxAPI = self.settings.value("pythonConsole/preloadAPI", True, type=bool)
         chekBoxPreparedAPI = self.settings.value("pythonConsole/usePreparedAPIFile", False, type=bool)
         if chekBoxAPI:
-            self.api.loadPrepared(QgsApplication.pkgDataPath() + "/python/qsci_apis/pyqgis_master.pap")
+            apisdir = os.path.join(QgsApplication.pkgDataPath(), "python", "qsci_apis")
+            pap = os.path.join(apisdir, "pyqgis.pap")
+            mpap = os.path.join(apisdir, "pyqgis-master.pap")
+            if os.path.exists(mpap):  # override installed with master .pap build
+                pap = mpap
+            if QgsApplication.isRunningFromBuildDir():
+                bdir = os.path.dirname(QgsApplication.buildOutputPath())
+                bpap = os.path.join(bdir, "python", "qsci_apis", "pyqgis-master.pap")
+                if os.path.exists(bpap):
+                    # if not generated .pap exists, else fall back to preprepared one
+                    pap = bpap
+            self.api.loadPrepared(pap)
         elif chekBoxPreparedAPI:
             self.api.loadPrepared(self.settings.value("pythonConsole/preparedAPIFile"))
         else:

python/qsci_apis/generate_console_pap.py

@@ -0,0 +1,96 @@
+# -*- coding:utf-8 -*-
+"""
+/***************************************************************************
+Module to generate prepared APIs for calltips and auto-completion.
+                             -------------------
+begin                : 2013-08-29
+copyright            : (C) 2013 Larry Shaffer
+email                : larrys (at) dakotacarto (dot) com
+ ***************************************************************************/
+
+/***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+Portions of this file contain code from Eric4 APIsManager module.
+"""
+
+import sys
+import os
+
+from PyQt4.Qsci import QsciLexerPython, QsciAPIs
+from PyQt4.QtGui import QApplication
+from PyQt4.QtCore import QObject
+
+
+class PrepareAPIs(QObject):
+    def __init__(self, api_lexer, api_files, pap_file):
+        QObject.__init__(self)
+        self._api = None
+        self._api_files = api_files
+        self._api_lexer = api_lexer
+        self._pap_file = pap_file
+
+    def _clearLexer(self):
+        self.qlexer = None
+
+    def _stopPreparation(self):
+        if self._api is not None:
+            self._api.cancelPreparation()
+        self._api = None
+        sys.exit(1)
+
+    def _preparationFinished(self):
+        self._clearLexer()
+        try:
+            if os.path.exists(self._pap_file):
+                os.remove(self._pap_file)
+            prepd = self._api.savePrepared(unicode(self._pap_file))
+            self._api = None
+            sys.exit(0 if prepd else 1)
+        except Exception, err:
+            self._api = None
+            sys.exit(1)
+
+    def prepareAPI(self):
+        try:
+            self._api = QsciAPIs(self._api_lexer)
+            self._api.apiPreparationFinished.connect(self._preparationFinished)
+            for api_file in self._api_files:
+                self._api.load(unicode(api_file))
+            self._api.prepare()
+        except Exception, err:
+            self._api = None
+            sys.exit(1)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 4:
+        print 'Usage: python <script> <pap_name_w-ext> ' \
+              '<APIs_dir_path> <output_dir>'
+        sys.exit(1)
+    pap_name = sys.argv[1]
+    api_src_dir = sys.argv[2]
+    output_dir = sys.argv[3]
+
+    api_files = [
+        os.path.join(output_dir, 'PyQGIS.api'),
+        os.path.join(api_src_dir, 'Python-2.7.api'),
+        os.path.join(api_src_dir, 'PyQt4-4.7.4.api'),
+        os.path.join(api_src_dir, 'OSGeo_GDAL-OGR-1.9.1.api')
+    ]
+    pap_file= os.path.join(output_dir, pap_name)
+
+    # print api_files.__repr__()
+    # print pap_file.__repr__()
+
+    app = QApplication(sys.argv, False)  # just start a non-gui console app
+    api_lexer = QsciLexerPython()
+    prepap = PrepareAPIs(api_lexer, api_files, pap_file)
+    prepap.prepareAPI()
+
+    sys.exit(app.exec_())