diff --git a/.travis.yml b/.travis.yml index 04a1efc..d5604e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,23 +11,16 @@ env: matrix: - QGIS_VERSION_TAG=master language: python -cache: - directories: - - "$HOME/.cache/pip" + python: - '3.6' branches: only: - master_qgis3 -addons: - apt: - packages: - - git - - python-software-properties + before_install: - docker pull ${IMAGE}:${QGIS_VERSION_TAG} install: -- pip install --upgrade pip - docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}:/tests_directory -e LDI_LINZ_KEY -e LDI_MFE_KEY -e LDI_NZDF_KEY -e DISPLAY=:99 ${IMAGE}:${QGIS_VERSION_TAG} - sleep 10 diff --git a/README.md b/README.md index be17c8d..168e595 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ the plugin is started. The left hand panel allows users to filter by service / protocol types (either, All, WFS, WMS, WMTS). All column headers can be toggled to allow ascending or descending ordering of their data. Text can be entered in the "Filter Data Sets" search bar to filter the datasets by keyword. -## Source Code, Further Documentation and Feedback +## Source Code and Feedback Please see the [LINZ-Data-Importer](https://github.com/linz/linz-data-importer/) repository on GitHub. ## Dev Notes diff --git a/linz-data-importer/plugin_upload.py b/linz-data-importer/plugin_upload.py deleted file mode 100644 index 8a23854..0000000 --- a/linz-data-importer/plugin_upload.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -"""This script uploads a plugin package on the server. - Authors: A. Pasotti, V. Picavet - git sha : $TemplateVCSFormat -""" - -import sys -import getpass -import xmlrpclib -from optparse import OptionParser - -# Configuration -PROTOCOL = 'http' -SERVER = 'plugins.qgis.org' -PORT = '80' -ENDPOINT = '/plugins/RPC2/' -VERBOSE = False - - -def main(parameters, arguments): - """Main entry point. - - :param parameters: Command line parameters. - :param arguments: Command line arguments. - """ - address = "%s://%s:%s@%s:%s%s" % ( - PROTOCOL, - parameters.username, - parameters.password, - parameters.server, - parameters.port, - ENDPOINT) - print "Connecting to: %s" % hide_password(address) - - server = xmlrpclib.ServerProxy(address, verbose=VERBOSE) - - try: - plugin_id, version_id = server.plugin.upload( - xmlrpclib.Binary(open(arguments[0]).read())) - print "Plugin ID: %s" % plugin_id - print "Version ID: %s" % version_id - except xmlrpclib.ProtocolError, err: - print "A protocol error occurred" - print "URL: %s" % hide_password(err.url, 0) - print "HTTP/HTTPS headers: %s" % err.headers - print "Error code: %d" % err.errcode - print "Error message: %s" % err.errmsg - except xmlrpclib.Fault, err: - print "A fault occurred" - print "Fault code: %d" % err.faultCode - print "Fault string: %s" % err.faultString - - -def hide_password(url, start=6): - """Returns the http url with password part replaced with '*'. - - :param url: URL to upload the plugin to. - :type url: str - - :param start: Position of start of password. - :type start: int - """ - start_position = url.find(':', start) + 1 - end_position = url.find('@') - return "%s%s%s" % ( - url[:start_position], - '*' * (end_position - start_position), - url[end_position:]) - - -if __name__ == "__main__": - parser = OptionParser(usage="%prog [options] plugin.zip") - parser.add_option( - "-w", "--password", dest="password", - help="Password for plugin site", metavar="******") - parser.add_option( - "-u", "--username", dest="username", - help="Username of plugin site", metavar="user") - parser.add_option( - "-p", "--port", dest="port", - help="Server port to connect to", metavar="80") - parser.add_option( - "-s", "--server", dest="server", - help="Specify server name", metavar="plugins.qgis.org") - options, args = parser.parse_args() - if len(args) != 1: - print "Please specify zip file.\n" - parser.print_help() - sys.exit(1) - if not options.server: - options.server = SERVER - if not options.port: - options.port = PORT - if not options.username: - # interactive mode - username = getpass.getuser() - print "Please enter user name [%s] :" % username, - res = raw_input() - if res != "": - options.username = res - else: - options.username = username - if not options.password: - # interactive mode - options.password = getpass.getpass() - main(options, args) diff --git a/linz-data-importer/tests/__init__.py b/linz-data-importer/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/linz-data-importer/Makefile b/linz_data_importer/Makefile similarity index 100% rename from linz-data-importer/Makefile rename to linz_data_importer/Makefile diff --git a/linz-data-importer/__init__.py b/linz_data_importer/__init__.py similarity index 100% rename from linz-data-importer/__init__.py rename to linz_data_importer/__init__.py diff --git a/linz-data-importer/about.html b/linz_data_importer/about.html similarity index 100% rename from linz-data-importer/about.html rename to linz_data_importer/about.html diff --git a/linz-data-importer/gui/Service_dialog.py b/linz_data_importer/gui/Service_dialog.py similarity index 95% rename from linz-data-importer/gui/Service_dialog.py rename to linz_data_importer/gui/Service_dialog.py index 14e0817..73b86a5 100644 --- a/linz-data-importer/gui/Service_dialog.py +++ b/linz_data_importer/gui/Service_dialog.py @@ -3,7 +3,7 @@ import os - +#from .ExtendedCombobox import ExtendedCombobox from PyQt5 import QtGui, QtWidgets, uic FORM_CLASS, _ = uic.loadUiType(os.path.join( @@ -44,4 +44,4 @@ def __init__(self, parent=None): # } # """ # ) - \ No newline at end of file + diff --git a/linz-data-importer/gui/Service_dialog_base.ui b/linz_data_importer/gui/Service_dialog_base.ui similarity index 99% rename from linz-data-importer/gui/Service_dialog_base.ui rename to linz_data_importer/gui/Service_dialog_base.ui index 17964e4..f3c7b16 100644 --- a/linz-data-importer/gui/Service_dialog_base.ui +++ b/linz_data_importer/gui/Service_dialog_base.ui @@ -146,7 +146,7 @@ - + 160 @@ -819,9 +819,15 @@ + + + ExtendedCombobox + QComboBox +
linz_data_importer.tablemodel.h
+ + uTextFilter - uCRSCombo uBtnImport uListOptions uTableView diff --git a/linz_data_importer/gui/__init__.py b/linz_data_importer/gui/__init__.py new file mode 100644 index 0000000..7837115 --- /dev/null +++ b/linz_data_importer/gui/__init__.py @@ -0,0 +1,4 @@ +import sys +from os.path import dirname, abspath +sys.path.append(dirname(dirname(dirname(abspath(__file__))))) + diff --git a/linz-data-importer/icons/about.png b/linz_data_importer/icons/about.png similarity index 100% rename from linz-data-importer/icons/about.png rename to linz_data_importer/icons/about.png diff --git a/linz-data-importer/icons/add.png b/linz_data_importer/icons/add.png similarity index 100% rename from linz-data-importer/icons/add.png rename to linz_data_importer/icons/add.png diff --git a/linz-data-importer/icons/all.png b/linz_data_importer/icons/all.png similarity index 100% rename from linz-data-importer/icons/all.png rename to linz_data_importer/icons/all.png diff --git a/linz-data-importer/icons/icon.png b/linz_data_importer/icons/icon.png similarity index 100% rename from linz-data-importer/icons/icon.png rename to linz_data_importer/icons/icon.png diff --git a/linz-data-importer/icons/remove.png b/linz_data_importer/icons/remove.png similarity index 100% rename from linz-data-importer/icons/remove.png rename to linz_data_importer/icons/remove.png diff --git a/linz-data-importer/icons/settings.png b/linz_data_importer/icons/settings.png similarity index 100% rename from linz-data-importer/icons/settings.png rename to linz_data_importer/icons/settings.png diff --git a/linz-data-importer/icons/wfs.png b/linz_data_importer/icons/wfs.png similarity index 100% rename from linz-data-importer/icons/wfs.png rename to linz_data_importer/icons/wfs.png diff --git a/linz-data-importer/icons/wms.png b/linz_data_importer/icons/wms.png similarity index 100% rename from linz-data-importer/icons/wms.png rename to linz_data_importer/icons/wms.png diff --git a/linz-data-importer/icons/wmts.png b/linz_data_importer/icons/wmts.png similarity index 100% rename from linz-data-importer/icons/wmts.png rename to linz_data_importer/icons/wmts.png diff --git a/linz-data-importer/linz_data_importer.py b/linz_data_importer/linz_data_importer.py similarity index 97% rename from linz-data-importer/linz_data_importer.py rename to linz_data_importer/linz_data_importer.py index 9e57c47..e9df99d 100644 --- a/linz-data-importer/linz_data_importer.py +++ b/linz_data_importer/linz_data_importer.py @@ -14,7 +14,6 @@ * see the LICENSE file for more information * ***************************************************************************/ """ -from __future__ import absolute_import # This program is released under the terms of the 3 clause BSD license. See the # LICENSE file for more information. @@ -24,7 +23,7 @@ from qgis.PyQt.QtCore import QSettings, QTranslator, qVersion, QCoreApplication, Qt, QRegExp, QSize, Qt from qgis.PyQt.QtWidgets import QAction, QListWidgetItem, QHeaderView, QMenu, QToolButton -from qgis.PyQt.QtGui import QIcon, QPixmap, QImage +from qgis.PyQt.QtGui import QIcon, QPixmap, QImage, QStandardItemModel, QStandardItem from qgis.PyQt.QtCore import QSortFilterProxyModel from qgis.core import (QgsRasterLayer, QgsVectorLayer, QgsProject, QgsCoordinateReferenceSystem, Qgis) @@ -238,7 +237,9 @@ def initGui(self): self.dlg.uListOptions.itemClicked.connect(self.showSelectedOption) self.dlg.uListOptions.itemClicked.emit(self.dlg.uListOptions.item(0)) self.curr_list_wid_index=0 - + + model = QStandardItemModel() + self.dlg.uCRSCombo.setModel(model) self.dlg.uCRSCombo.currentIndexChanged.connect(self.layerCrsSelected) self.dlg.uLabelWarning.setStyleSheet('color:red') @@ -541,9 +542,17 @@ def showSelectedOption(self, item): self.dlg.uStackedWidget.setCurrentIndex(2) def layerCrsSelected(self): - self.selected_crs = str(self.dlg.uCRSCombo.currentText()) - if self.selected_crs: - self.selected_crs_int = int(self.selected_crs.strip('EPSG:')) + """ + Track the user selected crs. Check validity to + ensure only well formed crs are provided. + """ + + valid = re.compile('^EPSG\:\d+') + crs_text = self.dlg.uCRSCombo.currentText() + if valid.match(crs_text): + self.selected_crs = str(self.dlg.uCRSCombo.currentText()) + if self.selected_crs: + self.selected_crs_int = int(self.selected_crs.strip('EPSG:')) def getPreview(self, res, res_timeout): """ @@ -706,7 +715,7 @@ def importDataset(self): Import the selected dataset to QGIS """ - if not self.layers_loaded: + if not self.layers_loaded and not self.data_type == 'table': self.setProjectSRID() if self.service == "WFS": diff --git a/linz-data-importer/metadata.txt b/linz_data_importer/metadata.txt similarity index 100% rename from linz-data-importer/metadata.txt rename to linz_data_importer/metadata.txt diff --git a/linz-data-importer/resources.py b/linz_data_importer/resources.py similarity index 100% rename from linz-data-importer/resources.py rename to linz_data_importer/resources.py diff --git a/linz-data-importer/resources.qrc b/linz_data_importer/resources.qrc similarity index 100% rename from linz-data-importer/resources.qrc rename to linz_data_importer/resources.qrc diff --git a/linz-data-importer/service_data.py b/linz_data_importer/service_data.py similarity index 93% rename from linz-data-importer/service_data.py rename to linz_data_importer/service_data.py index 22de951..00c053b 100644 --- a/linz-data-importer/service_data.py +++ b/linz_data_importer/service_data.py @@ -24,7 +24,7 @@ from owslib.wmts import WebMapTileService from owslib.util import ServiceException -from qgis.core import QgsMessageLog, QgsApplication +from qgis.core import QgsApplication import os.path @@ -40,6 +40,12 @@ from qgis.PyQt.QtCore import QSettings + +#temp +from qgis.core import QgsMessageLog + + + class ApiKey(object): """ Store API Keys for each domain. Required to @@ -166,8 +172,6 @@ def delAllLocalServiceXML(self, services=['wms','wfs','wmts']): file = os.path.join(dir, f) self.delLocalSeviceXML(file) - - def purgeCache(self): """ Delete all cached documents but the @@ -281,7 +285,7 @@ def isEnabled(self): def getServiceData(self): """ - Select method to obtain capbilties doc. + Select method to obtain capabilities doc. Either via localstore or internet """ @@ -381,30 +385,45 @@ def getServiceXml(self): elif hasattr(e, 'code'): self.err = 'Error: ({0}) {1}'.format(self.domain, e.reason) - + + def sortCrs(self): + # wms returns some no valid crs values + valid = re.compile('^EPSG\:\d+') + self.crs = [s for s in self.crs if valid.match(s)] + # sort + self.crs.sort(key = lambda x: int(x.split(':')[1])) + def formatForUI(self): """ Format the service data to display in the UI """ - + + wms_crs = [] service_data = [] cont = self.obj.contents - for dataset_id, dataset_obj in cont.items(): - crs=[] + self.crs=[] full_id = re.search(r'([aA-zZ]+\\.[aA-zZ]+\\.[aA-zZ]+\\.[aA-zZ]+\\:)?(?P[aA-zZ]+)-(?P[0-9]+)', dataset_obj.id) type = full_id.group('type') id = full_id.group('id') # Get and standarise espg codes if self.service == 'wmts': - crs = dataset_obj.tilematrixsets + self.crs = dataset_obj.tilematrixsets + self.sortCrs() elif self.service in ('wfs'): - crs = dataset_obj.crsOptions - crs = ['EPSG:{0}'.format(item.code) for item in crs] + self.crs = dataset_obj.crsOptions + self.crs = ['EPSG:{0}'.format(item.code) for item in self.crs] + self.sortCrs() elif self.service in ('wms'): - crs = dataset_obj.crsOptions - crs = ['EPSG:{0}'.format(item.strip('urn:ogc:def:crs:EPSG::')) for item in crs] + if wms_crs: + self.crs = wms_crs + else: + self.crs = dataset_obj.crsOptions + self.sortCrs() + wms_crs = self.crs + service_data.append([self.domain, type, self.service.upper(), id, - dataset_obj.title, dataset_obj.abstract, crs]) + dataset_obj.title, dataset_obj.abstract, self.crs]) self.info = service_data + diff --git a/linz-data-importer/tablemodel.py b/linz_data_importer/tablemodel.py similarity index 64% rename from linz-data-importer/tablemodel.py rename to linz_data_importer/tablemodel.py index f1f8c2c..666a3c5 100644 --- a/linz-data-importer/tablemodel.py +++ b/linz_data_importer/tablemodel.py @@ -16,11 +16,11 @@ """ from builtins import str -from qgis.PyQt.QtCore import QAbstractTableModel, Qt -from qgis.PyQt.QtWidgets import QTableView +from qgis.PyQt.QtCore import QAbstractTableModel, Qt, QSortFilterProxyModel +from qgis.PyQt.QtWidgets import QTableView, QComboBox, QApplication, QCompleter +from qgis.PyQt.QtGui import QStandardItem import sys - class TableView(QTableView): """ @@ -62,6 +62,7 @@ def __init__(self, data = [[]], headers = [], parent=None): :param parent: None :param parent: None """ + QAbstractTableModel.__init__(self, parent) self.arraydata = data self.header = headers @@ -166,3 +167,90 @@ def flags(self, index): """ return Qt.ItemIsEnabled | Qt.ItemIsSelectable + +class ExtendedCombobox( QComboBox ): + """ + Overwrite combobox to provide text filtering of + combobox list. + """ + + def __init__(self, parent): + """ + Initialise ExtendedCombobox + + :param parent: Parent of combobox + :type parent: PyQt5.QtWidgets.QWidget + """ + + super(ExtendedCombobox, self).__init__(parent) + + self.setFocusPolicy(Qt.StrongFocus) + self.setEditable(True) + self.completer = QCompleter(self) + + # always show all completions + self.completer.setCompletionMode(QCompleter.UnfilteredPopupCompletion) + self.pFilterModel = QSortFilterProxyModel(self) + self.pFilterModel.setFilterCaseSensitivity(Qt.CaseInsensitive) + self.completer.setPopup(self.view()) + self.setCompleter(self.completer) + self.lineEdit().textEdited.connect(self.pFilterModel.setFilterFixedString) + self.completer.activated.connect(self.setTextIfCompleterIsClicked) + + def setModel(self, model): + """ + Set the model to use the Filter model + + :param model: The model to be used by the combobox + :type model: PyQt5.QtGui.QStandardItemModel + """ + + super(ExtendedCombobox, self).setModel(model) + self.pFilterModel.setSourceModel(model) + self.completer.setModel(self.pFilterModel) + + def setModelColumn(self, column): + """ + :param model: The model to be used by the combobox + :type model: PyQt5.QtGui.QStandardItemModel + """ + + self.completer.setCompletionColumn(column) + self.pFilterModel.setFilterKeyColumn(column) + super(ExtendedCombobox, self).setModelColumn(column) + + def view(self): + """ + A QListView of items stored in the model + + :return: items stored in the model + :rtype: PyQt5.QtWidgets.QListView + """ + + + return self.completer.popup() + + def index(self): + """ + Index of the current item in the combobox. + + :return: index of the current item + :rtype: int + """ + + return self.currentIndex() + + def setTextIfCompleterIsClicked(self, text): + """ + :param text: The current text of the qlineedit + :type text: str + + If the combobx lineedit is clicked, set the lineedits + current item as the combobox's current item + """ + + if text: + index = self.findText(text) + self.setCurrentIndex(index) + + diff --git a/linz-data-importer/gui/__init__.py b/linz_data_importer/tests/__init__.py similarity index 100% rename from linz-data-importer/gui/__init__.py rename to linz_data_importer/tests/__init__.py diff --git a/linz-data-importer/tests/data/53158.png b/linz_data_importer/tests/data/53158.png similarity index 100% rename from linz-data-importer/tests/data/53158.png rename to linz_data_importer/tests/data/53158.png diff --git a/linz-data-importer/tests/data/53309.png b/linz_data_importer/tests/data/53309.png similarity index 100% rename from linz-data-importer/tests/data/53309.png rename to linz_data_importer/tests/data/53309.png diff --git a/linz-data-importer/tests/data/data.linz.govt.nz_wmts_corrupt.xml b/linz_data_importer/tests/data/data.linz.govt.nz_wmts_corrupt.xml similarity index 100% rename from linz-data-importer/tests/data/data.linz.govt.nz_wmts_corrupt.xml rename to linz_data_importer/tests/data/data.linz.govt.nz_wmts_corrupt.xml diff --git a/linz-data-importer/tests/run_tests.py b/linz_data_importer/tests/run_tests.py similarity index 100% rename from linz-data-importer/tests/run_tests.py rename to linz_data_importer/tests/run_tests.py diff --git a/linz-data-importer/tests/test_ldi_integration.py b/linz_data_importer/tests/test_ldi_integration.py similarity index 95% rename from linz-data-importer/tests/test_ldi_integration.py rename to linz_data_importer/tests/test_ldi_integration.py index 4f6867e..68f95a3 100644 --- a/linz-data-importer/tests/test_ldi_integration.py +++ b/linz_data_importer/tests/test_ldi_integration.py @@ -71,7 +71,7 @@ def setUp(self): """ #Get reference to plugin - self.ldi=plugins.get('linz-data-importer') + self.ldi=plugins.get('linz_data_importer') # Dont run cache update self.ldi.services_loaded=False @@ -170,7 +170,7 @@ def setUp(self): # 1. create six files # 2. three suffixed with one date three with the other # 3. Run the purge. Should only be the newest left. - self.ldi=plugins.get('linz-data-importer') + self.ldi=plugins.get('linz_data_importer') self.pl_settings_dir=os.path.join(QgsApplication.qgisSettingsDirPath(), "linz-data-importer") self.old_file1='data.govt.test.nz_wfs_000000000000001.xml' @@ -234,7 +234,7 @@ def setUp(self): Runs before each test """ - self.ldi=plugins.get('linz-data-importer') + self.ldi=plugins.get('linz_data_importer') self.ldi.update_cache=False self.ldi.services_loaded=False @@ -333,6 +333,17 @@ def test_all_services(self): self.assertEqual(sorted([u'WMS', u'WFS', u'WMTS']), sorted(list(data_types))) + + def test_crs_combo_filter(self): + """ + Test the importing of WMS layers into QGIS + """ + + #set text + self.ldi.dlg.uCRSCombo.lineEdit().setText('ESPG:2193') + #check that the lineEdit set the correct item in combobox + self.assertEqual('ESPG:2193', self.ldi.dlg.uCRSCombo.currentText()) + # def suite(): # suite = unittest.TestSuite() # suite.addTests(unittest.makeSuite(ApiKeyTest, 'test')) diff --git a/linz-data-importer/tests/test_ldi_plugin.py b/linz_data_importer/tests/test_ldi_plugin.py similarity index 99% rename from linz-data-importer/tests/test_ldi_plugin.py rename to linz_data_importer/tests/test_ldi_plugin.py index 6b472e3..00467d1 100644 --- a/linz-data-importer/tests/test_ldi_plugin.py +++ b/linz_data_importer/tests/test_ldi_plugin.py @@ -89,7 +89,7 @@ def setUp(self): self.domain2='data.linz.govt.nz' self.copyTestData() - self.ldi=plugins.get('linz-data-importer') + self.ldi=plugins.get('linz_data_importer') self.ldi.selectionModel.blockSignals(True) self.api_key_instance = self.ldi.api_key_instance self.api_key_instance.setApiKeys({self.domain1:API_KEYS[self.domain1]}) diff --git a/linz-data-importer/tests/test_ldi_ui_elements.py b/linz_data_importer/tests/test_ldi_ui_elements.py similarity index 99% rename from linz-data-importer/tests/test_ldi_ui_elements.py rename to linz_data_importer/tests/test_ldi_ui_elements.py index c29100d..b20e908 100644 --- a/linz-data-importer/tests/test_ldi_ui_elements.py +++ b/linz_data_importer/tests/test_ldi_ui_elements.py @@ -37,7 +37,7 @@ def setUp(self): Runs before each test. """ - self.ldi=plugins.get('linz-data-importer') + self.ldi=plugins.get('linz_data_importer') self.ldi.actions[0].trigger() def tearDown(self):