From 283ccca70a1e7591fe07b9ac09d7f23f51661f30 Mon Sep 17 00:00:00 2001 From: afabiani Date: Fri, 14 Sep 2018 10:09:25 +0200 Subject: [PATCH] - Version 2.14.x --- .gitignore | 12 +- setup.py | 2 +- src/wps_remote.egg-info/PKG-INFO | 105 ------------------ src/wps_remote.egg-info/SOURCES.txt | 52 --------- src/wps_remote.egg-info/dependency_links.txt | 1 - src/wps_remote.egg-info/requires.txt | 17 --- src/wps_remote.egg-info/top_level.txt | 1 - src/wpsremote/processbot.py | 6 + src/wpsremote/resource_cleaner.py | 1 + src/wpsremote/resource_monitor.py | 46 ++++---- .../xmpp_data/configs/logger.properties | 2 +- .../xmpp_data/configs/myservice/code/test.py | 12 +- .../configs/myservice/logger.properties | 2 +- .../configs/myservice/service.config | 9 +- src/wpsremote/xmpp_data/configs/remote.config | 16 +-- 15 files changed, 66 insertions(+), 218 deletions(-) delete mode 100644 src/wps_remote.egg-info/PKG-INFO delete mode 100644 src/wps_remote.egg-info/SOURCES.txt delete mode 100644 src/wps_remote.egg-info/dependency_links.txt delete mode 100644 src/wps_remote.egg-info/requires.txt delete mode 100644 src/wps_remote.egg-info/top_level.txt diff --git a/.gitignore b/.gitignore index 390789c..2e77f68 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,14 @@ *.py[cod] *.suo +/build/ RemoteWPS1.pyproj RemoteWPS1.sln -Include -Lib/ -Scripts/ +/Include +/Lib/ +/Scripts/ + +/RemoteWPS.pyproj +/RemoteWPS.sln +/.vs/config/applicationhost.config +/dist/ diff --git a/setup.py b/setup.py index c1b9029..4c3bfc9 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ setup( name = "wps-remote", - version = "2.12.0", + version = "2.14.0", author = "GeoServer Developers", author_email = "geoserver-devel@lists.sourceforge.net", description = "A library that allows users to publish their executables as GeoServer WPS Processes through the XMPP protocol", diff --git a/src/wps_remote.egg-info/PKG-INFO b/src/wps_remote.egg-info/PKG-INFO deleted file mode 100644 index 8fa363a..0000000 --- a/src/wps_remote.egg-info/PKG-INFO +++ /dev/null @@ -1,105 +0,0 @@ -Metadata-Version: 1.1 -Name: wps-remote -Version: 2.12.0 -Summary: A library that allows users to publish their executables as GeoServer WPS Processes through the XMPP protocol -Home-page: https://github.com/geoserver/wps-remote -Author: GeoServer Developers -Author-email: geoserver-devel@lists.sourceforge.net -License: GPL -Description: [![Build Status](https://travis-ci.org/geoserver/wps-remote.svg?branch=master)](https://travis-ci.org/geoserver/wps-remote) - [![Coverage Status](https://coveralls.io/repos/github/geoserver/wps-remote/badge.svg?branch=master)](https://coveralls.io/github/geoserver/wps-remote?branch=master) - - RemoteWPS - ========= - - [RemoteWPS] is an open source software written in Python part of the GeoServer (http://geoserver.org) - project that allows users to publish their executables as GeoServer WPS Processes through the XMPP protocol. - - RemoteWPS is meant to be used along with the GeoServer "wps-remote" plugin for the WPS - (https://github.com/geoserver/geoserver/tree/master/src/community/wps-remote). - - Being a community-driven project, RemoteWPS is developed, tested, and supported by a diverse group of - individuals and organizations from around the world. - - ## License - - RemoteWPS licensed under the [GPL](http://www.gnu.org/licenses/old-licenses/gpl-2.0.html). - - ## Using - - You can install the "wps-remote" Python package by using the PyPi distribution :: - - pip install wps-remote==2.12.0 - - Please refer to the [user guide](http://docs.geoserver.org/latest/en/user/community/remote-wps/index.html) for detailed - information on how to install and use RemoteWPS. - - ## API Reference Doc - - http://docs.geoserver.org/latest/en/user/community/remote-wps/install_python.html#annex-a-remote-wps-python-wrapper-reference - - ## Bugs - - Through GeoServer JIRA. - - GeoServer uses [JIRA](https://osgeo-org.atlassian.net/projects/GEOS), hosted by - [Atlassian](https://www.atlassian.com/), for issue tracking. - - ## Mailing Lists - - Through GeoServer Mailing Lists. - - The [mailing list page](http://geoserver.org/comm/) on the GeoServer web site provides - access to the various mailing list, as well as some indication of the [code of conduct](http://geoserver.org/comm/userlist-guidelines.html) when posting to the lists - - ## Contributing - - Through GeoServer Process. - - Please read [the contribution guidelines](https://github.com/geoserver/geoserver/blob/master/CONTRIBUTING.md) before contributing pull requests to the GeoServer project. - - ## More Information - - Visit the [website](http://geoserver.org/) or read the [docs](http://docs.geoserver.org/). - - Change Log - ========== - 2.12.0 [2017-07-03 12:13:10] (aligned with GeoServer Remote WPS Plugin - 2.12.x) - - Fix for: Inputs application/json mapping error - - Fix for: Inputs application/vnd.geo+json support - - Fix for: Date input / output parsing - - Fix for: JSON ID error - - Improved robustness of wps agent - - Allowed "-" in string input parameters - - Fix for: Catch resource clean exception - - Fix for: Input / output parameters are not sorted according to the process definition - - 2.11.2 [2017-02-16 10:12:23] - - SFTP/FTP pluggable uploaders support - - Minor refactoring and code cleanup - - 2.9.3 [2016-03-31 17:24:47] - - Added "processbot" XMPP connection checks - - Added "servicebot" check "processbot" status error and send error message impersonating the underlying process - - Added "NetCDF Output" example on service.config - - 2.9.2 [2016-03-17 17:33:19] - - Added "process_blacklist" property to "service.config" file - - Added "proc_is_running(self, proc_names)" to "ResourceMonitor(threading.Thread)" background thread - - 2.9.1 [2016-03-16 11:09:56] - - Added "GetLoadAverageMessage" XMPP message for resource monitoring - - Added "ResourceMonitor(threading.Thread)" background thread collecting resource consuption data - - On GeoServer side the "remoteConfig.properties" file on GEOSERVER_DATA must be updated with the properties - xmpp_cpu_perc_threshold = 80.0 - xmpp_mem_perc_threshold = 80.0 - -Keywords: XMPP Beckend for GeoServer Remote WPS ProcessFactory. -Platform: UNKNOWN -Classifier: Environment :: Web Environment -Classifier: License :: OSI Approved :: GNU General Public License v2 (GPLv2) -Classifier: Intended Audience :: Developers -Classifier: Intended Audience :: Science/Research -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Topic :: Scientific/Engineering :: GIS diff --git a/src/wps_remote.egg-info/SOURCES.txt b/src/wps_remote.egg-info/SOURCES.txt deleted file mode 100644 index 1aed3cd..0000000 --- a/src/wps_remote.egg-info/SOURCES.txt +++ /dev/null @@ -1,52 +0,0 @@ -setup.cfg -setup.py -src/wps_remote.egg-info/PKG-INFO -src/wps_remote.egg-info/SOURCES.txt -src/wps_remote.egg-info/dependency_links.txt -src/wps_remote.egg-info/requires.txt -src/wps_remote.egg-info/top_level.txt -src/wpsremote/ConfigParser.py -src/wpsremote/__init__.py -src/wpsremote/action.py -src/wpsremote/bus.py -src/wpsremote/busIndipendentMessages.py -src/wpsremote/command_line_parameter.py -src/wpsremote/computation_job_const.py -src/wpsremote/computation_job_input.py -src/wpsremote/computation_job_inputs.py -src/wpsremote/computation_job_param.py -src/wpsremote/computational_job_input_action.py -src/wpsremote/computational_job_input_action_cmd_param.py -src/wpsremote/computational_job_input_action_copyfile.py -src/wpsremote/computational_job_input_action_create_json_file.py -src/wpsremote/computational_job_input_action_update_ini_file.py -src/wpsremote/computational_job_input_action_update_json_file.py -src/wpsremote/computational_job_input_actions.py -src/wpsremote/configInstance.py -src/wpsremote/config_file_parameter.py -src/wpsremote/config_file_parameter_cmre_oaa.py -src/wpsremote/decryptor.py -src/wpsremote/encryptor.py -src/wpsremote/ftpUpload.py -src/wpsremote/ftpsUpload.py -src/wpsremote/input_parameter.py -src/wpsremote/input_parameters.py -src/wpsremote/introspection.py -src/wpsremote/mockutils.py -src/wpsremote/output_file_parameter.py -src/wpsremote/output_parameters.py -src/wpsremote/path.py -src/wpsremote/process_test.py -src/wpsremote/processbot.py -src/wpsremote/resource_cleaner.py -src/wpsremote/resource_monitor.py -src/wpsremote/serviceTester.py -src/wpsremote/servicebot.py -src/wpsremote/sftpUpload.py -src/wpsremote/termination_handler.py -src/wpsremote/upload.py -src/wpsremote/wpsagent.py -src/wpsremote/xmppBus.py -src/wpsremote/xmppMessages.py -test/test_computation_job_inputs.py -test/test_process_input_parameters.py \ No newline at end of file diff --git a/src/wps_remote.egg-info/dependency_links.txt b/src/wps_remote.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/src/wps_remote.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/wps_remote.egg-info/requires.txt b/src/wps_remote.egg-info/requires.txt deleted file mode 100644 index 5022cc3..0000000 --- a/src/wps_remote.egg-info/requires.txt +++ /dev/null @@ -1,17 +0,0 @@ -astroid==1.4.4 -colorama==0.3.6 -flake8==2.5.4 -functools32 -jsonschema==2.5.1 -lazy-object-proxy==1.2.1 -mccabe==0.4.0 -paramiko -pep8==1.7.0 -psutil>=4.0.0 -pycrypto -pyflakes==1.0.0 -pylint==1.5.4 -six==1.10.0 -sleekxmpp>=1.3.1 -wheel==0.24.0 -wrapt==1.10.6 diff --git a/src/wps_remote.egg-info/top_level.txt b/src/wps_remote.egg-info/top_level.txt deleted file mode 100644 index f1c9241..0000000 --- a/src/wps_remote.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -wpsremote diff --git a/src/wpsremote/processbot.py b/src/wpsremote/processbot.py index 165c823..218deca 100644 --- a/src/wpsremote/processbot.py +++ b/src/wpsremote/processbot.py @@ -75,10 +75,16 @@ def __init__(self, remote_config_filepath, service_config_filepath, execute_mess self._executable_path = serviceConfig.get("DEFAULT", "executable_path") self._executable_cmd = serviceConfig.get("DEFAULT", "executable_cmd") + if not os.path.isabs(self._executable_path): + full_executable_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), self._executable_path) + self._executable_cmd = self._executable_cmd.replace(self._executable_path, full_executable_path) + self._executable_path = full_executable_path self._stdout_parser = serviceConfig.get_list("Logging", "stdout_parser") self._stdout_action = serviceConfig.get_list("Logging", "stdout_action") self._output_dir = serviceConfig.get_path("DEFAULT", "output_dir") + if not os.path.isabs(self._output_dir): + self._output_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), self._output_dir) self._max_running_time = datetime.timedelta(seconds = serviceConfig.getint("DEFAULT", "max_running_time_seconds")) #create the concrete uploader object diff --git a/src/wpsremote/resource_cleaner.py b/src/wpsremote/resource_cleaner.py index 965d18f..c9a22e2 100644 --- a/src/wpsremote/resource_cleaner.py +++ b/src/wpsremote/resource_cleaner.py @@ -20,6 +20,7 @@ import logging import traceback + class Resource(object): """ Identify a process generated by a remote WPS call. diff --git a/src/wpsremote/resource_monitor.py b/src/wpsremote/resource_monitor.py index ef4ddea..085dba9 100644 --- a/src/wpsremote/resource_monitor.py +++ b/src/wpsremote/resource_monitor.py @@ -35,27 +35,31 @@ def __init__(self, load_average_scan_minutes): ResourceMonitor.lock.release() def proc_is_running(self, proc_names): - for proc in psutil.process_iter(): - - process = psutil.Process(proc.pid).as_dict() # Get the process info using PID - - pid = str(process["pid"]) - ppid = str(process["ppid"]) - status = process["status"] - - cpu_percent = process["cpu_percent"] - mem_percent = process["memory_percent"] - - rss = str(process["memory_info"].rss) - vms = str(process["memory_info"].vms) - username = process["username"] - name = process["name"] # Here is the process name - path = process["cwd"] - - for proc_name in proc_names: - if status.lower() == "running" and proc_name in name.lower(): - return True - + for proc in psutil.process_iter(): + try: + process = psutil.Process(proc.pid).as_dict() # Get the process info using PID + + pid = str(process["pid"]) + ppid = str(process["ppid"]) + status = process["status"] + + cpu_percent = process["cpu_percent"] + mem_percent = process["memory_percent"] + + rss = str(process["memory_info"].rss) + vms = str(process["memory_info"].vms) + username = process["username"] + name = process["name"] # Here is the process name + path = process["cwd"] + + for proc_name in proc_names: + if status.lower() == "running" and proc_name in name.lower(): + return True + except: + import traceback + tb = traceback.format_exc() + # print(tb) + continue return False def run(self): diff --git a/src/wpsremote/xmpp_data/configs/logger.properties b/src/wpsremote/xmpp_data/configs/logger.properties index 7ab79e0..a620f37 100644 --- a/src/wpsremote/xmpp_data/configs/logger.properties +++ b/src/wpsremote/xmpp_data/configs/logger.properties @@ -30,7 +30,7 @@ interval=midnight backupCount=5 formatter=simpleFormatter level=DEBUG -args=('/share/xmpp_data/service.log',) +args=('./xmpp_data/service.log',) [formatter_simpleFormatter] format=%(asctime)s - %(name)s - %(levelname)s - %(message)s diff --git a/src/wpsremote/xmpp_data/configs/myservice/code/test.py b/src/wpsremote/xmpp_data/configs/myservice/code/test.py index d1ce654..707a552 100644 --- a/src/wpsremote/xmpp_data/configs/myservice/code/test.py +++ b/src/wpsremote/xmpp_data/configs/myservice/code/test.py @@ -21,8 +21,9 @@ #id = os.urandom(10) id = str(uuid.uuid4()) gdalContour = r'/usr/bin/gdal_contour' -src = r'/share/xmpp_data/resource_dir/srtm_39_04/srtm_39_04_c.tif' dst = r'contour_'+id[:13] +src = '%s/../../../resource_dir/srtm_39_04/srtm_39_04_c.tif' % os.path.dirname(os.path.abspath(__file__)) +trg = '%s/../../../output/%s.shp' % (os.path.dirname(os.path.abspath(__file__)), dst) cmd = '-a elev' # just for example! interval = '-i' @@ -35,7 +36,7 @@ def __init__(self, args): def run(self): #fullCmd = ' '.join([gdalContour, cmd, self.youCanQuoteMe(src), self.youCanQuoteMe(dst), interval, self.args.interval]) - fullCmd = ' '.join([gdalContour, cmd, src, self.args.workdir+'/'+dst+'.shp', interval, self.args.interval]) + fullCmd = ' '.join([gdalContour, cmd, src, trg, interval, self.args.interval]) self.logger.debug("Running command > " + fullCmd) proc=subprocess.Popen(fullCmd.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False) for line in proc.stdout: @@ -46,8 +47,11 @@ def run(self): ret = proc.returncode if (ret == 0): - zipf = zipfile.ZipFile(self.args.workdir+'/contour.zip', 'w') - self.zipdir(self.args.workdir+'/', zipf) + # zipf = zipfile.ZipFile(self.args.workdir+'/contour.zip', 'w') + # self.zipdir(self.args.workdir+'/', zipf) + output_dir = '%s/../../../output/' % os.path.dirname(os.path.abspath(__file__)) + zipf = zipfile.ZipFile(output_dir+'/contour.zip', 'w') + self.zipdir(output_dir+'/', zipf) zipf.close() self.logger.info("ProgressInfo:100%") diff --git a/src/wpsremote/xmpp_data/configs/myservice/logger.properties b/src/wpsremote/xmpp_data/configs/myservice/logger.properties index bccee05..5c468fb 100644 --- a/src/wpsremote/xmpp_data/configs/myservice/logger.properties +++ b/src/wpsremote/xmpp_data/configs/myservice/logger.properties @@ -38,4 +38,4 @@ datefmt= [formatter_consoleFormatter] format=%(asctime)s [%(levelname)s] %(message)s -datefmt= \ No newline at end of file +datefmt= diff --git a/src/wpsremote/xmpp_data/configs/myservice/service.config b/src/wpsremote/xmpp_data/configs/myservice/service.config index 0a2608e..1c71469 100644 --- a/src/wpsremote/xmpp_data/configs/myservice/service.config +++ b/src/wpsremote/xmpp_data/configs/myservice/service.config @@ -15,9 +15,9 @@ service = GdalContour namespace = default description = GDAL Contour Remote Service -executable_path = /share/xmpp_data/configs/myservice/code +executable_path = ./xmpp_data/configs/myservice/code executable_cmd = python %(executable_path)s/test.py -output_dir = /share/xmpp_data/output/ +output_dir = ./xmpp_data/output/ unique_execution_id = %(unique_exe_id)s workdir = %(output_dir)s/%(unique_execution_id)s active = True @@ -74,6 +74,7 @@ template = -name value [Output1] name = result1 type = application/zip +output_mime_type = application/zip description = WPS Resource Binary File title = SRTM filepath = %(workdir)s/contour.zip @@ -98,6 +99,7 @@ publish_layer_name = contour [Output2] name = result2 type = application/x-netcdf +output_mime_type = application/x-netcdf description = NetCDF Binary File title = flexpart filepath = %(output_dir)s/flexpart.nc @@ -122,11 +124,12 @@ publish_layer_name = flexpart [Output3] name = result3 type = application/owc +output_mime_type = application/xml description = WPS OWC Json MapContext layers_to_publish = result2 publish_as_layer = true publish_layer_name = owc_json_ctx -publish_metadata = /share/xmpp_data/resource_dir/owc_json_ctx.json +publish_metadata = ./xmpp_data/resource_dir/owc_json_ctx.json # ########################################### # # Logging Options Declaration # diff --git a/src/wpsremote/xmpp_data/configs/remote.config b/src/wpsremote/xmpp_data/configs/remote.config index fa5f9ce..523e1a9 100644 --- a/src/wpsremote/xmpp_data/configs/remote.config +++ b/src/wpsremote/xmpp_data/configs/remote.config @@ -19,9 +19,9 @@ user = default.GdalContour password = R3m0T3wP5 mucService = conference.%(domain)s -mucServicePassword = admin +mucServicePassword = geoserver -resource_file_dir = /share/xmpp_data/resource_dir +resource_file_dir = ./xmpp_data/resource_dir # . Configure this option (along with 'backup_on_wps_execution_shared_dir' # . on single outputs of 'service.config') in order to make a copy @@ -41,10 +41,10 @@ resource_file_dir = /share/xmpp_data/resource_dir # ftpsUpload.FtpsUpload # . S-FTP Protocol (based on paramiko Python lib) # sftpUpload.SFtpUpload -uploader_class_name = ftpUpload.FtpUpload -uploader_host = ftp.: -uploader_username = -uploader_password = +#uploader_class_name = ftpUpload.FtpUpload +#uploader_host = ftp.: +#uploader_username = +#uploader_password = # . "encryptor" you can use encrypted passwords with a private/public key couple # @@ -59,5 +59,5 @@ uploader_password = # # . To double check the password is correct use the following utility # python decrypt.py password path/to/rsakey.pem passphrase -uploader_private_rsa_key = /share/xmpp_data/ssl/myTestKey.pem -uploader_passphrase = f00bar \ No newline at end of file +uploader_private_rsa_key = ./xmpp_data/ssl/myTestKey.pem +uploader_passphrase = f00bar