Merge pull request #458 from HXLStandard/dev

Release 1.29

CHANGELOG

@@ -1,3 +1,10 @@
+2023-04-06 Release 1.29
+	- /api/source-info now supports all formats (not just XLSX and XLS)
+	- renamed 'hashtag_hash' property from /api/source-info to 'hxl-hashtag-hash'
+
+2023-03-29 Release 1.28.1
+	- temporarily deactivate Python-layer timeout (causing random request failures)
+
 2023-03-20 Release 1.28
 	- remove controller support for adding or editing saved recipes
 	- request code stops running after a configurable timeout (default: 30 seconds)

Makefile

@@ -1,37 +1,24 @@
 ########################################################################
 # Makefile to automate common tasks
 #
-# Test-related targets:
-#
-# (These require the Python3 venv module)
-#
 # build-venv - (re)build the Python virtual environment if needed
-# test - run the unit tests (building a virtual environment if needed)
-# test-install - test a fresh installation in a temporary virtual environment
-#
-# Git-related targets:
-#
-# (All of these check out the dev branch at the end)
-#
-# close-issue - merge the current issue branch into dev and delete
-# push-dev - push current dev branch to upstream
-# merge-test - merge the dev branch into the test branch and push
-# merge-master - merge the test branch into the master branch and push
-#
-# Other:
-#
+# test - run the unit tests
+# test-failed - rerun just the failed unit tests
+# test-install - test a fresh installation in a temporary venv
+# test-docker - test a docker build
+# run-local - launch a temporary local server
+# browser-tests-local - run in-browser tests on the local server
+# browser-tests-dev - run browser tests on the dev server
+# browser-tests-staging - run browser tests on the staging server
+# browser-tests-prod - run browser tests on the production server
+# publish-pypi - publish a new release to PyPi
 # tags - build an Emacs TAGS file
-# restart - touch hxl-proxy.wsgi to restart the app
 ########################################################################
 
 
-# figure out what branch we're on currently
-BRANCH=$(shell git symbolic-ref --short HEAD)
-
 # activation script for the Python virtual environment
 VENV=venv/bin/activate
 
-
 # run unit tests
 test: $(VENV)
 	. $(VENV) && pytest
@@ -45,51 +32,36 @@ build-venv: $(VENV)
 
 # (re)build the virtual environment if it's missing, or whenever setup.py changes
 $(VENV): setup.py requirements.txt
-	rm -rf venv \
-		&& python3 -m venv venv \
-		&& . $(VENV) && cd ../libhxl-python \
-		&& pip3 install -r requirements.txt \
-		&& python setup.py develop \
-		&& cd ../hxl-proxy && python setup.py develop
-
-# close the current issue branch and merge into dev
-close-issue:
-	git checkout dev && git merge "$(BRANCH)" && git branch -d "$(BRANCH)"
-
-# push the dev branch to origin
-push-dev:
-	git checkout dev && git push
-
-# merge the dev branch into test and push both to origin
-merge-test: push-dev
-	git checkout test && git merge -m 'merge dev to test' dev && git push && git checkout dev
-
-# merge the test branch into master and push both to origin
-merge-main: merge-test
-	git checkout main && git merge -m 'merge test to main' test && git push && git checkout dev
+	rm -rf venv
+	python3 -m venv venv
+	. $(VENV) && cd ../libhxl-python \
+	  && pip3 install -r requirements.txt \
+	  && python setup.py develop \
+	  && cd ../hxl-proxy \
+	  && python3 setup.py develop
 
 # do a cold install in a temporary virtual environment and run unit tests
 test-install:
 	pip3 cache remove '*'
-	rm -rf venv-test && python3 -m venv venv-test && . venv-test/bin/activate && python setup.py install && python setup.py test
+	rm -rf venv-test
+	python3 -m venv venv-test
+	. venv-test/bin/activate \
+	  && python setup.py install \
+	  && pytest
 	rm -rf venv-test # make sure we clean up
 
 # Add target for docker build
 test-docker:
 	pkexec docker build --no-cache -t "unocha/hxl-proxy:local" `pwd`
 
+# run local dev (needs config in local/config.py)
+run-local: $(VENV)
+	. $(VENV) && HXL_PROXY_CONFIG=../local/config.py python run-server.py
+
 # browser tests for dev.proxy.hxlstandard.org
 browser-tests-local:
 	cat tests/browser-tests/local-urls.txt | xargs -d '\n' firefox
 
-# browser tests for dev.proxy.hxlstandard.org
-browser-tests-local-server:
-	cat tests/browser-tests/local-server-urls.txt | xargs -d '\n' firefox
-
-# browser tests for beta.proxy.hxlstandard.org
-browser-tests-beta:
-	cat tests/browser-tests/beta-urls.txt | xargs -d '\n' firefox
-
 # browser tests for dev.proxy-hxlstandard-org.ahconu.org
 browser-tests-dev:
 	cat tests/browser-tests/dev-urls.txt | xargs -d '\n' firefox
@@ -102,25 +74,21 @@ browser-tests-staging:
 browser-tests-prod:
 	cat tests/browser-tests/prod-urls.txt | xargs -d '\n' firefox
 
-# run local dev (needs config in local/config.py)
-run-dev: $(VENV)
-	. $(VENV) && HXL_PROXY_CONFIG=../local/config.py python run-server.py
-
-# Make sure we're in sync with upstream (merge down rather than up, in case main or test have changed)
-sync:
-	git checkout main && git pull && git push && git checkout test && git pull && git merge main && git push && git checkout dev && git pull && git merge test && git push
-
 # publish a new release on PyPi
 publish-pypi: $(VENV)
-	. $(VENV) && pip install twine && rm -rf dist/* && python setup.py sdist && twine upload dist/*
+	rm -rf dist/*
+	. $(VENV) \
+	  && pip install twine \
+	  && python setup.py sdist \
+	  && twine upload dist/*
 
 # (re)generate emacs TAGS file
 tags:
-	find hxl_proxy tests -name '*.py' -o -name '*.csv' -o -name '*.html' -o -name '*.j2' -o -name '*.js'| grep -v static/jquery | grep -v static/bootstrap | grep -v static/compat | xargs etags
-
-# restart the web app by touching the WSGI file (depends on the platform)
-restart:
-	touch hxl-proxy.wsgi
+	find hxl_proxy tests -name '*.py' -o -name '*.csv' -o -name '*.html' -o -name '*.j2' -o -name '*.js' \
+	  | grep -v static/jquery \
+	  | grep -v static/bootstrap \
+	  | grep -v static/compat \
+	  | xargs etags
 
 clean:
 	rm -rf venv

hxl_proxy/__init__.py

@@ -8,7 +8,7 @@
 
 """
 
-__version__="1.28"
+__version__="1.29"
 """Module version number
 See https://www.python.org/dev/peps/pep-0396/
 

hxl_proxy/controllers.py

@@ -36,7 +36,7 @@ def handle_alarm_signal(signum, frame):
     logup('Request timed out', level='info')
     raise TimeoutError()
 
-signal.signal(signal.SIGALRM, handle_alarm_signal)
+# signal.signal(signal.SIGALRM, handle_alarm_signal) # temporarily deactivated
 
 
 
@@ -187,7 +187,7 @@ def before_request():
         timeout = int(app.config.get('TIMEOUT', 30))
     except ValueError:
         timeout = 30
-    signal.alarm(timeout)
+    # signal.alarm(timeout) # temporarily deactivated
 
 
 
@@ -1422,18 +1422,11 @@ def make_info():
         raise ValueError("Parameter 'url' is required")
 
     # Open the dataset (not necessarily hxlated)
-    try:
-        info = util.hxl_make_input(url, util.make_input_options(flask.request.args)).info()
-        return flask.Response(
-            json.dumps(info, indent=4),
-            mimetype="application/json"
-        )
-    except NotImplementedError:
-        return flask.Response(
-            { "error": "dataset format not supported" },
-            mimetype="application/json",
-            status=400
-        )
+    info = hxl.input.info(util.hxl_make_input(url, util.make_input_options(flask.request.args)))
+    return flask.Response(
+        json.dumps(info, indent=4),
+        mimetype="application/json"
+    )
 
 
 ########################################################################

requirements.txt

@@ -1,6 +1,6 @@
 urllib3>=1.21.1<1.27 # version required by requests
-#git+https://github.com/HXLStandard/libhxl-python.git@4.27.3#egg=libhxl # for development
-libhxl==4.28 # for release
+#git+https://github.com/HXLStandard/libhxl-python.git@dev#egg=libhxl # for development
+libhxl==4.29 # for release
 ckanapi>=3.5
 flask-caching
 flask>=2.1.2

setup.py

@@ -16,7 +16,7 @@
     name = 'hxl-proxy',
     packages = ['hxl_proxy'],
     package_data={'hxl_proxy': ['*.sql']},
-    version = "1.28",
+    version = "1.29",
     description = 'Flask-based web proxy for HXL',
     long_description=long_description,
     long_description_content_type="text/markdown",
@@ -27,8 +27,8 @@
     zip_safe = False,
     install_requires=[
         'urllib3>=1.21.1,<1.27', # version required by requests
-        #'libhxl @ git+https://github.com/HXLStandard/libhxl-python.git@4.27.3', # for development
-        'libhxl==4.28', # for release
+        #'libhxl @ git+https://github.com/HXLStandard/libhxl-python.git@dev', # for development
+        'libhxl==4.29', # for release
         'ckanapi>=3.5',
         'flask-caching',
         'flask>=2.1.2',

tests/files/input-info-hxl.csv

@@ -0,0 +1,6 @@
+Registro,Sector/Cluster,Subsector,Organización,Hombres,Mujeres,País,Departamento/Provincia/Estado,
+,#sector+es,#subsector+es,#org+es,#targeted+f,#targeted+m,#country,#adm1,#date+reported
+001,WASH,Higiene,ACNUR,100,100,Panamá,Los Santos,1 March 2015
+002,Salud,Vacunación,UNICEF,105,110,Colombia,Cauca,1 March 2015
+003,Educación,Formación de enseñadores,UNICEF,250,300,Colombia,Chocó,1 March 2015
+004,WASH,Urbano,OMS,80,95,Venezuela,Amazonas,1 March 2015

tests/files/input-info-hxl.json

@@ -0,0 +1,8 @@
+[
+    ["Registro", "Sector/Cluster", "Subsector", "Organización", "Hombres", "Mujeres", "País", "Departamento/Provincia/Estado", ""],
+    ["", "#sector+es", "#subsector+es", "#org+es", "#targeted+f", "#targeted+m", "#country", "#adm1", "#date+reported"],
+    ["001", "WASH", "Higiene", "ACNUR", "100", "100", "Panamá", "Los Santos", "1 March 2015"],
+    ["002", "Salud", "Vacunación", "UNICEF", "105", "110", "Colombia", "Cauca", "1 March 2015"],
+    ["003", "Educación", "Formación de enseñadores", "UNICEF", "250", "300", "Colombia", "Chocó", "1 March 2015"],
+    ["004", "WASH", "Urbano", "OMS", "80", "95", "Venezuela", "Amazonas", "1 March 2015"]
+]

tests/files/input-info-nohxl.csv

@@ -0,0 +1,5 @@
+Registro,Sector/Cluster,Subsector,Organización,Hombres,Mujeres,País,Departamento/Provincia/Estado,
+001,WASH,Higiene,ACNUR,100,100,Panamá,Los Santos,1 March 2015
+002,Salud,Vacunación,UNICEF,105,110,Colombia,Cauca,1 March 2015
+003,Educación,Formación de enseñadores,UNICEF,250,300,Colombia,Chocó,1 March 2015
+004,WASH,Urbano,OMS,80,95,Venezuela,Amazonas,1 March 2015

tests/files/input-info-nohxl.json

@@ -0,0 +1,7 @@
+[
+    ["Registro", "Sector/Cluster", "Subsector", "Organización", "Hombres", "Mujeres", "País", "Departamento/Provincia/Estado", ""],
+    ["001", "WASH", "Higiene", "ACNUR", "100", "100", "Panamá", "Los Santos", "1 March 2015"],
+    ["002", "Salud", "Vacunación", "UNICEF", "105", "110", "Colombia", "Cauca", "1 March 2015"],
+    ["003", "Educación", "Formación de enseñadores", "UNICEF", "250", "300", "Colombia", "Chocó", "1 March 2015"],
+    ["004", "WASH", "Urbano", "OMS", "80", "95", "Venezuela", "Amazonas", "1 March 2015"]
+]