-
Notifications
You must be signed in to change notification settings - Fork 34
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
#61 first stab at Dockerfile and tests, not yet operational
- Loading branch information
Showing
7 changed files
with
179 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| FROM python:2.7.14-alpine3.6 | ||
|
|
||
| LABEL maintainer "Just van den Broecke <justb4@gmail.com>" | ||
|
|
||
| # These are default values, | ||
| # Override when running container via docker(-compose) | ||
|
|
||
| # General ENV settings | ||
| ENV LC_ALL "en_US.UTF-8" | ||
| ENV LANG "en_US.UTF-8" | ||
| ENV LANGUAGE "en_US.UTF-8" | ||
|
|
||
| ARG TZ="Europe/Amsterdam" | ||
|
|
||
| RUN apk add --no-cache --virtual .build-deps gcc build-base linux-headers postgresql-dev \ | ||
| && apk add --no-cache \ | ||
| bash \ | ||
| vim \ | ||
| tzdata \ | ||
| && echo $TZ > /etc/timezone \ | ||
| && cp /usr/share/zoneinfo/$TZ /etc/localtime \ | ||
| && rm -rf /var/cache/apk/* /tmp/* /var/tmp/* | ||
|
|
||
| RUN apk add --no-cache \ | ||
| expat \ | ||
| libxml2-dev \ | ||
| libxslt-dev \ | ||
| postgresql-client \ | ||
| py-lxml \ | ||
| proj4-dev \ | ||
| geos-dev \ | ||
| gdal-dev \ | ||
| py-gdal \ | ||
| gdal \ | ||
| --repository http://nl.alpinelinux.org/alpine/edge/testing \ | ||
| && rm -rf /var/cache/apk/* /tmp/* /var/tmp/* | ||
|
|
||
| RUN pip install gdal==2.1.3 \ | ||
| && pip install Jinja2 \ | ||
| && pip install lxml \ | ||
| && pip install nose2 \ | ||
| && pip install mock | ||
|
|
||
| # Add Source Code | ||
| ADD . /stetl | ||
|
|
||
| # Install and Remove build-related packages for smaller image size | ||
| RUN cd /stetl \ | ||
| && python setup.py install \ | ||
| && apk del .build-deps | ||
|
|
||
| # Allow docker run | ||
| # ENTRYPOINT ["/usr/local/bin/stetl"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| <GMLFeatureClassList> | ||
| <GMLFeatureClass> | ||
| <Name>City</Name> | ||
| <ElementPath>City</ElementPath> | ||
| <GeometryName>geometry</GeometryName> | ||
| <GeometryElementPath>geometry</GeometryElementPath> | ||
| <GeometryType>1</GeometryType> | ||
| <SRSName>GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]</SRSName> | ||
| <DatasetSpecificInfo> | ||
| <FeatureCount>3</FeatureCount> | ||
| <ExtentXMin>4.90000</ExtentXMin> | ||
| <ExtentXMax>12.50000</ExtentXMax> | ||
| <ExtentYMin>41.90000</ExtentYMin> | ||
| <ExtentYMax>52.40000</ExtentYMax> | ||
| </DatasetSpecificInfo> | ||
| <PropertyDefn> | ||
| <Name>name</Name> | ||
| <ElementPath>name</ElementPath> | ||
| <Type>String</Type> | ||
| <Width>9</Width> | ||
| </PropertyDefn> | ||
| </GMLFeatureClass> | ||
| </GMLFeatureClassList> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| <?xml version='1.0' encoding='utf-8'?> | ||
| <ogr:FeatureCollection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ogr="http://ogr.maptools.org/" xmlns:gml="http://www.opengis.net/gml" xsi:schemaLocation="http://ogr.maptools.org/ ../gmlcities.xsd http://www.opengis.net/gml http://schemas.opengis.net/gml/2.1.2/feature.xsd"> | ||
| <gml:boundedBy> | ||
| <gml:Box> | ||
| <gml:coord> | ||
| <gml:X>-180.0</gml:X> | ||
| <gml:Y>-90.0</gml:Y> | ||
| </gml:coord> | ||
| <gml:coord> | ||
| <gml:X>180.0</gml:X> | ||
| <gml:Y>90.0</gml:Y> | ||
| </gml:coord> | ||
| </gml:Box> | ||
| </gml:boundedBy> | ||
| <gml:featureMember> | ||
| <ogr:City> | ||
| <ogr:name>Amsterdam</ogr:name> | ||
| <ogr:geometry> | ||
| <gml:Point srsName="urn:ogc:def:crs:EPSG:4326"> | ||
| <gml:coordinates>52.4,4.9</gml:coordinates> | ||
| </gml:Point> | ||
| </ogr:geometry> | ||
| </ogr:City> | ||
| </gml:featureMember> | ||
| <gml:featureMember> | ||
| <ogr:City> | ||
| <ogr:name>Bonn</ogr:name> | ||
| <ogr:geometry> | ||
| <gml:Point srsName="urn:ogc:def:crs:EPSG:4326"> | ||
| <gml:coordinates>50.7,7.1</gml:coordinates> | ||
| </gml:Point> | ||
| </ogr:geometry> | ||
| </ogr:City> | ||
| </gml:featureMember> | ||
| <gml:featureMember> | ||
| <ogr:City> | ||
| <ogr:name>Rome</ogr:name> | ||
| <ogr:geometry> | ||
| <gml:Point srsName="urn:ogc:def:crs:EPSG:4326"> | ||
| <gml:coordinates>41.9,12.5</gml:coordinates> | ||
| </gml:Point> | ||
| </ogr:geometry> | ||
| </ogr:City> | ||
| </gml:featureMember> | ||
| </ogr:FeatureCollection> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| # Trivial example of an ETL Chain that just copies a file to standard output. | ||
|
|
||
| [etl] | ||
| chains = input_ogr|output_std | ||
|
|
||
| [input_ogr] | ||
| class = inputs.ogrinput.OgrInput | ||
| data_source = tests/data/cities.gml | ||
| source_format = GML | ||
| source_options = {'GDAL_CACHEMAX': '64', 'CPL_DEBUG': 'OFF'} | ||
| output_format = ogr_feature | ||
| file_path = tests/data/cities.xml | ||
| element_tags = city | ||
|
|
||
| [output_std] | ||
| class = outputs.standardoutput.StandardOutput |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| import os | ||
| import sys | ||
|
|
||
| from stetl.etl import ETL | ||
| from stetl.inputs.ogrinput import OgrInput | ||
| from tests.stetl_test_case import StetlTestCase | ||
|
|
||
| class OgrInputTest(StetlTestCase): | ||
| """Unit tests for OgrInput""" | ||
| pass | ||
|
|
||
| def setUp(self): | ||
| super(OgrInputTest, self).setUp() | ||
|
|
||
| # Initialize Stetl | ||
| curr_dir = os.path.dirname(os.path.realpath(__file__)) | ||
| cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/ogrinput.cfg')} | ||
| self.etl = ETL(cfg_dict) | ||
|
|
||
| def test_class(self): | ||
| chain = StetlTestCase.get_chain(self.etl) | ||
| section = StetlTestCase.get_section(chain) | ||
| class_name = self.etl.configdict.get(section, 'class') | ||
|
|
||
| self.assertEqual('inputs.ogrinput.OgrInput', class_name) | ||
|
|
||
| def test_instance(self): | ||
| chain = StetlTestCase.get_chain(self.etl) | ||
|
|
||
| self.assertTrue(isinstance(chain.first_comp, OgrInput)) | ||
|
|
||
| def test_execute(self): | ||
| chain = StetlTestCase.get_chain(self.etl) | ||
| chain.run() | ||
|
|
||
| result = sys.stdout.getvalue().strip().split('\n') | ||
| self.assertEqual(len(result), 3) |