Merge 4c9b96f into 9ce39c0

.dockerignore

@@ -1,6 +1,10 @@
+.tox/
+.cache/
+.github/
 .git/
 .idea/
 datapacakge_pipelines_measure.egg-info/
 tests/
 .dpp.db
-
+downloads/
+measure.sublime-project

.gitignore

@@ -44,6 +44,7 @@ nosetests.xml
 coverage.xml
 *,cover
 .hypothesis/
+.pytest_cache/
 
 # Translations
 *.mo
@@ -92,3 +93,4 @@ ENV/
 .idea/
 *iml
 .dpp.db
+.dpp/

.travis.yml

@@ -4,11 +4,6 @@ sudo:
 dist:
   trusty
 
-addons:
-  apt:
-    packages:
-      - pandoc
-
 language:
   python
 
@@ -28,11 +23,6 @@ script:
 after_success:
   - coveralls
 
-before_deploy:
-  - pandoc --version
-  - pandoc -f markdown_github -t rst -o README.rst README.md
-  - mv README.rst README.md
-
 deploy:
   provider: pypi
   user: okfn

Dockerfile

@@ -1,10 +1,11 @@
 FROM frictionlessdata/datapackage-pipelines:latest
 
+RUN apk add --update postgresql-client
+
 ADD . /app
 
 WORKDIR /app
 RUN pip install .
-RUN apk add --update postgresql-client
 
 WORKDIR /app/projects
 

datapackage_pipelines_measure/config.py

@@ -1,7 +1,7 @@
 from distutils.util import strtobool
 import os
 import collections
-from dotenv import load_dotenv
+from dotenv import load_dotenv, find_dotenv
 
 
 NAMESPACE = 'MEASURE_'
@@ -45,5 +45,5 @@ def __delitem__(self, key):
         del self.store[key]
 
 
-load_dotenv()
+load_dotenv(find_dotenv())
 settings = Config(os.environ.items(), NAMESPACE)

datapackage_pipelines_measure/pipeline_steps/code_hosting.py

@@ -14,12 +14,13 @@ def add_steps(steps: list, pipeline_id: str,
     for repo in config['github']['repositories']:
         steps.append(('measure.add_github_resource', {
             'name': slugify(repo).lower(),
-            'repo': repo
+            'repo': repo.lower()
         }))
 
     steps.append(('concatenate', {
         'sources':
-            [slugify(repo) for repo in config['github']['repositories']],
+            [slugify(repo).lower()
+             for repo in config['github']['repositories']],
         'target': {
             'name': 'code-hosting',
             'path': 'data/code-hosting.json'},

datapackage_pipelines_measure/processors/add_discourse_resource.py

@@ -25,14 +25,15 @@ def _page_active_users(domain):
         '''Request active users by page until an empty array is return, or we
         reach a user created after the last 24hr'''
         current_page = 1  # /admin/users/list paging starts at one
+        day_in_seconds = 24 * 60 * 60
         while True:
             users = _request_users_from_discourse(domain, 'active',
                                                   current_page)
             if len(users) == 0:
                 raise StopIteration
             current_page = current_page + 1
             for user in users:
-                if not user['last_seen_age'].endswith(('h', 'm')):
+                if user['last_seen_age'] > day_in_seconds:
                     raise StopIteration
                 yield user
 

datapackage_pipelines_measure/processors/discourse_utils.py

@@ -38,7 +38,7 @@ def request_data_from_discourse(domain, endpoint, **kwargs):
 
     try:
         json_response = response.json()
-    except simplejson.scanner.JSONDecodeError as e:
+    except simplejson.scanner.JSONDecodeError:
         log.error('Expected JSON in response from: {}'.format(url))
         raise ValueError('Expected JSON in response from: {}'.format(url))
 

projects/frictionlessdata/measure.source-spec.yaml

@@ -63,7 +63,7 @@ config:
         - "frictionlessdata/ckanext-validation"
         - "frictionlessdata/datapackage-pipelines-ckan"
         - "frictionlessdata/tableschema-ui"
-        - "ODIQueensland/data-curator"
+        - "odiqueensland/data-curator"
 
   website-analytics:
     ga:

projects/openfiscal/measure.source-spec.yaml

@@ -44,7 +44,5 @@ config:
       domains:
         - url: 'community.openspending.org'
           viewid: '74190894'
-        - url: 'openspending.org'
-          viewid: '44042637'
         - url: 'next.openspending.org'
           viewid: '131861793'

pylama.ini

@@ -1,5 +1,6 @@
 [pylama]
 linters = pyflakes,mccabe,pep8
+ignore = E0100
 
 [pylama:*/__init__.py]
 ignore = W0611

setup.py

@@ -47,6 +47,7 @@ def read(*paths):
     extras_require={'develop': TESTS_REQUIRE},
     zip_safe=False,
     long_description=README,
+    long_description_content_type="text/markdown",
     description='{{ DESCRIPTION }}',
     author='Open Knowledge International',
     author_email='info@okfn.org',

tests/test_discourse_processor.py

@@ -38,12 +38,13 @@
 }
 
 ACTIVE_USERS_RESPONSE = [
-    {'json': [{'last_seen_age': '3m'},
-              {'last_seen_age': '45h'},
-              {'last_seen_age': '2h'}], 'status_code': 200},
-    {'json': [{'last_seen_age': '3h'},
-              {'last_seen_age': '1d'},
-              {'last_seen_age': '25d'}], 'status_code': 200},
+    {'json': [{'last_seen_age': 3 * 60},  # 3 mins
+              {'last_seen_age': 23 * 60 * 60},  # 23hr
+              {'last_seen_age': 2 * 60 * 60}], 'status_code': 200},  # 2hrs
+    {'json': [{'last_seen_age': 3 * 60 * 60},  # 3hrs
+              {'last_seen_age': (24 * 60 * 60) + 1},  # 24hrs + 1 sec
+              {'last_seen_age': 25 * 24 * 60 * 60}],  # 25d
+        'status_code': 200},
     {'json': [], 'status_code': 200},
 ]