Patch setup

.gitignore

@@ -12,16 +12,17 @@ doc/usersguide/build/
 liam2/build
 liam2/*.c
 build
-tests/functional/error.log
-tests/functional/person_*.csv
-tests/functional/*.h5
-tests/functional/*.png
-tests/functional/*.pdf
-tests/functional/empty_file.csv
-tests/examples/simulation.h5
-tests/examples/demography_*.png
-tests/examples/error.log
-tests/examples/person_*.csv
+liam2/tests/functional/*.h5
+!liam2/tests/functional/small.h5
+liam2/tests/output/error.log
+liam2/tests/output/person_*.csv
+liam2/tests/output/*.h5
+liam2/tests/output/*.png
+liam2/tests/output/*.pdf
+liam2/tests/output/empty_file.csv
+liam2/tests/output/simulation.h5
+liam2/tests/output/demography_*.png
+liam2/tests/output/demography_*.csv
 doc/website/blog
 doc/website/objects.inv
 doc/website/pages/download.rst

.travis.yml

@@ -0,0 +1,16 @@
+env:
+  - USE_TRAVIS=true
+language: python
+python: "2.7"
+virtualenv:
+  system_site_packages: true
+before_install:
+  - sudo apt-get update -qq
+  - sudo apt-get install -qq cython python-matplotlib python-numexpr python-numpy python-yaml python-sip
+  - sudo apt-get build-dep python-tables
+install:
+  - pip install flake8  # Not yet available in Debian stable
+  - pip install tables  # For version > 3
+  - pip install -e .
+  - python setup.py build_ext --inplace
+script: make test-ci

INSTALL

@@ -17,7 +17,7 @@ LIAM2 uses a number of other open-source packages.
 Required dependencies:
 - Python 2.7.x (32b or 64b) - http://www.python.org/
 - Numpy 1.7.0 or later - http://www.numpy.org/
-- PyTables 2.2.1 or later - http://www.pytables.org
+- PyTables 3 or later - http://www.pytables.org
 - Numexpr 2.0 or later - https://github.com/pydata/numexpr
 - PyYAML 3.0.8 or later - http://pyyaml.org
 

Makefile

@@ -0,0 +1,35 @@
+TESTS_DIR=liam2/tests
+
+all: flake8 test
+
+check-syntax-errors:
+	@# This is a hack around flake8 not displaying E910 errors with the select option.
+	@# Do not analyse .gitignored files.
+	@# `make` needs `$$` to output `$`. Ref: http://stackoverflow.com/questions/2382764.
+	test -z "`flake8 --first $(shell git ls-files | grep "\.py$$") | grep E901`"
+
+clean: clean-build clean-cython clean-pyc
+
+clean-build:
+	rm -rf build
+
+clean-cython:
+	rm liam2/cpartition.c liam2/cutils.c cpartition.so cutils.so
+
+clean-pyc:
+	find . -name '*.pyc' -exec rm \{\} \;
+
+ctags:
+	ctags --recurse=yes .
+
+flake8: clean-pyc
+	flake8
+
+test: check-syntax-errors
+	nosetests $(TESTS_DIR) --exe --stop --with-doctest
+
+test-ci: check-syntax-errors
+	nosetests $(TESTS_DIR) --exe --with-doctest
+
+test-with-coverage:
+	nosetests $(TESTS_DIR) --exe --stop --with-coverage --cover-package=openfisca_france --cover-erase --cover-branches --cover-html

doc/usersguide/source/import.rst

@@ -178,10 +178,10 @@ file.
 
     entities:
         household:
-            path: input\household.csv
+            path: input/household.csv
 
         person:
-            path: input\person.csv
+            path: input/person.csv
 
 This will try to load all the fields of the household and person entities in 
 "*household.csv*" and "person.csv" in the "input" sub-directory of the
@@ -198,10 +198,10 @@ directory where the description file is.
 
     entities:
         household:
-            path: input\household.csv
+            path: input/household.csv
 
         person:
-            path: input\person.csv
+            path: input/person.csv
             fields:
                 - age:        int
                 - gender:     bool
@@ -245,7 +245,7 @@ If the fields of an entity are scattered in several files, you can use the
                         value: workstate
                 # person.csv should have at least 4 columns:
                 # period, id, age and gender
-                - param\person.csv:
+                - param/person.csv:
                     newnames:
                         # we override the "global" renaming
                         period: period

liam2/expr.py

@@ -45,7 +45,7 @@ def evaluate(expr, globals_dict, locals_dict=None, **kwargs):
 timings = Counter()
 
 type_to_idx = {bool: 0, np.bool_: 0,
-               int: 1, np.int8: 1, np.int16: 1, np.int32: 1, np.int64: 1, np.intc: 1, np.longlong: 1,
+               int: 1, np.int8: 1, np.int16: 1, np.int32: 1, np.int64: 1, np.intc: 1, np.long: 1, np.longlong: 1,
                float: 2, np.float64: 2}
 idx_to_type = [bool, int, float]
 

liam2/importer.py

@@ -379,7 +379,6 @@ def union1d(arrays):
 
 
 def interpolate(target, arrays, id_periods, fields):
-    assert bcolz is not None, 'bcolz package is required to use interpolate'
     print(" * indexing...")
     periods = np.unique(id_periods['period'])
     max_id = np.max(id_periods['id'])
@@ -412,11 +411,15 @@ def interpolate(target, arrays, id_periods, fields):
     del lastrow_for_id
 
     size = sum(row_for_id[period].nbytes for period in periods)
-    print(" * compressing index (%.2f Mb)..." % (size / MB), end=' ')
-    for period in periods:
-        row_for_id[period] = bcolz.carray(row_for_id[period])
-    csize = sum(row_for_id[period].cbytes for period in periods)
-    print("done. (%.2f Mb)" % (csize / MB))
+
+    if bcolz is not None:
+        print(" * compressing index (%.2f Mb)..." % (size / MB), end=' ')
+        for period in periods:
+            row_for_id[period] = bcolz.carray(row_for_id[period])
+        csize = sum(row_for_id[period].cbytes for period in periods)
+        print("done. (%.2f Mb)" % (csize / MB))
+    else:
+        print('bcolz package not found (bcolz is required to use compression in interpolate)')
 
     print(" * interpolating...")
     for values in arrays:
@@ -685,7 +688,7 @@ def load_def(localdir, ent_name, section_def, required_fields):
         return 'table', (target_fields, total_lines, iter(target), None)
 
 
-def csv2h5(fpath, buffersize=10 * 2 ** 20):
+def csv2h5(fpath = None, buffersize=10 * 2 ** 20):
     with open(fpath) as f:
         content = yaml.load(f)
 

tests/examples/demo_import.yml → liam2/tests/examples/demo_import.yml

@@ -5,16 +5,16 @@ output: simple2001.h5
 
 globals:
     periodic:
-        path: input\globals_transposed.csv
+        path: input/globals_transposed.csv
         transposed: True
 
 entities:
     household:
-        path: input\household.csv
+        path: input/household.csv
         # if fields are not specified, they are all imported
 
     person:
-        path: input\person.csv
+        path: input/person.csv
         fields:
             # period and id are implicit
             - age:           int

tests/functional/import.yml → liam2/tests/functional/import.yml

@@ -3,22 +3,22 @@ output: small.h5
 
 globals:
     ARRAY:
-        path: param\mig.csv
+        path: param/mig.csv
         type: float
 
     othertable_noperiod:
-        path: param\othertable.csv
+        path: param/othertable.csv
         fields:
             # PERIOD is *not* implicit
             - INTFIELD: int
             - FLOATFIELD: float
 
 entities:
     region:
-        path: param\region.csv
+        path: param/region.csv
 
     household:
-        path: param\household.csv
+        path: param/household.csv
 
     person:
         fields:
@@ -35,7 +35,7 @@ entities:
 
         # deprecated? no, let's just say it's a shortcut if there is only one
         # file. And it could serve as a prefix for individual files path.
-        path: param\person.csv
+        path: param/person.csv
 
         # deprecated? no. it could be useful even in the case of several files
         # if you want fields renamed the same way across all files
@@ -54,16 +54,16 @@ entities:
 ##        transposed: True
 #
         files:
-            - param\person.csv:
+            - param/person.csv:
 #                oldnames:
 #                    period: period
 #                    dead: co_alive
                 newnames:
                     period: period
                     co_alive: dead
 
-            - param\p_age.txt: {}
-            - param\p_work.txt: {}
+            - param/p_age.txt: {}
+            - param/p_work.txt: {}
 
 #        interpolate:
 #            work: previous_value

tests/functional/import_issue154.yml → liam2/tests/functional/import_issue154.yml

@@ -2,7 +2,7 @@ output: import_issue154.h5
 
 entities:
     dummy:
-        path: param\period0.csv
+        path: param/period0.csv
         fields:
             # period and id are implicit
             - period:   int

tests/functional/imported1.yml → liam2/tests/functional/imported1.yml

@@ -8,19 +8,20 @@ entities:
             - imported_field1: {type: int, initialdata: False}
 
         macros:
-            IMPORTED_MACRO: imported_field1 == 1 
+            IMPORTED_MACRO: imported_field1 == 1
             ISCHILD: age < 17
 
         processes:
-            agegroup: trunc(age / 10)
+            compute_agegroup:
+                - agegroup: trunc(age / 10)
 
             imported_process:
                 - imported_field1: age + 1
 
 simulation:
     # overridden by simulation.yml
     processes:
-        - person: [agegroup, imported_process]
+        - person: [compute_agegroup, imported_process]
 
     periods: 55
 
@@ -31,4 +32,3 @@ simulation:
     # not overridden (present in neither imported2.yml nor simulation.yml)
     input:
         file: small.h5
-

tests/functional/imported2.yml → liam2/tests/functional/imported2.yml

@@ -4,22 +4,23 @@ entities:
             - imported_field2: {type: int, initialdata: False}
 
         macros:
-            IMPORTED_MACRO: imported_field2 == 2 
+            IMPORTED_MACRO: imported_field2 == 2
             ISCHILD: age < 16
 
         processes:
-            agegroup: trunc(age / 20)
+            compute_agegroup:
+                - agegroup: trunc(age / 20)
 
             imported_process:
                 - imported_field2: age + 2
 
 simulation:
     # overrides imported1.yml and is overridden by simulation.yml
     periods: 1
-    
+
     # overrides imported1.yml and is not overridden
     start_period: 2002   # first simulated period
     default_entity: person
 
     # not defined elsewhere thus not overridden
-    random_seed: 0    
+    random_seed: 0