Merge branch 'change-track'

.gitignore

@@ -13,3 +13,6 @@ ynab.yaml
 nosetests*.xml
 tests/*.json
 testscripts/*.json
+\.pytest_cache/
+
+\.cache/v/cache/

.travis.yml

@@ -7,7 +7,7 @@ python:
 install:
 - PACKAGE_VERSION=`python setup.py --version`
 - TAG_NAME=v$PACKAGE_VERSION
-- pip install tox-travis
+- pip install -r dev-requirements.txt
 before_install:
 - export TRAVIS_COMMIT_MESSAGE=$(git log --format=%B -n 1 $TRAVIS_COMMIT)
 - echo "$TRAVIS_COMMIT_MESSAGE"

dev-requirements.txt

@@ -3,3 +3,4 @@ coveralls
 pytest
 jsbeautifier
 python-dotenv
+tox-travis

live_tox.ini

@@ -1,11 +1,15 @@
 [tox]
-envlist = {py27,py34,py35}-test_live
-
-[testenv:test_live]
-commands=
-    py.test test_live/
+envlist = {py27,py34,py35}-{test,test_live}
 
 [testenv]
 passenv = TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH YNAB_* NYNAB_*
 deps=
     -rdev-requirements.txt
+
+[testenv:test]
+commands =
+    coverage run -m py.test -x tests/
+
+[testenv:test_live]
+commands =
+    coverage run -m py.test -x test_live/

pynYNAB/ClientFactory.py

@@ -1,15 +1,12 @@
 import logging
 
-import os
-
-import yaml
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
 from pynYNAB.ObjClient import RootObjClient
 from pynYNAB.connection import nYnabConnection
 from pynYNAB.exceptions import NoBudgetNameException, BudgetNotFound, NoCredentialsException
-from pynYNAB.schema import Base
+from pynYNAB.schema import Base, Catalog, Budget
 
 LOG = logging.getLogger(__name__)
 
@@ -22,7 +19,7 @@ def extra(self):
     opname = 'syncCatalogData'
 
     def __init__(self, client):
-        super(CatalogClient, self).__init__(client.catalog, client)
+        super(CatalogClient, self).__init__(client.catalog, client, Catalog)
 
 
 class BudgetClient(RootObjClient):
@@ -33,7 +30,32 @@ def extra(self):
     opname = 'syncBudgetData'
 
     def __init__(self, client):
-        super(BudgetClient, self).__init__(client.budget, client)
+        super(BudgetClient, self).__init__(client.budget, client, Budget)
+
+    def get_changed_apidict(self):
+        changed_api_dict = super(BudgetClient, self).get_changed_apidict()
+        if 'be_transactions' in changed_api_dict:
+            changed_api_dict['be_transaction_groups'] = []
+            for transaction_dict in changed_api_dict.pop('be_transactions'):
+                transaction_id = transaction_dict['id']
+                subtransactions = []
+                if 'be_subtransactions' in changed_api_dict:
+                    for subtransaction_dic in changed_api_dict['be_subtransactions']:
+                        if subtransaction_dic['entities_transaction_id'] == transaction_id:
+                            subtransactions.append(subtransaction_dic)
+                    for subtransaction in subtransactions:
+                        changed_api_dict['be_subtransactions'].remove(subtransaction)
+                if not subtransactions:
+                    subtransactions = None
+                group = dict(
+                    id=transaction_id,
+                    be_transaction=transaction_dict,
+                    be_subtransactions=subtransactions,
+                    be_matched_transaction=None)
+                changed_api_dict['be_transaction_groups'].append(group)
+        if changed_api_dict.get('be_subtransactions') is not None:
+            del changed_api_dict['be_subtransactions']
+        return changed_api_dict
 
 
 class nYnabClientFactory(object):
@@ -89,14 +111,22 @@ def postprocessed_client(cl):
             LOG.error('No budget by the name %s found in nYNAB' % budget_name)
             raise
 
+    @classmethod
+    def from_args(cls, args, sync=True):
+        return nYnabClientFactory().create_client(args.email, args.password, args.budget_name, args.connection, sync)
+
+    @classmethod
+    def from_kwargs(cls, **kwargs):
+        return nYnabClientFactory().create_client(kwargs.get("email", None),
+                                                  kwargs.get("password", None),
+                                                  kwargs.get("budget_name", None),
+                                                  kwargs.get("connection", None),
+                                                  kwargs.get("sync", True))
+
 
 def clientfromargs(args, sync=True):
-    return nYnabClientFactory().create_client(args.email, args.password, args.budget_name, args.connection, sync)
+    return nYnabClientFactory.from_args(args, sync)
 
 
 def clientfromkwargs(**kwargs):
-    return nYnabClientFactory().create_client(kwargs.get("email", None),
-                                              kwargs.get("password", None),
-                                              kwargs.get("budget_name", None),
-                                              kwargs.get("connection", None),
-                                              kwargs.get("sync", True))
+    return nYnabClientFactory.from_kwargs(**kwargs)

pynYNAB/ObjClient.py

@@ -1,38 +1,104 @@
+import itertools
 import logging
-from abc import abstractproperty,ABCMeta
+from abc import abstractproperty, ABCMeta
 
-import itertools
+from sqlalchemy import event, inspect
 
-from pynYNAB.schema import fromapi_conversion_functions_table
+from pynYNAB.schema import fromapi_conversion_functions_table, listfields, scalarfields
 
 LOG = logging.getLogger(__name__)
 
+
 def split_seq(iterable, size):
     it = iter(iterable)
     item = list(itertools.islice(it, size))
     while item:
         yield item
         item = list(itertools.islice(it, size))
 
-class RootObjClient():
-    __metaclass__ = ABCMeta
 
-    @abstractproperty
+class RootObjClient(object):
+
     def extra(self):
         return {}
 
-    @abstractproperty
     def opname(self):
         return ''
 
-    def __init__(self, obj, client):
+    def __init__(self, obj, client, cls):
         self.obj = obj
+        self.cls = cls
+        self.changed = cls()
         self.client = client
         self.connection = client.connection
         self.session = client.session
         self.server_entities = {}
         self.synced = False
 
+        self.listfields = listfields(cls)
+        self.scalarfields = scalarfields(cls)
+        self.rev_listfields = {v: k for k, v in self.listfields.items()}
+
+        mapper_obj = inspect(cls)
+        for rel_attr in mapper_obj.relationships:
+            if rel_attr.key in self.listfields:
+                self.collection_listener(rel_attr)
+
+        for container,cls in self.listfields.items():
+            for col_attr in inspect(cls).column_attrs:
+                self.attribute_track_listener(col_attr)
+
+    def get_changed_entities(self):
+        returnvalue = {}
+        for k in self.obj.listfields:
+            v = getattr(self.changed,k)
+            if v:
+                returnvalue[k] = {el.id:el for el in v}
+        return returnvalue
+
+    def get_changed_apidict(self):
+        returnvalue = {}
+        changed = self.get_changed_entities()
+        for key, values in changed.items():
+            for k, v in values.items():
+                returnvalue.setdefault(key, []).append(v.dict_to_apidict(v.get_dict()))
+
+        return returnvalue
+
+    def clear_changed_entities(self):
+        self.changed = self.cls()
+
+    def collection_listener(self, rel_attr):
+        @event.listens_for(rel_attr, 'append')
+        def append(target, value, initiator):
+            if target == self.obj:
+                print('c append %s' % value.id)
+                container = getattr(self.changed, rel_attr.key)
+                if value in container:
+                    if value.is_tombstone:
+                        container.remove(value)
+                else:
+                    container.append(value)
+
+        @event.listens_for(rel_attr, 'remove')
+        def remove(target, value, initiator):
+            if target == self.obj:
+                print('c remove %s' % value.id)
+                container = getattr(self.changed, rel_attr.key)
+                if value in container:
+                    container.remove(value)
+                else:
+                    value.is_tombstone = True
+                    if value not in container:
+                        container.append(value)
+
+    def attribute_track_listener(self, col_attr):
+        @event.listens_for(col_attr, "set")
+        def receive_set(target, value, oldvalue, initiator):
+            if hasattr(target, 'parent') and target.parent  is not None and target.parent == self.obj:
+                print('c attr set %s %s' % (target.id, initiator.key))
+                getattr(self.changed, self.rev_listfields[target.__class__]).append(target)
+
     def update_from_api_changed_entitydicts(self, changed_entitydicts, update_keys=None):
         if update_keys is None:
             update_keys = list(self.obj.listfields.keys())
@@ -48,7 +114,7 @@ def update_from_api_changed_entitydicts(self, changed_entitydicts, update_keys=N
             modified_entitydicts[listfield_name] = newlist
         for scalarfield_name in self.obj.scalarfields:
             if scalarfield_name in changed_entitydicts:
-                typ = self.obj.scalarfields[scalarfield_name]
+                typ = self.scalarfields[scalarfield_name]
                 conversion_function = fromapi_conversion_functions_table.get(typ, lambda t, x: x)
                 modified_entitydicts[scalarfield_name] = conversion_function(typ, changed_entitydicts[scalarfield_name])
         self.update_from_changed_entities(modified_entitydicts)
@@ -84,11 +150,9 @@ def update_from_changed_entities(self, changed_entities):
             getattr(self.obj,name).dirty=True
         self.session.commit()
 
-
     def update_from_sync_data(self, sync_data, update_keys=None):
         self.update_from_api_changed_entitydicts(sync_data['changed_entities'],update_keys)
 
-
     def sync(self, update_keys=None):
         if self.connection is None:
             return
@@ -99,7 +163,7 @@ def sync(self, update_keys=None):
         LOG.debug('current_server_knowledge ' + str(sync_data['current_server_knowledge']))
         self.update_from_sync_data(sync_data,update_keys)
         self.session.commit()
-        self.obj.clear_changed_entities()
+        self.clear_changed_entities()
 
         server_knowledge_of_device = sync_data['server_knowledge_of_device']
         current_server_knowledge = sync_data['current_server_knowledge']
@@ -119,7 +183,7 @@ def sync(self, update_keys=None):
         self.synced = True
 
     def push(self, update_from_sync_data=True, update_keys=None):
-        changed_entities = self.obj.get_changed_apidict()
+        changed_entities = self.get_changed_apidict()
         request_data = dict(starting_device_knowledge=self.client.starting_device_knowledge,
                             ending_device_knowledge=self.client.ending_device_knowledge,
                             device_knowledge_of_server=self.obj.knowledge.device_knowledge_of_server,
@@ -128,7 +192,7 @@ def push(self, update_from_sync_data=True, update_keys=None):
 
         def validate():
             self.session.commit()
-            self.obj.clear_changed_entities()
+            self.clear_changed_entities()
         if self.connection is not None:
             sync_data = self.connection.dorequest(request_data, self.opname)
             LOG.debug('server_knowledge_of_device ' + str(sync_data['server_knowledge_of_device']))

pynYNAB/connection.py

@@ -5,10 +5,11 @@
 from time import sleep
 
 import requests
+from aenum import Enum
 from requests.cookies import RequestsCookieJar
 
 from pynYNAB.KeyGenerator import generateuuid
-from pynYNAB.schema.Entity import ComplexEncoder
+from pynYNAB.schema import Entity
 from pynYNAB.utils import rate_limited, pp_json
 
 LOG = logging.getLogger(__name__)
@@ -94,3 +95,12 @@ def errorout(message):
         else:
             errorout('Unknown API Error \"%s\" was returned from the API when sending request (%s)' % (error['id'], params))
 
+
+class ComplexEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, Entity):
+            return obj.get_apidict()
+        elif isinstance(obj, Enum):
+            return obj.value
+        else:
+            return json.JSONEncoder.default(self, obj)

pynYNAB/schema/Client.py

@@ -40,10 +40,6 @@ class nYnabClient_(Base):
     def user_id(self):
         return self.id
 
-    @property
-    def online(self):
-        return self.connection is not None
-
     def add_missing(self):
         self.catalog = Catalog()
         self.catalog.knowledge = Knowledge()
@@ -53,27 +49,25 @@ def add_missing(self):
         self.session.add(self.budget)
         self.session.commit()
 
-    def init_internal_db(self):
-        Base.metadata.create_all(self.engine)
-        self.Session = sessionmaker(bind=self.engine)
-        self.session = self.Session()
-
     def sync(self, update_keys=None):
         LOG.debug('Client.sync')
 
         self.catalogClient.sync(update_keys)
         self.select_budget(self.budget_name)
         self.budgetClient.sync(update_keys)
 
+        self.catalogClient.clear_changed_entities()
+        self.budgetClient.clear_changed_entities()
+
         if self.budget_version_id is None and self.budget_name is not None:
             raise BudgetNotFound()
 
     def push(self, expected_delta=1):
         # ending-starting represents the number of modifications that have been done to the data ?
         LOG.debug('Client.push')
 
-        catalog_changed_entities = self.catalog.get_changed_apidict()
-        budget_changed_entities = self.budget.get_changed_apidict()
+        catalog_changed_entities = self.catalogClient.get_changed_apidict()
+        budget_changed_entities = self.budgetClient.get_changed_apidict()
 
         delta = sum(len(l) for k, l in catalog_changed_entities.items()) + \
             sum(len(l) for k, l in budget_changed_entities.items())
@@ -86,6 +80,8 @@ def push(self, expected_delta=1):
 
         self.catalogClient.push()
         self.budgetClient.push()
+        self.catalogClient.clear_changed_entities()
+        self.budgetClient.clear_changed_entities()
 
         self.starting_device_knowledge = self.ending_device_knowledge
         self.session.commit()