Merge pull request #420 from OpenBazaar/stats-collection

Auditing to Database

api/restapi.py

@@ -4,6 +4,7 @@
 import os
 import obelisk
 import nacl.encoding
+import time
 from binascii import unhexlify
 from collections import OrderedDict
 from functools import wraps
@@ -172,7 +173,8 @@ def parse_profile(profile, temp_handle=None):
                         "pgp_key": profile.pgp_key.public_key,
                         "avatar_hash": profile.avatar_hash.encode("hex"),
                         "header_hash": profile.header_hash.encode("hex"),
-                        "social_accounts": {}
+                        "social_accounts": {},
+                        "last_modified": profile.last_modified
                     }
                 }
                 if temp_handle:
@@ -230,7 +232,8 @@ def parse_listings(listings):
                         "currency_code": l.currency_code,
                         "nsfw": l.nsfw,
                         "origin": str(CountryCode.Name(l.origin)),
-                        "ships_to": []
+                        "ships_to": [],
+                        "last_modified": l.last_modified
                     }
                     if l.contract_type != 0:
                         listing_json["contract_type"] = str(objects.Listings.ContractType.Name(l.contract_type))
@@ -445,6 +448,7 @@ def update_profile(self, request):
                 key.public_key = self.keychain.verify_key.encode()
                 key.signature = self.keychain.signing_key.sign(key.public_key)[:64]
                 u.guid_key.MergeFrom(key)
+            u.last_modified = int(time.time())
             p.update(u)
             request.write(json.dumps({"success": True}))
             request.finish()

db/datastore.py

@@ -2,6 +2,7 @@
 
 import os
 import sqlite3 as lite
+import time
 from api.utils import sanitize_html
 from collections import Counter
 from config import DATA_FOLDER
@@ -10,14 +11,14 @@
 from protos import objects
 from protos.objects import Listings, Followers, Following
 from os.path import join
-from db.migrations import migration1, migration2, migration3, migration4, migration5, migration6
+from db.migrations import migration1, migration2, migration3, migration4, migration5, migration6, migration7
 
 
 class Database(object):
 
     __slots__ = ['PATH', 'filemap', 'profile', 'listings', 'keys', 'follow', 'messages',
                  'notifications', 'broadcasts', 'vendors', 'moderators', 'purchases', 'sales',
-                 'cases', 'ratings', 'transactions', 'settings']
+                 'cases', 'ratings', 'transactions', 'settings', 'audit_shopping']
 
     def __init__(self, testnet=False, filepath=None):
         object.__setattr__(self, 'PATH', self._database_path(testnet, filepath))
@@ -37,6 +38,7 @@ def __init__(self, testnet=False, filepath=None):
         object.__setattr__(self, 'ratings', Ratings(self.PATH))
         object.__setattr__(self, 'transactions', Transactions(self.PATH))
         object.__setattr__(self, 'settings', Settings(self.PATH))
+        object.__setattr__(self, 'audit_shopping', ShoppingEvents(self.PATH))
 
         self._initialize_datafolder_tree()
         self._initialize_database(self.PATH)
@@ -174,6 +176,17 @@ def _create_database(database_path):
     smtpNotifications INTEGER, smtpServer TEXT, smtpSender TEXT, smtpRecipient TEXT, smtpUsername TEXT,
     smtpPassword TEXT)''')
 
+        cursor.execute('''CREATE TABLE IF NOT EXISTS audit_shopping (
+            audit_shopping_id integer PRIMARY KEY NOT NULL,
+            shopper_guid text NOT NULL,
+            contract_hash text NOT NULL,
+            "timestamp" integer NOT NULL,
+            action_id integer NOT NULL
+          );''')
+        cursor.execute('''CREATE INDEX IF NOT EXISTS shopper_guid_index ON audit_shopping
+                          (audit_shopping_id ASC);''')
+        cursor.execute('''CREATE INDEX IF NOT EXISTS action_id_index ON audit_shopping (audit_shopping_id ASC);''')
+
         conn.commit()
         conn.close()
 
@@ -190,23 +203,30 @@ def _run_migrations(self):
             migration3.migrate(self.PATH)
             migration4.migrate(self.PATH)
             migration5.migrate(self.PATH)
+            migration6.migrate(self.PATH)
         elif version == 1:
             migration2.migrate(self.PATH)
             migration3.migrate(self.PATH)
             migration4.migrate(self.PATH)
             migration5.migrate(self.PATH)
+            migration6.migrate(self.PATH)
         elif version == 2:
             migration3.migrate(self.PATH)
             migration4.migrate(self.PATH)
             migration5.migrate(self.PATH)
+            migration6.migrate(self.PATH)
         elif version == 3:
             migration4.migrate(self.PATH)
             migration5.migrate(self.PATH)
+            migration6.migrate(self.PATH)
         elif version == 4:
             migration5.migrate(self.PATH)
             migration6.migrate(self.PATH)
         elif version == 5:
             migration6.migrate(self.PATH)
+            migration7.migrate(self.PATH)
+        elif version == 6:
+            migration7.migrate(self.PATH)
 
 
 class HashMap(object):
@@ -1360,3 +1380,39 @@ def get_credentials(self):
         ret = cursor.fetchone()
         conn.close()
         return ret
+
+class ShoppingEvents(object):
+    """
+    Stores audit events for shoppers on your storefront
+    """
+
+    def __init__(self, database_path):
+        self.PATH = database_path
+
+    def set(self, shopper_guid, action_id, contract_hash=None):
+        conn = Database.connect_database(self.PATH)
+        with conn:
+            cursor = conn.cursor()
+            timestamp = int(time.time())
+            if not contract_hash:
+                contract_hash = ''
+            cursor.execute('''INSERT INTO audit_shopping(shopper_guid, timestamp, contract_hash, action_id) VALUES
+                              (?,?,?,?)''', (shopper_guid, timestamp, contract_hash, action_id))
+            conn.commit()
+        conn.close()
+
+    def get(self):
+        conn = Database.connect_database(self.PATH)
+        cursor = conn.cursor()
+        cursor.execute('''SELECT * FROM audit_shopping''')
+        ret = cursor.fetchall()
+        conn.close()
+        return ret
+
+    def get_events_by_id(self, event_id):
+        conn = Database.connect_database(self.PATH)
+        cursor = conn.cursor()
+        cursor.execute('''SELECT * FROM audit_shopping WHERE event_id=?''', event_id)
+        ret = cursor.fetchall()
+        conn.close()
+        return ret

db/migrations/migration7.py

@@ -0,0 +1,24 @@
+import sqlite3
+
+
+def migrate(database_path):
+    print "migrating to db version 7"
+    conn = sqlite3.connect(database_path)
+    conn.text_factory = str
+    cursor = conn.cursor()
+
+    # create new table
+    cursor.execute('''CREATE TABLE IF NOT EXISTS audit_shopping (
+      audit_shopping_id integer PRIMARY KEY NOT NULL,
+      shopper_guid text NOT NULL,
+      contract_hash text,
+      "timestamp" integer NOT NULL,
+      action_id integer NOT NULL
+    );''')
+    cursor.execute('''CREATE INDEX IF NOT EXISTS shopper_guid_index ON audit_shopping (audit_shopping_id ASC);''')
+    cursor.execute('''CREATE INDEX IF NOT EXISTS action_id_index ON audit_shopping (audit_shopping_id ASC);''')
+
+    # update version
+    cursor.execute('''PRAGMA user_version = 7''')
+    conn.commit()
+    conn.close()

market/audit.py

@@ -0,0 +1,31 @@
+__author__ = 'hoffmabc'
+
+from log import Logger
+
+
+class Audit(object):
+    """
+    A class for handling audit information
+    """
+
+    def __init__(self, db):
+        self.db = db
+
+        self.log = Logger(system=self)
+
+        self.action_ids = {
+            "GET_PROFILE": 0,
+            "GET_CONTRACT": 1,
+            "GET_LISTINGS": 2,  # Click Store tab
+            "GET_FOLLOWING": 3,
+            "GET_FOLLOWERS": 4,
+            "GET_RATINGS": 5
+        }
+
+    def record(self, guid, action_id, contract_hash=None):
+        self.log.info("Recording Audit Event [%s]" % action_id)
+
+        if action_id in self.action_ids:
+            self.db.audit_shopping.set(guid, self.action_ids[action_id], contract_hash)
+        else:
+            self.log.error("Could not identify this action id")

market/contracts.py

@@ -149,7 +149,8 @@ def create(self,
                         "metadata": {
                             "version": "1",
                             "category": metadata_category.lower(),
-                            "category_sub": "fixed price"
+                            "category_sub": "fixed price",
+                            "last_modified": int(time.time())
                         },
                         "id": {
                             "guid": self.keychain.guid.encode("hex"),
@@ -1059,6 +1060,7 @@ def save(self):
             data.contract_type = listings.DIGITAL_GOOD
         elif self.contract["vendor_offer"]["listing"]["metadata"]["category"].lower() == "service":
             data.contract_type = listings.SERVICE
+        data.last_modified = int(time.time())
 
         # save the mapping of the contract file path and contract hash in the database
         self.db.filemap.insert(data.contract_hash.encode("hex"), file_path[len(DATA_FOLDER):])

market/protocol.py

@@ -10,6 +10,7 @@
 from interfaces import MessageProcessor, BroadcastListener, MessageListener, NotificationListener
 from keys.bip32utils import derive_childkey
 from log import Logger
+from market.audit import Audit
 from market.contracts import Contract
 from market.moderation import process_dispute, close_dispute
 from market.profile import Profile
@@ -33,6 +34,7 @@ def __init__(self, node, router, signing_key, database):
         self.node = node
         RPCProtocol.__init__(self, node, router)
         self.log = Logger(system=self)
+        self.audit = Audit(db=database)
         self.multiplexer = None
         self.db = database
         self.signing_key = signing_key
@@ -50,6 +52,7 @@ def add_listener(self, listener):
 
     def rpc_get_contract(self, sender, contract_hash):
         self.log.info("serving contract %s to %s" % (contract_hash.encode('hex'), sender))
+        self.audit.record(sender.id.encode("hex"), "GET_CONTRACT", contract_hash.encode('hex'))
         self.router.addContact(sender)
         try:
             with open(self.db.filemap.get_file(contract_hash.encode("hex")), "r") as filename:
@@ -75,6 +78,7 @@ def rpc_get_image(self, sender, image_hash):
 
     def rpc_get_profile(self, sender):
         self.log.info("serving profile to %s" % sender)
+        self.audit.record(sender.id.encode("hex"), "GET_PROFILE")
         self.router.addContact(sender)
         try:
             proto = Profile(self.db).get(True)
@@ -101,6 +105,7 @@ def rpc_get_user_metadata(self, sender):
 
     def rpc_get_listings(self, sender):
         self.log.info("serving store listings to %s" % sender)
+        self.audit.record(sender.id.encode("hex"), "GET_LISTINGS")
         self.router.addContact(sender)
         try:
             p = Profile(self.db).get()
@@ -186,6 +191,7 @@ def rpc_unfollow(self, sender, signature):
 
     def rpc_get_followers(self, sender, start=None):
         self.log.info("serving followers list to %s" % sender)
+        self.audit.record(sender.id.encode("hex"), "GET_FOLLOWERS")
         self.router.addContact(sender)
         if start is not None:
             ser = self.db.follow.get_followers(int(start))
@@ -195,6 +201,7 @@ def rpc_get_followers(self, sender, start=None):
 
     def rpc_get_following(self, sender):
         self.log.info("serving following list to %s" % sender)
+        self.audit.record(sender.id.encode("hex"), "GET_FOLLOWING")
         self.router.addContact(sender)
         ser = self.db.follow.get_following()
         if ser is None:
@@ -346,6 +353,7 @@ def rpc_dispute_close(self, sender, pubkey, encrypted):
     def rpc_get_ratings(self, sender, listing_hash=None):
         a = "ALL" if listing_hash is None else listing_hash.encode("hex")
         self.log.info("serving ratings for contract %s to %s" % (a, sender))
+        self.audit.record(sender.id.encode("hex"), "GET_RATINGS", listing_hash.encode("hex"))
         self.router.addContact(sender)
         try:
             ratings = []

protos/objects.proto

@@ -74,6 +74,7 @@ message Profile {
     PublicKey pgp_key             = 21; // pgp signature covers guid
     bytes avatar_hash             = 22;
     bytes header_hash             = 23;
+    uint64 last_modified          = 24;
 
     // Social media account for the profile
     message SocialAccount {
@@ -121,6 +122,7 @@ message Listings {
         bytes avatar_hash             = 10;
         string handle                 = 11;
         ContractType contract_type    = 12;
+        uint64 last_modified          = 13;
     }
 
     enum ContractType {