Skip to content

Commit

Permalink
Merge pull request #83 from Storj/audits
Browse files Browse the repository at this point in the history
Add Audits
  • Loading branch information
super3 committed Oct 23, 2015
2 parents 6e49ea5 + 52a6fa8 commit b317e6b
Show file tree
Hide file tree
Showing 7 changed files with 237 additions and 37 deletions.
38 changes: 8 additions & 30 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -140,32 +140,6 @@ Fail Examples:
Status Code: 404
Text: Ping Failed: Farmer not found.

Online Status - Readable
************************

This API call was built to be human readable rather than machine readable. We get a simple
list of the all the farmers, their addresses, and their advertised heights. All of this is ordered by height.
We only display farmers that have done a ping in the last `online_time` minutes, which by default
is 15 minutes.

::

GET /api/online

Success Example:

::

GET /api/online
RESPONSE:
Status Code: 200
Text:
18RZNu2nxTdeNyuDCwAMq8aBpgC3FFERPp | Last Seen: 3 second(s) | Height: 7634
137x69jwmcyy4mYCBtQUVoxa21p9Fxyss5 | Last Seen: 7 second(s) | Height: 6234
14wLMb2A9APqrdXJhTQArYLyivmEAf7Y1r | Last Seen: 10 second(s) | Height: 431
1CgLoZT1ZuSHPBp3H4rLTXJvEUDV3kK7QK | Last Seen: 13 second(s) | Height: 245
1QACy1Tx5JFzGDyPd8J3oU8SrjhkZkru4H | Last Seen: 14 second(s) | Height: 88
1NeV1z5BMmFpCXgotwVeZjuN5k124W76MA | Last Seen: 14 second(s) | Height: 10

Online Status - JSON
********************
Expand Down Expand Up @@ -193,13 +167,17 @@ Success Example:
"btc_addr": "1JdEaubcd36ufmT64drdVsGu5SN65A3Z1L",
"height": 0,
"last_seen": 30,
"uptime": 100
"payout_addr": "1GTfrYEi9cRzMNAsz6DESXihTnaJpYxJot",
"reg_time": 1445459786,
"uptime": 59.0
},
{
"btc_addr": "1JdEaubcM36ufmT64drdVsGu5SN65A3Z1A",
"btc_addr": "1GTfrYEi9cRzMNAsz6DESXihTnaJpYxJot",
"height": 0,
"last_seen": 2,
"uptime": 93
"last_seen": 58,
"payout_addr": "1GTfrYEi9cRzMNAsz6DESXihTnaJpYxJot",
"reg_time": 1445459756,
"uptime": 99.0
}
]
}
Expand Down
64 changes: 64 additions & 0 deletions dataserv/Audit.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
from datetime import datetime
from sqlalchemy import DateTime
from dataserv.run import db, app
from btctxstore import BtcTxStore
from dataserv.Farmer import Farmer
from dataserv.Validator import is_sha256

from dataserv.config import logging
logger = logging.getLogger(__name__)
is_btc_address = BtcTxStore().validate_address


class Audit(db.Model):
id = db.Column(db.Integer, primary_key=True)

btc_addr = db.Column(db.String(35))
block = db.Column(db.Integer, index=True)
submit_time = db.Column(DateTime, default=datetime.utcnow)

response = db.Column(db.String(60))

def __init__(self, btc_addr, block, response=None):
if not is_btc_address(btc_addr):
msg = "Invalid BTC Address: {0}".format(btc_addr)
logger.warning(msg)
raise ValueError(msg)
if not Farmer(btc_addr).exists():
msg = "Farmer Not Found: {0}".format(btc_addr)
logger.warning(msg)
raise LookupError(msg)
if not response is None and not is_sha256(response):
msg = "Invalid Response: {0}".format(response)
logger.warning(msg)
raise TypeError(msg)

self.btc_addr = btc_addr
self.block = block
self.response = response

def __eq__(self, other):
return self.btc_addr == other.btc_addr and self.block == other.block \
and self.response == other.response

def save(self):
db.session.add(self)
db.session.commit()

def exists(self):
"""Check to see if this address is already listed."""
response = Audit.query.filter(Audit.btc_addr == self.btc_addr,
Audit.block == self.block).count() > 0
return response

def lookup(self):
"""Return the Farmer object for the bitcoin address passed."""
audit = Audit.query.filter_by(btc_addr=self.btc_addr,
block=self.block).first()
if not audit:
msg = "Block {0} Audit Missing: {1}".format(self.block,
self.btc_addr)
logger.warning(msg)
raise LookupError(msg)

return audit
4 changes: 1 addition & 3 deletions dataserv/Farmer.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
import json
import hashlib
import storjcore
from datetime import datetime
from datetime import timedelta
from sqlalchemy import DateTime
from dataserv.run import db, app
from btctxstore import BtcTxStore
from datetime import datetime, timedelta


from dataserv.config import logging
Expand Down Expand Up @@ -134,7 +133,6 @@ def ping(self, before_commit_callback=None):
before_commit_callback()
db.session.commit()

# TODO: Actually do an audit.
def audit(self):
"""
Complete a cryptographic audit of files stored on the farmer. If
Expand Down
44 changes: 43 additions & 1 deletion dataserv/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))

from sqlalchemy import desc
from dataserv.Audit import Audit
from dataserv.Farmer import Farmer
from dataserv.config import logging
from dataserv.run import app, db, cache
Expand Down Expand Up @@ -207,5 +208,46 @@ def set_height(btc_addr, height):
return make_response(error_msg.format(msg), 401)


if __name__ == '__main__':
@app.route('/api/audit/<btc_addr>/<int:block_height>/<response>',
methods=["GET"])
def audit(btc_addr, block_height, response):
logger.info("CALLED /api/audit/{0}/{1}/{2}".format(btc_addr, block_height,
response))
error_msg = "Audit failed: {0}"

try:
user = Farmer(btc_addr)
user.authenticate(dict(request.headers))

audit_msg = Audit(btc_addr, block_height, response)
if audit_msg.exists():
msg = "Duplicate audit: Block {0}".format(block_height)
logger.warning(msg)
return make_response(msg, 409)
else:
audit_msg.save()
return make_response("Audit accepted.", 201)

except TypeError:
msg = "Invalid response."
logger.warning(msg)
return make_response(msg, 400)

except ValueError:
msg = "Invalid Bitcoin address."
logger.warning(msg)
return make_response(msg, 400)

except LookupError:
msg = "Farmer not found."
logger.warning(msg)
return make_response(msg, 404)

except storjcore.auth.AuthError:
msg = "Invalid authentication headers."
logger.warning(msg)
return make_response(error_msg.format(msg), 401)


if __name__ == '__main__': # pragma: no cover
app.run(debug=True)
6 changes: 3 additions & 3 deletions dataserv/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,15 @@


# MAX_PING is the most a client may ping
if os.environ.get("DATASERV_MAX_PING"):
if os.environ.get("DATASERV_MAX_PING"): # pragma: no cover
MAX_PING = int(os.environ.get("DATASERV_MAX_PING"))
else:
MAX_PING = 60 # default seconds


# db setup
# example `export DATASERV_DATABASE_URI="postgresql:///dataserv"`
if os.environ.get("DATASERV_DATABASE_URI"):
if os.environ.get("DATASERV_DATABASE_URI"): # pragma: no cover
SQLALCHEMY_DATABASE_URI = os.environ.get("DATASERV_DATABASE_URI")
else: # default to sqlite
SQLALCHEMY_DATABASE_URI = "sqlite:///dataserv.db"
Expand All @@ -33,7 +33,7 @@
level=logging.DEBUG)
TOTAL_UPDATE = 30 # minutes

if os.environ.get("DATASERV_CACHING_TIME"):
if os.environ.get("DATASERV_CACHING_TIME"): # pragma: no cover
CACHING_TIME = int(os.environ.get("DATASERV_CACHING_TIME"))
else:
CACHING_TIME = 30 # seconds
Expand Down
47 changes: 47 additions & 0 deletions tests/test_App.py
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,53 @@ def test_height_limit(self):
self.assertEqual(rv.status_code, 200)


class AuditTest(TemplateTest):

def test_first_audit(self):
btc_addr = self.gen_wallet()
self.app.get('/api/register/{0}'.format(btc_addr))

ha = 'c059c8035bbd74aa81f4c787c39390b57b974ec9af25a7248c46a3ebfe0f9dc8'

# accepted audit
rv = self.app.get('/api/audit/{0}/{1}/{2}'.format(btc_addr, 0, ha))
self.assertEqual(rv.status_code, 201)

# duplicate audit
rv = self.app.get('/api/audit/{0}/{1}/{2}'.format(btc_addr, 0, ha))
self.assertEqual(rv.status_code, 409)

def test_invalid_response(self):
btc_addr = self.gen_wallet()
self.app.get('/api/register/{0}'.format(btc_addr))

ha = 'invalid hash'
rv = self.app.get('/api/audit/{0}/{1}/{2}'.format(btc_addr, 0, ha))
self.assertEqual(rv.status_code, 400)

def test_invalid_address(self):
ha = 'c059c8035bbd74aa81f4c787c39390b57b974ec9af25a7248c46a3ebfe0f9dc8'
rv = self.app.get('/api/audit/{0}/{1}/{2}'.format('bad', 0, ha))
self.assertEqual(rv.status_code, 400)

def test_farmer_not_found(self):
btc_addr = self.gen_wallet()
ha = 'c059c8035bbd74aa81f4c787c39390b57b974ec9af25a7248c46a3ebfe0f9dc8'
rv = self.app.get('/api/audit/{0}/{1}/{2}'.format(btc_addr, 0, ha))
self.assertEqual(rv.status_code, 404)

def test_auth(self):
app.config["SKIP_AUTHENTICATION"] = False

btc_addr = self.gen_wallet()
self.app.get('/api/register/{0}'.format(btc_addr))

ha = 'c059c8035bbd74aa81f4c787c39390b57b974ec9af25a7248c46a3ebfe0f9dc8'
rv = self.app.get('/api/audit/{0}/{1}/{2}'.format(btc_addr, 0, ha))

self.assertEqual(rv.status_code, 401)


class AppAuthenticationHeadersTest(unittest.TestCase):

def setUp(self):
Expand Down
71 changes: 71 additions & 0 deletions tests/test_Audit.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
import unittest
from dataserv.Audit import Audit
from dataserv.app import db, app
from btctxstore import BtcTxStore
from dataserv.Farmer import Farmer


class AuditTest(unittest.TestCase):

def setUp(self):
app.config["SKIP_AUTHENTICATION"] = True # monkey patch
app.config["DISABLE_CACHING"] = True

self.btctxstore = BtcTxStore()
self.bad_addr = 'notvalidaddress'

db.create_all()

def tearDown(self):
db.session.remove()
db.drop_all()

def gen_btc_addr(self):
return self.btctxstore.get_address(self.btctxstore.get_key(
self.btctxstore.create_wallet()))

def test_register_audit(self):
btc_addr = self.gen_btc_addr()
btc_addr2 = self.gen_btc_addr()

# register farmer and test db
farmer1 = Farmer(btc_addr)
self.assertFalse(farmer1.exists())
farmer1.register()
self.assertTrue(farmer1.exists())

# do callbacks to properly test errors
def callback_a():
Audit(self.bad_addr, 0)

def callback_b():
Audit(btc_addr2, 0)

audit = Audit(btc_addr, 0)
self.assertFalse(audit.exists())
audit.save()
audit2 = Audit(btc_addr, 0)
self.assertTrue(audit2.exists())

def callback_c():
Audit(btc_addr, 1, 'invalid_sha')

self.assertRaises(ValueError, callback_a)
self.assertRaises(LookupError, callback_b)
self.assertRaises(TypeError, callback_c)

def test_lookup(self):
btc_addr = self.gen_btc_addr()
Farmer(btc_addr).register()

audit = Audit(btc_addr, 0)
audit.save()

def callback_a():
Audit(btc_addr, 1).lookup()

self.assertRaises(LookupError, callback_a)

audit2 = Audit(btc_addr, 0).lookup()
audit3 = Audit(btc_addr, 0)
self.assertEqual(audit2, audit3)

0 comments on commit b317e6b

Please sign in to comment.