Merge pull request #51 from dashpay/develop

release Sentinel version 1.2.0

.travis.yml

@@ -13,4 +13,4 @@ script:
   - py.test -svv test/unit/
 
   # style guide check
-  - find ./lib ./test ./bin -name \*.py -exec pycodestyle --show-source --ignore=E501,E402,E722 {} +
+  - find ./lib ./test ./bin -name \*.py -exec pycodestyle --show-source --ignore=E501,E402,E722,E129 {} +

README.md

@@ -4,11 +4,11 @@ An all-powerful toolset for Dash.
 
 [![Build Status](https://travis-ci.org/dashpay/sentinel.svg?branch=master)](https://travis-ci.org/dashpay/sentinel)
 
-Sentinel is an autonomous agent for persisting, processing and automating Dash V12.1 governance objects and tasks, and for expanded functions in the upcoming Dash V13 release (Evolution).
+Sentinel is an autonomous agent for persisting, processing and automating Dash governance objects and tasks, and for expanded functions in the upcoming Dash V13 release (Evolution).
 
-Sentinel is implemented as a Python application that binds to a local version 12.1 dashd instance on each Dash V12.1 Masternode.
+Sentinel is implemented as a Python application that binds to a local version 12 dashd instance on each Dash Masternode.
 
-This guide covers installing Sentinel onto an existing 12.1 Masternode in Ubuntu 14.04 / 16.04.
+This guide covers installing Sentinel onto an existing Masternode in Ubuntu 14.04 / 16.04.
 
 ## Installation
 
@@ -47,7 +47,7 @@ In the crontab editor, add the lines below, replacing '/home/YOURUSERNAME/sentin
 
 ### 4. Test the Configuration
 
-Test the config by runnings all tests from the sentinel folder you cloned into
+Test the config by running all tests from the sentinel folder you cloned into
 
     $ ./venv/bin/py.test ./test
 
@@ -67,11 +67,11 @@ To view debug output, set the `SENTINEL_DEBUG` environment variable to anything
 
 ## Contributing
 
-Please follow the [DashCore guidelines for contributing](https://github.com/dashpay/dash/blob/v0.12.1.x/CONTRIBUTING.md).
+Please follow the [DashCore guidelines for contributing](https://github.com/dashpay/dash/blob/master/CONTRIBUTING.md).
 
 Specifically:
 
-* [Contributor Workflow](https://github.com/dashpay/dash/blob/v0.12.1.x/CONTRIBUTING.md#contributor-workflow)
+* [Contributor Workflow](https://github.com/dashpay/dash/blob/master/CONTRIBUTING.md#contributor-workflow)
 
     To contribute a patch, the workflow is as follows:
 

bin/dbtest.py

@@ -6,7 +6,7 @@
 import os
 sys.path.append(os.path.normpath(os.path.join(os.path.dirname(__file__), '../lib')))
 import config
-from models import Superblock, Proposal, GovernanceObject, Setting, Signal, Vote, Outcome, Watchdog
+from models import Superblock, Proposal, GovernanceObject, Setting, Signal, Vote, Outcome
 from models import VoteSignals, VoteOutcomes
 from peewee import PeeweeException  # , OperationalError, IntegrityError
 from dashd import DashDaemon

bin/sentinel.py

@@ -6,7 +6,7 @@
 import config
 import misc
 from dashd import DashDaemon
-from models import Superblock, Proposal, GovernanceObject, Watchdog
+from models import Superblock, Proposal, GovernanceObject
 from models import VoteSignals, VoteOutcomes, Transient
 import socket
 from misc import printdbg
@@ -24,42 +24,6 @@ def perform_dashd_object_sync(dashd):
     GovernanceObject.sync(dashd)
 
 
-# delete old watchdog objects, create new when necessary
-def watchdog_check(dashd):
-    printdbg("in watchdog_check")
-
-    # delete expired watchdogs
-    for wd in Watchdog.expired(dashd):
-        printdbg("\tFound expired watchdog [%s], voting to delete" % wd.object_hash)
-        wd.vote(dashd, VoteSignals.delete, VoteOutcomes.yes)
-
-    # now, get all the active ones...
-    active_wd = Watchdog.active(dashd)
-    active_count = active_wd.count()
-
-    # none exist, submit a new one to the network
-    if 0 == active_count:
-        # create/submit one
-        printdbg("\tNo watchdogs exist... submitting new one.")
-        wd = Watchdog(created_at=int(time.time()))
-        wd.submit(dashd)
-
-    else:
-        wd_list = sorted(active_wd, key=lambda wd: wd.object_hash)
-
-        # highest hash wins
-        winner = wd_list.pop()
-        printdbg("\tFound winning watchdog [%s], voting VALID" % winner.object_hash)
-        winner.vote(dashd, VoteSignals.valid, VoteOutcomes.yes)
-
-        # if remaining Watchdogs exist in the list, vote delete
-        for wd in wd_list:
-            printdbg("\tFound losing watchdog [%s], voting DELETE" % wd.object_hash)
-            wd.vote(dashd, VoteSignals.delete, VoteOutcomes.yes)
-
-    printdbg("leaving watchdog_check")
-
-
 def prune_expired_proposals(dashd):
     # vote delete for old proposals
     for proposal in Proposal.expired(dashd.superblockcycle()):
@@ -110,7 +74,8 @@ def attempt_superblock_creation(dashd):
     budget_max = dashd.get_superblock_budget_allocation(event_block_height)
     sb_epoch_time = dashd.block_height_to_epoch(event_block_height)
 
-    sb = dashlib.create_superblock(proposals, event_block_height, budget_max, sb_epoch_time)
+    maxgovobjdatasize = dashd.govinfo['maxgovobjdatasize']
+    sb = dashlib.create_superblock(proposals, event_block_height, budget_max, sb_epoch_time, maxgovobjdatasize)
     if not sb:
         printdbg("No superblock created, sorry. Returning.")
         return
@@ -207,9 +172,6 @@ def main():
 
     if dashd.has_sentinel_ping:
         sentinel_ping(dashd)
-    else:
-        # delete old watchdog objects, create a new if necessary
-        watchdog_check(dashd)
 
     # auto vote network objects as valid/invalid
     # check_object_validity(dashd)

lib/config.py

@@ -10,16 +10,19 @@
 )
 sentinel_config_file = os.environ.get('SENTINEL_CONFIG', default_sentinel_config)
 sentinel_cfg = DashConfig.tokenize(sentinel_config_file)
-sentinel_version = "1.1.0"
+sentinel_version = "1.2.0"
 min_dashd_proto_version_with_sentinel_ping = 70207
 
 
 def get_dash_conf():
-    home = os.environ.get('HOME')
+    if sys.platform == 'win32':
+        dash_conf = os.path.join(os.getenv('APPDATA'), "DashCore/dash.conf")
+    else:
+        home = os.environ.get('HOME')
 
-    dash_conf = os.path.join(home, ".dashcore/dash.conf")
-    if sys.platform == 'darwin':
-        dash_conf = os.path.join(home, "Library/Application Support/DashCore/dash.conf")
+        dash_conf = os.path.join(home, ".dashcore/dash.conf")
+        if sys.platform == 'darwin':
+            dash_conf = os.path.join(home, "Library/Application Support/DashCore/dash.conf")
 
     dash_conf = sentinel_cfg.get('dash_conf', dash_conf)
 
@@ -30,6 +33,10 @@ def get_network():
     return sentinel_cfg.get('network', 'mainnet')
 
 
+def get_rpchost():
+    return sentinel_cfg.get('rpchost', '127.0.0.1')
+
+
 def sqlite_test_db_name(sqlite_file_path):
     (root, ext) = os.path.splitext(sqlite_file_path)
     test_sqlite_file_path = root + '_test' + ext
@@ -81,4 +88,5 @@ def get_db_conn():
 
 dash_conf = get_dash_conf()
 network = get_network()
+rpc_host = get_rpchost()
 db = get_db_conn()

lib/dashd.py

@@ -36,26 +36,19 @@ def from_dash_conf(self, dash_dot_conf):
         config_text = DashConfig.slurp_config_file(dash_dot_conf)
         creds = DashConfig.get_rpc_creds(config_text, config.network)
 
+        creds[u'host'] = config.rpc_host
+
         return self(**creds)
 
     def rpc_command(self, *params):
         return self.rpc_connection.__getattr__(params[0])(*params[1:])
 
     # common RPC convenience methods
-    def is_testnet(self):
-        return self.rpc_command('getinfo')['testnet']
 
     def get_masternodes(self):
         mnlist = self.rpc_command('masternodelist', 'full')
         return [Masternode(k, v) for (k, v) in mnlist.items()]
 
-    def get_object_list(self):
-        try:
-            golist = self.rpc_command('gobject', 'list')
-        except JSONRPCException as e:
-            golist = self.rpc_command('mnbudget', 'show')
-        return golist
-
     def get_current_masternode_vin(self):
         from dashlib import parse_masternode_status_vin
 
@@ -89,12 +82,6 @@ def govinfo(self):
     def superblockcycle(self):
         return self.govinfo['superblockcycle']
 
-    def governanceminquorum(self):
-        return self.govinfo['governanceminquorum']
-
-    def proposalfee(self):
-        return self.govinfo['proposalfee']
-
     def last_superblock_height(self):
         height = self.rpc_command('getblockcount')
         cycle = self.superblockcycle()
@@ -191,15 +178,6 @@ def we_are_the_winner(self):
 
         return (winner == my_vin)
 
-    @property
-    def MASTERNODE_WATCHDOG_MAX_SECONDS(self):
-        # note: self.govinfo is already memoized
-        return self.govinfo['masternodewatchdogmaxseconds']
-
-    @property
-    def SENTINEL_WATCHDOG_MAX_SECONDS(self):
-        return (self.MASTERNODE_WATCHDOG_MAX_SECONDS // 2)
-
     def estimate_block_time(self, height):
         import dashlib
         """

lib/dashlib.py

@@ -92,7 +92,7 @@ def parse_masternode_status_vin(status_vin_string):
     return vin
 
 
-def create_superblock(proposals, event_block_height, budget_max, sb_epoch_time):
+def create_superblock(proposals, event_block_height, budget_max, sb_epoch_time, maxgovobjdatasize):
     from models import Superblock, GovernanceObject, Proposal
     from constants import SUPERBLOCK_FUDGE_WINDOW
 
@@ -102,9 +102,10 @@ def create_superblock(proposals, event_block_height, budget_max, sb_epoch_time):
         return None
 
     budget_allocated = Decimal(0)
-    fudge = SUPERBLOCK_FUDGE_WINDOW  # fudge-factor to allow for slighly incorrect estimates
+    fudge = SUPERBLOCK_FUDGE_WINDOW  # fudge-factor to allow for slightly incorrect estimates
 
     payments = []
+
     for proposal in proposals:
         fmt_string = "name: %s, rank: %4d, hash: %s, amount: %s <= %s"
 
@@ -151,12 +152,25 @@ def create_superblock(proposals, event_block_height, budget_max, sb_epoch_time):
             )
         )
 
-        # else add proposal and keep track of total budget allocation
-        budget_allocated += proposal.payment_amount
-
         payment = {'address': proposal.payment_address,
                    'amount': "{0:.8f}".format(proposal.payment_amount),
                    'proposal': "{}".format(proposal.object_hash)}
+
+        # calculate current sb data size
+        sb_temp = Superblock(
+            event_block_height=event_block_height,
+            payment_addresses='|'.join([pd['address'] for pd in payments]),
+            payment_amounts='|'.join([pd['amount'] for pd in payments]),
+            proposal_hashes='|'.join([pd['proposal'] for pd in payments])
+        )
+        data_size = len(sb_temp.dashd_serialise())
+
+        if data_size > maxgovobjdatasize:
+            printdbg("MAX_GOVERNANCE_OBJECT_DATA_SIZE limit reached!")
+            break
+
+        # else add proposal and keep track of total budget allocation
+        budget_allocated += proposal.payment_amount
         payments.append(payment)
 
     # don't create an empty superblock
@@ -179,57 +193,28 @@ def create_superblock(proposals, event_block_height, budget_max, sb_epoch_time):
     return sb
 
 
-# shims 'til we can fix the dashd side
+# shims 'til we can fix the JSON format
 def SHIM_serialise_for_dashd(sentinel_hex):
-    from models import DASHD_GOVOBJ_TYPES
+    from models import GOVOBJ_TYPE_STRINGS
+
     # unpack
     obj = deserialise(sentinel_hex)
 
     # shim for dashd
-    govtype = obj[0]
-
-    # add 'type' attribute
-    obj[1]['type'] = DASHD_GOVOBJ_TYPES[govtype]
+    govtype_string = GOVOBJ_TYPE_STRINGS[obj['type']]
 
     # superblock => "trigger" in dashd
-    if govtype == 'superblock':
-        obj[0] = 'trigger'
+    if govtype_string == 'superblock':
+        govtype_string = 'trigger'
 
-    # dashd expects an array (even though there is only a 1:1 relationship between govobj->class)
-    obj = [obj]
+    # dashd expects an array (will be deprecated)
+    obj = [(govtype_string, obj,)]
 
     # re-pack
     dashd_hex = serialise(obj)
     return dashd_hex
 
 
-# shims 'til we can fix the dashd side
-def SHIM_deserialise_from_dashd(dashd_hex):
-    from models import DASHD_GOVOBJ_TYPES
-
-    # unpack
-    obj = deserialise(dashd_hex)
-
-    # shim from dashd
-    # only one element in the array...
-    obj = obj[0]
-
-    # extract the govobj type
-    govtype = obj[0]
-
-    # superblock => "trigger" in dashd
-    if govtype == 'trigger':
-        obj[0] = govtype = 'superblock'
-
-    # remove redundant 'type' attribute
-    if 'type' in obj[1]:
-        del obj[1]['type']
-
-    # re-pack
-    sentinel_hex = serialise(obj)
-    return sentinel_hex
-
-
 # convenience
 def deserialise(hexdata):
     json = binascii.unhexlify(hexdata)

lib/gobject_json.py

@@ -0,0 +1,33 @@
+import simplejson
+
+
+def valid_json(input):
+    """ Return true/false depending on whether input is valid JSON """
+    is_valid = False
+    try:
+        simplejson.loads(input)
+        is_valid = True
+    except:
+        pass
+
+    return is_valid
+
+
+def extract_object(json_input):
+    """
+    Given either an old-style or new-style Proposal JSON string, extract the
+    actual object used (ignore old-style multi-dimensional array and unused
+    string for object type)
+    """
+    if not valid_json(json_input):
+        raise Exception("Invalid JSON input.")
+
+    obj = simplejson.loads(json_input, use_decimal=True)
+
+    if (isinstance(obj, list) and
+        isinstance(obj[0], list) and
+        (isinstance(obj[0][0], str) or (isinstance(obj[0][0], unicode))) and
+        isinstance(obj[0][1], dict)):
+        obj = obj[0][1]
+
+    return obj