Merge pull request #59 from dashpay/develop

Release Sentinel v1.3.0

.travis.yml

@@ -13,4 +13,4 @@ script:
   - py.test -svv test/unit/
 
   # style guide check
-  - find ./lib ./test ./bin -name \*.py -exec pycodestyle --show-source --ignore=E501,E402,E722,E129 {} +
+  - find ./lib ./test ./bin -name \*.py -exec pycodestyle --show-source --ignore=E501,E402,E722,E129,W503,W504 {} +

README.md

@@ -1,18 +1,26 @@
 # Dash Sentinel
 
-An all-powerful toolset for Dash.
-
 [![Build Status](https://travis-ci.org/dashpay/sentinel.svg?branch=master)](https://travis-ci.org/dashpay/sentinel)
 
-Sentinel is an autonomous agent for persisting, processing and automating Dash governance objects and tasks, and for expanded functions in the upcoming Dash V13 release (Evolution).
+> An automated governance helper for Dash Masternodes.
+
+Sentinel is an autonomous agent for persisting, processing and automating Dash governance objects and tasks. It is a Python application which runs alongside the DashCore instance on each Dash Masternode.
 
-Sentinel is implemented as a Python application that binds to a local version 12 dashd instance on each Dash Masternode.
+## Table of Contents
+- [Install](#install)
+  - [Dependencies](#dependencies)
+- [Usage](#usage)
+- [Configuration](#configuration)
+- [Troubleshooting](#troubleshooting)
+- [Maintainer](#maintainer)
+- [Contributing](#contributing)
+- [License](#license)
 
-This guide covers installing Sentinel onto an existing Masternode in Ubuntu 14.04 / 16.04.
+## Install
 
-## Installation
+These instructions cover installing Sentinel on Ubuntu 16.04 / 18.04.
 
-### 1. Install Prerequisites
+### Dependencies
 
 Make sure Python version 2.7.x or above is installed:
 
@@ -23,31 +31,35 @@ Update system packages and ensure virtualenv is installed:
     $ sudo apt-get update
     $ sudo apt-get -y install python-virtualenv
 
-Make sure the local Dash daemon running is at least version 12.1 (120100)
+Make sure the local DashCore daemon running is at least version 12.1 (120100)
 
     $ dash-cli getinfo | grep version
 
-### 2. Install Sentinel
+### Install Sentinel
 
 Clone the Sentinel repo and install Python dependencies.
 
     $ git clone https://github.com/dashpay/sentinel.git && cd sentinel
     $ virtualenv ./venv
     $ ./venv/bin/pip install -r requirements.txt
 
-### 3. Set up Cron
+## Usage
+
+Sentinel is "used" as a script called from cron every minute.
+
+### Set up Cron
 
 Set up a crontab entry to call Sentinel every minute:
 
     $ crontab -e
 
-In the crontab editor, add the lines below, replacing '/home/YOURUSERNAME/sentinel' to the path where you cloned sentinel to:
+In the crontab editor, add the lines below, replacing '/path/to/sentinel' to the path where you cloned sentinel to:
 
-    * * * * * cd /home/YOURUSERNAME/sentinel && ./venv/bin/python bin/sentinel.py >/dev/null 2>&1
+    * * * * * cd /path/to/sentinel && ./venv/bin/python bin/sentinel.py >/dev/null 2>&1
 
-### 4. Test the Configuration
+### Test Configuration
 
-Test the config by running all tests from the sentinel folder you cloned into
+Test the config by running tests:
 
     $ ./venv/bin/py.test ./test
 
@@ -65,6 +77,10 @@ To view debug output, set the `SENTINEL_DEBUG` environment variable to anything
 
     $ SENTINEL_DEBUG=1 ./venv/bin/python bin/sentinel.py
 
+## Maintainer
+
+[@nmarley](https://github.com/nmarley)
+
 ## Contributing
 
 Please follow the [DashCore guidelines for contributing](https://github.com/dashpay/dash/blob/master/CONTRIBUTING.md).
@@ -83,6 +99,6 @@ Specifically:
 
     Commit messages should be verbose by default, consisting of a short subject line (50 chars max), a blank line and detailed explanatory text as separate paragraph(s); unless the title alone is self-explanatory (like "Corrected typo in main.cpp") then a single title line is sufficient. Commit messages should be helpful to people reading your code in the future, so explain the reasoning for your decisions. Further explanation [here](http://chris.beams.io/posts/git-commit/).
 
-### License
+## License
 
 Released under the MIT license, under the same terms as DashCore itself. See [LICENSE](LICENSE) for more info.

bin/sentinel.py

@@ -74,8 +74,7 @@ def attempt_superblock_creation(dashd):
     budget_max = dashd.get_superblock_budget_allocation(event_block_height)
     sb_epoch_time = dashd.block_height_to_epoch(event_block_height)
 
-    maxgovobjdatasize = dashd.govinfo['maxgovobjdatasize']
-    sb = dashlib.create_superblock(proposals, event_block_height, budget_max, sb_epoch_time, maxgovobjdatasize)
+    sb = dashlib.create_superblock(proposals, event_block_height, budget_max, sb_epoch_time)
     if not sb:
         printdbg("No superblock created, sorry. Returning.")
         return

lib/config.py

@@ -10,7 +10,7 @@
 )
 sentinel_config_file = os.environ.get('SENTINEL_CONFIG', default_sentinel_config)
 sentinel_cfg = DashConfig.tokenize(sentinel_config_file)
-sentinel_version = "1.2.0"
+sentinel_version = "1.3.0"
 min_dashd_proto_version_with_sentinel_ping = 70207
 
 

lib/dash_config.py

@@ -15,7 +15,7 @@ def slurp_config_file(self, filename):
         f = io.open(filename)
         lines = []
         for line in f:
-            if re.match('^\s*#', line):
+            if re.match(r'^\s*#', line):
                 continue
             lines.append(line)
         f.close()

lib/dashlib.py

@@ -73,13 +73,13 @@ def elect_mn(**kwargs):
 
 
 def parse_masternode_status_vin(status_vin_string):
-    status_vin_string_regex = re.compile('CTxIn\(COutPoint\(([0-9a-zA-Z]+),\\s*(\d+)\),')
+    status_vin_string_regex = re.compile(r'CTxIn\(COutPoint\(([0-9a-zA-Z]+),\s*(\d+)\),')
 
     m = status_vin_string_regex.match(status_vin_string)
 
     # To Support additional format of string return from masternode status rpc.
     if m is None:
-        status_output_string_regex = re.compile('([0-9a-zA-Z]+)\-(\d+)')
+        status_output_string_regex = re.compile(r'([0-9a-zA-Z]+)-(\d+)')
         m = status_output_string_regex.match(status_vin_string)
 
     txid = m.group(1)
@@ -92,9 +92,10 @@ def parse_masternode_status_vin(status_vin_string):
     return vin
 
 
-def create_superblock(proposals, event_block_height, budget_max, sb_epoch_time, maxgovobjdatasize):
+def create_superblock(proposals, event_block_height, budget_max, sb_epoch_time):
     from models import Superblock, GovernanceObject, Proposal
     from constants import SUPERBLOCK_FUDGE_WINDOW
+    import copy
 
     # don't create an empty superblock
     if (len(proposals) == 0):
@@ -104,7 +105,7 @@ def create_superblock(proposals, event_block_height, budget_max, sb_epoch_time,
     budget_allocated = Decimal(0)
     fudge = SUPERBLOCK_FUDGE_WINDOW  # fudge-factor to allow for slightly incorrect estimates
 
-    payments = []
+    payments_list = []
 
     for proposal in proposals:
         fmt_string = "name: %s, rank: %4d, hash: %s, amount: %s <= %s"
@@ -152,69 +153,48 @@ def create_superblock(proposals, event_block_height, budget_max, sb_epoch_time,
             )
         )
 
-        payment = {'address': proposal.payment_address,
-                   'amount': "{0:.8f}".format(proposal.payment_amount),
-                   'proposal': "{}".format(proposal.object_hash)}
+        payment = {
+            'address': proposal.payment_address,
+            'amount': "{0:.8f}".format(proposal.payment_amount),
+            'proposal': "{}".format(proposal.object_hash)
+        }
+
+        temp_payments_list = copy.deepcopy(payments_list)
+        temp_payments_list.append(payment)
 
-        # calculate current sb data size
+        # calculate size of proposed Superblock
         sb_temp = Superblock(
             event_block_height=event_block_height,
-            payment_addresses='|'.join([pd['address'] for pd in payments]),
-            payment_amounts='|'.join([pd['amount'] for pd in payments]),
-            proposal_hashes='|'.join([pd['proposal'] for pd in payments])
+            payment_addresses='|'.join([pd['address'] for pd in temp_payments_list]),
+            payment_amounts='|'.join([pd['amount'] for pd in temp_payments_list]),
+            proposal_hashes='|'.join([pd['proposal'] for pd in temp_payments_list])
         )
-        data_size = len(sb_temp.dashd_serialise())
-
-        if data_size > maxgovobjdatasize:
-            printdbg("MAX_GOVERNANCE_OBJECT_DATA_SIZE limit reached!")
-            break
+        proposed_sb_size = len(sb_temp.serialise())
 
-        # else add proposal and keep track of total budget allocation
+        # add proposal and keep track of total budget allocation
         budget_allocated += proposal.payment_amount
-        payments.append(payment)
+        payments_list.append(payment)
 
     # don't create an empty superblock
-    if not payments:
+    if not payments_list:
         printdbg("No proposals made the cut!")
         return None
 
     # 'payments' now contains all the proposals for inclusion in the
     # Superblock, but needs to be sorted by proposal hash descending
-    payments.sort(key=lambda k: k['proposal'], reverse=True)
+    payments_list.sort(key=lambda k: k['proposal'], reverse=True)
 
     sb = Superblock(
         event_block_height=event_block_height,
-        payment_addresses='|'.join([pd['address'] for pd in payments]),
-        payment_amounts='|'.join([pd['amount'] for pd in payments]),
-        proposal_hashes='|'.join([pd['proposal'] for pd in payments]),
+        payment_addresses='|'.join([pd['address'] for pd in payments_list]),
+        payment_amounts='|'.join([pd['amount'] for pd in payments_list]),
+        proposal_hashes='|'.join([pd['proposal'] for pd in payments_list]),
     )
     printdbg("generated superblock: %s" % sb.__dict__)
 
     return sb
 
 
-# shims 'til we can fix the JSON format
-def SHIM_serialise_for_dashd(sentinel_hex):
-    from models import GOVOBJ_TYPE_STRINGS
-
-    # unpack
-    obj = deserialise(sentinel_hex)
-
-    # shim for dashd
-    govtype_string = GOVOBJ_TYPE_STRINGS[obj['type']]
-
-    # superblock => "trigger" in dashd
-    if govtype_string == 'superblock':
-        govtype_string = 'trigger'
-
-    # dashd expects an array (will be deprecated)
-    obj = [(govtype_string, obj,)]
-
-    # re-pack
-    dashd_hex = serialise(obj)
-    return dashd_hex
-
-
 # convenience
 def deserialise(hexdata):
     json = binascii.unhexlify(hexdata)

lib/governance_class.py

@@ -35,8 +35,7 @@ def vote_validity(self, dashd):
             self.vote(dashd, models.VoteSignals.valid, models.VoteOutcomes.no)
 
     def get_submit_command(self):
-        import dashlib
-        obj_data = dashlib.SHIM_serialise_for_dashd(self.serialise())
+        obj_data = self.serialise()
 
         # new objects won't have parent_hash, revision, etc...
         cmd = ['gobject', 'submit', '0', '1', str(int(time.time())), obj_data]
@@ -66,10 +65,6 @@ def serialise(self):
 
         return binascii.hexlify(simplejson.dumps(self.get_dict(), sort_keys=True).encode('utf-8')).decode('utf-8')
 
-    def dashd_serialise(self):
-        import dashlib
-        return dashlib.SHIM_serialise_for_dashd(self.serialise())
-
     @classmethod
     def serialisable_fields(self):
         # Python is so not very elegant...

requirements.txt

@@ -1,6 +1,6 @@
 peewee==2.8.3
 py==1.4.31
-pycodestyle==2.3.1
+pycodestyle==2.4.0
 pytest==3.0.1
 python-bitcoinrpc==1.0
 simplejson==3.8.2

test/unit/models/test_superblocks.py

@@ -235,9 +235,7 @@ def test_deterministic_superblock_creation(go_list_proposals):
     max_budget = 60
     prop_list = Proposal.approved_and_ranked(proposal_quorum=1, next_superblock_max_budget=max_budget)
 
-    # MAX_GOVERNANCE_OBJECT_DATA_SIZE defined in governance-object.h
-    maxgovobjdatasize = 16 * 1024
-    sb = dashlib.create_superblock(prop_list, 72000, max_budget, misc.now(), maxgovobjdatasize)
+    sb = dashlib.create_superblock(prop_list, 72000, max_budget, misc.now())
 
     assert sb.event_block_height == 72000
     assert sb.payment_addresses == 'yYe8KwyaUu5YswSYmB3q3ryx8XTUu9y7Ui|yTC62huR4YQEPn9AJHjnQxxreHSbgAoatV'
@@ -247,32 +245,6 @@ def test_deterministic_superblock_creation(go_list_proposals):
     assert sb.hex_hash() == 'bb3f33ccf95415c396bd09d35325dbcbc7b067010d51c7ccf772a9e839c1e414'
 
 
-def test_superblock_size_limit(go_list_proposals):
-    import dashlib
-    import misc
-    from dashd import DashDaemon
-    dashd = DashDaemon.from_dash_conf(config.dash_conf)
-    for item in go_list_proposals:
-        (go, subobj) = GovernanceObject.import_gobject_from_dashd(dashd, item)
-
-    max_budget = 60
-    prop_list = Proposal.approved_and_ranked(proposal_quorum=1, next_superblock_max_budget=max_budget)
-
-    maxgovobjdatasize = 469
-    sb = dashlib.create_superblock(prop_list, 72000, max_budget, misc.now(), maxgovobjdatasize)
-
-    # two proposals in the list, but...
-    assert len(prop_list) == 2
-
-    # only one should have been included in the SB, because the 2nd one is over the limit
-    assert sb.event_block_height == 72000
-    assert sb.payment_addresses == 'yYe8KwyaUu5YswSYmB3q3ryx8XTUu9y7Ui'
-    assert sb.payment_amounts == '25.75000000'
-    assert sb.proposal_hashes == 'dfd7d63979c0b62456b63d5fc5306dbec451180adee85876cbf5b28c69d1a86c'
-
-    assert sb.hex_hash() == '6b8cababf797644f1d62003e4cc68c1c40a8c1873c8a68ed0fc88772ea77cc44'
-
-
 def test_deterministic_superblock_selection(go_list_superblocks):
     from dashd import DashDaemon
     dashd = DashDaemon.from_dash_conf(config.dash_conf)

test/unit/test_dash_config.py

@@ -62,7 +62,23 @@ def test_get_rpc_creds():
     assert creds.get('port') == 19998
 
 
-# ensure dash network (mainnet, testnet) matches that specified in config
-# requires running dashd on whatever port specified...
-#
-# This is more of a dashd/jsonrpc test than a config test...
+def test_slurp_config_file():
+    import tempfile
+
+    dash_config = """# basic settings
+#testnet=1 # TESTNET
+server=1
+printtoconsole=1
+txindex=1 # enable transaction index
+"""
+
+    expected_stripped_config = """server=1
+printtoconsole=1
+txindex=1 # enable transaction index
+"""
+
+    with tempfile.NamedTemporaryFile(mode='w') as temp:
+        temp.write(dash_config)
+        temp.flush()
+        conf = DashConfig.slurp_config_file(temp.name)
+        assert conf == expected_stripped_config

test/unit/test_dashd_data_shims.py → test/unit/test_gobject_json.py

@@ -31,36 +31,6 @@ def trigger_hex_new():
     return "7b226576656e745f626c6f636b5f686569676874223a2036323530302c20227061796d656e745f616464726573736573223a2022795965384b77796155753559737753596d42337133727978385854557539793755697c795443363268755234595145506e39414a486a6e517878726548536267416f617456222c20227061796d656e745f616d6f756e7473223a2022357c33222c202274797065223a20327d"
 
 
-# TODO: remove fixtures below here once test_SHIM_serialise_for_dashd removed
-
-
-@pytest.fixture
-def sentinel_proposal_hex():
-    return '7b22656e645f65706f6368223a20313439313032323830302c20226e616d65223a2022626565722d7265696d62757273656d656e742d37222c20227061796d656e745f61646472657373223a2022795965384b77796155753559737753596d4233713372797838585455753979375569222c20227061796d656e745f616d6f756e74223a20372e30303030303030302c202273746172745f65706f6368223a20313438333235303430302c202274797065223a20312c202275726c223a202268747470733a2f2f6461736863656e7472616c2e636f6d2f626565722d7265696d62757273656d656e742d37227d'
-
-
-@pytest.fixture
-def sentinel_superblock_hex():
-    return '7b226576656e745f626c6f636b5f686569676874223a2036323530302c20227061796d656e745f616464726573736573223a2022795965384b77796155753559737753596d42337133727978385854557539793755697c795443363268755234595145506e39414a486a6e517878726548536267416f617456222c20227061796d656e745f616d6f756e7473223a2022357c33222c202274797065223a20327d'
-
-
-@pytest.fixture
-def dashd_proposal_hex():
-    return '5b5b2270726f706f73616c222c207b22656e645f65706f6368223a20313439313336383430302c20226e616d65223a2022626565722d7265696d62757273656d656e742d39222c20227061796d656e745f61646472657373223a2022795965384b77796155753559737753596d4233713372797838585455753979375569222c20227061796d656e745f616d6f756e74223a2034392e30303030303030302c202273746172745f65706f6368223a20313438333235303430302c202274797065223a20312c202275726c223a202268747470733a2f2f7777772e6461736863656e7472616c2e6f72672f702f626565722d7265696d62757273656d656e742d39227d5d5d'
-
-
-@pytest.fixture
-def dashd_superblock_hex():
-    return '5b5b2274726967676572222c207b226576656e745f626c6f636b5f686569676874223a2036323530302c20227061796d656e745f616464726573736573223a2022795965384b77796155753559737753596d42337133727978385854557539793755697c795443363268755234595145506e39414a486a6e517878726548536267416f617456222c20227061796d656e745f616d6f756e7473223a2022357c33222c202274797065223a20327d5d5d'
-
-# ========================================================================
-
-
-def test_SHIM_serialise_for_dashd(sentinel_proposal_hex, sentinel_superblock_hex):
-    assert dashlib.SHIM_serialise_for_dashd(sentinel_proposal_hex) == '5b5b2270726f706f73616c222c207b22656e645f65706f6368223a20313439313032323830302c20226e616d65223a2022626565722d7265696d62757273656d656e742d37222c20227061796d656e745f61646472657373223a2022795965384b77796155753559737753596d4233713372797838585455753979375569222c20227061796d656e745f616d6f756e74223a20372e30303030303030302c202273746172745f65706f6368223a20313438333235303430302c202274797065223a20312c202275726c223a202268747470733a2f2f6461736863656e7472616c2e636f6d2f626565722d7265696d62757273656d656e742d37227d5d5d'
-    assert dashlib.SHIM_serialise_for_dashd(sentinel_superblock_hex) == '5b5b2274726967676572222c207b226576656e745f626c6f636b5f686569676874223a2036323530302c20227061796d656e745f616464726573736573223a2022795965384b77796155753559737753596d42337133727978385854557539793755697c795443363268755234595145506e39414a486a6e517878726548536267416f617456222c20227061796d656e745f616d6f756e7473223a2022357c33222c202274797065223a20327d5d5d'
-
-
 def test_valid_json():
     import binascii