-
Notifications
You must be signed in to change notification settings - Fork 44
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Feat/testing language #19
Changes from all commits
0c47705
5fe420a
3ffd581
9b3db15
f158021
70c5f20
6d4afa7
e54e5d5
3d9f964
29992e0
f7ed7a9
bec1274
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -43,3 +43,15 @@ def init(): | |
REPORT_SUBJECTIVE_VIEWS = False | ||
|
||
init() | ||
|
||
|
||
def update(val_weights): | ||
global NUM_VALIDATORS | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. shouldn't have to redeclare these global vars There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Think we need to do this to be able to modify existing global variables. See here: https://stackoverflow.com/questions/423379/using-global-variables-in-a-function-other-than-the-one-that-created-them |
||
global VALIDATOR_NAMES | ||
global WEIGHTS | ||
global TOTAL_WEIGHT | ||
|
||
NUM_VALIDATORS = len(val_weights) | ||
VALIDATOR_NAMES = set(range(NUM_VALIDATORS)) | ||
WEIGHTS = {i: val_weights[i] for i in VALIDATOR_NAMES} | ||
TOTAL_WEIGHT = sum(val_weights) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,81 @@ | ||
from testing_language import TestLangCBC | ||
from block import Block | ||
from justification import Justification | ||
import unittest | ||
import settings as s | ||
import copy | ||
|
||
|
||
class TestUtils(unittest.TestCase): | ||
|
||
def test_equality_of_copies_off_genesis(self): | ||
s.update([10]) # necessary due to assertions during block creation | ||
block = Block(None, Justification(), 0) | ||
|
||
shallow_copy = copy.copy(block) | ||
deep_copy = copy.deepcopy(block) | ||
|
||
self.assertEqual(block, shallow_copy) | ||
self.assertEqual(block, deep_copy) | ||
self.assertEqual(shallow_copy, deep_copy) | ||
|
||
def test_equality_of_copies_of_non_genesis(self): | ||
test_string = "B0-A S1-A B1-B S0-B B0-C S1-C B1-D S0-D H0-D" | ||
testLang = TestLangCBC(test_string, [10, 11]) | ||
testLang.parse() | ||
|
||
for b in testLang.blocks: | ||
shallow_copy = copy.copy(b) | ||
deep_copy = copy.deepcopy(b) | ||
|
||
self.assertEqual(b, shallow_copy) | ||
self.assertEqual(b, deep_copy) | ||
self.assertEqual(shallow_copy, deep_copy) | ||
|
||
def test_non_equality_of_copies_off_genesis(self): | ||
s.update([10, 11]) | ||
block_0 = Block(None, Justification(), 0) | ||
block_1 = Block(None, Justification(), 1) | ||
|
||
self.assertNotEqual(block_0, block_1) | ||
|
||
def test_unique_block_creation_in_test_lang(self): | ||
test_string = "B0-A S1-A B1-B S0-B B0-C S1-C B1-D S0-D H0-D" | ||
testLang = TestLangCBC(test_string, [10, 11]) | ||
testLang.parse() | ||
|
||
num_equal = 0 | ||
for b in testLang.blocks: | ||
for b1 in testLang.blocks: | ||
if b1 == b: | ||
num_equal += 1 | ||
continue | ||
|
||
self.assertNotEqual(b, b1) | ||
|
||
self.assertEqual(num_equal, len(testLang.blocks)) | ||
|
||
def test_is_in_blockchain__separate_genesis(self): | ||
s.update([10, 11]) | ||
block_0 = Block(None, Justification(), 0) | ||
block_1 = Block(None, Justification(), 1) | ||
|
||
self.assertFalse(block_0.is_in_blockchain(block_1)) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Might call this There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Run the assertion the other way too, |
||
self.assertFalse(block_1.is_in_blockchain(block_0)) | ||
|
||
def test_is_in_blockchain__test_lang(self): | ||
test_string = "B0-A S1-A B1-B S0-B B0-C S1-C B1-D S0-D H0-D" | ||
testLang = TestLangCBC(test_string, [11, 10]) | ||
testLang.parse() | ||
|
||
prev = testLang.blocks['A'] | ||
for b in ['B', 'C', 'D']: | ||
block = testLang.blocks[b] | ||
self.assertTrue(prev.is_in_blockchain(block)) | ||
self.assertFalse(block.is_in_blockchain(prev)) | ||
|
||
prev = block | ||
|
||
|
||
if __name__ == "__main__": | ||
unittest.main() |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,127 @@ | ||
from testing_language import TestLangCBC | ||
import unittest | ||
import forkchoice | ||
import random as r | ||
|
||
|
||
class TestForkchoice(unittest.TestCase): | ||
|
||
def test_single_validator_correct_forkchoice(self): | ||
""" This tests that a single validator remains on their own chain """ | ||
test_string = "" | ||
for i in xrange(100): | ||
test_string += "B0-" + str(i) + " " + "H0-" + str(i) + " " | ||
test_string = test_string[:-1] | ||
|
||
testLang = TestLangCBC(test_string, [10]) | ||
testLang.parse() | ||
|
||
def test_two_validators_round_robin_forkchoice(self): | ||
test_string = "B0-A S1-A B1-B S0-B B0-C S1-C B1-D S0-D H0-D R" | ||
testLang = TestLangCBC(test_string, [10, 11]) | ||
testLang.parse() | ||
|
||
def test_many_val_round_robin_forkchoice(self): | ||
""" | ||
Tests that during a perfect round robin, | ||
validators choose the one chain as their fork choice | ||
""" | ||
test_string = "" | ||
for i in xrange(100): | ||
test_string += "B" + str(i % 10) + "-" + str(i) + " " \ | ||
+ "S" + str((i+1) % 10) + "-" + str(i) + " " \ | ||
+ "H" + str((i+1) % 10) + "-" + str(i) + " " | ||
test_string = test_string[:-1] | ||
|
||
testLang = TestLangCBC( | ||
test_string, | ||
[x + r.random() for x in xrange(10, 0, -1)] | ||
) | ||
testLang.parse() | ||
|
||
def test_fail_on_tie(self): | ||
""" | ||
Tests that if there are two subsets of the validator | ||
set with the same weight, the forkchoice fails | ||
""" | ||
test_string = "B1-A S0-A B0-B S1-B S2-A B2-C S1-C H1-C" | ||
testLang = TestLangCBC(test_string, [5, 6, 5]) | ||
with self.assertRaises(AssertionError): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This construct hides what assertion actually fails in the testLang. When I read this test, I'm not actually sure which component of the test_string us supposed to fail, and you as the tester can't be sure where it fails either. Might be worth having particular Open to other ways of solving this. Let me know your thoughts. Goal would be to have more transparency as to what exactly you are testing. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yeah, this is a good point - some reporting here would def be good here, as we don't even know what is causing this test to fail. Once we move to pytest, we could also likely use the assert statements and 'sys' library to parse the error message that comes with it (see here) - but this is pretty messy. Adding user-defined errors seems like the best way of accomplishing this (and they would only need to exist in the testing world). There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @djrtwo Ah, so I misinterpreted a bit what errors we would need here. In this case, this test throws an asserting error b/c of some safety check in the forkchoice itself. I was thinking that we would add errors to the testing language (which I think is a great idea and what you were suggesting), but do you think we should also add these to the forkchoice as well? Not sure what the best practice is here :~) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Throwing particular errors (ones native to python or our own) allows for us or any developer using this stuff to more easily handle the errors as they please. Assertions should be used for sanity checks and debugging. This particular example lies somewhere in between. Because we are testing for this and because there are a ton of other assertions currently throughout the codebase, I think we should raise a more specific error. |
||
testLang.parse() | ||
|
||
def test_ignore_zero_weight_validator(self): | ||
""" | ||
Tests that a validator with zero weight | ||
will not affect the forkchoice | ||
""" | ||
test_string = "B0-A S1-A B1-B S0-B H1-A H0-A" | ||
testLang = TestLangCBC(test_string, [1, 0]) | ||
testLang.parse() | ||
|
||
def test_ignore_zero_weight_block(self): | ||
""" Tests that the forkchoice ignores zero weight blocks """ | ||
# for more info about test, see | ||
# https://gist.github.com/naterush/8d8f6ec3509f50939d7911d608f912f4 | ||
test_string = ( | ||
"B0-A1 B0-A2 H0-A2 B1-B1 B1-B2 S3-B2 B3-D1 H3-D1 " | ||
"S3-A2 H3-A2 B3-D2 S2-B1 H2-B1 B2-C1 H2-C1 S1-D1 " | ||
"S1-D2 S1-C1 H1-B2" | ||
) | ||
testLang = TestLangCBC(test_string, [10, 9, 8, .5]) | ||
testLang.parse() | ||
|
||
def test_reverse_message_arrival_order_forkchoice_four_val(self): | ||
test_string = ( | ||
"B0-A S1-A B1-B S0-B B0-C S1-C B1-D S0-D B1-E S0-E " | ||
"S2-E H2-E S3-A S3-B S3-C S3-D S3-E H3-E" | ||
) | ||
testLang = TestLangCBC(test_string, [5, 6, 7, 8.1]) | ||
testLang.parse() | ||
|
||
def test_different_message_arrival_order_forkchoice_many_val(self): | ||
# TODO | ||
pass | ||
|
||
def test_max_weight_indexes(self): | ||
weight = {i: i for i in xrange(10)} | ||
max_weight_indexes = forkchoice.get_max_weight_indexes(weight) | ||
self.assertEqual(len(max_weight_indexes), 1) | ||
self.assertEqual(max_weight_indexes.pop(), 9) | ||
|
||
weight = {i: 9 - i for i in xrange(10)} | ||
max_weight_indexes = forkchoice.get_max_weight_indexes(weight) | ||
self.assertEqual(len(max_weight_indexes), 1) | ||
self.assertEqual(max_weight_indexes.pop(), 0) | ||
|
||
weight = dict() | ||
for i in xrange(5): | ||
weight[i] = i | ||
weight[9 - i] = i | ||
|
||
max_weight_indexes = forkchoice.get_max_weight_indexes(weight) | ||
self.assertEqual(len(max_weight_indexes), 2) | ||
self.assertEqual(set([4, 5]), max_weight_indexes) | ||
|
||
def test_max_weight_indexes_empty(self): | ||
weight = dict() | ||
with self.assertRaises(ValueError): | ||
forkchoice.get_max_weight_indexes(weight) | ||
|
||
def test_max_weight_indexes_zero_score(self): | ||
weight = {i: 0 for i in xrange(10)} | ||
with self.assertRaises(AssertionError): | ||
forkchoice.get_max_weight_indexes(weight) | ||
|
||
def test_max_weight_indexes_tie(self): | ||
weight = dict() | ||
for i in xrange(10): | ||
weight[i] = 10 | ||
|
||
max_weight_indexes = forkchoice.get_max_weight_indexes(weight) | ||
|
||
self.assertEqual(len(max_weight_indexes), 10) | ||
self.assertEqual(set(weight.keys()), max_weight_indexes) | ||
|
||
|
||
if __name__ == "__main__": | ||
unittest.main() |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
from testing_language import TestLangCBC | ||
import unittest | ||
|
||
|
||
class TestUtils(unittest.TestCase): | ||
|
||
def test_round_robin_safety(self): | ||
test_string = ( | ||
'R B0-A S1-A RR1-B RR1-C RR1-D RR1-E S2-E ' | ||
'S3-E S4-E H0-E H1-E H2-E H3-E H4-E C0-A ' | ||
'C1-A C2-A C3-A C4-A R' | ||
) | ||
test = TestLangCBC(test_string, [9.3, 8.2, 7.1, 6, 5]) | ||
test.parse() | ||
|
||
def test_majority_fork_safe(self): | ||
test_string = ( | ||
# create right hand side of fork and check for safety | ||
'R B1-A S0-A B0-L0 S1-L0 B1-L1 S0-L1 B0-L2 S1-L2 ' | ||
'B1-L3 S0-L3 B0-L4 S1-L4 H1-L4 C1-L0 H0-L4 C0-L0 R ' | ||
# other fork shows safe fork blocks, but they remain stuck | ||
'S2-A B2-R0 S0-R0 H0-L4 S1-R0 H0-L4 R' | ||
) | ||
|
||
test = TestLangCBC(test_string, [5, 6, 7]) | ||
test.parse() | ||
|
||
def test_no_majority_fork_unsafe(self): | ||
test_string = ( | ||
# create right hand side of fork and check for no safety | ||
'R B2-A S1-A B1-L0 S0-L0 B0-L1 S1-L1 B1-L2 S0-L2 ' | ||
'B0-L3 S1-L3 B1-L4 S0-L4 H0-L4 U0-L0 H1-L4 U1-L0 R ' | ||
# now, left hand side as well. still no safety | ||
'S3-A B3-R0 S4-R0 B4-R1 S3-R1 B3-R2 S4-R2 B4-R3 ' | ||
'S3-R3 B3-R4 S4-R4 H4-R4 U4-R0 H3-R4 U3-R0 R' | ||
) | ||
test = TestLangCBC(test_string, [5, 4.5, 6, 4, 5.25]) | ||
test.parse() | ||
|
||
def test_no_majority_fork_safe_after_union(self): | ||
test_string = ( | ||
# generate both sides of an extended fork | ||
'R B2-A S1-A B1-L0 S0-L0 B0-L1 S1-L1 B1-L2 S0-L2 ' | ||
'B0-L3 S1-L3 B1-L4 S0-L4 H0-L4 U0-L0 H1-L4 U1-L0 R ' | ||
'S3-A B3-R0 S4-R0 B4-R1 S3-R1 B3-R2 S4-R2 B4-R3 ' | ||
'S3-R3 B3-R4 S4-R4 H4-R4 U4-R0 H3-R4 U3-R0 R ' | ||
# show all validators all blocks | ||
'S0-R4 S1-R4 S2-R4 S2-L4 S3-L4 S4-L4 ' | ||
# check all have correct forkchoice | ||
'H0-L4 H1-L4 H2-L4 H3-L4 H4-L4 ' | ||
# two rounds of round robin, check have safety on the correct fork | ||
'RR0-J0 RR0-J1 C0-L0 R' | ||
) | ||
test = TestLangCBC(test_string, [5, 4.5, 6, 4, 5.25]) | ||
test.parse() | ||
|
||
|
||
if __name__ == "__main__": | ||
unittest.main() |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
import unittest | ||
import settings as s | ||
import random as r | ||
import utils | ||
|
||
|
||
class TestUtils(unittest.TestCase): | ||
|
||
def test_get_weight_increasing(self): | ||
weights = [i for i in xrange(10)] | ||
s.update(weights) | ||
self.assertEqual(utils.get_weight(s.VALIDATOR_NAMES), 45) | ||
|
||
def test_get_weight_decreasing(self): | ||
weights = [i for i in xrange(9, -1, -1)] | ||
s.update(weights) | ||
|
||
self.assertEqual(utils.get_weight(s.VALIDATOR_NAMES), 45) | ||
|
||
def test_get_weight_random(self): | ||
weights = [r.random() for i in xrange(10)] | ||
s.update(weights) | ||
|
||
self.assertEqual(utils.get_weight(s.VALIDATOR_NAMES), sum(weights)) | ||
|
||
def test_get_weight_partial_set(self): | ||
weights = [i*2 for i in xrange(10)] | ||
s.update(weights) | ||
|
||
subset = set([0, 1, 2, 3]) | ||
self.assertEqual(utils.get_weight(subset), 12) | ||
|
||
def test_get_weight_partial_list(self): | ||
weights = [i*2 for i in xrange(10)] | ||
s.update(weights) | ||
|
||
self.assertEqual(utils.get_weight([0, 1, 2, 3]), 12) | ||
|
||
def test_get_weight_none(self): | ||
weight = utils.get_weight(None) | ||
self.assertEqual(weight, 0) | ||
|
||
def test_get_weight_empty(self): | ||
weight = utils.get_weight(set()) | ||
self.assertEqual(weight, 0) | ||
|
||
|
||
if __name__ == "__main__": | ||
unittest.main() |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this hangs indefinitely w/ no output, and makes a graph with no values.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If you repeatedly kill the graphs as they come up, it moves through the tests. Going to disable plotting for testing by default before merge. Will add option to run plotting as a param when migrate to py.test
@naterush Disable plotting before merge