Merge pull request #20 from thombashi/develop

Develop

requirements/requirements.txt

@@ -1,6 +1,7 @@
-ipaddress
 DataProperty>=0.2.1
+ipaddress
 netifaces
 pyparsing
 six
-thutils>=0.1.19
+thutils>=0.1.21
+voluptuous

requirements/test_requirements.txt

@@ -1,2 +1,2 @@
 pytest
-pingparsing>=0.2.4
+pingparsing>=0.2.6

tcconfig/__init__.py

@@ -4,7 +4,7 @@
 @author: Tsuyoshi Hombashi
 '''
 
-VERSION = "0.6.0"
+VERSION = "0.6.1"
 
 
 def verify_network_interface(device):

tcconfig/parser.py

@@ -8,6 +8,14 @@
 
 import dataproperty
 import pyparsing as pp
+import six
+
+
+def _to_unicode(text):
+    try:
+        return text.decode("ascii")
+    except AttributeError:
+        return text
 
 
 class TcFilterParser(object):
@@ -20,7 +28,7 @@ class FilterMatchId:
     __FILTER_PATTERN = (
         pp.Literal("filter parent") +
         pp.SkipTo("flowid", include=True) +
-        pp.Word(pp.alphanums + ":")
+        pp.Word(pp.hexnums + ":")
     )
     __FILTER_MATCH_PATTERN = (
         pp.Literal("match") +
@@ -83,7 +91,7 @@ def parse_incoming_device(self, text):
             return None
 
         match = re.search(
-            "Egress Redirect to device ifb[\d]+", text, re.MULTILINE)
+            "Egress Redirect to device ifb[\d]+", _to_unicode(text), re.MULTILINE)
         if match is None:
             return None
 
@@ -102,11 +110,13 @@ def __get_filter(self):
         }
 
     def __parse_flow_id(self, line):
-        parsed_list = self.__FILTER_PATTERN.parseString(line.lstrip())
+        parsed_list = self.__FILTER_PATTERN.parseString(
+            _to_unicode(line.lstrip()))
         self.__flow_id = parsed_list[-1]
 
     def __parse_filter(self, line):
-        parsed_list = self.__FILTER_MATCH_PATTERN.parseString(line)
+        parsed_list = self.__FILTER_MATCH_PATTERN.parseString(
+            _to_unicode(line))
         value_hex, mask_hex = parsed_list[1].split("/")
         match_id = int(parsed_list[3])
 
@@ -132,15 +142,16 @@ def __init__(self):
 
     def parse(self, text):
         for line in text.splitlines():
-            line = line.lstrip()
             if dataproperty.is_empty_string(line):
                 continue
 
+            line = _to_unicode(line.lstrip())
+
             if re.search("qdisc netem|qdisc tbf", line) is None:
                 continue
 
             if re.search("qdisc netem", line) is not None:
-                self.__parse_netem_param(line, "parent", pp.alphanums + ":")
+                self.__parse_netem_param(line, "parent", pp.hexnums + ":")
 
             self.__parse_netem_param(line, "delay", pp.nums + ".")
             self.__parse_netem_delay_distro(line)
@@ -170,7 +181,7 @@ def __parse_netem_param(self, line, parse_param_name, word_pattern):
             pp.Word(word_pattern))
 
         try:
-            result = pattern.parseString(line)[-1]
+            result = pattern.parseString(_to_unicode(line))[-1]
             if dataproperty.is_not_empty_string(result):
                 self.__parsed_param[parse_param_name] = result
         except pp.ParseException:

tcconfig/tcset.py

@@ -163,7 +163,6 @@ def main():
 
         for tcconfig_command in get_tcconfig_command_list(
                 load_tcconfig(options.config_file), options.overwrite):
-            # print tcconfig_command
             return_code |= subproc_wrapper.run(tcconfig_command)
 
         return return_code

tcconfig/traffic_control.py

@@ -67,7 +67,7 @@ def validate(self):
     def __get_device_qdisc_major_id(self):
         import hashlib
 
-        base_device_hash = hashlib.md5(self.__device).hexdigest()[:3]
+        base_device_hash = hashlib.md5(six.b(self.__device)).hexdigest()[:3]
         device_hash_prefix = "1"
 
         return int(device_hash_prefix + base_device_hash, 16)

test/test_parser.py

@@ -7,6 +7,8 @@
 from __future__ import absolute_import
 
 import pytest
+import six
+
 import tcconfig.parser
 
 
@@ -26,7 +28,7 @@ class Test_TcFilterParser_parse_filter:
         [None, []],
         ["", []],
         [
-            """
+            six.b("""
             filter parent 1: protocol ip pref 1 u32
             filter parent 1: protocol ip pref 1 u32 fh 801: ht divisor 1
             filter parent 1: protocol ip pref 1 u32 fh 801::800 order 2048 key ht 801 bkt 0 flowid 1:1
@@ -35,14 +37,14 @@ class Test_TcFilterParser_parse_filter:
             filter parent 1: protocol ip pref 2 u32 fh 800: ht divisor 1
             filter parent 1: protocol ip pref 2 u32 fh 800::800 order 2048 key ht 800 bkt 0 flowid 1:2
               match 00000000/00000000 at 16
-            """,
+            """),
             [
                 {'flowid': '1:1', 'network': '192.168.0.10/32', 'port': None},
                 {'flowid': '1:2', 'network': '0.0.0.0/0', 'port': None},
             ],
         ],
         [
-            """
+            six.b("""
             filter parent 1: protocol ip pref 1 u32
             filter parent 1: protocol ip pref 1 u32 fh 801: ht divisor 1
             filter parent 1: protocol ip pref 1 u32 fh 801::800 order 2048 key ht 801 bkt 0 flowid 1:1
@@ -52,14 +54,14 @@ class Test_TcFilterParser_parse_filter:
             filter parent 1: protocol ip pref 2 u32 fh 800: ht divisor 1
             filter parent 1: protocol ip pref 2 u32 fh 800::800 order 2048 key ht 800 bkt 0 flowid 1:2
               match 00000000/00000000 at 16
-            """,
+            """),
             [
                 {'flowid': '1:1', 'network': '192.168.0.0/24', 'port': 80},
                 {'flowid': '1:2', 'network': '0.0.0.0/0', 'port': None},
             ],
         ],
         [
-            """
+            six.b("""
             filter parent 1: protocol ip pref 1 u32
             filter parent 1: protocol ip pref 1 u32 fh 801: ht divisor 1
             filter parent 1: protocol ip pref 1 u32 fh 801::800 order 2048 key ht 801 bkt 0 flowid 1:3
@@ -69,7 +71,7 @@ class Test_TcFilterParser_parse_filter:
             filter parent 1: protocol ip pref 2 u32 fh 800: ht divisor 1
             filter parent 1: protocol ip pref 2 u32 fh 800::800 order 2048 key ht 800 bkt 0 flowid 1:2
               match 00000000/00000000 at 12
-            """,
+            """),
             [
                 {'flowid': '1:3', 'network': '192.168.0.10/32', 'port': 8080},
                 {'flowid': '1:2', 'network': '0.0.0.0/0', 'port': None},
@@ -86,12 +88,12 @@ class Test_TcFilterParser_parse_incoming_device:
         ["", None],
         [None, None],
         [
-            """filter parent ffff: protocol ip pref 49152 u32
+            six.b("""filter parent ffff: protocol ip pref 49152 u32
 filter parent ffff: protocol ip pref 49152 u32 fh 800: ht divisor 1
 filter parent ffff: protocol ip pref 49152 u32 fh 800::800 order 2048 key ht 800 bkt 0 flowid 1:
   match 00000000/00000000 at 0
         action order 1: mirred (Egress Redirect to device ifb0) stolen
-        index 3795 ref 1 bind 1""",
+        index 3795 ref 1 bind 1"""),
             "ifb0",
         ],
     ])
@@ -113,9 +115,9 @@ class Test_TcQdiscParser_parse:
     @pytest.mark.parametrize(["value", "expected"], [
         ["", {}],
         [
-            """qdisc prio 1: root refcnt 2 bands 3 priomap  1 2 2 2 1 2 0 0 1 1 1 1 1 1 1 1
+            six.b("""qdisc prio 1: root refcnt 2 bands 3 priomap  1 2 2 2 1 2 0 0 1 1 1 1 1 1 1 1
 qdisc netem 11: parent 1:1 limit 1000 delay 10.0ms loss 0.1% corrupt 0.1%
-qdisc tbf 20: parent 11:1 rate 100Mbit burst 100000b limit 10000b""",
+qdisc tbf 20: parent 11:1 rate 100Mbit burst 100000b limit 10000b"""),
             {
                 'parent': '1:1',
                 'corrupt': "0.1",
@@ -125,10 +127,10 @@ class Test_TcQdiscParser_parse:
             },
         ],
         [
-            """qdisc prio 1: dev eth0 root refcnt 2 bands 3 priomap  1 2 2 2 1 2 0 0 1 1 1 1 1 1 1 1
+            six.b("""qdisc prio 1: dev eth0 root refcnt 2 bands 3 priomap  1 2 2 2 1 2 0 0 1 1 1 1 1 1 1 1
 qdisc netem 11: dev eth0 parent 1:1 limit 1000 delay 10.0ms  2.0ms loss 0.1% corrupt 0.1%
 qdisc tbf 20: dev eth0 parent 11:1 rate 100Mbit burst 100000b limit 10000b
-qdisc pfifo_fast 0: dev ifb0 root refcnt 2 bands 3 priomap  1 2 2 2 1 2 0 0 1 1 1 1 1 1 1 1""",
+qdisc pfifo_fast 0: dev ifb0 root refcnt 2 bands 3 priomap  1 2 2 2 1 2 0 0 1 1 1 1 1 1 1 1"""),
             {
                 'parent': '1:1',
                 'corrupt': "0.1",
@@ -139,9 +141,9 @@ class Test_TcQdiscParser_parse:
             },
         ],
         [
-            """qdisc prio 1: root refcnt 2 bands 3 priomap  1 2 2 2 1 2 0 0 1 1 1 1 1 1 1 1
+            six.b("""qdisc prio 1: root refcnt 2 bands 3 priomap  1 2 2 2 1 2 0 0 1 1 1 1 1 1 1 1
 qdisc netem 11: parent 1:1 limit 1000 delay 1.0ms loss 0.01%
-qdisc tbf 20: parent 11:1 rate 250Kbit burst 1600b lat 268.8ms""",
+qdisc tbf 20: parent 11:1 rate 250Kbit burst 1600b lat 268.8ms"""),
             {
                 'parent': '1:1',
                 'delay': '1.0',

test/test_tcconfig.py

@@ -10,6 +10,7 @@
 import dataproperty
 import pingparsing
 import pytest
+import six
 import thutils
 import tcconfig
 
@@ -91,11 +92,11 @@ class NormalTestValue:
 
 class Test_tcconfig:
     """
-    Inappropriate tests to Travis CI.
-    Run locally with following command:
+    Tests of in this class are inappropriate for Travis CI.
+    Execute following command at the local environment  when running tests:
       python setup.py test --addopts --runxfail
 
-    These tests expected to execute on following environment:
+    These tests are expected to execute on following environment:
        - Linux(debian) w/ iputils-ping package
        - English environment (for parsing ping output)
     """
@@ -162,23 +163,25 @@ def test_config_file(self, tmpdir, subproc_wrapper, overwrite, expected):
 """
         p.write(config)
 
+        subproc_wrapper.run("tcdel --device " + DEVICE)
         command = " ".join(["tcset -f ", str(p), overwrite])
         assert subproc_wrapper.run(command) == expected
 
         proc = subproc_wrapper.popen_command("tcshow --device " + DEVICE)
         tcshow_stdout, _stderr = proc.communicate()
-        assert tcshow_stdout == config
+        assert thutils.loader.JsonLoader.loads(
+            tcshow_stdout) == thutils.loader.JsonLoader.loads(config)
 
         assert subproc_wrapper.run("tcdel --device " + DEVICE) == 0
 
 
 class Test_tcset_one_network:
     """
-    Inappropriate tests to Travis CI.
-    Run locally with following command:
+    Tests of in this class are inappropriate for Travis CI.
+    Execute following command at the local environment  when running tests:
       python setup.py test --addopts "--dst-host=<hostname or IP address>"
 
-    These tests expected to execute on following environment:
+    These tests are expected to execute on following environment:
        - Linux(debian) w/ iputils-ping package
        - English environment (for parsing ping output)
     """
@@ -307,12 +310,12 @@ def test_const_packet_loss(
 
 class Test_tcset_two_network:
     """
-    Inappropriate tests to Travis CI.
-    Run locally with following command:
+    Tests of in this class are inappropriate for Travis CI.
+    Execute following command at the local environment  when running tests:
       python setup.py test --addopts \
         "--dst-host=<hostname or IP address> --dst-host-ex=<hostname or IP address>"
 
-    These tests expected to execute on following environment:
+    These tests are expected to execute on following environment:
        - Linux(debian) w/ iputils-ping package
        - English environment (for parsing ping output)
     """

test/test_tcshow.py

@@ -18,8 +18,8 @@ def subproc_wrapper():
 
 class Test_tcshow:
     """
-    Inappropriate tests to Travis CI.
-    Run locally with following command:
+    Tests of in this class are inappropriate for Travis CI.
+    Execute following command at the local environment  when running tests:
       python setup.py test --addopts --runxfail
     """
 
@@ -53,7 +53,7 @@ def test_normal(self, subproc_wrapper):
         ])
         proc = subproc_wrapper.popen_command(command)
         stdout, _stderr = proc.communicate()
-        assert stdout == """{
+        assert thutils.loader.JsonLoader.loads(stdout) == thutils.loader.JsonLoader.loads("""{
     "eth0": {
         "outgoing": {
             "network=192.168.0.10/32, port=8080": {
@@ -73,6 +73,6 @@ def test_normal(self, subproc_wrapper):
         }
     }
 }
-"""
+""")
 
         assert subproc_wrapper.run("tcdel --device " + DEVICE) == 0