Merge 5d76d4e into 9038e9c

spinn_utilities/make_tools/file_converter.py

@@ -19,12 +19,14 @@
 
 TOKEN = chr(30)  # Record Separator
 
+COMMA_SPLIITER = re.compile(r'(?!\B"[^"]*),(?![^"]*"\B)')
 STRING_REGEXP = re.compile(r'"([^"]|\\"|(""))*"')
-FORMAT_EXP = re.compile(r"%\d*(?:\.\d+)?[cdfiksuxR]")
+FORMAT_EXP = re.compile(r"(?:^|[^%])(%\d*(?:\.\d+)?[cdfiksuxRF])")
 LOG_END_REGEX = re.compile(r'\)(\s)*;')
 END_COMMENT_REGEX = re.compile(r"/*/")
 LOG_START_REGEX = re.compile(
     r"log_((info)|(error)|(debug)|(warning))(\s)*\(")
+DOUBLE_HEX = ", double_to_upper({0}), double_to_lower({0})"
 
 # Status values
 NORMAL_CODE = 0
@@ -134,7 +136,9 @@ def _run(self, range_start):
                         if len(check) == 0 or check == "*":
                             self._too_many_lines -= 1
                             continue
+                    previous_status = self._status
                     if not self._process_line(dest_f, line_num, text):
+                        self._status = previous_status
                         self._process_chars(dest_f, line_num, text)
         # print (self._dest)
         return self._message_id
@@ -242,6 +246,9 @@ def _process_line_in_log_close_bracket(self, dest_f, line_num, text):
         :return: True if and only if the whole line was processed
         """
         stripped = text.strip()
+        if len(stripped) == 0:
+            self._log_lines += 1
+            return True
         if stripped[0] == ";":
             if stripped == ";":
                 self._log_full += (";")
@@ -292,33 +299,115 @@ def _process_line_normal_code(self, dest_f, line_num, text):
         # remove white spaces and save log command
         self._log_start = text.index(match.group(0))
         self._log = "".join(match.group(0).split())
+        start_len = self._log_start + len(self._log)
+
         self._status = IN_LOG
         self._log_full = ""  # text saved in process_line_in_log
         self._log_lines = 0
         # Now check for the end of log command
-        return self._process_line_in_log(dest_f, line_num, text)
+        return self._process_line_in_log(dest_f, line_num, text[start_len:])
+
+    def quote_part(self, text):
+        return (text.count('"') - text.count('\\"')) % 2 > 0
+
+    def bracket_count(self, text):
+        return (text.count('(') - text.count(')'))
+
+    def split_by_comma_plus(self, main, line_num):
+        try:
+            parts = main.split(",")
+            for i, part in enumerate(parts):
+                check = part.strip()
+                if check[0] == '"':
+                    # Dealing with a String
+                    if check[-1] == '"':
+                        if check[-2] != '\\':
+                            # Part is a full sting fine
+                            continue
+                    # Save start of String and get next part
+                    new_part = parts.pop(i)
+                    next_part = parts.pop(i)
+                    new_check = next_part.strip()
+                    while new_check[-1] != '"' or new_check[-2] == '\\':
+                        # Still not end of String so add and get next
+                        new_part += "," + next_part
+                        next_part = parts.pop(i)
+                        new_check = next_part.strip()
+                    # Add the end and put back new in the list
+                    new_part += "," + next_part
+                    parts.insert(i, new_part)
+                else:
+                    # Not a String so look for function
+                    count = self.bracket_count(part)
+                    if (count > 0):
+                        # More opening and closing brackets so in function
+                        new_part = parts.pop(i)
+                        # Keep combining parts until you find the last closing
+                        while count > 0:
+                            next_part = parts.pop(i)
+                            count += self.bracket_count(next_part)
+                            new_part += "," + next_part
+                        # Put the new part back into the list
+                        parts.insert(i, new_part)
+            if parts[0][0] == '"' and parts[0][-1] == '"':
+                parts[0] = parts[0][1:-1]
+            return parts
+
+        except Exception:
+            raise Exception("Unexpected line {} at {} in {}".format(
+                self._log_full, line_num, self._src))
 
-    def _shorten(self, original):
+    def _short_log(self, line_num):
         """ shortens the log string message and adds the id
 
         Assumes that self._message_id has already been updated
 
         :param original: Source log messages
         :return: new log message and the id
         """
-        count = original.count("%")
+        try:
+            match = LOG_END_REGEX.search(self._log_full)
+            main = self._log_full[:-len(match.group(0))]
+        except Exception:
+            raise Exception("Unexpected line {} at {} in {}".format(
+                self._log_full, line_num, self._src))
+        parts = self.split_by_comma_plus(main, line_num)
+        original = parts[0]
+        count = original.count("%") - original.count("%%")*2
         if count == 0:
-            return '"%u", {}'.format(self._message_id)
+            return original, '"%u", {});'.format(self._message_id)
         else:
-            result = '"%u'
+            front = '"%u'
+            back = ""
             matches = FORMAT_EXP.findall(original)
             if len(matches) != count:
                 raise Exception(
                     "Unexpected formatString in {}".format(original))
-            for match in matches:
-                result += TOKEN
-                result += match
-            return result + '", {}'.format(self._message_id)
+            if len(parts) < count + 1:
+                raise Exception("Too few parameters in line {} at {} in "
+                                "{}".format(self._log_full, line_num,
+                                            self._src))
+            if len(parts) > count + 1:
+                raise Exception("Too many parameters in line {} at {} in "
+                                "{}".format(self._log_full, line_num,
+                                            self._src))
+            for i, match in enumerate(matches):
+                front += TOKEN
+                if match.endswith("f"):
+                    front += "%x"
+                elif match.endswith("F"):
+                    front += "%x" + TOKEN + "%x"
+                else:
+                    front += match
+                if match.endswith("f"):
+                    back += ", float_to_int({})".format(parts[i + 1])
+                elif match.endswith("F"):
+                    back += DOUBLE_HEX.format(parts[i + 1])
+                else:
+                    back += ", {}".format(parts[i+1])
+            front += '", {}'.format(self._message_id)
+            back += ");"
+        return original, front + back
 
     def _write_log_method(self, dest_f, line_num, tail=""):
         """ Writes the log message and the dict value
@@ -342,31 +431,28 @@ def _write_log_method(self, dest_f, line_num, tail=""):
         """
         self._message_id += 1
         self._log_full = self._log_full.replace('""', '')
-        try:
-            original = STRING_REGEXP.search(self._log_full).group(0)
-        except Exception:
-            raise Exception("Unexpected line {} at {} in {}".format(
-                self._log_full, line_num, self._src))
-        replacement = self._shorten(original)
-        self._log_full = self._log_full.replace(self._log, MINIS[self._log])\
-            .replace(original, replacement)
+        original, short_log = self._short_log(line_num)
+
         dest_f.write(" " * self._log_start)
-        dest_f.write(self._log_full)
+        dest_f.write(MINIS[self._log])
+        dest_f.write(short_log)
         if self._log_lines == 0:
             # Writing an extra newline here so need to recover that ASAP
             self._too_many_lines += 1
         end = tail + "\n"
         if self._log_lines <= 1:
             dest_f.write("  /* ")
-            dest_f.write(original)
+            dest_f.write(self._log)
+            dest_f.write(self._log_full)
             dest_f.write("*/")
             dest_f.write(end)
         else:
             dest_f.write(tail)
             dest_f.write(end)
             dest_f.write(" " * self._log_start)
             dest_f.write("/* ")
-            dest_f.write(original)
+            dest_f.write(self._log)
+            dest_f.write(self._log_full)
             dest_f.write("*/")
             dest_f.write(end * (self._log_lines - 1))
         with open(self._dict, 'a') as mess_f:
@@ -376,7 +462,7 @@ def _write_log_method(self, dest_f, line_num, tail=""):
                 self._message_id, LEVELS[self._log],
                 os.path.basename(self._src).replace(",", ";"),
                 line_num + 1,
-                original[1:-1]))
+                original))
 
     def _process_chars(self, dest_f, line_num, text):
         """ Deals with complex lines that can not be handled in one go
@@ -400,6 +486,8 @@ def _process_chars(self, dest_f, line_num, text):
                 if text[pos+1] == "*":
                     if self._status == IN_LOG:
                         self._log_full += text[write_flag:pos].strip()
+                        if self._log_full[-1] == ")":
+                            self._status = IN_LOG_CLOSE_BRACKET
                         # NO change to self._log_lines as newline not removed
                     else:
                         dest_f.write(text[write_flag:pos])
@@ -487,9 +575,9 @@ def _process_chars(self, dest_f, line_num, text):
                     self._log_lines = 0
                     dest_f.write(text[write_flag:pos])
                     # written up to not including log_start
-                    write_flag = pos
-                    # skip to end of log start
+                    # skip to end of log instruction
                     pos = pos + len(match.group(0))
+                    write_flag = pos
                 else:
                     # Not a log start after all so treat as normal test
                     pos += 1

spinn_utilities/make_tools/replacer.py

@@ -15,6 +15,7 @@
 
 import logging
 import os
+import struct
 from spinn_utilities.log import FormatAdapter
 from .file_converter import FORMAT_EXP
 from .file_converter import TOKEN
@@ -52,12 +53,32 @@ def replace(self, short):
         replaced = six.b(original).decode("unicode_escape")
         if len(parts) > 1:
             matches = FORMAT_EXP.findall(original)
-            if len(matches) != len(parts) - 1:
-                # try removing any blanks due to double spacing
-                matches = [x for x in matches if x != ""]
-            if len(matches) != len(parts) - 1:
-                # wrong number of elements so not short after all
+            # Remove any blanks due to double spacing
+            matches = [x for x in matches if x != ""]
+            # Start at 0 so first i+1 puts you at 1 as part 0 is the short
+            i = 0
+            try:
+                for match in matches:
+                    i += 1
+                    if match.endswith("f"):
+                        replacement = str(self.hex_to_float(parts[i]))
+                    elif match.endswith("F"):
+                        replacement = str(self.hexes_to_double(
+                            parts[i], parts[i+1]))
+                        i += 1
+                    else:
+                        replacement = parts[i]
+                    replaced = replaced.replace(match, replacement, 1)
+            except Exception:
                 return short
-            for i in range(len(matches)):
-                replaced = replaced.replace(matches[i], parts[i+1], 1)
+
         return preface + replaced
+
+    def hex_to_float(self, hex):
+        return struct.unpack('!f', struct.pack("!I", int(hex, 16)))[0]
+
+    def hexes_to_double(self, upper, lower):
+        return struct.unpack(
+            '!d',
+            struct.pack("!I", int(upper, 16)) +
+            struct.pack("!I", int(lower, 16)))[0]

unittests/make_tools/mock_src/formats.c

@@ -0,0 +1,19 @@
+/*
+ * Copyright (c) 2013-2018 The University of Manchester
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+log_info("test u %u", 2);
+log_info("test f %f", 2.0f);

unittests/make_tools/mock_src/weird,file.c

@@ -21,10 +21,15 @@
 
 static String woops = "log_info(";
 
+    log_debug("%08x [%3d: (w: %5u (=",
+                synapse, i, synapse_row_sparse_weight(synapse));
+
     String naughty = "what is this /* nonsense"
 
     /* log_info("inside a comment */
 
+    log_info("test -three %f", -3.0f);
+
     log_info("this is ok");
 
     //log_info("this is just a comment");
@@ -36,7 +41,7 @@ static String woops = "log_info(";
     // a comment
              "after comment");
 
-    log_info("One line commted"); //blah blah
+    log_info("One line commented"); //blah blah
 
     log_info("this is for alan); so there!");
 
@@ -69,9 +74,45 @@ static String woops = "log_info(";
     ;
 
     log_info("then a standard comment on a middle line")
-    /* comment */
+    /* evil comment */
+    ;
+
+    log_info("then a empty line in the middle line")
+
     ;
+    log_info("neuron_initialise: starting");
+    log_info("test -two %f", -2.0f);
+    log_info("test -one %f", -1.0f);
+    log_info("test zero %x", 0.0f);
+    log_info("test one %x", 1.0f);
+    log_info("test two %x", 2.0f);
+    log_info("test string comma, %u is fluff ", 12);
+    log_info("test string %%s in string, %u fluff", 45);
+    log_info("test string quote \" in string, %u fluff", 45);
+    log_info("test string bacKslash %s fluff", "Rowley \" wins");
+    log_info("test string comma %s fluff ", "Rowley, wins");
+    log_info("test string comma, %u is fluff ", 12);
+    log_info("test string many comma %s fluff ",
+        "Rowley, wins, even more ( fluff");
+
+    log_info("magic = %08x, version = %d.%d", ds_regions->magic_number,
+            ds_regions->version >> VERSION_SHIFT,
+            ds_regions->version & VERSION_MASK);
 
     /* comment */ log_info("comment before");
 
-    fluff fluff
+    fluff fluff
+
+    two = 2; log_info("two %u", two);
+
+    log_info("this is a float %f fluff", 1.0);
+
+    log_debug("dumping into sorted at index %d proc %d, for key %d and "
+        "has redundant packet count of %d",
+        *sorted_bf_fill_loc, coverage[i]->processor_ids[bf_index],
+        coverage[i]->bit_field_addresses[bf_index]->key,
+        detect_redundant_packet_count(
+        *coverage[i]->bit_field_addresses[bf_index],
+        region_addresses));
+
+    the end

unittests/make_tools/test_convert.py

@@ -49,7 +49,9 @@ def test_replace(self):
         dict = os.path.join("modified_src", "test.dict")
         c = Converter(src, dest, dict)
         path = "/home/me/mock_src/FEC/c_common/fec/mock_src/"
+        path = path.replace("/", os.path.sep)
         new_path = "/home/me/mock_src/FEC/c_common/fec/modified_src/"
+        new_path = new_path.replace("/", os.path.sep)
         self.assertEqual(new_path, c._any_destination(path))
 
     def test_double_level(self):