m4 tokenizer seems to be working

tokenize/m4Tokenizer/m4.py

@@ -301,7 +301,7 @@ def parse(self, stream=sys.stdout):
         if options.verbose:
             print >> sys.stderr, 'Parsing ', filename
 
-        for tok in self._expand_tokens():
+        for tok in self.token_iter:
             if tok.value != '\n' and tok.value != "\t" and tok.value != ' ':
                 if options.location:
                     print "%d:%d\t%s" % (tok.line , tok.column, tok)
@@ -322,6 +322,8 @@ def parse(self, stream=sys.stdout):
 
 
 if __name__ == '__main__':
+
+    print "begin_unit|revision:0.1;language:m4;cregit-version:0.0.1"
     parser = OptionParser()
     parser.add_option("-v", "--verbose",
                   action="store_true", dest="verbose", default=False,
@@ -345,3 +347,4 @@ def parse(self, stream=sys.stdout):
 
     f = open(filename, "r")
     Parser(f.read()).parse()
+    print "end_unit"

tokenizeByBlobId/tokenBySha.pl

@@ -57,6 +57,8 @@
                "java" => 'Java',
                "md" => "Markdown",
                "yaml" => "Yaml",
+               "ac" => "M4",
+               "am" => "M4",
               );