From b9b368545f70b5b0c7d894cb66349b8baa5a5679 Mon Sep 17 00:00:00 2001
From: Muthiah Annamalai <ezhillang@gmail.com>
Date: Sat, 18 Aug 2018 09:50:50 -0700
Subject: [PATCH] Tamil Morse code using Project Madurai data

---
 examples/tamilmorse/madurai_unigram.json |   1 +
 examples/tamilmorse/tamil_morse_code.py  |  28 ++-
 solthiruthi/data/madurai_unigram.json    |   1 +
 solthiruthi/data/madurai_unigram.txt     | 293 +++++++++++++++++++++++
 4 files changed, 317 insertions(+), 6 deletions(-)
 create mode 100644 examples/tamilmorse/madurai_unigram.json
 create mode 100644 solthiruthi/data/madurai_unigram.json
 create mode 100644 solthiruthi/data/madurai_unigram.txt

diff --git a/examples/tamilmorse/madurai_unigram.json b/examples/tamilmorse/madurai_unigram.json
new file mode 100644
index 0000000..7044762
--- /dev/null
+++ b/examples/tamilmorse/madurai_unigram.json
@@ -0,0 +1 @@
+{"\u0bb7\u0bcc": 1.0, "\u0b85": 252521.0, "\u0bb9\u0bbf": 422.0, "\u0b89": 114865.0, "\u0bb7\u0bcb": 36.0, "\u0bb7\u0bca": 2.0, "\u0bb5\u0bcd": 21881.0, "\u0bb5\u0bcc": 529.0, "\u0b95": 774635.0, "\u0bb5\u0bc8": 48136.0, "\u0bb5\u0bcb": 11022.0, "\u0bb5\u0bca": 5789.0, "\u0b99": 1530.0, "\u0bb5\u0bc7": 76558.0, "\u0bb5\u0bc6": 53903.0, "\u0bb5\u0bc1": 75941.0, "\u0bb5\u0bc0": 26158.0, "\u0b9c\u0bc8": 438.0, "\u0bb5\u0bc2": 2542.0, "\u0ba4\u0bcc": 180.0, "\u0ba4\u0bcd": 540767.0, "\u0ba4\u0bca": 39591.0, "\u0ba9\u0bbf": 85378.0, "\u0ba4\u0bc8": 74444.0, "\u0baa\u0bcd": 345626.0, "\u0ba9": 242198.0, "\u0ba4\u0bc7": 60822.0, "\u0ba4\u0bc2": 15680.0, "\u0ba4\u0bc0": 22597.0, "\u0ba4\u0bc1": 395602.0, "\u0bb1": 189701.0, "\u0bb5": 472124.0, "\u0bae\u0bbe": 147356.0, "\u0bae\u0bbf": 73393.0, "\u0bb9": 1175.0, "\u0bb0\u0bca": 4135.0, "\u0b9c\u0bcb": 511.0, "\u0bb9\u0bc1": 98.0, "\u0bae\u0bc8": 73801.0, "\u0bae\u0bca": 18422.0, "\u0bae\u0bcb": 12860.0, "\u0bae\u0bcc": 649.0, "\u0bae\u0bcd": 621683.0, "\u0bae\u0bc0": 12201.0, "\u0bae\u0bc1": 166201.0, "\u0bae\u0bc2": 19880.0, "\u0bae\u0bc6": 38364.0, "\u0bae\u0bc7": 53458.0, "\u0ba4\u0bbe": 164244.0, "\u0ba4\u0bbf": 354355.0, "\u0b9e\u0bc7": 41.0, "\u0b9c\u0bc6": 538.0, "\u0bb5\u0bbf": 251004.0, "\u0bb5\u0bbe": 136684.0, "\u0bb1\u0bcc": 1.0, "\u0b9c\u0bc7": 173.0, "\u0b9c\u0bbe": 2153.0, "\u0b9c\u0bbf": 862.0, "\u0b9f\u0bc6": 5798.0, "\u0bb8\u0bbf": 845.0, "\u0b9c\u0bc2": 83.0, "\u0baf\u0bcd": 145377.0, "\u0baf\u0bcc": 66.0, "\u0baf\u0bcb": 24095.0, "\u0baf\u0bca": 9965.0, "\u0b86": 68845.0, "\u0baf\u0bc7": 41717.0, "\u0baf\u0bc6": 28882.0, "\u0b8a": 11458.0, "\u0baf\u0bc2": 4575.0, "\u0baf\u0bc1": 118962.0, "\u0b8e": 112762.0, "\u0b9e\u0bc8": 735.0, "\u0b9e\u0bca": 8.0, "\u0b92": 41876.0, "\u0bb8\u0bbe": 556.0, "\u0b9e\u0bcd": 61923.0, "\u0b9e\u0bc0": 197.0, "\u0b9e\u0bc1": 22.0, "\u0b9e\u0bc2": 59.0, "\u0b9a": 119945.0, "\u0b9e\u0bc6": 350.0, "\u0b9e": 3453.0, "\u0b9f\u0bbf": 199482.0, "\u0b9f\u0bbe": 53287.0, "\u0b9c\u0bcd": 1280.0, "\u0bb7\u0bc2": 63.0, "\u0bb7\u0bc1": 229.0, "\u0bb7\u0bc0": 37.0, "\u0b9c\u0bca": 99.0, "\u0bb7\u0bcd": 4559.0, "\u0baa": 370431.0, "\u0bb0\u0bbe": 61895.0, "\u0bb0\u0bbf": 139808.0, "\u0bae": 299013.0, "\u0b95\u0bcd": 527909.0, "\u0b95\u0bcc": 697.0, "\u0bb2": 199855.0, "\u0b95\u0bc8": 75850.0, "\u0b95\u0bcb": 36032.0, "\u0b95\u0bca": 97975.0, "\u0b95\u0bc7": 33291.0, "\u0b95\u0bc6": 17833.0, "\u0b95\u0bc1": 338725.0, "\u0b95\u0bc0": 7403.0, "\u0b94": 367.0, "\u0b95\u0bc2": 42581.0, "\u0b95\u0bbf": 212583.0, "\u0b95\u0bbe": 145079.0, "\u0bb9\u0bcb": 240.0, "\u0bb0\u0bc2": 3683.0, "\u0b9f\u0bcd": 212838.0, "\u0bb0\u0bc1": 391928.0, "\u0bb0\u0bc6": 10822.0, "\u0bb0\u0bc7": 13756.0, "\u0b9f\u0bc8": 104237.0, "\u0b9f\u0bc7": 11481.0, "\u0bb0\u0bcb": 9739.0, "\u0bb0\u0bc8": 73132.0, "\u0b9f\u0bc2": 1244.0, "\u0b9f\u0bc1": 243848.0, "\u0bb0\u0bcd": 441925.0, "\u0bb8\u0bca": 11.0, "\u0bb8\u0bcb": 44.0, "\u0bb8\u0bc8": 176.0, "\u0b9e\u0bbe": 7297.0, "\u0bb8\u0bcd": 10491.0, "\u0bb8\u0bc2": 119.0, "\u0bb7\u0bc6": 36.0, "\u0bb8\u0bc0": 143.0, "\u0bb8\u0bc1": 261.0, "\u0bb8\u0bc6": 85.0, "\u0bb0\u0bc0": 4011.0, "\u0baf\u0bbf": 169555.0, "\u0baf\u0bbe": 139367.0, "\u0b9f\u0bcc": 24.0, "\u0b9c\u0bc1": 231.0, "\u0b9f\u0bcb": 4948.0, "\u0b9f\u0bca": 3035.0, "\u0b83": 1861.0, "\u0bb9\u0bc2": 61.0, "\u0b87": 162272.0, "\u0ba9\u0bbe": 78966.0, "\u0b99\u0bc2": 2.0, "\u0b8f": 26171.0, "\u0bb1\u0bc1": 169031.0, "\u0bb1\u0bc0": 4199.0, "\u0b93": 16942.0, "\u0bb1\u0bc2": 1336.0, "\u0bb9\u0bc7": 74.0, "\u0bb1\u0bc7": 19030.0, "\u0bb1\u0bc6": 8828.0, "\u0bb1\u0bc8": 65921.0, "\u0bb1\u0bcb": 9447.0, "\u0bb1\u0bca": 5216.0, "\u0bb1\u0bcd": 261277.0, "\u0bb7\u0bc8": 525.0, "\u0b9f": 267257.0, "\u0bb9\u0bc8": 95.0, "\u0ba3": 88135.0, "\u0bb9\u0bca": 30.0, "\u0bb9\u0bcd": 180.0, "\u0bb9\u0bcc": 12.0, "\u0bb7\u0bbf": 1073.0, "\u0bb9\u0bc0": 93.0, "\u0bb2\u0bbe": 85850.0, "\u0bb2\u0bbf": 86421.0, "\u0b9e\u0bcb": 30.0, "\u0ba8\u0bc8": 1052.0, "\u0baf": 328267.0, "\u0bb9\u0bc6": 60.0, "\u0ba8\u0bca": 1900.0, "\u0ba8\u0bcb": 10117.0, "\u0bb3": 101777.0, "\u0bb4\u0bc0": 907.0, "\u0bb7": 6891.0, "\u0ba8\u0bcd": 369333.0, "\u0ba8\u0bc2": 8576.0, "\u0ba8\u0bc0": 36899.0, "\u0ba8\u0bc1": 9398.0, "\u0ba8\u0bc6": 23529.0, "\u0ba8\u0bc7": 10853.0, "\u0b99\u0bcb": 1.0, "\u0b99\u0bca": 11.0, "\u0b99\u0bcd": 241771.0, "\u0ba8\u0bbf": 84254.0, "\u0bb3\u0bcc": 3.0, "\u0b99\u0bc1": 18.0, "\u0b99\u0bbe": 5.0, "\u0b99\u0bc7": 4.0, "\u0bb2\u0bc6": 10211.0, "\u0bb2\u0bc7": 29612.0, "\u0bb2\u0bc0": 2042.0, "\u0bb2\u0bc1": 68471.0, "\u0bb2\u0bc2": 1916.0, "\u0bb9\u0bbe": 1042.0, "\u0bb2\u0bcc": 23.0, "\u0bb2\u0bcd": 403345.0, "\u0baf\u0bc0": 2946.0, "\u0bb2\u0bc8": 121144.0, "\u0bb2\u0bca": 4179.0, "\u0bb2\u0bcb": 10682.0, "\u0b9e\u0bcc": 10.0, "\u0bb8\u0bcc": 26.0, "\u0bb3\u0bca": 1759.0, "\u0bb0\u0bcc": 76.0, "\u0b9e\u0bbf": 130.0, "\u0ba4\u0bcb": 40740.0, "\u0bb1\u0bbf": 135924.0, "\u0bb1\u0bbe": 57305.0, "\u0ba9\u0bc8": 107583.0, "\u0ba9\u0bcb": 11248.0, "\u0ba9\u0bca": 4973.0, "\u0ba9\u0bcd": 657842.0, "\u0ba9\u0bcc": 11.0, "\u0bb7\u0bbe": 541.0, "\u0ba9\u0bc1": 74745.0, "\u0ba9\u0bc0": 3355.0, "\u0ba9\u0bc2": 1994.0, "\u0ba9\u0bc7": 27384.0, "\u0ba9\u0bc6": 13510.0, "\u0b9a\u0bcc": 584.0, "\u0b9a\u0bcd": 120634.0, "\u0b9a\u0bc8": 31470.0, "\u0b9a\u0bca": 30463.0, "\u0b9a\u0bcb": 11006.0, "\u0b88": 9613.0, "\u0b9a\u0bc6": 95350.0, "\u0b9a\u0bc7": 25194.0, "\u0b9a\u0bc0": 12270.0, "\u0b9a\u0bc1": 58661.0, "\u0b9a\u0bc2": 16193.0, "\u0bb8\u0bc7": 71.0, "\u0bb3\u0bcb": 5028.0, "\u0b90": 6967.0, "\u0bb3\u0bc8": 71868.0, "\u0ba3\u0bbf": 60469.0, "\u0ba3\u0bbe": 12196.0, "\u0bb3\u0bcd": 211262.0, "\u0bb3\u0bc2": 942.0, "\u0ba8\u0bbe": 62223.0, "\u0bb3\u0bc1": 49515.0, "\u0bb3\u0bc0": 987.0, "\u0bb3\u0bc7": 11079.0, "\u0bb3\u0bc6": 5783.0, "\u0bb4\u0bbe": 6186.0, "\u0bb4\u0bbf": 68970.0, "\u0b99\u0bbf": 1.0, "\u0ba4": 566469.0, "\u0b9c\u0bc0": 500.0, "\u0ba8": 92960.0, "\u0baf\u0bc8": 46276.0, "\u0b9c": 5714.0, "\u0baa\u0bcc": 387.0, "\u0bb0": 243877.0, "\u0ba8\u0bcc": 18.0, "\u0baa\u0bc8": 12780.0, "\u0bb4": 58801.0, "\u0baa\u0bca": 62712.0, "\u0baa\u0bcb": 82205.0, "\u0bb8": 2250.0, "\u0baa\u0bc6": 66538.0, "\u0baa\u0bc7": 30690.0, "\u0baa\u0bc0": 4219.0, "\u0baa\u0bc1": 146058.0, "\u0baa\u0bc2": 25286.0, "\u0bb7\u0bc7": 478.0, "\u0baa\u0bbe": 116682.0, "\u0baa\u0bbf": 118648.0, "\u0ba4\u0bc6": 47579.0, "\u0bb4\u0bcc": 2.0, "\u0bb4\u0bcd": 56156.0, "\u0bb4\u0bca": 270.0, "\u0bb4\u0bcb": 283.0, "\u0bb4\u0bc8": 23614.0, "\u0bb4\u0bc6": 493.0, "\u0bb4\u0bc7": 758.0, "\u0bb4\u0bc2": 298.0, "\u0b9f\u0bc0": 2902.0, "\u0bb4\u0bc1": 67835.0, "\u0ba3\u0bcb": 1325.0, "\u0ba3\u0bca": 611.0, "\u0ba3\u0bc8": 22128.0, "\u0bb3\u0bbf": 93496.0, "\u0bb3\u0bbe": 28154.0, "\u0ba3\u0bcd": 231490.0, "\u0ba3\u0bc2": 458.0, "\u0ba3\u0bc1": 17030.0, "\u0ba3\u0bc0": 2995.0, "\u0ba3\u0bc7": 1997.0, "\u0ba3\u0bc6": 2372.0, "\u0b9a\u0bbe": 40004.0, "\u0b9a\u0bbf": 123619.0}
\ No newline at end of file
diff --git a/examples/tamilmorse/tamil_morse_code.py b/examples/tamilmorse/tamil_morse_code.py
index 850789f..c2a37d0 100644
--- a/examples/tamilmorse/tamil_morse_code.py
+++ b/examples/tamilmorse/tamil_morse_code.py
@@ -13,8 +13,17 @@
 from solthiruthi import resources
 from huffman import huffman, print_huffman_code_cwl
 
-def build_morse_code():
+def TVU_morse_code():
+    # unigram data from Project Madurai
     unigram = TamilUnigramStats().unigram
+    build_morse_code(unigram)
+
+def Madurai_morse_code():
+    # unigram data from Project Madurai
+    unigram = TamilUnigramStats().unigram
+    build_morse_code(unigram)
+    
+def build_morse_code(unigram):
     v_keys = unigram.keys()
     p = [unigram[k] for k in v_keys]
     code,_ = huffman(v_keys,p)
@@ -32,10 +41,10 @@ def build_morse_code():
         fp.write( json.dumps(tamilmorse) )
     return
 
-class TamilUnigramStats:
-    def __init__(self):
+class UnigramStats:
+    def __init__(self,filename):
         self.unigram = {} # Tamil letter -> probability of occurence
-        self.unigram_file = resources.mk_path("tvu_unigram.txt")
+        self.unigram_file = resources.mk_path(filename)
         with codecs.open(self.unigram_file,"r","utf-8") as fp:
             for L in fp.readlines():
                 a,b = L.split("-")
@@ -45,8 +54,15 @@ def __init__(self):
             normalize = 1+sum(self.unigram.values())
             for k,v in self.unigram.items():
                 self.unigram[k] = v/normalize
-        
+
+class TamilUnigramStats(UnigramStats):
+    def __init__(self):
+        UnigramStats.__init__(self,"tvu_unigram.txt")
+
+class MaduraiUnigramStats(UnigramStats):
+    def __init__(self):
+        UnigramStats.__init__(self,"madurai_unigram.txt")
 
 if __name__ == u"__main__":
-    build_morse_code()
+    Madurai_morse_code()
 
diff --git a/solthiruthi/data/madurai_unigram.json b/solthiruthi/data/madurai_unigram.json
new file mode 100644
index 0000000..7044762
--- /dev/null
+++ b/solthiruthi/data/madurai_unigram.json
@@ -0,0 +1 @@
+{"\u0bb7\u0bcc": 1.0, "\u0b85": 252521.0, "\u0bb9\u0bbf": 422.0, "\u0b89": 114865.0, "\u0bb7\u0bcb": 36.0, "\u0bb7\u0bca": 2.0, "\u0bb5\u0bcd": 21881.0, "\u0bb5\u0bcc": 529.0, "\u0b95": 774635.0, "\u0bb5\u0bc8": 48136.0, "\u0bb5\u0bcb": 11022.0, "\u0bb5\u0bca": 5789.0, "\u0b99": 1530.0, "\u0bb5\u0bc7": 76558.0, "\u0bb5\u0bc6": 53903.0, "\u0bb5\u0bc1": 75941.0, "\u0bb5\u0bc0": 26158.0, "\u0b9c\u0bc8": 438.0, "\u0bb5\u0bc2": 2542.0, "\u0ba4\u0bcc": 180.0, "\u0ba4\u0bcd": 540767.0, "\u0ba4\u0bca": 39591.0, "\u0ba9\u0bbf": 85378.0, "\u0ba4\u0bc8": 74444.0, "\u0baa\u0bcd": 345626.0, "\u0ba9": 242198.0, "\u0ba4\u0bc7": 60822.0, "\u0ba4\u0bc2": 15680.0, "\u0ba4\u0bc0": 22597.0, "\u0ba4\u0bc1": 395602.0, "\u0bb1": 189701.0, "\u0bb5": 472124.0, "\u0bae\u0bbe": 147356.0, "\u0bae\u0bbf": 73393.0, "\u0bb9": 1175.0, "\u0bb0\u0bca": 4135.0, "\u0b9c\u0bcb": 511.0, "\u0bb9\u0bc1": 98.0, "\u0bae\u0bc8": 73801.0, "\u0bae\u0bca": 18422.0, "\u0bae\u0bcb": 12860.0, "\u0bae\u0bcc": 649.0, "\u0bae\u0bcd": 621683.0, "\u0bae\u0bc0": 12201.0, "\u0bae\u0bc1": 166201.0, "\u0bae\u0bc2": 19880.0, "\u0bae\u0bc6": 38364.0, "\u0bae\u0bc7": 53458.0, "\u0ba4\u0bbe": 164244.0, "\u0ba4\u0bbf": 354355.0, "\u0b9e\u0bc7": 41.0, "\u0b9c\u0bc6": 538.0, "\u0bb5\u0bbf": 251004.0, "\u0bb5\u0bbe": 136684.0, "\u0bb1\u0bcc": 1.0, "\u0b9c\u0bc7": 173.0, "\u0b9c\u0bbe": 2153.0, "\u0b9c\u0bbf": 862.0, "\u0b9f\u0bc6": 5798.0, "\u0bb8\u0bbf": 845.0, "\u0b9c\u0bc2": 83.0, "\u0baf\u0bcd": 145377.0, "\u0baf\u0bcc": 66.0, "\u0baf\u0bcb": 24095.0, "\u0baf\u0bca": 9965.0, "\u0b86": 68845.0, "\u0baf\u0bc7": 41717.0, "\u0baf\u0bc6": 28882.0, "\u0b8a": 11458.0, "\u0baf\u0bc2": 4575.0, "\u0baf\u0bc1": 118962.0, "\u0b8e": 112762.0, "\u0b9e\u0bc8": 735.0, "\u0b9e\u0bca": 8.0, "\u0b92": 41876.0, "\u0bb8\u0bbe": 556.0, "\u0b9e\u0bcd": 61923.0, "\u0b9e\u0bc0": 197.0, "\u0b9e\u0bc1": 22.0, "\u0b9e\u0bc2": 59.0, "\u0b9a": 119945.0, "\u0b9e\u0bc6": 350.0, "\u0b9e": 3453.0, "\u0b9f\u0bbf": 199482.0, "\u0b9f\u0bbe": 53287.0, "\u0b9c\u0bcd": 1280.0, "\u0bb7\u0bc2": 63.0, "\u0bb7\u0bc1": 229.0, "\u0bb7\u0bc0": 37.0, "\u0b9c\u0bca": 99.0, "\u0bb7\u0bcd": 4559.0, "\u0baa": 370431.0, "\u0bb0\u0bbe": 61895.0, "\u0bb0\u0bbf": 139808.0, "\u0bae": 299013.0, "\u0b95\u0bcd": 527909.0, "\u0b95\u0bcc": 697.0, "\u0bb2": 199855.0, "\u0b95\u0bc8": 75850.0, "\u0b95\u0bcb": 36032.0, "\u0b95\u0bca": 97975.0, "\u0b95\u0bc7": 33291.0, "\u0b95\u0bc6": 17833.0, "\u0b95\u0bc1": 338725.0, "\u0b95\u0bc0": 7403.0, "\u0b94": 367.0, "\u0b95\u0bc2": 42581.0, "\u0b95\u0bbf": 212583.0, "\u0b95\u0bbe": 145079.0, "\u0bb9\u0bcb": 240.0, "\u0bb0\u0bc2": 3683.0, "\u0b9f\u0bcd": 212838.0, "\u0bb0\u0bc1": 391928.0, "\u0bb0\u0bc6": 10822.0, "\u0bb0\u0bc7": 13756.0, "\u0b9f\u0bc8": 104237.0, "\u0b9f\u0bc7": 11481.0, "\u0bb0\u0bcb": 9739.0, "\u0bb0\u0bc8": 73132.0, "\u0b9f\u0bc2": 1244.0, "\u0b9f\u0bc1": 243848.0, "\u0bb0\u0bcd": 441925.0, "\u0bb8\u0bca": 11.0, "\u0bb8\u0bcb": 44.0, "\u0bb8\u0bc8": 176.0, "\u0b9e\u0bbe": 7297.0, "\u0bb8\u0bcd": 10491.0, "\u0bb8\u0bc2": 119.0, "\u0bb7\u0bc6": 36.0, "\u0bb8\u0bc0": 143.0, "\u0bb8\u0bc1": 261.0, "\u0bb8\u0bc6": 85.0, "\u0bb0\u0bc0": 4011.0, "\u0baf\u0bbf": 169555.0, "\u0baf\u0bbe": 139367.0, "\u0b9f\u0bcc": 24.0, "\u0b9c\u0bc1": 231.0, "\u0b9f\u0bcb": 4948.0, "\u0b9f\u0bca": 3035.0, "\u0b83": 1861.0, "\u0bb9\u0bc2": 61.0, "\u0b87": 162272.0, "\u0ba9\u0bbe": 78966.0, "\u0b99\u0bc2": 2.0, "\u0b8f": 26171.0, "\u0bb1\u0bc1": 169031.0, "\u0bb1\u0bc0": 4199.0, "\u0b93": 16942.0, "\u0bb1\u0bc2": 1336.0, "\u0bb9\u0bc7": 74.0, "\u0bb1\u0bc7": 19030.0, "\u0bb1\u0bc6": 8828.0, "\u0bb1\u0bc8": 65921.0, "\u0bb1\u0bcb": 9447.0, "\u0bb1\u0bca": 5216.0, "\u0bb1\u0bcd": 261277.0, "\u0bb7\u0bc8": 525.0, "\u0b9f": 267257.0, "\u0bb9\u0bc8": 95.0, "\u0ba3": 88135.0, "\u0bb9\u0bca": 30.0, "\u0bb9\u0bcd": 180.0, "\u0bb9\u0bcc": 12.0, "\u0bb7\u0bbf": 1073.0, "\u0bb9\u0bc0": 93.0, "\u0bb2\u0bbe": 85850.0, "\u0bb2\u0bbf": 86421.0, "\u0b9e\u0bcb": 30.0, "\u0ba8\u0bc8": 1052.0, "\u0baf": 328267.0, "\u0bb9\u0bc6": 60.0, "\u0ba8\u0bca": 1900.0, "\u0ba8\u0bcb": 10117.0, "\u0bb3": 101777.0, "\u0bb4\u0bc0": 907.0, "\u0bb7": 6891.0, "\u0ba8\u0bcd": 369333.0, "\u0ba8\u0bc2": 8576.0, "\u0ba8\u0bc0": 36899.0, "\u0ba8\u0bc1": 9398.0, "\u0ba8\u0bc6": 23529.0, "\u0ba8\u0bc7": 10853.0, "\u0b99\u0bcb": 1.0, "\u0b99\u0bca": 11.0, "\u0b99\u0bcd": 241771.0, "\u0ba8\u0bbf": 84254.0, "\u0bb3\u0bcc": 3.0, "\u0b99\u0bc1": 18.0, "\u0b99\u0bbe": 5.0, "\u0b99\u0bc7": 4.0, "\u0bb2\u0bc6": 10211.0, "\u0bb2\u0bc7": 29612.0, "\u0bb2\u0bc0": 2042.0, "\u0bb2\u0bc1": 68471.0, "\u0bb2\u0bc2": 1916.0, "\u0bb9\u0bbe": 1042.0, "\u0bb2\u0bcc": 23.0, "\u0bb2\u0bcd": 403345.0, "\u0baf\u0bc0": 2946.0, "\u0bb2\u0bc8": 121144.0, "\u0bb2\u0bca": 4179.0, "\u0bb2\u0bcb": 10682.0, "\u0b9e\u0bcc": 10.0, "\u0bb8\u0bcc": 26.0, "\u0bb3\u0bca": 1759.0, "\u0bb0\u0bcc": 76.0, "\u0b9e\u0bbf": 130.0, "\u0ba4\u0bcb": 40740.0, "\u0bb1\u0bbf": 135924.0, "\u0bb1\u0bbe": 57305.0, "\u0ba9\u0bc8": 107583.0, "\u0ba9\u0bcb": 11248.0, "\u0ba9\u0bca": 4973.0, "\u0ba9\u0bcd": 657842.0, "\u0ba9\u0bcc": 11.0, "\u0bb7\u0bbe": 541.0, "\u0ba9\u0bc1": 74745.0, "\u0ba9\u0bc0": 3355.0, "\u0ba9\u0bc2": 1994.0, "\u0ba9\u0bc7": 27384.0, "\u0ba9\u0bc6": 13510.0, "\u0b9a\u0bcc": 584.0, "\u0b9a\u0bcd": 120634.0, "\u0b9a\u0bc8": 31470.0, "\u0b9a\u0bca": 30463.0, "\u0b9a\u0bcb": 11006.0, "\u0b88": 9613.0, "\u0b9a\u0bc6": 95350.0, "\u0b9a\u0bc7": 25194.0, "\u0b9a\u0bc0": 12270.0, "\u0b9a\u0bc1": 58661.0, "\u0b9a\u0bc2": 16193.0, "\u0bb8\u0bc7": 71.0, "\u0bb3\u0bcb": 5028.0, "\u0b90": 6967.0, "\u0bb3\u0bc8": 71868.0, "\u0ba3\u0bbf": 60469.0, "\u0ba3\u0bbe": 12196.0, "\u0bb3\u0bcd": 211262.0, "\u0bb3\u0bc2": 942.0, "\u0ba8\u0bbe": 62223.0, "\u0bb3\u0bc1": 49515.0, "\u0bb3\u0bc0": 987.0, "\u0bb3\u0bc7": 11079.0, "\u0bb3\u0bc6": 5783.0, "\u0bb4\u0bbe": 6186.0, "\u0bb4\u0bbf": 68970.0, "\u0b99\u0bbf": 1.0, "\u0ba4": 566469.0, "\u0b9c\u0bc0": 500.0, "\u0ba8": 92960.0, "\u0baf\u0bc8": 46276.0, "\u0b9c": 5714.0, "\u0baa\u0bcc": 387.0, "\u0bb0": 243877.0, "\u0ba8\u0bcc": 18.0, "\u0baa\u0bc8": 12780.0, "\u0bb4": 58801.0, "\u0baa\u0bca": 62712.0, "\u0baa\u0bcb": 82205.0, "\u0bb8": 2250.0, "\u0baa\u0bc6": 66538.0, "\u0baa\u0bc7": 30690.0, "\u0baa\u0bc0": 4219.0, "\u0baa\u0bc1": 146058.0, "\u0baa\u0bc2": 25286.0, "\u0bb7\u0bc7": 478.0, "\u0baa\u0bbe": 116682.0, "\u0baa\u0bbf": 118648.0, "\u0ba4\u0bc6": 47579.0, "\u0bb4\u0bcc": 2.0, "\u0bb4\u0bcd": 56156.0, "\u0bb4\u0bca": 270.0, "\u0bb4\u0bcb": 283.0, "\u0bb4\u0bc8": 23614.0, "\u0bb4\u0bc6": 493.0, "\u0bb4\u0bc7": 758.0, "\u0bb4\u0bc2": 298.0, "\u0b9f\u0bc0": 2902.0, "\u0bb4\u0bc1": 67835.0, "\u0ba3\u0bcb": 1325.0, "\u0ba3\u0bca": 611.0, "\u0ba3\u0bc8": 22128.0, "\u0bb3\u0bbf": 93496.0, "\u0bb3\u0bbe": 28154.0, "\u0ba3\u0bcd": 231490.0, "\u0ba3\u0bc2": 458.0, "\u0ba3\u0bc1": 17030.0, "\u0ba3\u0bc0": 2995.0, "\u0ba3\u0bc7": 1997.0, "\u0ba3\u0bc6": 2372.0, "\u0b9a\u0bbe": 40004.0, "\u0b9a\u0bbf": 123619.0}
\ No newline at end of file
diff --git a/solthiruthi/data/madurai_unigram.txt b/solthiruthi/data/madurai_unigram.txt
new file mode 100644
index 0000000..e46eb7c
--- /dev/null
+++ b/solthiruthi/data/madurai_unigram.txt
@@ -0,0 +1,293 @@
+க - 774635
+ன் - 657842
+ம் - 621683
+த - 566469
+த் - 540767
+க் - 527909
+வ - 472124
+ர் - 441925
+ல் - 403345
+து - 395602
+ரு - 391928
+ப - 370431
+ந் - 369333
+தி - 354355
+ப் - 345626
+கு - 338725
+ய - 328267
+ம - 299013
+ட - 267257
+ற் - 261277
+அ - 252521
+வி - 251004
+ர - 243877
+டு - 243848
+ன - 242198
+ங் - 241771
+ண் - 231490
+ட் - 212838
+கி - 212583
+ள் - 211262
+ல - 199855
+டி - 199482
+ற - 189701
+யி - 169555
+று - 169031
+மு - 166201
+தா - 164244
+இ - 162272
+மா - 147356
+பு - 146058
+ய் - 145377
+கா - 145079
+ரி - 139808
+யா - 139367
+வா - 136684
+றி - 135924
+சி - 123619
+லை - 121144
+ச் - 120634
+ச - 119945
+யு - 118962
+பி - 118648
+பா - 116682
+உ - 114865
+எ - 112762
+னை - 107583
+டை - 104237
+ள - 101777
+கொ - 97975
+செ - 95350
+ளி - 93496
+ந - 92960
+ண - 88135
+லி - 86421
+லா - 85850
+னி - 85378
+நி - 84254
+போ - 82205
+னா - 78966
+வே - 76558
+வு - 75941
+கை - 75850
+னு - 74745
+தை - 74444
+மை - 73801
+மி - 73393
+ரை - 73132
+ளை - 71868
+ழி - 68970
+ஆ - 68845
+லு - 68471
+ழு - 67835
+பெ - 66538
+றை - 65921
+பொ - 62712
+நா - 62223
+ஞ் - 61923
+ரா - 61895
+தே - 60822
+ணி - 60469
+ழ - 58801
+சு - 58661
+றா - 57305
+ழ் - 56156
+வெ - 53903
+மே - 53458
+டா - 53287
+ளு - 49515
+வை - 48136
+தெ - 47579
+யை - 46276
+கூ - 42581
+ஒ - 41876
+யே - 41717
+தோ - 40740
+சா - 40004
+தொ - 39591
+மெ - 38364
+நீ - 36899
+கோ - 36032
+கே - 33291
+சை - 31470
+பே - 30690
+சொ - 30463
+லே - 29612
+யெ - 28882
+ளா - 28154
+னே - 27384
+ஏ - 26171
+வீ - 26158
+பூ - 25286
+சே - 25194
+யோ - 24095
+ழை - 23614
+நெ - 23529
+தீ - 22597
+ணை - 22128
+வ் - 21881
+மூ - 19880
+றே - 19030
+மொ - 18422
+கெ - 17833
+ணு - 17030
+ஓ - 16942
+சூ - 16193
+தூ - 15680
+ரே - 13756
+னெ - 13510
+மோ - 12860
+பை - 12780
+சீ - 12270
+மீ - 12201
+ணா - 12196
+டே - 11481
+ஊ - 11458
+னோ - 11248
+ளே - 11079
+வோ - 11022
+சோ - 11006
+நே - 10853
+ரெ - 10822
+லோ - 10682
+ஸ் - 10491
+லெ - 10211
+நோ - 10117
+யொ - 9965
+ரோ - 9739
+ஈ - 9613
+றோ - 9447
+நு - 9398
+றெ - 8828
+நூ - 8576
+கீ - 7403
+ஞா - 7297
+ஐ - 6967
+ஷ - 6891
+ழா - 6186
+டெ - 5798
+வொ - 5789
+ளெ - 5783
+ஜ - 5714
+றொ - 5216
+ளோ - 5028
+னொ - 4973
+டோ - 4948
+யூ - 4575
+ஷ் - 4559
+பீ - 4219
+றீ - 4199
+லொ - 4179
+ரொ - 4135
+ரீ - 4011
+ரூ - 3683
+ஞ - 3453
+னீ - 3355
+டொ - 3035
+ணீ - 2995
+யீ - 2946
+டீ - 2902
+வூ - 2542
+ணெ - 2372
+ஸ - 2250
+ஜா - 2153
+லீ - 2042
+ணே - 1997
+னூ - 1994
+லூ - 1916
+நொ - 1900
+ஃ - 1861
+ளொ - 1759
+ங - 1530
+றூ - 1336
+ணோ - 1325
+ஜ் - 1280
+டூ - 1244
+ஹ - 1175
+ஷி - 1073
+நை - 1052
+ஹா - 1042
+ளீ - 987
+ளூ - 942
+ழீ - 907
+ஜி - 862
+ஸி - 845
+ழே - 758
+ஞை - 735
+கௌ - 697
+மௌ - 649
+ணொ - 611
+சௌ - 584
+ஸா - 556
+ஷா - 541
+ஜெ - 538
+வௌ - 529
+ஷை - 525
+ஜோ - 511
+ஜீ - 500
+ழெ - 493
+ஷே - 478
+ணூ - 458
+ஜை - 438
+ஹி - 422
+பௌ - 387
+ஔ - 367
+ஞெ - 350
+ழூ - 298
+ழோ - 283
+ழொ - 270
+ஸு - 261
+ஹோ - 240
+ஜு - 231
+ஷு - 229
+ஞீ - 197
+தௌ - 180
+ஹ் - 180
+ஸை - 176
+ஜே - 173
+ஸீ - 143
+ஞி - 130
+ஸூ - 119
+ஜொ - 99
+ஹு - 98
+ஹை - 95
+ஹீ - 93
+ஸெ - 85
+ஜூ - 83
+ரௌ - 76
+ஹே - 74
+ஸே - 71
+யௌ - 66
+ஷூ - 63
+ஹூ - 61
+ஹெ - 60
+ஞூ - 59
+ஸோ - 44
+ஞே - 41
+ஷீ - 37
+ஷோ - 36
+ஷெ - 36
+ஹொ - 30
+ஞோ - 30
+ஸௌ - 26
+டௌ - 24
+லௌ - 23
+ஞு - 22
+ஙு - 18
+நௌ - 18
+ஹௌ - 12
+ஸொ - 11
+ஙொ - 11
+னௌ - 11
+ஞௌ - 10
+ஞொ - 8
+ஙா - 5
+ஙே - 4
+ளௌ - 3
+ஷொ - 2
+ஙூ - 2
+ழௌ - 2
+ஷௌ - 1
+றௌ - 1
+ஙி - 1
+ஙோ - 1