Permalink
Browse files

Adapting main modules for Python 3 compatibility. Dropped Python 2.5 …

…support
  • Loading branch information...
proycon committed Mar 29, 2013
1 parent e17ff35 commit 94c5e28a52b79688f19286da7418c2286a3e362b
Showing with 290 additions and 231 deletions.
  1. +16 −0 algorithms.py
  2. +30 −11 common.py
  3. +27 −8 datatypes.py
  4. +41 −60 evaluation.py
  5. +18 −10 net.py
  6. +67 −54 search.py
  7. +41 −35 statistics.py
  8. +47 −53 textprocessors.py
  9. +3 −0 web.py
View
@@ -1,4 +1,20 @@
+###############################################################9
+# PyNLPl - Algorithms
+# by Maarten van Gompel
+# Centre for Language Studies
+# Radboud University Nijmegen
+# http://www.github.com/proycon/pynlpl
+# proycon AT anaproy DOT nl
+#
+# Licensed under GPLv3
+#
+###############################################################
+
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division
+from __future__ import absolute_import
def sum_to_n(n, size, limit=None): #from http://stackoverflow.com/questions/2065553/python-get-all-numbers-that-add-up-to-a-number
"""Produce all lists of `size` positive integers in decreasing order
View
@@ -1,21 +1,27 @@
#!/usr/bin/env python
#-*- coding:utf-8 -*-
-###############################################################
-# PyNLPl - Common Library
-# by Maarten van Gompel (proycon)
-# http://ilk.uvt.nl/~mvgompel
-# Induction for Linguistic Knowledge Research Group
-# Universiteit van Tilburg
+###############################################################9
+# PyNLPl - Common functions
+# by Maarten van Gompel
+# Centre for Language Studies
+# Radboud University Nijmegen
+# http://www.github.com/proycon/pynlpl
+# proycon AT anaproy DOT nl
#
# Licensed under GPLv3
#
# This contains very common functions and language extensions
#
###############################################################
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division
+from __future__ import absolute_import
+
import datetime
-from sys import stderr
+from sys import stderr, version
## From http://code.activestate.com/recipes/413486/ (r7)
def Enum(*names):
@@ -55,6 +61,22 @@ def __repr__(self): return str(names[self.__value])
return EnumType
+def u(s, encoding = 'utf-8', errors='strict'):
+ #ensure s is properly unicode.. wrapper for python 2.6/2.7,
+ if version < '3':
+ #ensure the object is unicode
+ if isinstance(s, unicode):
+ return s
+ else:
+ return unicode(s, encoding,errors=errors)
+ else:
+ #will work on byte arrays
+ if isinstance(s, str):
+ return s
+ else:
+ return str(s,encoding,errors=errors)
+
+
def log(msg, **kwargs):
"""Generic log method. Will prepend timestamp.
@@ -80,10 +102,7 @@ def log(msg, **kwargs):
if 'indent' in kwargs:
s += ("\t" * int(kwargs['indent']))
- if isinstance(msg, unicode):
- s += msg.encode('utf-8')
- else:
- s += unicode(msg,'utf-8', errors='ignore').encode('utf-8')
+ s += u(msg)
if s[-1] != '\n':
s += '\n'
View
@@ -1,7 +1,9 @@
#---------------------------------------------------------------
# PyNLPl - Data Types
-# by Maarten van Gompel, ILK, Universiteit van Tilburg
-# http://ilk.uvt.nl/~mvgompel
+# by Maarten van Gompel
+# Centre for Language Studies
+# Radboud University Nijmegen
+# http://www.github.com/proycon/pynlpl
# proycon AT anaproy DOT nl
#
# Based in large part on MIT licensed code from
@@ -14,10 +16,16 @@
"""This library contains various extra data types, based to a certain extend on MIT-licensed code from Peter Norvig, AI: A Modern Appproach : http://aima.cs.berkeley.edu/python/utils.html"""
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division
+from __future__ import absolute_import
+from pynlpl.common import u
import bisect
import array
+
class Queue: #from AI: A Modern Appproach : http://aima.cs.berkeley.edu/python/utils.html
"""Queue is an abstract class/interface. There are three types:
Python List: A Last In First Out Queue (no Queue object necessary).
@@ -56,7 +64,7 @@ def pop(self):
"""Retrieve the next element in line, this will remove it from the queue"""
e = self.data[self.start]
self.start += 1
- if self.start > 5 and self.start > len(self.data)/2:
+ if self.start > 5 and self.start > len(self.data)//2:
self.data = self.data[self.start:]
self.start = 0
return e
@@ -240,21 +248,25 @@ def __len__(self):
else:
return len(self.children)
- def __nonzero__(self):
+ def __bool__(self):
return True
def __iter__(self):
+ """Iterate over all items in the tree"""
for c in self.children:
return c
def append(self, item):
+ """Add an item to the Tree"""
if not isinstance(item, Tree):
return ValueError("Can only append items of type Tree")
if not self.children: self.children = []
item.parent = self
self.children.append(item)
def __getitem__(self, index):
+ """Retrieve a specific item, by index, from the Tree"""
+ assert isinstance(index,int)
try:
return self.children[index]
except:
@@ -263,6 +275,9 @@ def __getitem__(self, index):
def __str__(self):
return str(value)
+ def __unicode__(self): #Python 2.x
+ return u(value)
+
@@ -281,6 +296,7 @@ def leaf(self):
return not self.children
def root(self):
+ """Returns True if this is the root of the Trie"""
return not self.parent
def __len__(self):
@@ -289,7 +305,7 @@ def __len__(self):
else:
return len(self.children)
- def __nonzero__(self):
+ def __bool__(self):
return True
def __iter__(self):
@@ -348,12 +364,14 @@ def size(self):
return 1
def path(self):
+ """Returns the path to the current node"""
if self.parent:
return (self,) + self.parent.path()
else:
return (self,)
def depth(self):
+ """Returns the depth of the current node"""
if self.parent:
return 1 + self.parent.depth()
else:
@@ -383,12 +401,13 @@ def walk(self, leavesonly=True, maxdepth=None, _depth = 0):
def containsnullbyte(i):
- while True:
+ assert isinstance(i,int)
+ while True:
r = i % 256
if i % 256 == 0:
return True
if i >= 256:
- i = i / 256
+ i = i // 256
else:
return False
@@ -403,7 +422,7 @@ def inttobytearray(i,bigendian=False, nonullbyte=False):
raise ValueError("Null byte encountered")
a.append(r)
if i >= 256:
- i = i / 256
+ i = i // 256
else:
break
if bigendian: a.reverse()
Oops, something went wrong.

0 comments on commit 94c5e28

Please sign in to comment.