Setup sphinx quickstart, reorg data load, adding docs

deepmatcher/batch.py

@@ -0,0 +1,54 @@
+from collections import namedtuple
+
+import torch
+
+AttrTensor_ = namedtuple('AttrTensor', ['data', 'lengths', 'word_probs', 'pc'])
+
+
+class AttrTensor(AttrTensor_):
+
+    @staticmethod
+    def __new__(cls, *args, **kwargs):
+        if len(kwargs) == 0:
+            return super(AttrTensor, cls).__new__(cls, *args)
+        else:
+            name = kwargs['name']
+            attr = kwargs['attr']
+            train_dataset = kwargs['train_dataset']
+            if isinstance(attr, tuple):
+                data = attr[0]
+                lengths = attr[1]
+            else:
+                data = attr
+                lengths = None
+            word_probs = None
+            if 'word_probs' in train_dataset.metadata:
+                raw_word_probs = train_dataset.metadata['word_probs'][name]
+                word_probs = torch.Tensor(
+                    [[raw_word_probs[w] for w in b] for b in data.data])
+                if data.is_cuda:
+                    word_probs = word_probs.cuda()
+            pc = None
+            if 'pc' in train_dataset.metadata:
+                pc = torch.Tensor(train_dataset.metadata['pc'][name])
+                if data.is_cuda:
+                    pc = pc.cuda()
+            return AttrTensor(data, lengths, word_probs, pc)
+
+    @staticmethod
+    def from_old_metadata(data, old_attrtensor):
+        return AttrTensor(data, *old_attrtensor[1:])
+
+
+class Batch(object):
+
+    def __init__(self, input, train_dataset):
+        copy_fields = train_dataset.all_text_fields
+        for name in copy_fields:
+            setattr(self, name,
+                    AttrTensor(
+                        name=name, attr=getattr(input, name),
+                        train_dataset=train_dataset))
+        for name in [train_dataset.label_field, train_dataset.id_field]:
+            if name is not None:
+                setattr(self, name, getattr(input, name))

deepmatcher/data/__init__.py

@@ -1,7 +1,6 @@
-from .torchtext_extensions import (AttrTensor, MatchingField, MatchingDataset,
-                                   MatchingBatch, MatchingIterator)
+from .field import MatchingField
+from .dataset import MatchingDataset
+from .iterator import MatchingIterator
 from .process import process
 
-__all__ = [
-    AttrTensor, MatchingField, MatchingDataset, MatchingBatch, MatchingIterator, process
-]
+__all__ = [MatchingField, MatchingDataset, MatchingIterator, process]