Rename exporter to indexer. Add dummy SQLite target writer.

lsif-py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-from lsif_exporter.script import main
+from lsif_indexer.script import main
 
 if __name__ == '__main__':
     main()

lsif_exporter/consts.py → lsif_indexer/consts.py

@@ -1,4 +1,4 @@
-EXPORTER_VERSION = '0.1.0'
+INDEXER_VERSION = '0.1.0'
 POSITION_ENCODING = 'utf-16'
 PROTOCOL_VERSION = '0.4.0'
 

lsif_exporter/emitter.py → lsif_indexer/emitter.py

@@ -3,12 +3,12 @@
 
 class Emitter:
     """
-    Emitter writes LSIF-dump data to the given file writer. There are
+    Emitter writes LSIF-dump data to the given writer. The location to
+    which dump data is written depends on the given writer. There are
     convenience methods to generate unique vertex and edge identifiers
-    and map positional arguments ot the correct names for the label type.
-
-    The majority of the methods in this class definition are added
-    dynamically via setattr (below).
+    and map positional arguments ot the correct names for the label
+    type. The majority of the methods in this class definition are
+    added dynamically via setattr (below).
     """
     def __init__(self, writer):
         self.writer = writer
@@ -22,10 +22,33 @@ def emit(self, **kwargs):
         """
         node_id = str(self._lines + 1)
         self._lines += 1
-        self.writer.write(json.dumps({'id': node_id, **kwargs}) + '\n')
+        self.writer.write({'id': node_id, **kwargs})
         return node_id
 
 
+class FileWriter:
+    """
+    FileWriter writes LSIF-dump data to the given file.
+    """
+    def __init__(self, file):
+        self.file = file
+
+    def write(self, data):
+        self.file.write(json.dumps(data) + '\n')
+
+
+class DBWriter:
+    """
+    DBWriter writes LSIF-dump data into a SQLite database.
+    """
+    def __init__(self):
+        pass
+
+    def write(self, data):
+        # TODO(efritz) - implement
+        raise RuntimeError('Unimplemented')
+
+
 # A map from vertex labels to the fields they support. Fields
 # are ordered based on their positional argument construction.
 VERTEX_FIELDS = {
@@ -75,5 +98,6 @@ def emitter(self, *args):
                 make_emitter(type_name, name, fields),
             )
 
+
 # Meta-construct the Emitter class
 add_emitters()

lsif_exporter/file_export.py → lsif_indexer/index.py

@@ -2,7 +2,11 @@
 import os
 
 from .analysis import get_names
-from .consts import INDENT, MAX_HIGHLIGHT_RANGE
+from .emitter import Emitter, FileWriter
+from .consts import (
+    INDENT, MAX_HIGHLIGHT_RANGE,
+    POSITION_ENCODING, PROTOCOL_VERSION,
+)
 
 
 class DefinitionMeta:
@@ -18,12 +22,12 @@ def __init__(self, range_id, result_set_id, contents):
         self.reference_range_ids = set()
 
 
-class FileExporter:
+class FileIndexer:
     """
     Analysis the definitions and uses in the given file and
     add an LSIF document to the emitter. As analysis is done
     on a per-file basis, this class holds the majority of the
-    exporter logic.
+    indexer logic.
     """
     def __init__(self, filename, emitter, project_id, verbose):
         self.filename = filename
@@ -32,8 +36,8 @@ def __init__(self, filename, emitter, project_id, verbose):
         self.verbose = verbose
         self.definition_metas = {}
 
-    def export(self):
-        print('Analyzing file {}'.format(self.filename))
+    def index(self):
+        print('Indexing file {}'.format(self.filename))
 
         with open(self.filename) as f:
             source = f.read()
@@ -239,6 +243,33 @@ def _debug_use(self, name, definition):
         ))
 
 
+def index(workspace, writer, verbose):
+    """
+    Read each python file (recursively) in the given path and
+    write the analysis of each source file as an LSIF-dump to
+    the given file writer.
+    """
+    uri = 'file://{}'.format(os.path.abspath(workspace))
+
+    emitter = Emitter(FileWriter(writer))
+    emitter.emit_metadata(PROTOCOL_VERSION, POSITION_ENCODING, uri)
+    project_id = emitter.emit_project('py')
+
+    file_count = 0
+    for root, dirs, files in os.walk(workspace):
+        for file in files:
+            _, ext = os.path.splitext(file)
+            if ext != '.py':
+                continue
+
+            file_count += 1
+            path = os.path.join(root, file)
+            FileIndexer(path, emitter, project_id, verbose).index()
+
+    if file_count == 0:
+        print('No files found to index')
+
+
 def make_ranges(name):
     """
     Return a start and end range values for a range vertex.

lsif_exporter/script.py → lsif_indexer/script.py

@@ -1,27 +1,27 @@
 import argparse
 import time
 
-from .consts import (EXPORTER_VERSION, PROTOCOL_VERSION)
-from .export import export
+from .consts import (INDEXER_VERSION, PROTOCOL_VERSION)
+from .index import index
 
 
 def main():
     args = parse_args()
     start = time.time()
 
     with open(args.o, 'w+') as f:
-        export(args.workspace, f, args.verbose)
+        index(args.workspace, f, args.verbose)
 
     print('\nProcessed in {0:.2f}ms'.format((time.time() - start) * 1000))
 
 
 def parse_args():
-    parser = argparse.ArgumentParser(description='lsif-py is an LSIF exporter for Python.')
+    parser = argparse.ArgumentParser(description='lsif-py is an LSIF indexer for Python.')
     parser.add_argument('workspace', help='set the path to the code, current directory by default')
     parser.add_argument('-o', help='change the output file, "data.lsif" by default', default='data.lsif')
     parser.add_argument('-v', '--verbose', action='store_true', help='Output verbose logs', default=False)
-    parser.add_argument('--version', action='version', version='Go LSIF exporter: {}, Protocol version: {}'.format(
-        EXPORTER_VERSION,
+    parser.add_argument('--version', action='version', version='Go LSIF indexer: {}, Protocol version: {}'.format(
+        INDEXER_VERSION,
         PROTOCOL_VERSION,
     ))