Rename src.ast⇒ast_util, find AST end points with asttokens, move rep…

…orting out of ast_parse and into functions which know src/dst explicitly (don't surmise and confuse the control flow), begin src.editor to execute edit agenda

README.md

@@ -80,22 +80,22 @@ The idea is to run a command like `mvdef src.py dst.py fn1 fn2 fn3` to do the fo
    - If not, it would just be a matter of testing this manually (i.e. not necessary to define test
      to use tool, but suggested best practice)
 - [x] Enumerate all import statements in `src.py` (nodes in the AST of type `ast.Import`)
-   - `src.ast`⠶`annotate_imports` returns this list, which gets assigned to `imports`
-     in `src.ast`⠶`parse_mv_funcs`
+   - `src.ast_util`⠶`annotate_imports` returns this list, which gets assigned to `imports`
+     in `src.ast_util`⠶`parse_mv_funcs`
 - [x] Enumerate all function definitions in `src.py` (nodes in the AST of type `ast.FunctionDef`)
    - `ast`⠶`parse` provides this as the `.body` nodes which are of type `ast.FunctionDef`.
-     - This subset of AST nodes is assigned to `defs` in `src.ast`⠶`ast_parse`.
+     - This subset of AST nodes is assigned to `defs` in `src.ast_util`⠶`ast_parse`.
 - [x] Find the following subsets:
    - [x] `mvdefs`: subset of all function definitions which are to be moved (`fn1`, `fn2`, `fn3`)
      - This subset is determined by cross-referencing the names of the `defs` (from previous step)
        against the `mv_list` (list of functions to move, such as `["fn1", "fn2", "fn3"]`),
-       in the dedicated function `src.ast`⠶`get_def_names`, then returned by `src.ast`⠶
-       `parse_mv_funcs` as a list, assigned to `mvdefs` in `src.ast`⠶`ast_parse`.
+       in the dedicated function `src.ast_util`⠶`get_def_names`, then returned by `src.ast_tools`⠶
+       `parse_mv_funcs` as a list, assigned to `mvdefs` in `src.ast_util`⠶`ast_parse`.
    - [x] `nonmvdefs`: subset of all function definitions **not** to be moved (not in `mvdefs`)
      - This subset is determined by negative cross-ref. to names of the `defs` against the
-       `mv_list` (such as `["fn4", "fn5", "fn6"]`), again using `src.ast`⠶`get_def_names`,
-       then returned by `src.ast`⠶`parse_mv_funcs` as a list, assigned to `nonmvdefs`
-       in `src.ast`⠶`ast_parse`.
+       `mv_list` (such as `["fn4", "fn5", "fn6"]`), again using `src.ast_util`⠶`get_def_names`,
+       then returned by `src.ast_util`⠶`parse_mv_funcs` as a list, assigned to `nonmvdefs`
+       in `src.ast_util`⠶`ast_parse`.
    - [x] `mv_imports`: Import statements used only by the functions in `mvdefs`
    - [x] `nonmv_imports`: Import statements used only by the functions in `nonmvdefs`
    - [x] `mutual_imports`: Import statements used by both functions in `mvdefs` and `nonmvdefs`
@@ -119,7 +119,7 @@ The idea is to run a command like `mvdef src.py dst.py fn1 fn2 fn3` to do the fo
      `src.demo`⠶`run_demo` has returned a parsed version of the source and destination files
      (which will only matter once the parameter `nochange` is set to `False` in `run_demo`,
      allowing it to propagate through the call to `src.demo`⠶`parse_example` into a call to
-     `src.ast`⠶`ast_parse(..., edit=True)` and ultimately carry out in-place editing of the
+     `src.ast_util`⠶`ast_parse(..., edit=True)` and ultimately carry out in-place editing of the
      source and/or destination file/s as required).
    - [ ] If they fail, ask to restore the backup and give the altered src/dst `.py` files
      `.py.mvdef_fix` suffixes (i.e. always permit the user to exit gracefully with no further

mvdef/example/demo_program.py

@@ -1,7 +1,11 @@
 import numpy as np
 from numpy import arange, pi
 import matplotlib.pyplot as plt
-from os.path import basename, sep as pathsep, islink
+from os.path import (
+    basename as aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,
+    sep as pathsep,
+    islink,
+)
 
 
 def show_line(n=4, suppress_display=False):

mvdef/src/ast_tokens.py

@@ -0,0 +1,44 @@
+from asttokens import ASTTokens
+from ast import Import as IType, ImportFrom as IFType, walk
+
+def get_imports(source_filepath, index_list=None):
+    with open(source_filepath, 'r') as f:
+        source = f.read()
+    fl = source.split('\n')
+    a = ASTTokens(source, parse=True)
+    imports = [t for t in walk(a.tree) if type(t) in (IType, IFType)]
+    if index_list is not None:
+        imports = []
+        for (n, n_i) in index_list:
+            return [imports[i] for i in index_list]
+    return imports
+
+def count_imported_names(nodes):
+    """
+    Return an integer for a single node (0 if not an import statement),
+    else return a list of integers for a list of AST nodes.
+    """
+    if type(nodes) is not list:
+        if type(nodes) in [IType, IFType]:
+            return len(nodes.names)
+        else:
+            assert ast.stmt in type(nodes).mro(), f"{nodes} is not an AST statement"
+            return 0
+    counts = []
+    for node in nodes:
+        if type(node) in [IType, IFType]:
+            c = len(node.names)
+            counts.append(c)
+        else:
+            assert ast.stmt in type(nodes).mro(), f"{nodes} is not an AST statement"
+            counts.append(0)
+    return counts
+
+def locate_import_ends(source_filepath, index_list=None):
+    ends = []
+    nodes = get_imports(source_filepath, index_list)
+    for n in nodes:
+        end = {}
+        end["line"], end["index"] = n.last_token.end[0]
+        ends.append(end)
+    return ends

mvdef/src/ast.py → mvdef/src/ast_util.py

@@ -3,6 +3,72 @@
 from collections import OrderedDict
 import builtins
 from src.display import colour_str as colour
+from asttokens import ASTTokens
+from src.editor import edit_defs
+
+
+def ast_parse(py_file, mv_list=[], report=True, edit=False, backup=True):
+    """
+    Build and arse the Abstract Syntax Tree (AST) of a Python file, and either return
+    a report of what changes would be required to move the mv_list of funcdefs out
+    of it, or a report of the imports and funcdefs in general if no mv_list is
+    provided (taken to indicate that the file is the target funcdefs are moving to),
+    or make changes to the file (either newly creating one if no such file exists,
+    or editing in place according to the reported import statement differences).
+
+    If the py_file doesn't exist, it's being newly created by the move and obviously
+    no report can be made on it: it has no funcdefs and no import statements, so
+    all the ones being moved will be newly created.
+
+    mv_list should be given if the file is the source of moved functions, and left
+    empty (defaulting to value of []) if the file is the destination to move them to.
+    
+    If report is True, returns a string describing the changes
+    to be made (if False, nothing is returned).
+    
+    If edit is True, files will be changed in place (note that this function does
+    not actually do the editing, it just returns the edit agenda and uses the edit
+    parameter as a sanity check to prevent wasted computation if neither edit nor
+    report is True).
+
+    If backup is True, files will be changed in place by calling src.backup.backup
+    (obviously, be careful switching this setting off if report is True, as any
+    changes made cannot be restored afterwards from this backup file).
+    """
+    assert True in (edit, report), "Nothing to do"
+    extant = py_file.exists() and py_file.is_file()
+    if extant:
+        with open(py_file, "r") as f:
+            fc = f.read()
+            # a = ast
+            nodes = ast.parse(fc).body
+
+        imports = [n for n in nodes if type(n) in [ast.Import, ast.ImportFrom]]
+        defs = [n for n in nodes if type(n) == ast.FunctionDef]
+        # return imports, funcdefs
+        edit_agenda = process_imports(py_file, mv_list, defs, imports, report)
+
+        if mv_list == [] and report:
+            # The mv_list is empty if it was not passed in at all, i.e. this indicates
+            # no files are to be moved from py_file, i.e. they are moving into py_file
+            # extant is True so non_mvdef is just all funcdefs for the file
+            print(f"⇒ No functions to move from {colour('light_gray', py_file)}")
+        elif mv_list != [] and report:
+            print(f"⇒ Functions moving from {colour('light_gray',py_file)}: {mv_list}")
+        elif report:
+            print(f"⇒ Functions moving to {colour('light_gray', py_file)}")
+        return edit_agenda
+    elif mv_list == [] and report:
+        # not extant so file doesn't exist (cannot produce a parsed AST)
+        # however mv_list is [] so file must be dst
+        print(
+            f"⇒ Functions will move to {colour('light_gray', py_file)}"
+            + " (it's being created from them)"
+        )
+        return
+    else:
+        raise ValueError(f"Can't move {mv_list} from {py_file} – it doesn't exist!")
+    return
 
 
 def annotate_imports(imports, report=True):
@@ -346,65 +412,6 @@ def process_imports(fp, mv_list, defs, imports, report=True, edit=False):
     return edit_agenda
 
 
-def ast_parse(py_file, mv_list=[], report=True, edit=False, backup=True):
-    """
-    Build and arse the Abstract Syntax Tree (AST) of a Python file, and either return
-    a report of what changes would be required to move the mv_list of funcdefs out
-    of it, or a report of the imports and funcdefs in general if no mv_list is
-    provided (taken to indicate that the file is the target funcdefs are moving to),
-    or make changes to the file (either newly creating one if no such file exists,
-    or editing in place according to the reported import statement differences).
-
-    If the py_file doesn't exist, it's being newly created by the move and obviously
-    no report can be made on it: it has no funcdefs and no import statements, so
-    all the ones being moved will be newly created.
-
-    mv_list should be given if the file is the source of moved functions, and left
-    empty (defaulting to value of []) if the file is the destination to move them to.
-    
-    If report is True, returns a string describing the changes
-    to be made (if False, nothing is returned).
-    
-    If edit is True, files will be changed in place.
-
-    If backup is True, files will be changed in place by calling src.backup.backup
-    (obviously, be careful switching this setting off if report is True, as any
-    changes made cannot be restored afterwards from this backup file).
-    """
-    assert True in (edit, report), "Nothing to do"
-    extant = py_file.exists() and py_file.is_file()
-    if extant:
-        with open(py_file, "r") as f:
-            fc = f.read()
-            nodes = ast.parse(fc).body
-
-        imports = [n for n in nodes if type(n) in [ast.Import, ast.ImportFrom]]
-        defs = [n for n in nodes if type(n) == ast.FunctionDef]
-        # return imports, funcdefs
-        edit_agenda = process_imports(py_file, mv_list, defs, imports, report)
-
-        if mv_list == [] and report:
-            # The mv_list is empty if it was not passed in at all, i.e. this indicates
-            # no files are to be moved from py_file, i.e. they are moving into py_file
-            # extant is True so non_mvdef is just all funcdefs for the file
-            print(f"⇒ No functions to move from {colour('light_gray', py_file)}")
-        elif mv_list != [] and report:
-            print(f"⇒ Functions moving from {colour('light_gray',py_file)}: {mv_list}")
-        elif report:
-            print(f"⇒ Functions moving to {colour('light_gray', py_file)}")
-
-        if edit:
-            # Act out the changes specified in edit_agenda
-            pass
-    elif mv_list == [] and report:
-        # not extant so file doesn't exist, however mv_list is [] so file must be dst
-        print(
-            f"⇒ Functions will move to {colour('light_gray', py_file)}"
-            + " (it's being created from them)"
-        )
-    return
-
-
 def spare_mvdef_func():
     """
     Wrote this on a misunderstanding of what mvdef_import should be, but might reuse

mvdef/src/backup.py

@@ -5,7 +5,7 @@ def backup(filepath, dry_run=False, suffix=".backup", hidden=True):
     an incrementing integer. If dry_run is True, all the checks for the possibility of
     creating the backup will be run, but no files will be opened or touched. (To report
     on the funcdef/import statements to be moved without having to adhere to these
-    requirements, don't call `src.backup.backup` before calling `src.ast.ast_parse`).
+    requirements, don't call `src.backup.backup` before calling `src.ast_util.ast_parse`).
     """
     fd = filepath.parent
     assert fd.exists() and fd.is_dir(), "Can't backup {filepath}: {fd} doesn't exist"

mvdef/src/demo.py

@@ -1,7 +1,8 @@
 from sys import path as syspath
-from src.ast import ast_parse
+from src.ast_util import ast_parse
 from src.backup import backup
 from src.__env__ import example_dir
+from src.display import colour_str as colour
 from example.test.test_demo_program import test_report
 
 # TODO: Move parse_example to AST once logic is figured out for the demo
@@ -10,6 +11,7 @@ def parse_example(src_p, dst_p, move_list, report=True, nochange=True):
     # and creating a hidden placeholder if the target doesn't exist yet
     assert backup(src_p, dry_run=nochange)
     assert backup(dst_p, dry_run=nochange)
+    # Create edit agendas from the parsed AST of source and destination files
     src_parsed = ast_parse(src_p, move_list, report=report, edit=(not nochange))
     dst_parsed = ast_parse(dst_p, report=report, edit=(not nochange))
     return src_parsed, dst_parsed
@@ -26,12 +28,11 @@ def main(mvdefs, dry_run=True, report=True):
     src_parsed, dst_parsed = parse_example(
         src_p, dst_p, move_list=mvdefs, report=report, nochange=dry_run
     )
-    if dst_parsed is None:
-        dst_parsed = "(Dst will take all src_parsed imports and funcdefs)"
     # src imports, src_funcdefs = src_parsed
-    src_ret = src_parsed
-    if type(dst_parsed) is str:
-        if report: print(dst_parsed)
+    assert src_parsed is not None, "The src file did not return a processed AST"
+    if dst_parsed is None:
+        if report:
+            print("(Dst will take all src_parsed imports and funcdefs)")
     else:
         dst_imports, dst_funcdefs = dst_parsed
     if dry_run:

mvdef/src/editor.py

@@ -0,0 +1,13 @@
+from src.ast_tokens import get_imports, count_imported_names, locate_import_ends
+
+def edit_defs(src_path, dst_path, imports, defs, edit_agenda):
+    for mvdef in edit_agenda.get("move"):
+        # Move the mvdefs out of the source file
+        pass
+    for cpdef in edit_agenda.get("copy"):
+        # Copy the cpdefs over
+        pass
+    for rmdef in edit_agenda.get("lose"):
+        # Just comment out the rmdefs for now
+        pass
+    return