Skip to content

Commit

Permalink
[cdd/compound/{exmod,exmod_utils}.py] WiP to get the latest TensorFlo…
Browse files Browse the repository at this point in the history 
…w working with SQLalchemy table emitter ; [README.md] Update all help text
  • Loading branch information
@SamuelMarks
SamuelMarks committed Mar 30, 2023
1 parent 8c4852f commit 6de6f80
Show file tree
Hide file tree
Showing 7 changed files with 396 additions and 93 deletions.
65 changes: 36 additions & 29 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -480,13 +480,12 @@ class Config(Base):
## CLI for this project

$ python -m cdd --help

usage: python -m cdd [-h] [--version]
{sync_properties,sync,gen,gen_routes,openapi,doctrans,exmod}
...

Open API to/fro routes, models, and tests. Convert between docstrings,
classes, methods, argparse, and SQLalchemy.
classes, methods, argparse, pydantic, and SQLalchemy.

positional arguments:
{sync_properties,sync,gen,gen_routes,openapi,doctrans,exmod}
Expand All @@ -511,13 +510,13 @@ class Config(Base):
### `sync`

$ python -m cdd sync --help

usage: python -m cdd sync [-h] [--argparse-function ARGPARSE_FUNCTIONS]
[--argparse-function-name ARGPARSE_FUNCTION_NAMES]
[--class CLASSES] [--class-name CLASS_NAMES]
[--function FUNCTIONS]
[--function-name FUNCTION_NAMES] --truth
{argparse_function,class,function}
[--function-name FUNCTION_NAMES] [--no-word-wrap]
--truth
{argparse_function,class,function,sqlalchemy,sqlalchemy_table}

options:
-h, --help show this help message and exit
Expand All @@ -532,14 +531,15 @@ class Config(Base):
--function-name FUNCTION_NAMES
Name of Function. If method, use Python resolution
syntax, i.e., ClassName.function_name
--truth {argparse_function,class,function}
--no-word-wrap Whether word-wrap is disabled (on emission). None
enables word-wrap. Defaults to None.
--truth {argparse_function,class,function,sqlalchemy,sqlalchemy_table}
Single source of truth. Others will be generated from
this. Will run with first found choice.

### `sync_properties`

$ python -m cdd sync_properties --help

usage: python -m cdd sync_properties [-h] --input-filename INPUT_FILENAME
--input-param INPUT_PARAMS [--input-eval]
--output-filename OUTPUT_FILENAME
Expand Down Expand Up @@ -568,17 +568,17 @@ class Config(Base):
### `gen`

$ python -m cdd gen --help

usage: python -m cdd gen [-h] --name-tpl NAME_TPL --input-mapping
INPUT_MAPPING [--prepend PREPEND]
[--imports-from-file IMPORTS_FROM_FILE]
[--parse {argparse,class,function,sqlalchemy,sqlalchemy_table}]
[--parse {argparse,class,function,json_schema,pydantic,sqlalchemy,sqlalchemy_table,infer}]
--emit
{argparse,class,function,sqlalchemy,sqlalchemy_table}
{argparse,class,function,json_schema,pydantic,sqlalchemy,sqlalchemy_table}
--output-filename OUTPUT_FILENAME [--emit-call]
[--decorator DECORATOR_LIST]
[--emit-and-infer-imports] [--no-word-wrap]
[--decorator DECORATOR_LIST] [--phase PHASE]

optional arguments:
options:
-h, --help show this help message and exit
--name-tpl NAME_TPL Template for the name, e.g., `{name}Config`.
--input-mapping INPUT_MAPPING
Expand All @@ -588,16 +588,23 @@ class Config(Base):
Extract imports from file and append to `output_file`.
If module or other symbol path given, resolve file
then use it.
--parse {argparse,class,function,sqlalchemy,sqlalchemy_table}
--parse {argparse,class,function,json_schema,pydantic,sqlalchemy,sqlalchemy_table,infer}
What type the input is.
--emit {argparse,class,function,sqlalchemy,sqlalchemy_table}
What type to generate.
--emit {argparse,class,function,json_schema,pydantic,sqlalchemy,sqlalchemy_table}
Which type to generate.
--output-filename OUTPUT_FILENAME, -o OUTPUT_FILENAME
Output file to write to.
--emit-call Whether to place all the previous body into a new
`__call__` internal function
--emit-and-infer-imports
Whether to emit and infer imports at the top of the
generated code
--no-word-wrap Whether word-wrap is disabled (on emission). None
enables word-wrap. Defaults to None.
--decorator DECORATOR_LIST
List of decorators.
--phase PHASE Which phase to run through. E.g., SQLalchemy may
require multiple phases to resolve foreign keys.


PS: If you're outputting JSON-schema and want a file per schema then:
Expand All @@ -607,7 +614,6 @@ PS: If you're outputting JSON-schema and want a file per schema then:
### `gen_routes`

$ python -m cdd gen_routes --help

usage: python -m cdd gen_routes [-h] --crud {CRUD,CR,C,R,U,D,CR,CU,CD,CRD}
[--app-name APP_NAME] --model-path MODEL_PATH
--model-name MODEL_NAME --routes-path
Expand All @@ -634,29 +640,26 @@ PS: If you're outputting JSON-schema and want a file per schema then:
### `openapi`

$ python -m cdd openapi --help

usage: python -m cdd openapi [-h] [--app-name APP_NAME] --model-paths
MODEL_PATHS --routes-paths
[ROUTES_PATHS [ROUTES_PATHS ...]]
MODEL_PATHS --routes-paths [ROUTES_PATHS ...]

optional arguments:
options:
-h, --help show this help message and exit
--app-name APP_NAME Name of app (e.g., `app_name = Bottle();
@app_name.get('/api') def slash(): pass`)
--model-paths MODEL_PATHS
Python module resolution (foo.models) or filepath
(foo/models)
--routes-paths [ROUTES_PATHS [ROUTES_PATHS ...]]
--routes-paths [ROUTES_PATHS ...]
Python module resolution 'foo.routes' or filepath
'foo/routes'

### `doctrans`

$ python -m cdd doctrans --help

usage: python -m cdd doctrans [-h] --filename FILENAME --format
{rest,google,numpydoc}
(--type-annotations | --no-type-annotations)
(--type-annotations | --no-type-annotations | --no-word-wrap)

options:
-h, --help show this help message and exit
Expand All @@ -669,22 +672,26 @@ PS: If you're outputting JSON-schema and want a file per schema then:
--no-type-annotations
Ensure all types are in docstring (rather than a
PEP484 type annotation)
--no-word-wrap Whether word-wrap is disabled (on emission). None
enables word-wrap. Defaults to None.

### `exmod`

$ python -m cdd exmod --help

usage: python -m cdd exmod [-h] --module MODULE --emit
{argparse,class,function,sqlalchemy,sqlalchemy_table}
[--blacklist BLACKLIST] [--whitelist WHITELIST]
--output-directory OUTPUT_DIRECTORY [--dry-run]
{argparse,class,function,json_schema,pydantic,sqlalchemy,sqlalchemy_table}
[--no-word-wrap] [--blacklist BLACKLIST]
[--whitelist WHITELIST] --output-directory
OUTPUT_DIRECTORY [--dry-run]

options:
-h, --help show this help message and exit
--module MODULE, -m MODULE
The module or fully-qualified name (FQN) to expose.
--emit {argparse,class,function,sqlalchemy,sqlalchemy_table}
What type to generate.
--emit {argparse,class,function,json_schema,pydantic,sqlalchemy,sqlalchemy_table}
Which type to generate.
--no-word-wrap Whether word-wrap is disabled (on emission). None
enables word-wrap. Defaults to None.
--blacklist BLACKLIST
Modules/FQN to omit. If unspecified will emit all
(unless whitelist).
Expand Down
116 changes: 62 additions & 54 deletions cdd/compound/exmod.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,28 @@
Not a dead module
"""

from ast import Assign, Expr, ImportFrom, List, Load, Module, Name, Store, alias
from collections import OrderedDict, deque
from ast import Assign, Expr, ImportFrom, List, Load, Module, Name, Store, alias, parse
from collections import deque
from functools import partial
from importlib import import_module
from inspect import getfile
from itertools import chain, groupby
from operator import itemgetter
from operator import attrgetter, itemgetter
from os import makedirs, path

import cdd.class_.parse
import cdd.shared.emit.file
from cdd.compound.exmod_utils import emit_file_on_hierarchy, get_module_contents
from cdd.shared.ast_utils import maybe_type_comment, set_value
from cdd.shared.pkg_utils import relative_filename
from cdd.shared.pure_utils import INIT_FILENAME
from cdd.compound.exmod_utils import emit_files_from_module_and_return_imports
from cdd.shared.ast_utils import (
construct_module_with_symbols,
maybe_type_comment,
set_value,
)
from cdd.shared.pure_utils import (
INIT_FILENAME,
find_module_filepath,
pp,
read_file_to_str,
rpartial,
)


def exmod(
Expand Down Expand Up @@ -86,24 +93,20 @@ def exmod(

emit_name = (
emit_name[0]
if emit_name is not None and len(emit_name) == 1 and isinstance(emit_name, list)
if emit_name is not None
and len(emit_name) == 1
and isinstance(emit_name, (list, tuple))
else emit_name
)
assert isinstance(
emit_name, (str, type(None))
), "Expected `str` got `{emit_name_type!r}`".format(emit_name_type=type(emit_name))

module_name, new_module_name = map(path.basename, (module, output_directory))
module = (
# partial(module_from_file, module_name=module_name)
# if path.isdir(module) else
import_module
)(module)

# if module.__file__ is None:
# raise ModuleNotFoundError(module_name)

module_root_dir = path.dirname(module.__file__) + path.sep
module_root_dir = (
path.dirname(find_module_filepath(*module.rsplit(".", 1))) + path.sep
)

mod_path = ".".join((path.basename(module_root_dir[:-1]), module_name))
blacklist, whitelist = map(
Expand All @@ -118,50 +121,55 @@ def exmod(
if not proceed:
return

_emit_file_on_hierarchy = partial(
emit_file_on_hierarchy,
emit_name=emit_name,
module_name=module_name,
_emit_files_from_module_and_return_imports = partial(
emit_files_from_module_and_return_imports,
new_module_name=new_module_name,
mock_imports=mock_imports,
filesystem_layout=filesystem_layout,
emit_name=emit_name,
output_directory=output_directory,
mock_imports=mock_imports,
no_word_wrap=no_word_wrap,
dry_run=dry_run,
filesystem_layout=filesystem_layout,
)

# Might need some `groupby` in case multiple files are in the one project; same for `get_module_contents`
imports = _emit_files_from_module_and_return_imports(
module_name=module_name, module=module, module_root_dir=module_root_dir
)
if not imports:
# Case: no obvious folder hierarchy, so parse the `__init__` file in root
with open(module_root_dir + "__init__{}py".format(path.extsep), "rt") as f:
mod = parse(f.read())

imports = list(
map(
_emit_file_on_hierarchy,
map(
lambda name_source: (
name_source[0],
(
lambda filename: filename[len(module_name) + 1 :]
if filename.startswith(module_name)
else filename
)(relative_filename(getfile(name_source[1]))),
{"params": OrderedDict(), "returns": OrderedDict()}
if dry_run
else cdd.class_.parse.class_(
name_source[1], merge_inner_function="__init__"
),
),
# TODO: Optimise these imports
imports = list(
chain.from_iterable(
map(
lambda name_source: (
name_source[0][len(module_name) + 1 :],
name_source[1],
lambda filepath_name_module: _emit_files_from_module_and_return_imports(
module_root_dir=filepath_name_module[0],
module_name=filepath_name_module[1],
module=filepath_name_module[2],
),
get_module_contents(
module, module_root_dir=module_root_dir
).items(),
),
),
),
)
assert len(imports), "Module contents are empty"
map(
lambda import_from: (
(
lambda module_filepath: (
module_filepath,
import_from.module,
construct_module_with_symbols(
parse(read_file_to_str(module_filepath)),
map(attrgetter("name"), import_from.names),
),
)
)(find_module_filepath(*import_from.module.rsplit(".", 1)))
),
filter(rpartial(isinstance, ImportFrom), mod.body),
),
)
)
)
pp(imports)

assert imports, "Module contents are empty"
modules_names = tuple(
map(
lambda name_module: (
Expand Down
Loading

0 comments on commit 6de6f80

Please sign in to comment.