Added lambda subcommand, closes #2

README.md

@@ -21,3 +21,22 @@ For example, if a row in the database has an `opened` column which contains `10/
 Will result in that value being replaced by `2019-10-10T20:10:00`.
 
 Using the `parsedate` subcommand here would result in `2019-10-10` instead.
+
+## lambda for executing your own code
+
+The `lambda` subcommand lets you specify Python code which will be executed against the column.
+
+Here's how to convert a column to uppercase:
+
+    $ sqlite-transform lambda my.db mytable mycolumn --code='return str(value).upper()'
+
+The code you provide will be compiled into a function that takes `value` as a single argument. You can break your function body into multiple lines:
+
+    $ sqlite-transform lambda my.db mytable mycolumn --code='value = str(value)
+    return value.upper()'
+
+You can also specify Python modules that should be imported and made available to your code using one or more `--import` options:
+
+    $ sqlite-transform lambda my.db mytable mycolumn \
+        --code='return "\n".join(textwrap.wrap(value, 10))' \
+        --import=textwrap

sqlite_transform/cli.py

@@ -3,6 +3,8 @@
 import sqlite3
 import tqdm
 
+sqlite3.enable_callback_tracebacks(True)
+
 
 @click.group()
 @click.version_option()
@@ -40,6 +42,33 @@ def parsedatetime(db_path, table, columns):
     _transform(db_path, table, columns, lambda v: parser.parse(v).isoformat())
 
 
+@cli.command(name="lambda")
+@click.argument(
+    "db_path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@click.argument("table", type=str)
+@click.argument("columns", type=str, nargs=-1)
+@click.option("--code", type=str, required=True)
+@click.option("--import", "imports", type=str, multiple=True)
+def lambda_(db_path, table, columns, code, imports):
+    """
+    Transform columns using Python code you supply
+    """
+    # First we need to build the code into a function body called fn(value)
+    new_code = ["def fn(value):"]
+    for line in code.split("\n"):
+        new_code.append("    {}".format(line))
+    code_o = compile("\n".join(new_code), "<string>", "exec")
+    locals = {}
+    globals = {}
+    for import_ in imports:
+        globals[import_] = __import__(import_)
+    exec(code_o, globals, locals)
+    _transform(db_path, table, columns, locals["fn"])
+
+
 def _transform(db_path, table, columns, fn):
     db = sqlite3.connect(db_path)
     count_sql = "select count(*) from [{}]".format(table)
@@ -51,10 +80,7 @@ def _transform_value(v):
             bar.update(1)
             if not v:
                 return v
-            try:
-                return fn(v)
-            except Exception as e:
-                return str(e)
+            return fn(v)
 
         db.create_function("transform", 1, _transform_value)
         sql = "update [{table}] set {sets};".format(

tests/conftest.py

@@ -0,0 +1,17 @@
+import pathlib
+import pytest
+import sqlite_utils
+
+
+@pytest.fixture
+def test_db(tmpdir):
+    db_path = str(pathlib.Path(tmpdir) / "data.db")
+    db = sqlite_utils.Database(db_path)
+    db["example"].insert_all(
+        [
+            {"id": 1, "dt": "5th October 2019 12:04"},
+            {"id": 2, "dt": "6th October 2019 00:05:06"},
+        ],
+        pk="id",
+    )
+    return db_path

tests/test_lambda.py

@@ -0,0 +1,62 @@
+from click.testing import CliRunner
+from sqlite_transform import cli
+import sqlite_utils
+
+
+def test_lambda_single_line(test_db):
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "lambda",
+            test_db,
+            "example",
+            "dt",
+            "--code",
+            "return value.replace('October', 'Spooktober')",
+        ],
+    )
+    assert 0 == result.exit_code, result.output
+    assert [
+        {"id": 1, "dt": "5th Spooktober 2019 12:04"},
+        {"id": 2, "dt": "6th Spooktober 2019 00:05:06"},
+    ] == list(sqlite_utils.Database(test_db)["example"].rows)
+
+
+def test_lambda_multiple_lines(test_db):
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "lambda",
+            test_db,
+            "example",
+            "dt",
+            "--code",
+            "v = value.replace('October', 'Spooktober')\nreturn v.upper()",
+        ],
+    )
+    assert 0 == result.exit_code, result.output
+    assert [
+        {"id": 1, "dt": "5TH SPOOKTOBER 2019 12:04"},
+        {"id": 2, "dt": "6TH SPOOKTOBER 2019 00:05:06"},
+    ] == list(sqlite_utils.Database(test_db)["example"].rows)
+
+
+def test_lambda_import(test_db):
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "lambda",
+            test_db,
+            "example",
+            "dt",
+            "--code",
+            "return re.sub('O..', 'OXX', value)",
+            "--import",
+            "re",
+        ],
+    )
+    assert 0 == result.exit_code, result.output
+    assert [
+        {"id": 1, "dt": "5th OXXober 2019 12:04"},
+        {"id": 2, "dt": "6th OXXober 2019 00:05:06"},
+    ] == list(sqlite_utils.Database(test_db)["example"].rows)

tests/test_parsedate.py

@@ -1,38 +1,20 @@
 from click.testing import CliRunner
 from sqlite_transform import cli
-import pathlib
-import pytest
 import sqlite_utils
 
 
-@pytest.fixture
-def db_with_dates(tmpdir):
-    db_path = str(pathlib.Path(tmpdir) / "data.db")
-    db = sqlite_utils.Database(db_path)
-    db["example"].insert_all(
-        [
-            {"id": 1, "dt": "5th October 2019 12:04"},
-            {"id": 2, "dt": "6th October 2019 00:05:06"},
-        ],
-        pk="id",
-    )
-    return db_path
-
-
-def test_parsedate(db_with_dates):
-    result = CliRunner().invoke(cli.cli, ["parsedate", db_with_dates, "example", "dt"])
+def test_parsedate(test_db):
+    result = CliRunner().invoke(cli.cli, ["parsedate", test_db, "example", "dt"])
     assert 0 == result.exit_code, result.output
     assert [{"id": 1, "dt": "2019-10-05"}, {"id": 2, "dt": "2019-10-06"}] == list(
-        sqlite_utils.Database(db_with_dates)["example"].rows
+        sqlite_utils.Database(test_db)["example"].rows
     )
 
 
-def test_parsedatetime(db_with_dates):
-    result = CliRunner().invoke(
-        cli.cli, ["parsedatetime", db_with_dates, "example", "dt"]
-    )
+def test_parsedatetime(test_db):
+    result = CliRunner().invoke(cli.cli, ["parsedatetime", test_db, "example", "dt"])
     assert 0 == result.exit_code, result.output
     assert [
         {"id": 1, "dt": "2019-10-05T12:04:00"},
         {"id": 2, "dt": "2019-10-06T00:05:06"},
-    ] == list(sqlite_utils.Database(db_with_dates)["example"].rows)
+    ] == list(sqlite_utils.Database(test_db)["example"].rows)