release 5.579.24081

.gitignore

@@ -19,3 +19,4 @@
 
 /sql/a.sql
 /sql/b.sql
+/MANIFEST.in

jx_sqlite/expressions/_utils.py

@@ -76,7 +76,7 @@ def multiop_to_sql(self, schema):
         temp = [SqlCoalesceOp(t, zero).partial_eval(SQLang).to_sql(schema).expr for t in self.terms]
         expr = iso(sign.join(sql_iso(t) for t in temp))
     else:
-        miss = OrOp(*(t.missing(SQLang) for t in self.terms), nulls=True)
+        miss = OrOp(*(t.missing(SQLang) for t in self.terms), nulls=False)
         expr = iso(sign.join(sql_iso(t.partial_eval(SQLang).to_sql(schema)) for t in self.terms))
 
     return SqlScript(jx_type=jx_type, expr=expr, frum=self, miss=miss, schema=schema)
@@ -103,7 +103,7 @@ def basic_multiop_to_sql(self, schema, many=False):
 
 
 _sql_operators = {
-    # (operator, zero-array default value) PAIR
+    # (operator, zero-array default value) TUPLE
     "add": (sql_iso, SQL_PLUS, ZERO, JX_NUMBER),
     "sum": (sql_iso, SQL_PLUS, ZERO, JX_NUMBER),
     "mul": (sql_iso, SQL_STAR, ONE, JX_NUMBER),

jx_sqlite/expressions/or_op.py

@@ -26,7 +26,7 @@ def to_sql(self, schema) -> SqlScript:
         return SqlScript(
             jx_type=JX_BOOLEAN,
             miss=FALSE,
-            expr=SqlOrOp(*(t.to_sql(schema) for t in self.terms)),
+            expr=SqlOrOp(*(t.to_sql(schema).expr for t in self.terms)),
             frum=self,
             schema=schema,
         )

jx_sqlite/insert.py

@@ -34,7 +34,7 @@
     relative_field, exists,
 )
 from mo_future import text, first, extend
-from mo_json import STRUCT, ARRAY, OBJECT, value_to_json_type, get_if_type
+from mo_json import STRUCT, ARRAY, OBJECT, value_to_json_type, get_if_type, jx_type_to_json_type, base_type
 from mo_logs import logger
 from mo_sql.utils import json_type_to_sql_type_key
 from mo_sqlite import (
@@ -93,27 +93,30 @@ def update(self, command):
             logger.error("Deep update not supported")
 
     # ADD NEW COLUMNS
-    where = jx_expression(command.where) or TRUE
+    where = jx_expression(command.where) if command.where else TRUE
     _vars = where.vars()
     _map = {v: c.es_column for v in _vars for _, c in self.schema.leaves(v) if c.json_type not in STRUCT}
     where_sql = where.map(_map).to_sql(self.schema)
     new_columns = set(command.set.keys()) - set(c.name for c in self.schema.columns)
     for new_column_name in new_columns:
-        nested_value = command.set[new_column_name]
-        ctype = value_to_json_type(nested_value)
+        expr = jx_expression(command.set[new_column_name])
+        json_type = jx_type_to_json_type(expr.jx_type)
         column = Column(
             name=new_column_name,
-            json_type=ctype,
+            json_type=json_type,
             es_index=self.name,
-            es_type=json_type_to_sqlite_type(ctype),
-            es_column=typed_column(new_column_name, ctype),
+            es_type=json_type_to_sqlite_type[json_type],
+            es_column=typed_column(new_column_name, json_type_to_sql_type_key[json_type]),
+            nested_path=[self.name],
+            multi=1,
+            cardinality=1,
             last_updated=Date.now(),
         )
-        self.add_column(column)
+        self.snowflake._add_column(column)
 
     # UPDATE THE ARRAY VALUES
-    for nested_column_name, nested_value in command.set.items():
-        if value_to_json_type(nested_value) == "nested":
+    for nested_column_name, expr in command.set.items():
+        if value_to_json_type(expr) == "nested":
             nested_table_name = concat_field(self.name, nested_column_name)
             nested_table = nested_tables[nested_column_name]
             self_primary_key = sql_list(quote_column(c.es_column) for u in self.uid for c in self.columns[u])
@@ -147,11 +150,11 @@ def update(self, command):
             self.container.db.execute(sql_command)
 
             # INSERT NEW RECORDS
-            if not nested_value:
+            if not expr:
                 continue
 
             doc_collection = {}
-            for d in listwrap(nested_value):
+            for d in listwrap(expr):
                 nested_table.flatten(d, Data(), doc_collection, path=nested_column_name)
 
             prefix = ConcatSQL(
@@ -238,15 +241,15 @@ def update(self, command):
         SQL_UPDATE,
         quote_column(self.name),
         SQL_SET,
-        sql_list(
-            [
-                ConcatSQL(quote_column(c.es_column), SQL_EQ, quote_value(get_if_type(v, c.json_type)),)
+        sql_list([
+            *(
+                ConcatSQL(quote_column(c.es_column), SQL_EQ, jx_expression(v).to_sql(self.schema))
                 for c in self.schema.columns
                 if c.json_type != ARRAY and len(c.nested_path) == 1
                 for v in [command.set[c.name]]
                 if v != None
-            ]
-            + [
+            ),
+            *(
                 ConcatSQL(quote_column(c.es_column), SQL_EQ, SQL_NULL)
                 for c in self.schema.columns
                 if (
@@ -255,8 +258,8 @@ def update(self, command):
                     and c.json_type != ARRAY
                     and len(c.nested_path) == 1
                 )
-            ]
-        ),
+            )
+        ]),
         SQL_WHERE,
         where_sql,
     )

jx_sqlite/meta_columns.py

@@ -10,15 +10,17 @@
 from copy import copy
 
 import jx_base
-from jx_base import Schema, Table, Container, Column, Snowflake
+from jx_base import Schema, Container, Column, Snowflake
 from jx_base.meta_columns import (
     META_COLUMNS_DESC,
     META_COLUMNS_NAME,
     SIMPLE_METADATA_COLUMNS,
+    META_TABLES_NAME,
 )
 from jx_python import jx
+from jx_sqlite.models.table import Table
 from jx_sqlite.utils import untyped_column, untype_field
-from mo_dots import Data, Null, coalesce, is_data, is_list, startswith_field, unwraplist, wrap, list_to_data
+from mo_dots import Data, Null, coalesce, is_data, is_list, startswith_field, unwraplist, list_to_data, to_data
 from mo_future import first
 from mo_json import STRUCT, IS_NULL
 from mo_json.typed_encoder import detype
@@ -54,7 +56,7 @@ def __init__(self, db):
             if hasattr(self, "db"):
                 return
             self.db = db
-        Table.__init__(self, META_COLUMNS_NAME)
+        Table.__init__(self, [META_COLUMNS_NAME], self)
         self.data = {}  # MAP FROM fact_name TO (abs_column_name to COLUMNS)
         self.locker = Lock()
         self._schema = None
@@ -445,9 +447,13 @@ def namespace(self):
         return self
 
     def get_table(self, table_name):
-        if table_name != META_COLUMNS_NAME:
-            Log.error("this container has only the " + META_COLUMNS_NAME)
-        return self
+        if table_name in (META_TABLES_NAME, META_COLUMNS_NAME):
+            return Table([table_name], self)
+        Log.error("this container has only the " + META_COLUMNS_NAME)
+
+    def find_snowflake(self, fact_name):
+        if fact_name in (META_TABLES_NAME, META_COLUMNS_NAME):
+            return [fact_name]
 
     def get_columns(self, table_name):
         if table_name != META_COLUMNS_NAME:
@@ -503,12 +509,12 @@ def denormalized(self):
         from jx_python.containers.list import ListContainer
 
         return ListContainer(
-            self.name, data=output, schema=jx_base.Schema(META_COLUMNS_NAME, SIMPLE_METADATA_COLUMNS),
+            self.name, data=output, schema=jx_base.Schema([META_COLUMNS_NAME], SIMPLE_METADATA_COLUMNS),
         )
 
 
 def doc_to_column(doc):
-    return Column(**wrap(detype(doc)))
+    return Column(**to_data(detype(doc)))
 
 
 def mark_as_deleted(col):

jx_sqlite/models/container.py

@@ -154,7 +154,7 @@ def drop(self, item):
             logger.error("do not know how to handle {item}", item=item)
 
     def drop_facts(self, fact_name):
-        paths = self.namespace.columns._snowflakes.get(fact_name)
+        paths = self.namespace.find_snowflake(fact_name)
         if paths:
             with self.db.transaction() as t:
                 for p in paths:
@@ -193,6 +193,8 @@ def get_or_create_facts(self, fact_name, uid=UID):
 
         return Facts(fact_name, self)
 
+    get_or_create_table = get_or_create_facts
+
     def get_table(self, table_name):
         nested_path = self.namespace.columns.get_nested_path(table_name)
         return Table(nested_path, self)

jx_sqlite/models/namespace.py

@@ -38,8 +38,11 @@ def rename_tables(self, name_map):
         output.columns = self.columns.rename_tables(name_map)
         return output
 
-
-
+    def find_snowflake(self, fact_name):
+        """
+        RETURN ALL PATHS IF EXISTS
+        """
+        return self.columns._snowflakes.get(fact_name)
 
     def get_facts(self, fact_name):
         snowflake = Snowflake(fact_name, self)

jx_sqlite/models/snowflake.py

@@ -57,7 +57,7 @@ class Snowflake(jx_base.Snowflake):
     """
 
     def __init__(self, fact_name, namespace):
-        if not namespace.columns._snowflakes.get(fact_name):
+        if not namespace.find_snowflake(fact_name):
             Log.error("{name} does not exist", name=fact_name)
         self.fact_name = fact_name  # THE CENTRAL FACT TABLE
         self.namespace = namespace
@@ -305,7 +305,7 @@ def new_nested_path(c):
             all_columns.add(c)
 
     def add_table(self, nested_path):
-        query_paths = self.namespace.columns._snowflakes[self.fact_name]
+        query_paths = self.namespace.find_snowflake(self.fact_name)
         if nested_path in query_paths:
             Log.error("table exists")
         query_paths.append(nested_path[0])

jx_sqlite/query.py

@@ -154,10 +154,9 @@ def get_table(self, table_name):
 
 @extend(Facts)
 def query_metadata(self, query):
-    frum, query["from"] = query["from"], self
-    columns = self.snowflake.columns
+    container = self.container.namespace.columns.denormalized()
+    query = QueryOp.wrap(query, container, JX)
 
-    query = QueryOp.wrap(query, self.container, JX)
     where = query.where
     table_name = None
     column_name = None
@@ -172,8 +171,10 @@ def query_metadata(self, query):
     else:
         raise Log.error('Only simple filters are expected like: "eq" on table and column name')
 
+
     tables = [concat_field(self.snowflake.fact_name, i) for i in self.tables.keys()]
 
+    columns = self.snowflake.columns
     metadata = []
     if columns[-1].es_column != GUID:
         columns.append(Column(

packaging/setup.py

@@ -8,13 +8,13 @@
     description='JSON query expressions using SQLite',
     extras_require={"tests":["mo-testing>=7.562.24075"]},
     include_package_data=True,
-    install_requires=["hjson","jx-python==4.576.24080","mo-sql==4.576.24080","mo-sqlite==2.576.24080","requests"],
+    install_requires=["hjson","jx-python==4.579.24081","mo-sql==4.579.24081","mo-sqlite==2.579.24081","requests"],
     license='MPL 2.0',
     long_description='# jx-sqlite \n\nJSON query expressions using SQLite\n\n\n[![PyPI Latest Release](https://img.shields.io/pypi/v/jx-sqlite.svg)](https://pypi.org/project/jx-sqlite/)\n[![Build Status](https://app.travis-ci.com/klahnakoski/jx-sqlite.svg?branch=master)](https://travis-ci.com/github/klahnakoski/jx-sqlite)\n[![Coverage Status](https://coveralls.io/repos/github/klahnakoski/jx-sqlite/badge.svg?branch=dev)](https://coveralls.io/github/klahnakoski/jx-sqlite?branch=dev)\n\n## Summary\n\nThis library will manage your database schema to store JSON documents. You get all the speed of a well-formed database schema without the schema migration headaches. \n\n\n## Status\n\nSignificant updates to the supporting libraries has broken this code.  It still works for the simple cases that require it\n\n**Jan 2024** - 118 of 334 tests ignored\n\n\n## Installation\n\n    pip install jx-sqlite\n\n## Code Example\n\nThe smoke test, found in the `tests` is a simple example of how to use this library.\n\n```python\nimport jx_sqlite\n\ntable = (\n    jx_sqlite\n    .Container(filename="my.db")\n    .get_or_create_facts("my_table")\n    .insert([{"os": "linux", "value": 42}])\n    .query({\n        "select": "os",\n        "where": {"gt": {"value": 0}}\n    })\n)\n```\n\n## More\n\nThis project is an attempt to store JSON documents in SQLite so that they are accessible via SQL. The hope is this will serve a basis for a general document-relational map (DRM), and leverage the database\'s query optimizer.\n`jx-sqlite` is responsible for expanding the schema dynamically as new JSON documents are encountered.  It also strives to ensure old queries against the new schema have the same meaning; the same results.\n\nThe most interesting, and most important feature is that we query nested object arrays as if they were just another table.  This is important for two reasons:\n\n1. Inner objects `{"a": {"b": 0}}` are a shortcut for nested arrays `{"a": [{"b": 0}]}`, plus\n2. Schemas can be expanded from one-to-one  to one-to-many `{"a": [{"b": 0}, {"b": 1}]}`.\n\n\n## Motivation\n\nJSON is a nice format to store data, and it has become quite prevalent. Unfortunately, databases do not handle it well, often a human is required to declare a schema that can hold the JSON before it can be queried. If we are not overwhelmed by the diversity of JSON now, we soon will be. There will be more JSON, of more different shapes, as the number of connected devices( and the information they generate) continues to increase.\n\n## Contributing\n\nContributions are always welcome! The best thing to do is find a failing test, and try to fix it.\n\nThese instructions will get you a copy of the project up and running on your local machine for development and testing purposes.\n\n    $ git clone https://github.com/klahnakoski/jx-sqlite\n    $ cd jx-sqlite\n\n### Running tests\n\nThere are over 300 tests used to confirm the expected behaviour: They test a variety of JSON forms, and the queries that can be performed on them. Most tests are further split into three different output formats ( list, table and cube).\n\nThe `requirements.lock` file is the last successful combination that passed all tests, despite the version conflicts.\n\n    python.exe -m pip install --no-deps -r tests\\requirements.lock\n\n\nLinux\n\n    export PYTHONPATH=.:vendor\n    python -m unittest discover -v -s tests\n\nWindows\n\n    set PYTHONPATH=.;vendor\n    python -m unittest discover -v -s tests\n\n\n### Technical Docs\n\n* [Json Query Expression](https://github.com/klahnakoski/ActiveData/blob/dev/docs/jx.md)\n* [Nomenclature](https://github.com/mozilla/jx-sqlite/blob/master/docs/Nomenclature.md)\n* [Snowflake](https://github.com/mozilla/jx-sqlite/blob/master/docs/Perspective.md)\n* [JSON in Database](https://github.com/mozilla/jx-sqlite/blob/master/docs/JSON%20in%20Database.md)\n* [The Future](https://github.com/mozilla/jx-sqlite/blob/master/docs/The%20Future.md)\n\n## License\n\nThis project is licensed under Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/.\n\n\n## History\n\n*Jan 2024* - Attempt to resurrect this project (118 of 334 tests ignored)\n\n*Sep 2018* - Upgrade libs, start refactoring to work with other libs\n\n*Dec 2017* - A number of tests were added, but they do not pass.\n\n*Sep 2017* - GSoC work completed, all but a few tests pass.\n \n\n## GSOC\n\nGood work by Rohit Kumar.  You may see the end result on [gsoc branch](https://github.com/klahnakoski/jx-sqlite/tree/gsoc).  Installation requires python2.7,  and will require some version fixing to get running.\n\nSee [the demonstration video](https://www.youtube.com/watch?v=0_YLzb7BegI&list=PLSE8ODhjZXja7K1hjZ01UTVDnGQdx5v5U&index=26&t=260s)\n\n\nWork done up to the deadline of GSoC\'17:\n\n* [Pull Requests](https://github.com/mozilla/jx-sqlite/pulls?utf8=%E2%9C%93&q=is%3Apr%20author%3Arohit-rk)\n* [Commits](https://github.com/mozilla/jx-sqlite/commits?author=rohit-rk)\n\n\n\n',
     long_description_content_type='text/markdown',
     name='jx-sqlite',
     package_dir={},
     packages=["jx_sqlite","jx_sqlite.models","jx_sqlite.expressions"],
     url='https://github.com/klahnakoski/jx-sqlite',
-    version='5.576.24080'
+    version='5.579.24081'
 )

packaging/setuptools.json

@@ -19,8 +19,8 @@
     "extras_require": {"tests": ["mo-testing>=7.562.24075"]},
     "include_package_data": true,
     "install_requires": [
-                         "hjson", "jx-python==4.576.24080",    "mo-sql==4.576.24080",
-        "mo-sqlite==2.576.24080",               "requests"
+                         "hjson", "jx-python==4.579.24081",    "mo-sql==4.579.24081",
+        "mo-sqlite==2.579.24081",               "requests"
     ],
     "license": "MPL 2.0",
     "long_description": {
@@ -160,5 +160,5 @@
     "package_dir": {},
     "packages": ["jx_sqlite", "jx_sqlite.models", "jx_sqlite.expressions"],
     "url": "https://github.com/klahnakoski/jx-sqlite",
-    "version": "5.576.24080"
+    "version": "5.579.24081"
 }