insert_all(..., alter=True) works for columns introduced after first …

…100 records

* Insert all columns for every chunk
* Update unit test to reflect new behaviour
* Test that exception is raised
* Update documentation

Closes #139. Thanks, Simon Wiles!

docs/python-api.rst

@@ -410,7 +410,7 @@ Use it like this:
         "is_good_dog": True,
     }], pk="id", column_order=("id", "twitter", "name"))
 
-The column types used in the ``CREATE TABLE`` statement are automatically derived from the types of data in that first batch of rows. Any additional or missing columns in subsequent batches will be ignored.
+The column types used in the ``CREATE TABLE`` statement are automatically derived from the types of data in that first batch of rows. Any additional columns in subsequent batches will cause a ``sqlite3.OperationalError`` exception to be raised unless the ``alter=True`` argument is supplied, in which case the new columns will be created.
 
 The function can accept an iterator or generator of rows and will commit them according to the batch size. The default batch size is 100, but you can specify a different size using the ``batch_size`` parameter:
 

sqlite_utils/db.py

@@ -1074,6 +1074,14 @@ def insert_all(
                 all_columns = list(sorted(all_columns))
                 if hash_id:
                     all_columns.insert(0, hash_id)
+            else:
+                all_columns += [
+                    column
+                    for record in chunk
+                    for column in record
+                    if column not in all_columns
+                ]
+
             validate_column_names(all_columns)
             first = False
             # values is the list of insert data that is passed to the

tests/test_create.py

@@ -707,13 +707,24 @@ def test_insert_thousands_using_generator(fresh_db):
     assert 10000 == fresh_db["test"].count
 
 
-def test_insert_thousands_ignores_extra_columns_after_first_100(fresh_db):
+def test_insert_thousands_raises_exception_wtih_extra_columns_after_first_100(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/139
+    with pytest.raises(Exception, match="table test has no column named extra"):
+        fresh_db["test"].insert_all(
+            [{"i": i, "word": "word_{}".format(i)} for i in range(100)]
+            + [{"i": 101, "extra": "This extra column should cause an exception"}],
+        )
+
+
+def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/139
     fresh_db["test"].insert_all(
         [{"i": i, "word": "word_{}".format(i)} for i in range(100)]
-        + [{"i": 101, "extra": "This extra column should cause an exception"}]
+        + [{"i": 101, "extra": "Should trigger ALTER"}],
+        alter=True,
     )
     rows = fresh_db.execute_returning_dicts("select * from test where i = 101")
-    assert [{"i": 101, "word": None}] == rows
+    assert [{"i": 101, "word": None, "extra": "Should trigger ALTER"}] == rows
 
 
 def test_insert_ignore(fresh_db):