Merge branch 'main' into feature/coveralls-main-branch

HISTORY.md

@@ -4,6 +4,16 @@ In the beginning, programmers created the databases. Now the databases were form
 
 And so [Salesforce.org](http://salesforce.org/) said “Let there be data,” and there was Snowfakery. And it was good.
 
+## Snowfakery 3.3
+
+Snowfakery has a `datetime_between` function (#779)
+
+Date-time values can now be coerced from strings and dates (#779)
+
+Fixes to documentation (thank you @BrettMN) (#727)
+
+`find_record` is now cached so that it only calls into Salesforce once. (#726)
+
 ## Snowfakery 3.2
 
 Snowfakery can now do `random_reference` to nicknames. (#639)

docs/index.md

@@ -690,7 +690,7 @@ The `date_between` function picks a random date in some date range. For example,
 ```yaml
 - object: OBJ
     fields:
-    date:
+      date:
         date_between:
             start_date: 2000-01-01
             end_date: today
@@ -726,6 +726,41 @@ The `date_between` function can also be used in formulas.
 wedding_date: Our big day is ${{date_between(start_date="2022-01-31", end_date="2022-12-31")}}
 ```
 
+### `datetime_between`
+
+`datetime_between` is similar to `date_between` but relates to [datetimes](#datetime).
+
+Some example of randomized datetimes:
+
+```yaml
+# tests/test_fake_datetimes.yml
+- object: OBJ
+  fields:
+    past:
+      datetime_between:
+        start_date: 1999-12-31 # party like its 1999!!
+        end_date: today
+    future:
+      datetime_between:
+        start_date: today
+        end_date: 2525-01-01 # if man is still alive!!
+    y2k:
+      datetime_between:
+        start_date: 1999-12-31 11:59:00
+        end_date: 2000-01-01 01:01:00
+    empty:
+      datetime_between:
+        start_date: 1999-12-31 11:59:00
+        end_date: 1999-12-31 11:59:00
+    westerly:
+      datetime_between:
+        start_date: 1999-12-31 11:59:00
+        end_date: now
+        timezone:
+          relativedelta:
+            hours: +8
+```
+
 ### `random_number`
 
 The `random_number` function picks a number in a range specified by `min` and `max`.
@@ -1161,15 +1196,34 @@ And these methods:
 #### `datetime`
 
 The `datetime` function can generate
-a new datetime object from year/month/day parts:
+a new datetime object from year/month/day parts, from a string
+or from a date object.
+
+A `datetime` combines both a date and a time into a single value. E.g. 11:03:21 on February 14, 2024. We can express that `datetime` as
+`2024-02-14 11:03:21`
+
+Datetimes default to using the UTC time-zone, but you can control that by
+adding a timezone after a plus sign: `2008-04-25 21:18:29+08:00`
 
 ```yaml
 # tests/test_datetime.yml
+- snowfakery_version: 3
 - object: Datetimes
   fields:
-    from_date: ${{datetime(year=2000, month=1, day=1)}}
-    from_datetime: ${{datetime(year=2000, month=1, day=1, hour=1, minute=1, second=1)}}
+    from_date_fields: ${{datetime(year=2000, month=1, day=1)}}
+    from_datetime_fields: ${{datetime(year=2000, month=1, day=1, hour=1, minute=1, second=1)}}
+    some_date: # a date, not a datetime, for conversion later
+      date_between:
+        start_date: today
+        end_date: +1y
+    from_date: ${{datetime(some_date)}}
+    from_string: ${{datetime("2000-01-01 01:01:01")}}
+    from_yaml:
+      datetime: 2000-01-01 01:01:01
     right_now: ${{now}}
+    also_right_now: ${{datetime()}}
+    also_also_right_now:
+      datetime: now
     hour: ${{now.hour}}
     minute: ${{now.minute}}
     second: ${{now.second}}

requirements/dev.txt

@@ -10,7 +10,7 @@ attrs==22.1.0
     #   pytest
 black==22.8.0
     # via -r requirements/dev.in
-certifi==2022.9.14
+certifi==2022.9.24
     # via
     #   -r requirements/prod.txt
     #   requests
@@ -34,13 +34,13 @@ coverage[toml]==6.4.4
     #   pytest-cov
 coveralls==3.3.1
     # via -r requirements/dev.in
-diff-cover==6.5.1
+diff-cover==7.0.1
     # via -r requirements/dev.in
 distlib==0.3.6
     # via virtualenv
 docopt==0.6.2
     # via coveralls
-faker==14.2.0
+faker==15.0.0
     # via
     #   -r requirements/prod.txt
     #   faker-microservice
@@ -73,7 +73,7 @@ importlib-resources==5.9.0
     # via tox-gh-actions
 iniconfig==1.1.1
     # via pytest
-jinja2==2.11.3
+jinja2==3.1.2
     # via
     #   -r requirements/prod.txt
     #   diff-cover
@@ -82,7 +82,7 @@ jsonschema==4.16.0
     # via -r requirements/dev.in
 markdown==3.4.1
     # via mkdocs
-markupsafe==2.0.1
+markupsafe==2.1.1
     # via
     #   -r requirements/prod.txt
     #   jinja2
@@ -182,7 +182,7 @@ tox==3.26.0
     # via
     #   -r requirements/dev.in
     #   tox-gh-actions
-tox-gh-actions==2.9.1
+tox-gh-actions==2.10.0
     # via -r requirements/dev.in
 typeguard==2.10.0
     # via -r requirements/dev.in
@@ -216,5 +216,5 @@ zipp==3.8.1
     #   importlib-resources
 
 # The following packages are considered to be unsafe in a requirements file:
-setuptools==65.3.0
+setuptools==65.4.0
     # via nodeenv

requirements/prod.in

@@ -1,13 +1,10 @@
 SQLAlchemy
-# Remove this when Faker/Docs building issue is cleared up.
 Faker
-jinja2<3        # until CCI is compatible
+jinja2
 PyYAML
 click
 python-dateutil
 gvgen
 pydantic
 python-baseconv
 requests
-# This whole line should be deleted when Jinja and MarkupSafe are compatible:
-MarkupSafe<2.1.0

requirements/prod.txt

@@ -4,26 +4,24 @@
 #
 #    pip-compile --allow-unsafe requirements/prod.in
 #
-certifi==2022.9.14
+certifi==2022.9.24
     # via requests
 charset-normalizer==2.1.1
     # via requests
 click==8.1.3
     # via -r requirements/prod.in
-faker==14.2.0
+faker==15.0.0
     # via -r requirements/prod.in
 greenlet==1.1.3
     # via sqlalchemy
 gvgen==1.0
     # via -r requirements/prod.in
 idna==3.4
     # via requests
-jinja2==2.11.3
+jinja2==3.1.2
     # via -r requirements/prod.in
-markupsafe==2.0.1
-    # via
-    #   -r requirements/prod.in
-    #   jinja2
+markupsafe==2.1.1
+    # via jinja2
 pydantic==1.10.2
     # via -r requirements/prod.in
 python-baseconv==1.2.2

snowfakery/output_streams.py

@@ -53,7 +53,7 @@ class OutputStream(ABC):
         int: int,
         float: float,
         datetime.date: noop,
-        datetime.datetime: format_datetime,
+        datetime.datetime: noop,
         type(None): noop,
         bool: int,
         Decimal: str,
@@ -280,6 +280,11 @@ def close(self, **kwargs) -> Optional[Sequence[str]]:
 class SqlDbOutputStream(OutputStream):
     """Output stream for talking to SQL Databases"""
 
+    encoders: Mapping[type, Callable] = {
+        **OutputStream.encoders,
+        datetime.datetime: format_datetime,  # format into Salesforce-friendly syntax
+    }
+
     should_close_session = False
 
     def __init__(self, engine: Engine, mappings: None = None, **kwargs):

snowfakery/template_funcs.py

@@ -58,6 +58,21 @@ def parse_date(d: Union[str, datetime, date]) -> date:
     return dateutil.parser.parse(d).date()
 
 
+@lru_cache(maxsize=512)
+def parse_datetimespec(d: Union[str, datetime, date]) -> datetime:
+    """Parse a string, datetime or date into a datetime."""
+    if isinstance(d, datetime):
+        return d
+    elif isinstance(d, str):
+        if d == "now":
+            return datetime.now()
+        elif d == "today":
+            return datetime.combine(date.today(), datetime.min.time())
+        return dateutil.parser.parse(d)
+    elif isinstance(d, date):
+        return datetime.combine(d, datetime.min.time())
+
+
 def render_boolean(context: PluginContext, value: FieldDefinition) -> bool:
     val = context.evaluate(value)
     if isinstance(val, str):
@@ -90,16 +105,21 @@ def date(
         ):
             """A YAML-embeddable function to construct a date from strings or integers"""
             if datespec:
+                if any((day, month, year)):
+                    raise DataGenError(
+                        "Should not specify a date specification and also day or month or year."
+                    )
                 return parse_date(datespec)
             else:
                 return date(year, month, day)
 
         def datetime(
             self,
+            datetimespec=None,
             *,
-            year: Union[str, int],
-            month: Union[str, int],
-            day: Union[str, int],
+            year: Union[str, int] = None,
+            month: Union[str, int] = None,
+            day: Union[str, int] = None,
             hour=0,
             minute=0,
             second=0,
@@ -108,11 +128,24 @@ def datetime(
         ):
             """A YAML-embeddable function to construct a datetime from strings or integers"""
             timezone = _normalize_timezone(timezone)
-            return datetime(
-                year, month, day, hour, minute, second, microsecond, tzinfo=timezone
-            )
+            if datetimespec:
+                if any((day, month, year, hour, minute, second, microsecond)):
+                    raise DataGenError(
+                        "Should not specify a date specification and also other parameters."
+                    )
+                dt = parse_datetimespec(datetimespec)
+                dt = dt.replace(tzinfo=timezone)
+            elif not (any((year, month, day, hour, minute, second, microsecond))):
+                # no dt specification provided at all...just use now()
+                dt = datetime.now(timezone)
+            else:
+                dt = datetime(
+                    year, month, day, hour, minute, second, microsecond, timezone
+                )
+
+            return dt
 
-        def date_between(self, *, start_date, end_date):
+        def date_between(self, *, start_date, end_date, timezone=UTCAsRelDelta):
             """A YAML-embeddable function to pick a date between two ranges"""
 
             def try_parse_date(d):
@@ -133,6 +166,20 @@ def try_parse_date(d):
                     raise
             # swallow empty range errors per Python conventions
 
+        def datetime_between(self, *, start_date, end_date, timezone=UTCAsRelDelta):
+            """A YAML-embeddable function to pick a datetime between two ranges"""
+
+            start_date = self.datetime(start_date)
+            end_date = self.datetime(end_date)
+
+            timezone = _normalize_timezone(timezone)
+            if end_date < start_date:
+                raise DataGenError("End date is before start date")
+
+            return self._faker_for_dates.date_time_between(
+                start_date, end_date, tzinfo=timezone
+            )
+
         def i18n_fake(self, locale: str, fake: str):
             # deprecated by still here for backwards compatibility
             faker = Faker(locale, use_weighting=False)

snowfakery/version.txt

@@ -1 +1 @@
-3.2.1
+3.3.0

tests/test_dates.py

@@ -26,7 +26,7 @@ def test_date_math__native_types(self, generated_rows):
         """
         generate_data(StringIO(yaml), plugin_options={"snowfakery_version": 3})
         date = generated_rows.table_values("OBJ", 1, "dateplus")
-        assert date == "2022-01-01T00:00:00+00:00"
+        assert str(date) == "2022-01-01 00:00:00+00:00"
 
     def test_date_math__native_types__error(self, generated_rows):
         yaml = """
@@ -38,3 +38,14 @@ def test_date_math__native_types__error(self, generated_rows):
         with pytest.raises(exc.DataGenValueError) as e:
             generate_data(StringIO(yaml), plugin_options={"snowfakery_version": 3})
         assert "dateplus" in str(e.value)
+
+    def test_date_time__too_many_params__error(self):
+        yaml = """
+        - object: OBJ
+          fields:
+            basedate: ${{datetime("2022-01-01", year=2000, month=1, day=1)}}
+            dateplus: ${{basedate + "XYZZY"}}
+        """
+        with pytest.raises(exc.DataGenValueError) as e:
+            generate_data(StringIO(yaml), plugin_options={"snowfakery_version": 3})
+        assert "date specification" in str(e.value)

tests/test_datetime.yml

@@ -1,8 +1,20 @@
+- snowfakery_version: 3
 - object: Datetimes
   fields:
-    from_date: ${{datetime(year=2000, month=1, day=1)}}
-    from_datetime: ${{datetime(year=2000, month=1, day=1, hour=1, minute=1, second=1)}}
+    from_date_fields: ${{datetime(year=2000, month=1, day=1)}}
+    from_datetime_fields: ${{datetime(year=2000, month=1, day=1, hour=1, minute=1, second=1)}}
+    some_date: # a date, not a datetime, for conversion later
+      date_between:
+        start_date: today
+        end_date: +1y
+    from_date: ${{datetime(some_date)}}
+    from_string: ${{datetime("2000-01-01 01:01:01")}}
+    from_yaml:
+      datetime: 2000-01-01 01:01:01
     right_now: ${{now}}
+    also_right_now: ${{datetime()}}
+    also_also_right_now:
+      datetime: now
     hour: ${{now.hour}}
     minute: ${{now.minute}}
     second: ${{now.second}}