From f3d0f9ee8ebb90e8ea9e699c645d92ec603f85e0 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Sat, 15 Dec 2018 16:47:31 +0000 Subject: [PATCH 1/2] [AIRFLOW-3447] Add 2 options for ts_nodash Macro Add Option to use `ts_nodash` and `ts_nodash_with_tz` macro --- airflow/models.py | 4 +++- docs/code.rst | 13 +++++++------ tests/core.py | 3 ++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/airflow/models.py b/airflow/models.py index 06a79b8af5489..1089970b654b6 100755 --- a/airflow/models.py +++ b/airflow/models.py @@ -1869,7 +1869,8 @@ def get_template_context(self, session=None): prev_ds_nodash = prev_ds.replace('-', '') ds_nodash = ds.replace('-', '') - ts_nodash = ts.replace('-', '').replace(':', '') + ts_nodash = self.execution_date.strftime('%Y%m%dT%H%M%S') + ts_nodash_with_tz = ts.replace('-', '').replace(':', '') yesterday_ds_nodash = yesterday_ds.replace('-', '') tomorrow_ds_nodash = tomorrow_ds.replace('-', '') @@ -1939,6 +1940,7 @@ def __repr__(self): 'ds_nodash': ds_nodash, 'ts': ts, 'ts_nodash': ts_nodash, + 'ts_nodash_with_tz': ts_nodash_with_tz, 'yesterday_ds': yesterday_ds, 'yesterday_ds_nodash': yesterday_ds_nodash, 'tomorrow_ds': tomorrow_ds, diff --git a/docs/code.rst b/docs/code.rst index 996f702a0eb79..61414ecbd6633 100644 --- a/docs/code.rst +++ b/docs/code.rst @@ -269,7 +269,7 @@ Sensors .. _macros: Macros ---------- +------ Here's a list of variables and macros that can be used in templates @@ -284,19 +284,20 @@ Variable Description ``{{ ds }}`` the execution date as ``YYYY-MM-DD`` ``{{ ds_nodash }}`` the execution date as ``YYYYMMDD`` ``{{ prev_ds }}`` the previous execution date as ``YYYY-MM-DD`` - if ``{{ ds }}`` is ``2016-01-08`` and ``schedule_interval`` is ``@weekly``, + if ``{{ ds }}`` is ``2018-01-08`` and ``schedule_interval`` is ``@weekly``, ``{{ prev_ds }}`` will be ``2016-01-01`` ``{{ prev_ds_nodash }}`` the previous execution date as ``YYYYMMDD`` if exists, else ``None` ``{{ next_ds }}`` the next execution date as ``YYYY-MM-DD`` - if ``{{ ds }}`` is ``2016-01-01`` and ``schedule_interval`` is ``@weekly``, - ``{{ next_ds }}`` will be ``2016-01-08`` + if ``{{ ds }}`` is ``2018-01-01`` and ``schedule_interval`` is ``@weekly``, + ``{{ next_ds }}`` will be ``2018-01-08`` ``{{ next_ds_nodash }}`` the next execution date as ``YYYYMMDD`` if exists, else ``None` ``{{ yesterday_ds }}`` the day before the execution date as ``YYYY-MM-DD`` ``{{ yesterday_ds_nodash }}`` the day before the execution date as ``YYYYMMDD`` ``{{ tomorrow_ds }}`` the day after the execution date as ``YYYY-MM-DD`` ``{{ tomorrow_ds_nodash }}`` the day after the execution date as ``YYYYMMDD`` -``{{ ts }}`` same as ``execution_date.isoformat()`` -``{{ ts_nodash }}`` same as ``ts`` without ``-`` and ``:`` +``{{ ts }}`` same as ``execution_date.isoformat()``. Example: ``2018-01-01T00:00:00+00:00`` +``{{ ts_nodash }}`` same as ``ts`` without ``-``, ``:`` and TimeZone info. Example: ``20180101T000000`` +``{{ ts_nodash_with_tz }}`` same as ``ts`` without ``-`` and ``:``. Example: ``20180101T000000+0000`` ``{{ execution_date }}`` the execution_date, (datetime.datetime) ``{{ prev_execution_date }}`` the previous execution date (if available) (datetime.datetime) ``{{ next_execution_date }}`` the next execution date (datetime.datetime) diff --git a/tests/core.py b/tests/core.py index b297edcbcd4c9..3cf4c4e18f540 100644 --- a/tests/core.py +++ b/tests/core.py @@ -667,7 +667,8 @@ def test_task_get_template(self): self.assertEquals(context['prev_ds_nodash'], '20141231') self.assertEquals(context['ts'], '2015-01-01T00:00:00+00:00') - self.assertEquals(context['ts_nodash'], '20150101T000000+0000') + self.assertEquals(context['ts_nodash'], '20150101T000000') + self.assertEquals(context['ts_nodash_with_tz'], '20150101T000000+0000') self.assertEquals(context['yesterday_ds'], '2014-12-31') self.assertEquals(context['yesterday_ds_nodash'], '20141231') From 554a1875063b5ce5ecdfd7ae712ee6e4df3d255e Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Sat, 15 Dec 2018 19:22:28 +0000 Subject: [PATCH 2/2] Add to updating.md --- UPDATING.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/UPDATING.md b/UPDATING.md index 814e2e107dc0c..986d3a23c1f26 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -24,6 +24,13 @@ assists users migrating to a new version. ## Airflow Master +### Modification to `ts_nodash` macro +`ts_nodash` previously contained TimeZone information alongwith execution date. For Example: `20150101T000000+0000`. This is not user-friendly for file or folder names which was a popular use case for `ts_nodash`. Hence this behavior has been changed and using `ts_nodash` will no longer contain TimeZone information, restoring the pre-1.10 behavior of this macro. And a new macro `ts_nodash_with_tz` has been added which can be used to get a string with execution date and timezone info without dashes. + +Examples: + * `ts_nodash`: `20150101T000000` + * `ts_nodash_with_tz`: `20150101T000000+0000` + ### New `dag_processor_manager_log_location` config option The DAG parsing manager log now by default will be log into a file, where its location is