Only show latest dataset event timestamp after last run (apache#38340)

* Only show latest dataset event timestamp after last run * Move date filter to join statement in query * Update airflow/www/views.py Co-authored-by: Tzu-ping Chung <uranusjr@gmail.com> --------- Co-authored-by: Tzu-ping Chung <uranusjr@gmail.com>
astronomer · Apr 22, 2024 · 818a0cb · 818a0cb
1 parent 6e66d78
commit 818a0cb
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 2 deletions.
diff --git a/airflow/www/templates/airflow/dataset_next_run_modal.html b/airflow/www/templates/airflow/dataset_next_run_modal.html
@@ -42,7 +42,7 @@ <h4 class="modal-title" id="datasetNextRunModalLabel">
             <thead>
               <tr>
                 <th>Dataset URI</th>
-                <th>Latest Update</th>
+                <th>Latest Update since last Dag Run</th>
               </tr>
             </thead>
             <tbody id="datasets_tbody">

diff --git a/airflow/www/views.py b/airflow/www/views.py
@@ -3290,6 +3290,8 @@ def next_run_datasets(self, dag_id):
         with create_session() as session:
             dag_model = DagModel.get_dagmodel(dag_id, session=session)
 
+            latest_run = dag_model.get_last_dagrun(session=session)
+
             events = [
                 dict(info)
                 for info in session.execute(
@@ -3311,7 +3313,12 @@ def next_run_datasets(self, dag_id):
                     )
                     .join(
                         DatasetEvent,
-                        DatasetEvent.dataset_id == DatasetModel.id,
+                        and_(
+                            DatasetEvent.dataset_id == DatasetModel.id,
+                            DatasetEvent.timestamp >= latest_run.execution_date
+                            if latest_run and latest_run.execution_date
+                            else True,
+                        ),
                         isouter=True,
                     )
                     .where(DagScheduleDatasetReference.dag_id == dag_id, ~DatasetModel.is_orphaned)