From f3220c253eff66c16d269b0af401b94085a40135 Mon Sep 17 00:00:00 2001 From: Artjoms Iskovs Date: Tue, 18 Oct 2022 19:43:51 +0100 Subject: [PATCH] Disable usage of placeholder dates in Singer (has own failure handling) --- splitgraph/hooks/data_source/base.py | 13 +++++++++++-- splitgraph/ingestion/singer/data_source.py | 6 +++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/splitgraph/hooks/data_source/base.py b/splitgraph/hooks/data_source/base.py index fdcb8ad4..e7233167 100644 --- a/splitgraph/hooks/data_source/base.py +++ b/splitgraph/hooks/data_source/base.py @@ -288,13 +288,17 @@ def prepare_new_image( hash_or_tag: Optional[str], comment: str = "Singer tap ingestion", copy_latest: bool = True, + use_placeholder_dt: bool = False, ) -> Tuple[Optional[Image], str]: new_image_hash = "{:064x}".format(getrandbits(256)) if repository_exists(repository) and copy_latest: # Clone the base image and delta compress against it base_image: Optional[Image] = repository.images[hash_or_tag] if hash_or_tag else None repository.images.add( - parent_id=None, image=new_image_hash, comment=comment, created=PLACEHOLDER_DATE + parent_id=None, + image=new_image_hash, + comment=comment, + created=PLACEHOLDER_DATE if use_placeholder_dt else None, ) if base_image: repository.engine.run_sql( @@ -311,7 +315,12 @@ def prepare_new_image( ) else: base_image = None - repository.images.add(parent_id=None, image=new_image_hash, comment=comment) + repository.images.add( + parent_id=None, + image=new_image_hash, + comment=comment, + created=PLACEHOLDER_DATE if use_placeholder_dt else None, + ) return base_image, new_image_hash diff --git a/splitgraph/ingestion/singer/data_source.py b/splitgraph/ingestion/singer/data_source.py index 18f606a1..83bfff27 100644 --- a/splitgraph/ingestion/singer/data_source.py +++ b/splitgraph/ingestion/singer/data_source.py @@ -15,7 +15,6 @@ from splitgraph.hooks.data_source.base import ( SyncableDataSource, get_ingestion_state, - make_image_latest, prepare_new_image, ) from splitgraph.ingestion.common import add_timestamp_tags @@ -129,7 +128,9 @@ def sync( catalog = self._run_singer_discovery(config) catalog = self.build_singer_catalog(catalog, tables) - base_image, new_image_hash = prepare_new_image(repository, image_hash) + base_image, new_image_hash = prepare_new_image( + repository, image_hash, use_placeholder_dt=False + ) state = get_ingestion_state(repository, image_hash) if use_state else None logging.info("Current ingestion state: %s", state) @@ -159,7 +160,6 @@ def sync( store_ingestion_state(repository, new_image_hash, state, latest_state) add_timestamp_tags(repository, new_image_hash) - make_image_latest(repository, new_image_hash) repository.commit_engines() if failure: