diff --git a/Jenkinsfile b/Jenkinsfile index aa700f28..e72c6d4c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -77,9 +77,9 @@ pipeline { script { if (env.BRANCH_NAME != 'main') { echo "Transforming with --s3_test since we aren't on main/master branch" - sh '. venv/bin/activate && env && python3.8 run.py --s3_test --bucket fake_bucket' + sh '. venv/bin/activate && env && python3.8 run.py --s3_test --bucket fake_bucket --no_dl_progress --force_index_refresh' } else { - sh '. venv/bin/activate && env && python3.8 run.py --bucket kg-hub-public-data --no_dl_progress --force_index_refresh' + sh '. venv/bin/activate && env && python3.8 run.py --bucket kg-hub-public-data --no_dl_progress' } } } diff --git a/kg_obo/transform.py b/kg_obo/transform.py index 7a975606..8cda4490 100644 --- a/kg_obo/transform.py +++ b/kg_obo/transform.py @@ -436,6 +436,7 @@ def run_transform(skip: list = [], get_only: list = [], bucket="bucket", # If requested, refresh the root index.html if force_index_refresh and not s3_test: + print(f"Refreshing root index on {bucket}") if kg_obo.upload.upload_index_files(bucket, remote_path, data_dir, data_dir, update_root=True): kg_obo_logger.info(f"Refreshed root index at {remote_path}") else: diff --git a/kg_obo/upload.py b/kg_obo/upload.py index 1c2f19ff..a002e676 100644 --- a/kg_obo/upload.py +++ b/kg_obo/upload.py @@ -248,7 +248,7 @@ def mock_upload_dir_to_s3(local_directory: str, s3_bucket: str, s3_bucket_dir: s for bucket_object in conn.Bucket(s3_bucket).objects.all(): print(bucket_object.key) -def upload_index_files(bucket: str, remote_path: str, local_path: str, data_dir: str, update_root=False) -> bool: +def upload_index_files(bucket: str, remote_path: str, local_path: str, data_dir: str, update_root=False, refresh=False) -> bool: """ Checks the obo directory and version directory, creating index.html where it does not exist. @@ -260,6 +260,7 @@ def upload_index_files(bucket: str, remote_path: str, local_path: str, data_dir: :param versioned_obo_path: str of directory containing the files to create index for :param data_dir: str of the data directory, so we can get the relative path :param update_root: bool, True to update root index (in this case, versioned_obo_path will be the data_dir) + :param refresh: bool, True to run without checking local files (i.e., create an empty local data dir) :return: bool returns True if all index files created successfully """ @@ -293,9 +294,12 @@ def upload_index_files(bucket: str, remote_path: str, local_path: str, data_dir: else: # Update root index check_dirs = [local_path] + + if refresh: + os.mkdir(data_dir) for dir in check_dirs: - + # Get the list of local files current_path = os.path.join(dir,ifilename) current_files = os.listdir(dir)