Merge pull request #81 from Knowledge-Graph-Hub/fix_force_index_refre…

…sh_run_jenkins

Fixes for force_index_refresh

Jenkinsfile

@@ -77,9 +77,9 @@ pipeline {
                             script {
                                 if (env.BRANCH_NAME != 'main') {
                                     echo "Transforming with --s3_test since we aren't on main/master branch"
-                                    sh '. venv/bin/activate && env && python3.8 run.py --s3_test --bucket fake_bucket'
+                                    sh '. venv/bin/activate && env && python3.8 run.py --s3_test --bucket fake_bucket --no_dl_progress --force_index_refresh'
                                 } else {
-                                    sh '. venv/bin/activate && env && python3.8 run.py --bucket kg-hub-public-data --no_dl_progress --force_index_refresh'
+                                    sh '. venv/bin/activate && env && python3.8 run.py --bucket kg-hub-public-data --no_dl_progress'
                                 }
                             }
                     }

kg_obo/transform.py

@@ -436,6 +436,7 @@ def run_transform(skip: list = [], get_only: list = [], bucket="bucket",
 
     # If requested, refresh the root index.html
     if force_index_refresh and not s3_test:
+        print(f"Refreshing root index on {bucket}")
         if kg_obo.upload.upload_index_files(bucket, remote_path, data_dir, data_dir, update_root=True):
             kg_obo_logger.info(f"Refreshed root index at {remote_path}")
         else:

kg_obo/upload.py

@@ -248,7 +248,7 @@ def mock_upload_dir_to_s3(local_directory: str, s3_bucket: str, s3_bucket_dir: s
     for bucket_object in conn.Bucket(s3_bucket).objects.all():
         print(bucket_object.key)
 
-def upload_index_files(bucket: str, remote_path: str, local_path: str, data_dir: str, update_root=False) -> bool:
+def upload_index_files(bucket: str, remote_path: str, local_path: str, data_dir: str, update_root=False, refresh=False) -> bool:
     """
     Checks the obo directory and version directory,
     creating index.html where it does not exist.
@@ -260,6 +260,7 @@ def upload_index_files(bucket: str, remote_path: str, local_path: str, data_dir:
     :param versioned_obo_path: str of directory containing the files to create index for
     :param data_dir: str of the data directory, so we can get the relative path
     :param update_root: bool, True to update root index (in this case, versioned_obo_path will be the data_dir)
+    :param refresh: bool, True to run without checking local files (i.e., create an empty local data dir)
     :return: bool returns True if all index files created successfully
     """
 
@@ -293,9 +294,12 @@ def upload_index_files(bucket: str, remote_path: str, local_path: str, data_dir:
     else:
         # Update root index
         check_dirs = [local_path]
+
+    if refresh:
+        os.mkdir(data_dir)
 
     for dir in check_dirs:
-
+        
         # Get the list of local files
         current_path = os.path.join(dir,ifilename)
         current_files = os.listdir(dir)