Skip to content

Use _archive_file() for dorado and i2map #189

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MBARIMike merged 1 commit into from
May 22, 2026
Merged

Use _archive_file() for dorado and i2map #189

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 5 additions & 53 deletions src/data/archive.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,19 +111,7 @@ def copy_to_AUVTCD(self, nc_file_base: Path, freq: str = FREQ) -> None: # noqa:
ftypes = (f"{freq}.nc", "cal.nc4", "align.nc4")
for ftype in ftypes:
src_file = Path(f"{nc_file_base}_{ftype}")
dst_file = Path(surveynetcdfs_dir, src_file.name)
if self.clobber:
if dst_file.exists():
self.logger.info("Removing %s", dst_file)
dst_file.unlink()
if src_file.exists():
shutil.copyfile(src_file, dst_file)
self.logger.info("copyfile %s %s done.", src_file, surveynetcdfs_dir)
elif src_file.exists():
self.logger.info(
"%26s exists, but is not being archived because --clobber is not specified.", # noqa: E501
src_file.name,
)
self._archive_file(src_file, Path(surveynetcdfs_dir, src_file.name))

if not self.resample:
# Copy intermediate files to AUVCTD/missionnetcdfs/YYYY/YYYYJJJ
Expand All @@ -141,15 +129,7 @@ def copy_to_AUVTCD(self, nc_file_base: Path, freq: str = FREQ) -> None: # noqa:
# so that lopc.nc is archived along with the other netcdf versions of the log files.
for log in [*LOG_FILES, "lopc.log"]:
src_file = Path(src_dir, f"{log.replace('.log', '')}.nc")
if self.clobber:
if src_file.exists():
shutil.copyfile(src_file, missionnetcdfs_dir / src_file.name)
self.logger.info("copyfile %s %s done.", src_file, missionnetcdfs_dir)
elif src_file.exists():
self.logger.info(
"%26s exists, but is not being archived because --clobber is not specified.", # noqa: E501
src_file.name,
)
self._archive_file(src_file, missionnetcdfs_dir / src_file.name)

# Copy files created by create_products.py
self.logger.info("Archiving product files")
Expand All @@ -163,26 +143,8 @@ def copy_to_AUVTCD(self, nc_file_base: Path, freq: str = FREQ) -> None: # noqa:
if Path(src_dir).exists():
dst_dir = Path(surveys_dir, year, dst_dir) # noqa: PLW2901
Path(dst_dir).mkdir(parents=True, exist_ok=True)
if self.clobber:
# Copy files individually to avoid permission issues with copytree.
# This will not copy subdirectories, but we don't expect any.
for src_file in src_dir.glob("*"):
dst_file = Path(dst_dir, src_file.name)
if dst_file.exists():
self.logger.info("Removing %s", dst_file)
dst_file.unlink()
shutil.copyfile(src_file, dst_file)
self.logger.info("copyfile %s %s done.", src_file, dst_dir)

# shutil.copytree(
# src_dir, dst_dir, dirs_exist_ok=True, copy_function=shutil.copy
# )
# self.logger.info("copytree %s/* %s done.", src_dir, dst_dir)
elif src_dir.exists():
self.logger.info(
"%26s exists, but is not being archived because --clobber is not specified.", # noqa: E501
src_dir.name,
)
for src_file in src_dir.glob("*"):
self._archive_file(src_file, Path(dst_dir, src_file.name))
else:
self.logger.debug("%s not found", src_dir)
if self.create_products or self.resample:
Expand All @@ -194,17 +156,7 @@ def copy_to_AUVTCD(self, nc_file_base: Path, freq: str = FREQ) -> None: # noqa:
else:
# Copy the processing.log file last so that we get everything
src_file = Path(f"{nc_file_base}_{LOG_NAME}")
dst_file = Path(surveynetcdfs_dir, src_file.name)
if src_file.exists():
if self.clobber:
self.logger.info("copyfile %s %s", src_file, surveynetcdfs_dir)
shutil.copyfile(src_file, dst_file)
self.logger.info("copyfile %s %s done.", src_file, surveynetcdfs_dir)
elif src_file.exists():
self.logger.info(
"%26s exists, but is not being archived because --clobber is not specified.", # noqa: E501
src_file.name,
)
self._archive_file(src_file, Path(surveynetcdfs_dir, src_file.name))

def copy_to_M3(self, resampled_nc_file: str) -> None:
pass
Expand Down
Loading