From a5ee84167f5540db9bad4fdfc20bfd75ee66e443 Mon Sep 17 00:00:00 2001 From: Jesus Pulido Date: Sun, 16 Nov 2025 11:13:59 -0700 Subject: [PATCH 1/3] first uuid implementation --- dsi/core.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/dsi/core.py b/dsi/core.py index 6ebd8f29..47223ba5 100644 --- a/dsi/core.py +++ b/dsi/core.py @@ -13,6 +13,7 @@ import re import tarfile import subprocess +import uuid from contextlib import redirect_stdout class Terminal(): @@ -1434,11 +1435,12 @@ def index(self, local_loc, remote_loc, isVerbose=False): st_dict['created_time'] = [] st_dict['accessed_time'] = [] st_dict['mode'] = [] - #st_dict['inode'] = [] + st_dict['inode'] = [] st_dict['device'] = [] st_dict['n_links'] = [] st_dict['uid'] = [] st_dict['gid'] = [] + st_dict['uuid'] = [] st_dict['file_remote'] = [] for file in file_list: @@ -1455,11 +1457,12 @@ def index(self, local_loc, remote_loc, isVerbose=False): st_dict['created_time'].append(st.st_ctime) st_dict['accessed_time'].append(st.st_atime) st_dict['mode'].append(st.st_mode) - #st_dict['inode'].append(st.st_ino) + st_dict['inode'].append(st.st_ino) st_dict['device'].append(st.st_dev) st_dict['n_links'].append(st.st_nlink) st_dict['uid'].append(st.st_uid) st_dict['gid'].append(st.st_gid) + st_dict['uuid'].append(self.gen_uuid(st)) st_dict['file_remote'].append(rfilepath) st_list.append(st) @@ -1683,6 +1686,22 @@ def get(self, project_name = "Project"): ''' True + def gen_uuid(self, st): + ''' + Generates a unique file identifier using the os.stat data object as the input + + ''' + inode=st.st_ino + ctime=st.st_ctime + unique_str = f"{inode}-{ctime}" + + file_uuid = uuid.uuid5(uuid.NAMESPACE_URL, unique_str) + print(f"UUID:{file_uuid}") + return file_uuid + + + + class TarFile(): def __init__(self, tar_name, local_files, local_tmp_dir = 'tmp'): self.tar_name = tar_name From ca01527df14771651116ab062d2c735de5d3c488 Mon Sep 17 00:00:00 2001 From: Jesus Pulido Date: Sun, 16 Nov 2025 11:19:49 -0700 Subject: [PATCH 2/3] added pfcp support --- dsi/core.py | 44 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/dsi/core.py b/dsi/core.py index 47223ba5..080b0f20 100644 --- a/dsi/core.py +++ b/dsi/core.py @@ -1629,7 +1629,7 @@ def copy(self, tool="copy", isVerbose=False, **kwargs): stdout, stderr = process.communicate() returncode = process.communicate() - print( " DSI submitted Conduit job. ") + print( " DSI submitted Conduit data movement job. ") # Database Movement if isVerbose: @@ -1641,12 +1641,50 @@ def copy(self, tool="copy", isVerbose=False, **kwargs): stdout, stderr = process.communicate() returncode = process.communicate() - print( " DSI submitted Conduit job. ") + print( " DSI submitted Conduit data movement job. ") except subprocess.CalledProcessError as e: print(f"Command failed with error: {e.stderr} ") + elif tool == "pfcp": + env = os.environ.copy() + + if not os.path.exists(self.remote_location): + if isVerbose: + print( " mkdir " + self.remote_location) + path = Path(self.remote_location) + try: + path.mkdir(parents=True) + except Exception: + print(f"Unable to create folder {abspath} . Do you have access rights?") + raise + + try: + #subprocess.call(["pfcp", "-r", self.local_location, self.remote_location], env=env, shell=True) + # File Movement + if isVerbose: + print( "pfcp -r " + self.local_location + " " + os.path.join(self.remote_location, self.project_name) ) + cmd = ['pfcp','-r',self.local_location, os.path.join(self.remote_location, self.project_name)] + process = subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='latin-1') + + stdout, stderr = process.communicate() + returncode = process.communicate() + + print( " DSI submitted pfcp data movement job. ") + # Database Movement + if isVerbose: + print( " pfcp " + str(self.project_name+".db") + " " + os.path.join(self.remote_location, self.project_name, self.project_name+".db" ) ) + + cmd = ['pfcp','-r', str(self.project_name+".db"), os.path.join(self.remote_location, self.project_name, self.project_name+".db" )] + process = subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='latin-1') + + stdout, stderr = process.communicate() + returncode = process.communicate() + + print( " DSI submitted pfcp data movement job. ") + except subprocess.CalledProcessError as e: + print(f"Command failed with error: {e.stderr} ") elif tool == "ftp": True elif tool == "git": @@ -1654,8 +1692,6 @@ def copy(self, tool="copy", isVerbose=False, **kwargs): else: raise TypeError(f"Data movement format not supported:, Type: {tool}") - - def dircrawl(self,filepath): """ From 2a77004092f445e1b1c9c321b378b118ba4fdf13 Mon Sep 17 00:00:00 2001 From: Jesus Pulido Date: Wed, 19 Nov 2025 11:17:15 -0700 Subject: [PATCH 3/3] fixed recursive pfcp --- dsi/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dsi/core.py b/dsi/core.py index 080b0f20..30583382 100644 --- a/dsi/core.py +++ b/dsi/core.py @@ -1663,8 +1663,8 @@ def copy(self, tool="copy", isVerbose=False, **kwargs): # File Movement if isVerbose: - print( "pfcp -r " + self.local_location + " " + os.path.join(self.remote_location, self.project_name) ) - cmd = ['pfcp','-r',self.local_location, os.path.join(self.remote_location, self.project_name)] + print( "pfcp -R " + self.local_location + " " + os.path.join(self.remote_location, self.project_name) ) + cmd = ['pfcp','-R',self.local_location, os.path.join(self.remote_location, self.project_name)] process = subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='latin-1') stdout, stderr = process.communicate() @@ -1676,7 +1676,7 @@ def copy(self, tool="copy", isVerbose=False, **kwargs): if isVerbose: print( " pfcp " + str(self.project_name+".db") + " " + os.path.join(self.remote_location, self.project_name, self.project_name+".db" ) ) - cmd = ['pfcp','-r', str(self.project_name+".db"), os.path.join(self.remote_location, self.project_name, self.project_name+".db" )] + cmd = ['pfcp', str(self.project_name+".db"), os.path.join(self.remote_location, self.project_name, self.project_name+".db" )] process = subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='latin-1') stdout, stderr = process.communicate()