added support to install rclone without sudo

ctuning · Mar 3, 2024 · 53e4bc2 · 53e4bc2
1 parent 8ad5582
commit 53e4bc2
Show file tree

Hide file tree

Showing 9 changed files with 81 additions and 13 deletions.
diff --git a/cm-mlops/script/get-ml-model-gptj/_cm.json b/cm-mlops/script/get-ml-model-gptj/_cm.json
@@ -89,7 +89,8 @@
         "CM_PACKAGE_URL": "https://cloud.mlcommons.org/index.php/s/QAZ2oM94MkFtbQx/download",
         "CM_RCLONE_CONFIG_CMD": "rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com",
         "CM_RCLONE_URL": "mlc-inference:mlcommons-inference-wg-public/gpt-j"
-      }
+      },
+      "required_disk_space": 22700
     },
     "pytorch,fp32,wget": {
       "add_deps_recursive": {

diff --git a/cm-mlops/script/get-ml-model-stable-diffusion/_cm.json b/cm-mlops/script/get-ml-model-stable-diffusion/_cm.json
@@ -110,10 +110,14 @@
       },
       "group": "framework"
     },
+    "pytorch,fp16": {
+      "required_disk_space": 6500
+    },
     "pytorch,fp32": {
       "env": {
         "CM_ML_MODEL_STARTING_WEIGHTS_FILENAME": "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0"
-      }
+      },
+      "required_disk_space": 13000
     },
     "uint8": {
       "env": {

diff --git a/cm-mlops/script/get-rclone/_cm.json b/cm-mlops/script/get-rclone/_cm.json
@@ -3,8 +3,13 @@
   "automation_alias": "script",
   "automation_uid": "5b4e0237da074764",
   "cache": true,
-  "category": "Cloud automation",
-  "clean_files": [],
+  "deps": [
+    {
+      "tags": "detect,os"
+    }
+  ],
+  "category": "Detection or installation of tools and artifacts",
+  "default_version": "1.65.2",
   "new_env_keys": [
     "CM_RCLONE_CACHE_TAGS",
     "CM_RCLONE_BIN_WITH_PATH",
@@ -21,6 +26,11 @@
       "env": {
         "CM_RCLONE_GDRIVE": "yes"
       }
+    },
+    "system": {
+      "env": {
+        "CM_RCLONE_SYSTEM": "yes"
+      }
     }
   }
 }
diff --git a/cm-mlops/script/get-rclone/customize.py b/cm-mlops/script/get-rclone/customize.py
@@ -16,14 +16,48 @@ def preprocess(i):
     run_script_input = i['run_script_input']
     automation = i['automation']
 
+    need_version = env.get('CM_VERSION','')
+
+    host_os_machine = ''
+    if os_info['platform'] != 'windows':
+        host_os_machine = env['CM_HOST_OS_MACHINE'] # ABI
+
     r = automation.detect_version_using_script({
                'env': env,
                'run_script_input': run_script_input,
                'recursion_spaces':recursion_spaces})
 
     if r['return'] >0:
         if r['return'] == 16:
-            r = automation.run_native_script({'run_script_input':run_script_input, 'env':env, 'script_name':'install'})
+            install_script = 'install'
+            if os_info['platform'] != 'windows' and env.get('CM_RCLONE_SYSTEM','')=='yes':
+                install_script += '-system'
+            else:
+                if os_info['platform'] != 'windows':
+                    x1 = 'arm64' if host_os_machine.startswith('arm') or host_os_machine.startswith('aarch') else 'amd64'
+
+                    filebase = 'rclone-v{}-{}-{}'
+                    urlbase = 'https://downloads.rclone.org/v{}/{}'
+
+                    if os_info['platform'] == 'darwin':
+                        filename = filebase.format(need_version, 'osx', x1)
+                    elif os_info['platform'] == 'linux':
+                        filename = filebase.format(need_version, 'linux', x1)
+
+                    env['CM_RCLONE_URL'] = urlbase.format(need_version, filename+'.zip')
+                    env['CM_RCLONE_ARCHIVE'] = filename
+                    env['CM_RCLONE_ARCHIVE_WITH_EXT'] = filename+'.zip'
+
+                    print(recursion_spaces + 'Downloading {}'.format(env['CM_RCLONE_URL']))
+
+                cur_dir = os.getcwd()
+                path_bin = os.path.join(cur_dir, file_name)
+                env['CM_RCLONE_BIN_WITH_PATH'] = path_bin
+                env['+PATH']=[cur_dir]
+
+            r = automation.run_native_script({'run_script_input':run_script_input, 
+                                              'env':env, 
+                                              'script_name':install_script})
             if r['return']>0: return r
         else:
             return r
@@ -77,9 +111,11 @@ def postprocess(i):
 
     env['CM_RCLONE_CACHE_TAGS'] = 'version-'+version
 
-    if os_info['platform'] == 'windows':
+    file_name = 'rclone.exe' if os_info['platform'] == 'windows' else 'rclone'
+
+    if os_info['platform'] == 'windows' or env.get('CM_RCLONE_SYSTEM','')!='yes':
         cur_dir = os.getcwd()
-        path_bin = os.path.join(cur_dir, 'rclone.exe')
+        path_bin = os.path.join(cur_dir, file_name)
         if os.path.isfile(path_bin):
             # Was downloaded and extracted by CM
             env['CM_RCLONE_BIN_WITH_PATH'] = path_bin

diff --git a/cm-mlops/script/get-rclone/install-macos.sh → ...script/get-rclone/install-system-macos.sh b/cm-mlops/script/get-rclone/install-macos.sh → ...script/get-rclone/install-system-macos.sh
diff --git a/cm-mlops/script/get-rclone/install-system.sh b/cm-mlops/script/get-rclone/install-system.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+sudo -v ; curl -k https://rclone.org/install.sh | sudo bash
+test $? -eq 0 || exit 1
diff --git a/cm-mlops/script/get-rclone/install.bat b/cm-mlops/script/get-rclone/install.bat
@@ -1,12 +1,12 @@
-del /Q /S rclone-v1.65.2-windows-amd64.zip > NUL 2>&1
+del /Q /S rclone-v%CM_VERSION%-windows-amd64.zip > NUL 2>&1
 
-wget --no-check-certificate https://downloads.rclone.org/v1.65.2/rclone-v1.65.2-windows-amd64.zip
+wget --no-check-certificate https://downloads.rclone.org/v%CM_VERSION%/rclone-v%CM_VERSION%-windows-amd64.zip
 IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%
 
-unzip -o rclone-v1.65.2-windows-amd64.zip
+unzip -o rclone-v%CM_VERSION%-windows-amd64.zip
 IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%
 
-move /Y rclone-v1.65.2-windows-amd64\* .
+move /Y rclone-v%CM_VERSION%-windows-amd64\* .
 
-del /Q /S rclone-v1.65.2-windows-amd64.zip > NUL 2>&1
+del /Q /S rclone-v%CM_VERSION%-windows-amd64.zip > NUL 2>&1
 
diff --git a/cm-mlops/script/get-rclone/install.sh b/cm-mlops/script/get-rclone/install.sh
@@ -1,3 +1,13 @@
 #!/bin/bash
-sudo -v ; curl -k https://rclone.org/install.sh | sudo bash
+
+rm -rf ${CM_RCLONE_ARCHIVE_WITH_EXT}
+rm -rf rclone
+
+wget ${CM_RCLONE_URL} --no-check-certificate
+test $? -eq 0 || exit 1
+
+unzip ${CM_RCLONE_ARCHIVE_WITH_EXT}
+test $? -eq 0 || exit 1
+
+mv ${CM_RCLONE_ARCHIVE}/rclone .
 test $? -eq 0 || exit 1
diff --git a/cm-mlops/script/get-rclone/run.sh b/cm-mlops/script/get-rclone/run.sh
@@ -1,6 +1,10 @@
 #!/bin/bash
+
+echo ${PATH}
+
 if ! command -v rclone &> /dev/null
 then
+  echo "rclone was not detected"
   exit 1
 fi
 rclone --version > tmp-ver.out