From 1d2fce01a1d5f4892a5c43c530d6c78adcaf9391 Mon Sep 17 00:00:00 2001 From: Justin Timperio Date: Sun, 24 Nov 2019 12:02:47 -0500 Subject: [PATCH 1/4] Readme fixes, doc fixes, version bump --- README.md | 17 +++++++++-------- core/setup.py | 4 +--- core/warp.py | 2 +- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 146f173..6e3118f 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Warp-CLI shortens this command to:\ Warp.py can also be imported into any python3 script and then used independently of the CLI to send and receive directories. -## Usage +## CLI Usage Warp-CLI features a number of shortcuts that attempt to make sending files as trivial and intuitive as possible. ### Transfer Modes @@ -37,13 +37,14 @@ Warp-CLI provides three core transfer modes: - `-tr, --threads` - default=8: In most cases, 8 threads is sufficient to saturate the connection. You may want to raise or lower this depending on your hardware. - `-ri, --report_interval`- default=5000: This limits the heartbeat report to 5000 milliseconds(5 seconds). - `-ts, --throttle_speed` - default=-1: This setting throttles the transfer to an average mbytes per second. -- `-ow, --overwrite` - default=false: Allow the receiver to overwrite existing files. +- `-ow, --overwrite`: Allow the receiver to overwrite existing files. +- `-sym, --follow_sym`: Let WDT follow symlinks during transfer. ### Macros Warp-CLI also includes a macro system for repeating custom transfers with a single command. Macros are stored transfer commands (stored in ~/warp-cli/macros) that are invoked with `warp -m macro_name`. To generate a macro:\ - `warp -gm daily_backup -f source_ssh /dir/to/backup /dir/to/store/backup -tr 16 -ri 10000 -ow true -cp '-skip_writes=true -start_port=12345'` + `warp -gm daily_backup -f source_ssh /dir/to/backup /dir/to/store/backup -ow -tr 16 -ri 10000 -cp '-skip_writes=true -start_port=12345'` This macro can now be called with:\ `warp -m daily_backup` @@ -56,7 +57,7 @@ Warp-CLI provides a number of utilities to streamline the daily use of WTD when - -m, --macro: Execute a custom macro from ~/warp-cli/config/ by name.\ `warp -m macro_name` - -gm, --gen_macro: Enter your transfer command as normal and include the gen_macro with a name for your new macro.\ - `warp -gm macro_name -f source_ssh /dir/to/fetch /dir/to/receive -tr 16 -ri 10000 -ow true` + `warp -gm macro_name -f source_ssh /dir/to/fetch /dir/to/receive -tr 16 -ri 10000 -ow` - -d, --daemon: Start a permanent receiver daemon on a local directory and export a file containing the connection URL and meta-data.\ `warp --daemon /dir/to/receive` - -i, --install: Attempt to install WDT and dependencies.\ @@ -81,14 +82,14 @@ So far, automatic installation is available on: - Debian 10.x and 9.x - Fedora 30, 29 and 28 Workstation and Server -WDT requires CMAKE version > 3.2 or greater, making it incompatible on: -- CentOS 7 -- Debian 8 - ### Uninstall Warp-CLI will remove itself from the machine but WDT will remain installed.\ `warp --uninstall` +### WDT Incompatible OS's +WDT requires CMAKE version > 3.2 or greater, making it incompatible on: +- CentOS 7 +- Debian 8 ### OpenSSH for URL Sharing Warp uses ssh to securely share connection URLs via a standard Linux pipe. It expects the use of an RSA key, which does not require a user password. While it is possible to use PAM authentication or key passwords, I have not yet added this as a feature. diff --git a/core/setup.py b/core/setup.py index 7da4c79..55ea003 100755 --- a/core/setup.py +++ b/core/setup.py @@ -1,6 +1,6 @@ #! /usr/bin/env python3 #### WDT Wrapper - https://github.com/facebook/wdt -## Version 2.1.2 +version = '2.1.3' from python_scripts import * ############ @@ -32,7 +32,6 @@ def build_wdt(base_dir): mkdir(base_dir + '/build', sudo=False) os.system('cd ' + base_dir + '/build && git clone https://github.com/facebook/folly.git') os.system('cd ' + base_dir + '/build/folly && git checkout v2019.09.02.00') - ### unused command while folly is broke # os.system('cd ' + base_dir + '/build/folly && git checkout "$(git describe --abbrev=0 --always)"') os.system('cd ' + base_dir + '/build && git clone https://github.com/facebook/wdt.git') mkdir(base_dir + '/build/wdt/_build', sudo=False) @@ -47,7 +46,6 @@ def setup_warp(base_dir): mkdir(base_dir + '/macros', sudo=False) os.system('sudo chmod 777 ' + base_dir + '/pool ' + base_dir + '/macros') ### link warp in bash to warp.py - rm_file('/usr/bin/warp', sudo=True) os.system('sudo ln -s ' + base_dir + '/core/warp.py /usr/bin/warp') print('Warp-CLI is Now Setup and Registered in /usr/bin!') diff --git a/core/warp.py b/core/warp.py index 7a8edf8..3c3fbf4 100755 --- a/core/warp.py +++ b/core/warp.py @@ -1,6 +1,6 @@ #! /usr/bin/env python3 #### WDT Wrapper - https://github.com/facebook/wdt -version = '2.1.2' +version = '2.1.3' from python_scripts import * import argparse From 35ddaef10240f0b33b512454b79a034c239a0ce4 Mon Sep 17 00:00:00 2001 From: Justin Timperio Date: Sun, 24 Nov 2019 17:12:15 -0500 Subject: [PATCH 2/4] Doc additions and error fixes --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6e3118f..ab38d65 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Warp-CLI provides three core transfer modes: `warp -p /dir/to/push dest_ssh /dir/to/receive` ### Flags -- `-tr, --threads` - default=8: In most cases, 8 threads is sufficient to saturate the connection. You may want to raise or lower this depending on your hardware. +- `-tr, --threads` - default=8: You may want to raise or lower this depending on your hardware. - `-ri, --report_interval`- default=5000: This limits the heartbeat report to 5000 milliseconds(5 seconds). - `-ts, --throttle_speed` - default=-1: This setting throttles the transfer to an average mbytes per second. - `-ow, --overwrite`: Allow the receiver to overwrite existing files. @@ -62,8 +62,8 @@ Warp-CLI provides a number of utilities to streamline the daily use of WTD when `warp --daemon /dir/to/receive` - -i, --install: Attempt to install WDT and dependencies.\ `warp --install` -- -ri, --remote_install: Attempt to install WDT and dependencies on a remote machine.\ - `warp -ri ssh.alias /dir/to/install` +- -ir, --install_remote: Attempt to install WDT and dependencies on a remote machine.\ + `warp -ir ssh.alias /dir/to/install` - -rm, --uninstall: Uninstall Warp-CLI and config files.\ `warp --uninstall` From 1af0b1acf58bf788a410b7b4e8db06160e78c536 Mon Sep 17 00:00:00 2001 From: Justin Timperio Date: Sun, 24 Nov 2019 17:14:00 -0500 Subject: [PATCH 3/4] -sym default switch --- core/warp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/warp.py b/core/warp.py index 3c3fbf4..8cddc40 100755 --- a/core/warp.py +++ b/core/warp.py @@ -90,7 +90,7 @@ def start_recv_daemon(recv_path): parser.add_argument("-ri", "--report_interval", default="3000", metavar='INT', help="Update interval in milliseconds for transfer report updates.") parser.add_argument("-ts", "--throttle_speed", default="-1", metavar='INT', help=" Throttle the transfer to an average mbytes per second.") parser.add_argument("-ow", "--overwrite", action='store_true', help="Allow the receiver to overwrite existing files in a directory.") -parser.add_argument("-sym", "--follow_sym", action='store_false', help="Let WDT follow symlinks during transfer.") +parser.add_argument("-sym", "--follow_sym", action='store_true', help="Let WDT follow symlinks during transfer.") parser.add_argument("-cp", "--custom_parms", default="", metavar="-CUSTOM_PARM value", help="Inject any additional parameters available in `wdt --help`.") ### utilities parser.add_argument("-d", "--daemon", metavar='/DIR/FOR/DAEMON', help="Start a receiver daemon on a directory. Returns a connection url to ~/warp-cli/macros.") From 53d6ef7ddbeec1e558ea00c11865fdfd77e72074 Mon Sep 17 00:00:00 2001 From: jtimperio Date: Sun, 24 Nov 2019 17:25:11 -0500 Subject: [PATCH 4/4] Update README.md --- README.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ab38d65..35a275a 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,13 @@ -# Warp-CLI - _Alpha v2.1_ +# Warp-CLI - _Alpha v2.1_ TLDR: Warp is a CLI tool designed to make interacting with Facebook's [Warp Speed Data Transfer (WDT)](https://github.com/facebook/wdt) pain-free. ## Abstract [WDT](https://github.com/facebook/wdt) is designed to provide the highest possible speed when transferring files(to be only hardware and network limited). WDT provides many advantages over most file transfer protocols including: native concurrency, end-to-end encryption, IPV6 support, and the ability to easily achieve +40Gbit speeds when supported. Unlike most file transfer solutions (Except [NORM](https://www.nrl.navy.mil/itd/ncs/products/norm)) WDT provides a native parallel solution for transferring files by separating files into chunks then queuing them across an arbitrary number of threads and TCP connections. In most cases, file transfer times are dramatically reduced compared to traditional methods like FTP or HTTP. -While WDT provides several benefits, it requires comparatively lengthy build process. Additionally, if you are already using a modified version of SSH such as [HPN-SSH](https://www.psc.edu/hpn-ssh), you are likely to see smaller performance gains. Since WDT is designed to fully saturate even the highest-end enterprise hardware, it is likely to overwhelm your network. Please consider this when transferring more than a 1TB of files. +While WDT provides several benefits, it requires a comparatively lengthy build process. Additionally, if you are already using a modified version of SSH such as [HPN-SSH](https://www.psc.edu/hpn-ssh), you are likely to see smaller performance gains. Since WDT is designed to fully saturate even the highest-end enterprise hardware, it will overwhelm your network. Please consider this when transferring more than a 1TB of files. ### [Performance Gains](https://github.com/JustinTimperio/warp-cli/blob/master/Performance.md) +Click the above link to get a detailed comparison of WDT performance compared to traditional file transfer protocols. ## Design Warp-CLI is mainly a wrapper for the limited existing [CLI app provided by WDT](https://github.com/facebook/wdt/wiki/Getting-Started-with-the-WDT-command-line). While the tool works extremely well, building performant commands for daily use is often unwieldy. @@ -22,7 +23,7 @@ Warp-CLI shortens this command to:\ Warp.py can also be imported into any python3 script and then used independently of the CLI to send and receive directories. ## CLI Usage -Warp-CLI features a number of shortcuts that attempt to make sending files as trivial and intuitive as possible. +Warp-CLI features several shortcuts that attempt to make sending files as trivial and intuitive as possible. ### Transfer Modes Warp-CLI provides three core transfer modes: @@ -34,9 +35,9 @@ Warp-CLI provides three core transfer modes: `warp -p /dir/to/push dest_ssh /dir/to/receive` ### Flags -- `-tr, --threads` - default=8: You may want to raise or lower this depending on your hardware. -- `-ri, --report_interval`- default=5000: This limits the heartbeat report to 5000 milliseconds(5 seconds). -- `-ts, --throttle_speed` - default=-1: This setting throttles the transfer to an average mbytes per second. +- `-tr, --threads`: default=8: You may want to raise or lower this depending on your hardware. +- `-ri, --report_interval`: default=5000: This limits the heartbeat report to 5000 milliseconds(5 seconds). +- `-ts, --throttle_speed`: default=-1: This setting throttles the transfer to an average mbytes per second. - `-ow, --overwrite`: Allow the receiver to overwrite existing files. - `-sym, --follow_sym`: Let WDT follow symlinks during transfer. @@ -50,7 +51,7 @@ This macro can now be called with:\ `warp -m daily_backup` ### Utilities -Warp-CLI provides a number of utilities to streamline the daily use of WTD when sending files in high frequency. +Warp-CLI provides a number of utilities and custom options to assist in more complicated transfers and WDT deployment. - -cp, --custom_parms: Inject any additional parameters available from `wdt --help`.\ `warp -f /dir/to/receive source_ssh /dir/to/send -cp '-skip_writes=true -start_port=12345'` @@ -58,7 +59,7 @@ Warp-CLI provides a number of utilities to streamline the daily use of WTD when `warp -m macro_name` - -gm, --gen_macro: Enter your transfer command as normal and include the gen_macro with a name for your new macro.\ `warp -gm macro_name -f source_ssh /dir/to/fetch /dir/to/receive -tr 16 -ri 10000 -ow` -- -d, --daemon: Start a permanent receiver daemon on a local directory and export a file containing the connection URL and meta-data.\ +- -d, --daemon: Start a permanent receiver daemon on a local directory and export a file containing connection meta-data.\ `warp --daemon /dir/to/receive` - -i, --install: Attempt to install WDT and dependencies.\ `warp --install` @@ -68,7 +69,7 @@ Warp-CLI provides a number of utilities to streamline the daily use of WTD when `warp --uninstall` ## Setup - _STILL UNDER DEVELOPMENT_ -Since WDT requires multiple dependencies, Warp-CLI attempts to provide a fully automated installation process for as many linux flavors as possible. If your flavor is not supported, please refer to the [manual install documentation](https://github.com/facebook/wdt/blob/master/build/BUILD.md). Once you install WDT and its dependencies Warp-CLI will function normally. +Since WDT requires multiple dependencies, Warp-CLI attempts to provide a fully automated installation process for as many Linux flavors as possible. If your flavor is not supported, please refer to the [manual install documentation](https://github.com/facebook/wdt/blob/master/build/BUILD.md). Once you install WDT and its dependencies Warp-CLI will function normally. ### Automatic Installation To install WDT and Warp-CLI automatically on your machine: @@ -76,7 +77,7 @@ To install WDT and Warp-CLI automatically on your machine: 2. `git clone --recurse-submodules https://github.com/JustinTimperio/warp-cli.git /usr/app/warp-cli` 3. `python3 /usr/app/warp-cli/core/warp.py --install` -So far, automatic installation is available on: +**So far, automatic installation is available on:** - Arch Linux - Ubuntu 19.xx and 18.xx Workstation and Server - Debian 10.x and 9.x