diff --git a/README.md b/README.md index 077e554e..32590530 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,8 @@ For using mergin client with its dependencies packaged locally run: python3 setup.py sdist bdist_wheel mkdir -p mergin/deps pip wheel -r mergin_client.egg-info/requires.txt -w mergin/deps - unzip mergin/deps/pygeodiff-*.whl -d mergin/deps + unzip mergin/deps/pygeodiff-*.whl -d mergin/deps + pip install --editable . ### Tests For running test do: diff --git a/mergin/cli.py b/mergin/cli.py index 6065ba51..1203e689 100755 --- a/mergin/cli.py +++ b/mergin/cli.py @@ -7,24 +7,44 @@ pip puts these tools). """ +from datetime import datetime, timezone +import click import json import os import sys +import time import traceback -import click from mergin import ( ClientError, - MerginClient, - MerginProject, InvalidProject, LoginError, + MerginClient, + MerginProject, +) +from mergin.client_pull import ( + download_project_async, + download_project_cancel, + download_project_finalize, + download_project_is_running, ) -from mergin.client_pull import download_project_async, download_project_is_running, download_project_finalize, download_project_cancel from mergin.client_pull import pull_project_async, pull_project_is_running, pull_project_finalize, pull_project_cancel from mergin.client_push import push_project_async, push_project_is_running, push_project_finalize, push_project_cancel +class OptionPasswordIfUser(click.Option): + """Custom option class for getting a password only if the --username option was specified.""" + + def handle_parse_result(self, ctx, opts, args): + self.has_username = "username" in opts + return super(OptionPasswordIfUser, self).handle_parse_result(ctx, opts, args) + + def prompt_for_value(self, ctx): + if self.has_username: + return super(OptionPasswordIfUser, self).prompt_for_value(ctx) + return None + + def get_changes_count(diff): attrs = ["added", "removed", "updated"] return sum([len(diff[attr]) for attr in attrs]) @@ -34,15 +54,12 @@ def pretty_diff(diff): added = diff["added"] removed = diff["removed"] updated = diff["updated"] - if removed: click.secho("\n>>> Removed:", fg="cyan") click.secho("\n".join("- " + f["path"] for f in removed), fg="red") - if added: click.secho("\n>>> Added:", fg="cyan") click.secho("\n".join("+ " + f["path"] for f in added), fg="green") - if updated: click.secho("\n>>> Modified:", fg="cyan") click.secho("\n".join("M " + f["path"] for f in updated), fg="yellow") @@ -51,132 +68,185 @@ def pretty_diff(diff): def pretty_summary(summary): for k, v in summary.items(): click.secho("Details " + k) - click.secho("".join("layer name - " + d["table"] + ": inserted: " + str(d["insert"]) + ", modified: " + - str(d["update"]) + ", deleted: " + str(d["delete"]) + "\n" for d in v['geodiff_summary'] if d["table"] != "gpkg_contents")) - - -def _init_client(): - url = os.environ.get('MERGIN_URL') - auth_token = os.environ.get('MERGIN_AUTH') - if auth_token is None: - click.secho("Missing authorization token: please run 'login' first and set MERGIN_AUTH env variable", fg='red') + click.secho( + "".join( + "layer name - " + + d["table"] + + ": inserted: " + + str(d["insert"]) + + ", modified: " + + str(d["update"]) + + ", deleted: " + + str(d["delete"]) + + "\n" + for d in v["geodiff_summary"] + if d["table"] != "gpkg_contents" + ) + ) + + +def get_token(url, username, password, show_token=True): + """Get authorization token for given user and password.""" + mc = MerginClient(url) + if not mc.is_server_compatible(): + click.secho(str("This client version is incompatible with server, try to upgrade"), fg="red") return None - - mc = MerginClient(url, auth_token='Bearer {}'.format(auth_token)) - - # check whether the token has not expired already (normally it expires in 12 hours) - from datetime import datetime, timezone - delta = mc._auth_session['expire'] - datetime.now(timezone.utc) - if delta.total_seconds() < 0: - click.secho("Access token has expired: please run 'login' again and set MERGIN_AUTH env variable", fg='red') + try: + session = mc.login(username, password) + except LoginError as e: + click.secho("Unable to log in: " + str(e), fg="red") + return None + if show_token: + click.secho(f'To set the MERGIN_AUTH variable run:\nexport MERGIN_AUTH="{session["token"]}"') + return session["token"] + + +def get_client(url=None, auth_token=None, username=None, password=None): + """Return Mergin client.""" + if auth_token is not None: + mc = MerginClient(url, auth_token=f"Bearer {auth_token}") + # Check if the token has expired or is just about to expire + delta = mc._auth_session["expire"] - datetime.now(timezone.utc) + if delta.total_seconds() > 5: + return mc + if username and password: + auth_token = get_token(url, username, password) + if auth_token is None: + return None + mc = MerginClient(url, auth_token=f"Bearer {auth_token}") + else: + click.secho( + "Missing authorization data.\n" + "Either set environment variables (MERGIN_USERNAME and MERGIN_PASSWORD) " + "or specify --username / --password options.\n" + "Note: if --username is specified but password is missing you will be prompted for password.", + fg="red", + ) return None - return mc def _print_unhandled_exception(): """ Outputs details of an unhandled exception that is being handled right now """ - click.secho("Unhandled exception!", fg='red') + click.secho("Unhandled exception!", fg="red") for line in traceback.format_exception(*sys.exc_info()): click.echo(line) @click.group() -def cli(): - pass +@click.option( + "--url", + envvar="MERGIN_URL", + default=MerginClient.default_url(), + help=f"Mergin server URL. Default is: {MerginClient.default_url()}", +) +@click.option("--auth-token", envvar="MERGIN_AUTH", help="Mergin authentication token string") +@click.option("--username", envvar="MERGIN_USERNAME") +@click.option("--password", cls=OptionPasswordIfUser, prompt=True, hide_input=True, envvar="MERGIN_PASSWORD") +@click.pass_context +def cli(ctx, url, auth_token, username, password): + """ + Command line interface for the Mergin client module. + For user authentication on server there are two options: + 1. authorization token environment variable (MERGIN_AUTH) is defined, or + 2. username and password need to be given either as environment variables (MERGIN_USERNAME, MERGIN_PASSWORD), + or as command options (--username, --password). + Run `mergin --username login` to see how to set the token variable manually. + """ + mc = get_client(url=url, auth_token=auth_token, username=username, password=password) + ctx.obj = {"client": mc} @cli.command() -@click.argument('url', required=False) -@click.option('--login', prompt=True) -@click.option('--password', prompt=True, hide_input=True) -def login(url, login, password): - """Fetch new authentication token. If no URL is specified, the public Mergin instance will be used.""" - c = MerginClient(url) - click.echo("Mergin URL: " + c.url) - if not c.is_server_compatible(): - click.secho(str('This client version is incompatible with server, try to upgrade'), fg='red') - return - try: - session = c.login(login, password) - except LoginError as e: - click.secho('Unable to log in: ' + str(e), fg='red') - return - - print('export MERGIN_URL="%s"' % c.url) - print('export MERGIN_AUTH="%s"' % session['token']) +@click.pass_context +def login(ctx): + """Login to the service and see how to set the token environment variable.""" + mc = ctx.obj["client"] + if mc is not None: + click.secho("Login successful!", fg="green") @cli.command() -@click.argument('project') -@click.option('--public', is_flag=True, default=False, help='Public project, visible to everyone') -def create(project, public): - """Create a new project on Mergin server""" - - c = _init_client() - if c is None: +@click.argument("project") +@click.option("--public", is_flag=True, default=False, help="Public project, visible to everyone") +@click.pass_context +def create(ctx, project, public): + """Create a new project on Mergin server. `project` needs to be a combination of namespace/project.""" + mc = ctx.obj["client"] + if mc is None: return - + if "/" in project: + try: + namespace, project = project.split("/") + assert namespace, "No namespace given" + assert project, "No project name given" + except (ValueError, AssertionError) as e: + click.secho(f"Incorrect namespace/project format: {e}", fg="red") + return + else: + # namespace not specified, use current user namespace + namespace = mc.username() try: - c.create_project(project, is_public=public) - click.echo('Done') + mc.create_project(project, is_public=public, namespace=namespace) + click.echo("Remote project created") except ClientError as e: - click.secho('Error: ' + str(e), fg='red') + click.secho("Error: " + str(e), fg="red") return except Exception as e: _print_unhandled_exception() @cli.command() -@click.option('--flag', help="What kind of projects (e.g. 'created' for just my projects," - "'shared' for projects shared with me. No flag means returns all public projects.") -def list_projects(flag): +@click.option( + "--flag", + help="What kind of projects (e.g. 'created' for just my projects," + "'shared' for projects shared with me. No flag means returns all public projects.", +) +@click.pass_context +def list_projects(ctx, flag): """List projects on the server""" filter_str = "(filter flag={})".format(flag) if flag is not None else "(all public)" - click.echo('List of projects {}:'.format(filter_str)) - c = _init_client() - if c is None: + click.echo("List of projects {}:".format(filter_str)) + mc = ctx.obj["client"] + if mc is None: return - resp = c.paginated_projects_list(flag=flag) + resp = mc.paginated_projects_list(flag=flag) projects_list = resp["projects"] for project in projects_list: full_name = "{} / {}".format(project["namespace"], project["name"]) - click.echo(" {:40}\t{:6.1f} MB\t{}".format(full_name, project["disk_usage"]/(1024*1024), project['version'])) + click.echo( + " {:40}\t{:6.1f} MB\t{}".format(full_name, project["disk_usage"] / (1024 * 1024), project["version"]) + ) @cli.command() -@click.argument('project') -@click.argument('directory', type=click.Path(), required=False) -@click.option('--version', default=None, help='Version of project to download') -def download(project, directory, version): +@click.argument("project") +@click.argument("directory", type=click.Path(), required=False) +@click.option("--version", default=None, help="Version of project to download") +@click.pass_context +def download(ctx, project, directory, version): """Download last version of mergin project""" - - c = _init_client() - if c is None: + mc = ctx.obj["client"] + if mc is None: return directory = directory or os.path.basename(project) - - click.echo('Downloading into {}'.format(directory)) + click.echo("Downloading into {}".format(directory)) try: - job = download_project_async(c, project, directory, version) - - import time + job = download_project_async(mc, project, directory, version) with click.progressbar(length=job.total_size) as bar: last_transferred_size = 0 while download_project_is_running(job): - time.sleep(1/10) # 100ms + time.sleep(1 / 10) # 100ms new_transferred_size = job.transferred_size bar.update(new_transferred_size - last_transferred_size) # the update() needs increment only last_transferred_size = new_transferred_size - download_project_finalize(job) - - click.echo('Done') + click.echo("Done") except KeyboardInterrupt: - print("Cancelling...") + click.secho("Cancelling...") download_project_cancel(job) except ClientError as e: - click.secho("Error: " + str(e), fg='red') + click.secho("Error: " + str(e), fg="red") except Exception as e: _print_unhandled_exception() @@ -186,25 +256,23 @@ def num_version(name): @cli.command() -def status(): +@click.pass_context +def status(ctx): """Show all changes in project files - upstream and local""" - - c = _init_client() - if c is None: + mc = ctx.obj["client"] + if mc is None: return - try: - pull_changes, push_changes, push_changes_summary = c.project_status(os.getcwd()) + pull_changes, push_changes, push_changes_summary = mc.project_status(os.getcwd()) except InvalidProject as e: - click.secho('Invalid project directory ({})'.format(str(e)), fg='red') + click.secho("Invalid project directory ({})".format(str(e)), fg="red") return except ClientError as e: - click.secho('Error: ' + str(e), fg='red') + click.secho("Error: " + str(e), fg="red") return except Exception as e: _print_unhandled_exception() return - click.secho("### Server changes:", fg="magenta") pretty_diff(pull_changes) click.secho("### Local changes:", fg="magenta") @@ -214,207 +282,172 @@ def status(): @cli.command() -def push(): +@click.pass_context +def push(ctx): """Upload local changes into Mergin repository""" - - c = _init_client() - if c is None: + mc = ctx.obj["client"] + if mc is None: return directory = os.getcwd() - try: - job = push_project_async(c, directory) - - if job is not None: # if job is none, we don't upload any files, and the transaction is finished already - import time + job = push_project_async(mc, directory) + if job is not None: # if job is none, we don't upload any files, and the transaction is finished already with click.progressbar(length=job.total_size) as bar: last_transferred_size = 0 while push_project_is_running(job): - time.sleep(1/10) # 100ms + time.sleep(1 / 10) # 100ms new_transferred_size = job.transferred_size bar.update(new_transferred_size - last_transferred_size) # the update() needs increment only last_transferred_size = new_transferred_size - push_project_finalize(job) - - click.echo('Done') + click.echo("Done") except InvalidProject as e: - click.secho('Invalid project directory ({})'.format(str(e)), fg='red') + click.secho("Invalid project directory ({})".format(str(e)), fg="red") except ClientError as e: - click.secho('Error: ' + str(e), fg='red') + click.secho("Error: " + str(e), fg="red") return except KeyboardInterrupt: - print("Cancelling...") + click.secho("Cancelling...") push_project_cancel(job) except Exception as e: _print_unhandled_exception() @cli.command() -def pull(): +@click.pass_context +def pull(ctx): """Fetch changes from Mergin repository""" - - c = _init_client() - if c is None: + mc = ctx.obj["client"] + if mc is None: return directory = os.getcwd() - try: - job = pull_project_async(c, directory) - + job = pull_project_async(mc, directory) if job is None: - click.echo('Project is up to date') + click.echo("Project is up to date") return - - import time with click.progressbar(length=job.total_size) as bar: last_transferred_size = 0 while pull_project_is_running(job): - time.sleep(1/10) # 100ms + time.sleep(1 / 10) # 100ms new_transferred_size = job.transferred_size bar.update(new_transferred_size - last_transferred_size) # the update() needs increment only last_transferred_size = new_transferred_size - pull_project_finalize(job) - - click.echo('Done') + click.echo("Done") except InvalidProject as e: - click.secho('Invalid project directory ({})'.format(str(e)), fg='red') + click.secho("Invalid project directory ({})".format(str(e)), fg="red") except ClientError as e: - click.secho('Error: ' + str(e), fg='red') + click.secho("Error: " + str(e), fg="red") return except KeyboardInterrupt: - print("Cancelling...") + click.secho("Cancelling...") pull_project_cancel(job) except Exception as e: _print_unhandled_exception() @cli.command() -@click.argument('version') -def show_version(version): - """ Displays information about a single version of a project """ - - c = _init_client() - if c is None: +@click.argument("version") +@click.pass_context +def show_version(ctx, version): + """ Displays information about a single version of a project. `version` is 'v1', 'v2', etc. """ + mc = ctx.obj["client"] + if mc is None: return directory = os.getcwd() - mp = MerginProject(directory) project_path = mp.metadata["name"] - - version_info_dict = c.project_version_info(project_path, version)[0] - print("Project: " + version_info_dict['project']['namespace'] + "/" + version_info_dict['project']['name']) - print("Version: " + version_info_dict['name'] + " by " + version_info_dict['author']) - print("Time: " + version_info_dict['created']) - pretty_diff(version_info_dict['changes']) + # TODO: handle exception when version not found + version_info_dict = mc.project_version_info(project_path, version)[0] + click.secho("Project: " + version_info_dict["project"]["namespace"] + "/" + version_info_dict["project"]["name"]) + click.secho("Version: " + version_info_dict["name"] + " by " + version_info_dict["author"]) + click.secho("Time: " + version_info_dict["created"]) + pretty_diff(version_info_dict["changes"]) @cli.command() -@click.argument('path') -def show_file_history(path): +@click.argument("path") +@click.pass_context +def show_file_history(ctx, path): """ Displays information about a single version of a project """ - - c = _init_client() - if c is None: + mc = ctx.obj["client"] + if mc is None: return directory = os.getcwd() - mp = MerginProject(directory) project_path = mp.metadata["name"] - - info_dict = c.project_file_history_info(project_path, path) - history_dict = info_dict['history'] - - print("File history: " + info_dict['path']) - print("-----") + info_dict = mc.project_file_history_info(project_path, path) + # TODO: handle exception if history not found + history_dict = info_dict["history"] + click.secho("File history: " + info_dict["path"]) + click.secho("-----") for version, version_data in history_dict.items(): - diff_info = '' - if 'diff' in version_data: - diff_info = "diff ({} bytes)".format(version_data['diff']['size']) - print(" {:5} {:10} {}".format(version, version_data['change'], diff_info)) + diff_info = "" + if "diff" in version_data: + diff_info = "diff ({} bytes)".format(version_data["diff"]["size"]) + click.secho(" {:5} {:10} {}".format(version, version_data["change"], diff_info)) @cli.command() -@click.argument('path') -@click.argument('version') -def show_file_changeset(path, version): - """ Displays information about a single version of a project """ - - c = _init_client() - if c is None: +@click.argument("path") +@click.argument("version") +@click.pass_context +def show_file_changeset(ctx, path, version): + """ Displays information about project changes.""" + mc = ctx.obj["client"] + if mc is None: return directory = os.getcwd() - mp = MerginProject(directory) project_path = mp.metadata["name"] - - info_dict = c.project_file_changeset_info(project_path, path, version) - print(json.dumps(info_dict, indent=2)) + info_dict = mc.project_file_changeset_info(project_path, path, version) + # TODO: handle exception if Diff not found + click.secho(json.dumps(info_dict, indent=2)) @cli.command() -@click.argument('directory', required=False) -def modtime(directory): - """ - Show files modification time info. For debug purposes only. - """ - from datetime import datetime - directory = os.path.abspath(directory or os.getcwd()) - strip_len = len(directory) + 1 - for root, dirs, files in os.walk(directory, topdown=True): - for entry in dirs + files: - abs_path = os.path.abspath(os.path.join(root, entry)) - - click.echo(abs_path[strip_len:]) - # click.secho('atime %s' % datetime.fromtimestamp(os.path.getatime(abs_path)), fg="cyan") - click.secho('mtime %s' % datetime.fromtimestamp(os.path.getmtime(abs_path)), fg="cyan") - # click.secho('ctime %s' % datetime.fromtimestamp(os.path.getctime(abs_path)), fg="cyan") - click.echo() - - -@cli.command() -@click.argument('source_project_path', required=True) -@click.argument('cloned_project_name', required=True) -@click.argument('cloned_project_namespace', required=False) -def clone(source_project_path, cloned_project_name, cloned_project_namespace=None): +@click.argument("source_project_path", required=True) +@click.argument("cloned_project_name", required=True) +@click.argument("cloned_project_namespace", required=False) +@click.pass_context +def clone(ctx, source_project_path, cloned_project_name, cloned_project_namespace=None): """Clone project from server.""" - c = _init_client() - if c is None: + mc = ctx.obj["client"] + if mc is None: return try: - c.clone_project(source_project_path, cloned_project_name, cloned_project_namespace) - click.echo('Done') + mc.clone_project(source_project_path, cloned_project_name, cloned_project_namespace) + click.echo("Done") except Exception as e: _print_unhandled_exception() @cli.command() -@click.argument('project', required=False) -def remove(project): - """Remove project from server and locally (if exists).""" - local_info = None - if not project: - from mergin.client import inspect_project # TODO: no inspect_project defined in the client! +@click.argument("project", required=True) +@click.pass_context +def remove(ctx, project): + """Remove project from server.""" + mc = ctx.obj["client"] + if mc is None: + return + if "/" in project: try: - local_info = inspect_project(os.path.join(os.getcwd())) - project = local_info['name'] - except InvalidProject as e: - click.secho('Invalid project directory ({})'.format(str(e)), fg='red') + namespace, project = project.split("/") + assert namespace, "No namespace given" + assert project, "No project name given" + except (ValueError, AssertionError) as e: + click.secho(f"Incorrect namespace/project format: {e}", fg="red") return - - c = _init_client() - if c is None: - return + else: + # namespace not specified, use current user namespace + namespace = mc.username() try: - c.delete_project(project) - if local_info: - import shutil - shutil.rmtree(os.path.join(os.getcwd())) - click.echo('Done') + mc.delete_project(f"{namespace}/{project}") + click.echo("Remote project removed") except Exception as e: _print_unhandled_exception() -if __name__ == '__main__': +if __name__ == "__main__": cli()