Skip to content

Commit

Permalink
Use git-lfs push --stdin to push LFS blobs...
Browse files Browse the repository at this point in the history
... so as not to overflow the command-line buffer.
Falls back to the other way if git-lfs doesn't support it.
  • Loading branch information
olsen232 committed Aug 31, 2022
1 parent cad6039 commit 18b8f93
Showing 1 changed file with 50 additions and 12 deletions.
62 changes: 50 additions & 12 deletions kart/lfs_commands/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import pygit2
import subprocess
import sys
import tempfile

import click

Expand All @@ -12,6 +13,7 @@
from kart.object_builder import ObjectBuilder
from kart.rev_list_objects import rev_list_tile_pointer_files
from kart.repo import KartRepoState
from kart.subprocess_util import subprocess_tee

EMPTY_SHA = "0" * 40

Expand All @@ -23,6 +25,53 @@ def lfs_plus(ctx, **kwargs):
"""Git-LFS commands re-implemented in Kart to allow for spatial filtering."""


def push_lfs_oids(repo, remote_name, lfs_oids):
"""
Given a list of OIDs of LFS blobs (not the pointer files, but the LFS blobs themselves)
push all of those LFS blobs from the local cache to the given remote.
"""
# Older git-lfs doesn't support stdin so we fall back to using args if we somehow have an older version.
if not _push_lfs_oids_using_stdin(repo, remote_name, lfs_oids):
_push_lfs_oids_using_args(repo, remote_name, lfs_oids)


def _push_lfs_oids_using_stdin(repo, remote_name, lfs_oids):
# TODO - capture progress reporting and do our own.
with tempfile.TemporaryFile() as oid_file:
oid_file.write("\n".join(lfs_oids).encode("utf-8"))
oid_file.write(b"\n")
oid_file.seek(0)

returncode, stdout, stderr = subprocess_tee(
["git-lfs", "push", remote_name, "--object-id", "--stdin"],
env=tool_environment(),
cwd=repo.workdir_path,
stdin=oid_file,
)
if returncode == 0:
return True
elif b"unknown flag: --stdin" in stderr:
return False
else:
raise SubprocessError(
"There was a problem with git-lfs push", exit_code=returncode
)


def _push_lfs_oids_using_args(repo, remote_name, lfs_oids):
try:
# TODO - capture progress reporting and do our own.
subprocess.check_call(
["git-lfs", "push", remote_name, "--object-id", *lfs_oids],
env=tool_environment(),
cwd=repo.workdir_path,
)
except subprocess.CalledProcessError as e:
raise SubprocessError(
f"There was a problem with git-lfs push: {e}", called_process_error=e
)


@lfs_plus.command()
@click.pass_context
@click.option(
Expand Down Expand Up @@ -71,18 +120,7 @@ def pre_push(ctx, remote_name, remote_url, dry_run):
return

if lfs_oids:
try:
# TODO - chunk lfs_oids so that we don't overflow the maximum argument size.
# TODO - capture chunk progress and report our own overall progress
subprocess.check_call(
["git-lfs", "push", remote_name, "--object-id", *lfs_oids],
env=tool_environment(),
cwd=repo.workdir_path,
)
except subprocess.CalledProcessError as e:
raise SubprocessError(
f"There was a problem with git-lfs push: {e}", called_process_error=e
)
push_lfs_oids(repo, remote_name, lfs_oids)


def get_start_and_stop_commits(input_iter):
Expand Down

0 comments on commit 18b8f93

Please sign in to comment.