Skip to content
This repository was archived by the owner on Oct 21, 2022. It is now read-only.

Shallow clone the github head version for client libs #243

Merged
liyanhui1228 merged 1 commit into from
Feb 12, 2019
Merged

Shallow clone the github head version for client libs #243

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 41 additions & 18 deletions compatibility_server/pip_checker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@
PYPI_URL = 'https://pypi.org/pypi/'
CONTAINER_WITH_PKG = "checker"
TIME_OUT = 300 # seconds
GITHUB_CLIENTLIBS_PREFIX = 'git+git://github.com/googleapis/' \
Copy link
Copy Markdown
Contributor

@brianquinlan brianquinlan Feb 14, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this really called client libs? I thought that was https://github.com/googleapis/google-api-python-client

Copy link
Copy Markdown
Member Author

@liyanhui1228 liyanhui1228 Feb 14, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's https://github.com/googleapis/google-cloud-python actually, I just call it clientlibs for short. That one is the discovery based APIs.

'google-cloud-python.git#subdirectory='
CLIENTLIBS_GITHUB_URL = 'https://github.com/googleapis/google-cloud-python.git'

# Pattern for pip installation errors not related to the package being
# installed. See:
Expand Down Expand Up @@ -353,24 +356,44 @@ def _build_command(self, subcommands: List[str]):

def _install(self, container: docker.models.containers.Container):
"""Run pip install in the container."""
command = self._build_command(['install', '-U'] + self._packages)
returncode, output = self._run_command(
container,
command,
stdout=False,
stderr=True,
raise_on_failure=False)
if returncode:
# Checking for environment error
environment_error = PIP_ENVIRONMENT_ERROR_PATTERN.search(output)
if environment_error:
raise PipError(error_msg=environment_error.group('error'),
command=command,
returncode=returncode)

return PipCheckResult(self._packages,
PipCheckResultType.INSTALL_ERROR,
output)
# If there is cloud client libraries, install it from source.
# Else install it from PyPI.
for pkg in self._packages:
Copy link
Copy Markdown
Contributor

@brianquinlan brianquinlan Feb 14, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd structure this a bit differently to make it more readable.

def _clone_repo(self, container, repo):
 """<docstring>"""
  _, directory = self._run_command(
    container,
    "echo $(mktemp -d)"
    stderr=False,
    stdout=True,
    raise_on_failure=True)

  shallow_clone_command = ['git', 'clone', '--depth', '1', repo, directory]
  self._run_command(
    container,
    shallow_clone_command,
    stdout=False,
    stderr=True,
    raise_on_failure=True)
  return directory

def _install(self, container: docker.models.containers.Container):
if any(GITHUB_CLIENTLIBS_PREFIX in pkg for pkg in self._packages):
  # The https://github.com/googleapis/google-cloud-python repository is large,
  # has a large amount of history and contains many independent packages.
  # Installing packages from that repository is quite slow e.g.
  # $ time pip install git+git://...
  # <result>
  # As an optimization, if any provided package is installed from this repository,
  # it is first cloned (using --depth=1 to avoid unnecessary history) and then
  # installed from local disk e.g.
  # $ time git clone --depth=1 ... && pip install ...
  # <result>
  client_repo_directory = self._clone_repo(CLIENTLIBS_GITHUB_URL)

install_names = []
for pkg in self._packages:
  if GITHUB_CLIENTLIBS_PREFIX in pkg:
    install_subdirectory = pkg.split(GITHUB_CLIENTLIBS_PREFIX)[1]
    install_names.append(os.path.join(client_repo_directory, install_subdirectory))
  else:
    install_names.append(pkg)
command = self._build_command(['install', '-U'] + install_names)
# Rest is the same.

Copy link
Copy Markdown
Member Author

@liyanhui1228 liyanhui1228 Feb 15, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

if GITHUB_CLIENTLIBS_PREFIX in pkg:
# Shallow clone the repository
shallow_clone_command = ['git', 'clone', '--depth', '1',
CLIENTLIBS_GITHUB_URL]
self._run_command(
Copy link
Copy Markdown
Contributor

@brianquinlan brianquinlan Feb 14, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't like that we can run this multiple times and every time after the first will fail. I also don't like that we don't check for failures.

container,
shallow_clone_command,
stdout=False,
stderr=True,
raise_on_failure=False)
gh_clientlib = pkg.split(GITHUB_CLIENTLIBS_PREFIX)[1]
command = self._build_command(
['install', '-e',
'google-cloud-python/{}'.format(gh_clientlib)])
else:
command = self._build_command(['install', '-U'] + [pkg])

returncode, output = self._run_command(
container,
command,
stdout=False,
stderr=True,
raise_on_failure=False)

if returncode:
# Checking for environment error
environment_error = PIP_ENVIRONMENT_ERROR_PATTERN.search(
output)
if environment_error:
raise PipError(error_msg=environment_error.group('error'),
command=command,
returncode=returncode)
return PipCheckResult(self._packages,
PipCheckResultType.INSTALL_ERROR,
output)
return PipCheckResult(self._packages, PipCheckResultType.SUCCESS)

def _check(self, container: docker.models.containers.Container):
Expand Down