From 1c32c4df609946c99658449b4305e4ba7147b4ed Mon Sep 17 00:00:00 2001 From: Andrew Meyer Date: Tue, 12 Oct 2021 16:49:25 -0700 Subject: [PATCH] Support independently-installed pip Adds pip as a dependency to this buildpack. When BP_PIP_VERSION=latest, this version will be used. When unset, python's own pip module will be used. --- .../integration/deploy_a_python_app_test.go | 29 ++++++ src/python/supply/supply.go | 99 ++++++++++++++----- src/python/supply/supply_test.go | 68 +++++++++++++ 3 files changed, 174 insertions(+), 22 deletions(-) diff --git a/src/python/integration/deploy_a_python_app_test.go b/src/python/integration/deploy_a_python_app_test.go index 8c23f5124..71a479bcf 100644 --- a/src/python/integration/deploy_a_python_app_test.go +++ b/src/python/integration/deploy_a_python_app_test.go @@ -278,4 +278,33 @@ var _ = Describe("CF Python Buildpack", func() { Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!")) Eventually(app.Stdout.String).Should(MatchRegexp(`\[APP/PROC/WEB/0\] .* "GET / HTTP/1.1"`)) }) + + Context("specifying pip version", func() { + Context("default", func() { + BeforeEach(func() { + app = cutlass.New(Fixtures("flask")) + app.SetEnv("BP_PIP_VERSION", "") + }) + + It("uses python's pip module", func() { + PushAppAndConfirm(app) + Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!")) + Expect(app.Stdout.String()).To(ContainSubstring("Using python's pip module")) + }) + }) + + Context("latest", func() { + BeforeEach(func() { + app = cutlass.New(Fixtures("flask")) + app.SetEnv("BP_PIP_VERSION", "latest") + }) + + It("uses latest from manifest", func() { + PushAppAndConfirm(app) + Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!")) + Expect(app.Stdout.String()).To(ContainSubstring("Installing pip")) + Expect(app.Stdout.String()).To(MatchRegexp(`Successfully installed pip-\d+.\d+.\d+`)) + }) + }) + }) }) diff --git a/src/python/supply/supply.go b/src/python/supply/supply.go index 3b878bee2..044c6c9bf 100644 --- a/src/python/supply/supply.go +++ b/src/python/supply/supply.go @@ -21,31 +21,33 @@ import ( "github.com/kr/text" ) +const EnvPipVersion = "BP_PIP_VERSION" + type Stager interface { BuildDir() string CacheDir() string DepDir() string DepsIdx() string - LinkDirectoryInDepDir(string, string) error - WriteEnvFile(string, string) error - WriteProfileD(string, string) error + LinkDirectoryInDepDir(destDir, destSubDir string) error + WriteEnvFile(envVar, envVal string) error + WriteProfileD(scriptName, scriptContents string) error } type Manifest interface { - AllDependencyVersions(string) []string - DefaultVersion(string) (libbuildpack.Dependency, error) + AllDependencyVersions(depName string) []string + DefaultVersion(depName string) (libbuildpack.Dependency, error) IsCached() bool } type Installer interface { - InstallDependency(libbuildpack.Dependency, string) error - InstallOnlyVersion(string, string) error + InstallDependency(dep libbuildpack.Dependency, outputDir string) error + InstallOnlyVersion(depName, installDir string) error } type Command interface { - Execute(string, io.Writer, io.Writer, string, ...string) error + Execute(dir string, stdout io.Writer, stderr io.Writer, program string, args ...string) error Output(dir string, program string, args ...string) (string, error) - RunWithOutput(*exec.Cmd) ([]byte, error) + RunWithOutput(cmd *exec.Cmd) ([]byte, error) } type Supplier struct { @@ -95,6 +97,11 @@ func RunPython(s *Supplier) error { return err } + if err := s.InstallPip(); err != nil { + s.Log.Error("Could not install pip: %v", err) + return err + } + if err := s.InstallPipPop(); err != nil { s.Log.Error("Could not install pip pop: %v", err) return err @@ -198,7 +205,7 @@ func (s *Supplier) HandleMercurial() error { s.Log.Warning("Cloud Foundry does not support Pip Mercurial dependencies while in offline-mode. Vendor your dependencies if they do not work.") } - if err := s.Command.Execute(s.Stager.BuildDir(), indentWriter(os.Stdout), indentWriter(os.Stderr), "python", "-m", "pip", "install", "mercurial"); err != nil { + if err := s.runPipInstall("mercurial"); err != nil { return err } @@ -328,14 +335,43 @@ func (s *Supplier) RewriteShebangs() error { return nil } +func (s *Supplier) InstallPip() error { + pipVersion := os.Getenv(EnvPipVersion) + if pipVersion == "" { + s.Log.Info("Using python's pip module") + return nil + } + if pipVersion != "latest" { + return fmt.Errorf("invalid pip version: %s", pipVersion) + } + + tempPath := filepath.Join("/tmp", "pip") + if err := s.Installer.InstallOnlyVersion("pip", tempPath); err != nil { + return err + } + + if err := s.Command.Execute(s.Stager.BuildDir(), indentWriter(os.Stdout), indentWriter(os.Stderr), + "python", + "-m", "pip", + "install", "pip", + "--exists-action=w", + "--no-index", + "--ignore-installed", + fmt.Sprintf("--find-links=%s", tempPath), + ); err != nil { + return err + } + + return s.Stager.LinkDirectoryInDepDir(filepath.Join(s.Stager.DepDir(), "python", "bin"), "bin") +} + func (s *Supplier) InstallPipPop() error { tempPath := filepath.Join("/tmp", "pip-pop") if err := s.Installer.InstallOnlyVersion("pip-pop", tempPath); err != nil { return err } - if err := s.Command.Execute(s.Stager.BuildDir(), ioutil.Discard, ioutil.Discard, "python", "-m", "pip", "install", "pip-pop", "--exists-action=w", "--no-index", fmt.Sprintf("--find-links=%s", tempPath)); err != nil { - s.Log.Debug("******Path val: %s", os.Getenv("PATH")) + if err := s.runPipInstall("pip-pop", "--exists-action=w", "--no-index", fmt.Sprintf("--find-links=%s", tempPath)); err != nil { return err } @@ -386,7 +422,12 @@ func (s *Supplier) InstallPipEnv() error { s.Log.Info("Installing %s", dep) out := &bytes.Buffer{} stderr := &bytes.Buffer{} - if err := s.Command.Execute(s.Stager.BuildDir(), out, stderr, "python", "-m", "pip", "install", dep, "--exists-action=w", "--no-index", fmt.Sprintf("--find-links=%s", filepath.Join("/tmp", "pipenv"))); err != nil { + if err := s.runPipInstall( + dep, + "--exists-action=w", + "--no-index", + fmt.Sprintf("--find-links=%s", filepath.Join("/tmp", "pipenv")), + ); err != nil { return fmt.Errorf("Failed to install %s: %v.\nStdout: %v\nStderr: %v", dep, err, out, stderr) } } @@ -614,8 +655,13 @@ func (s *Supplier) RunPipUnvendored() error { return err } - installArgs := []string{"-m", "pip", "install", "-r", requirementsPath, "--ignore-installed", "--exists-action=w", "--src=" + filepath.Join(s.Stager.DepDir(), "src"), "--disable-pip-version-check"} - if err := s.Command.Execute(s.Stager.BuildDir(), indentWriter(os.Stdout), indentWriter(os.Stderr), "python", installArgs...); err != nil { + if err := s.runPipInstall( + "-r", requirementsPath, + "--ignore-installed", + "--exists-action=w", + "--src="+filepath.Join(s.Stager.DepDir(), "src"), + "--disable-pip-version-check", + ); err != nil { return fmt.Errorf("could not run pip: %v", err) } @@ -639,11 +685,7 @@ func (s *Supplier) RunPipVendored() error { } installArgs := []string{ - "-m", - "pip", - "install", - "-r", - requirementsPath, + "-r", requirementsPath, "--ignore-installed", "--exists-action=w", "--src=" + filepath.Join(s.Stager.DepDir(), "src"), @@ -672,7 +714,7 @@ func (s *Supplier) RunPipVendored() error { return fmt.Errorf("could not overwrite requirements file: %v", err) } - if err := s.Command.Execute(s.Stager.BuildDir(), indentWriter(os.Stdout), indentWriter(os.Stderr), "python", installArgs...); err != nil { + if err := s.runPipInstall(installArgs...); err != nil { s.Log.Info("Running pip install without indexes failed. Not all dependencies were vendored. Trying again with indexes.") if err := ioutil.WriteFile(requirementsPath, originalReqs, 0644); err != nil { @@ -797,6 +839,18 @@ func (s *Supplier) shouldRunPip() (bool, string, error) { return true, requirementsPath, nil } +func pipCommand() []string { + if os.Getenv(EnvPipVersion) != "" { + return []string{"pip"} + } + return []string{"python", "-m", "pip"} +} + +func (s *Supplier) runPipInstall(args ...string) error { + installCmd := append(append(pipCommand(), "install"), args...) + return s.Command.Execute(s.Stager.BuildDir(), indentWriter(os.Stdout), indentWriter(os.Stderr), installCmd[0], installCmd[1:]...) +} + func (s *Supplier) formatVersion(version string) string { verSlice := strings.Split(version, ".") @@ -814,7 +868,8 @@ func (s *Supplier) writeTempRequirementsTxt(content string) error { } func (s *Supplier) hasBuildOptions() bool { - err := s.Command.Execute(s.Stager.BuildDir(), nil, nil, "python", "-m", "pip", "install", "--no-build-isolation", "-h") + helpCommand := append(pipCommand(), "install", "--no-build-isolation", "-h") + err := s.Command.Execute(s.Stager.BuildDir(), nil, nil, helpCommand[0], helpCommand[1:]...) return nil == err } diff --git a/src/python/supply/supply_test.go b/src/python/supply/supply_test.go index eb02def8b..afe171941 100644 --- a/src/python/supply/supply_test.go +++ b/src/python/supply/supply_test.go @@ -130,6 +130,74 @@ var _ = Describe("Supply", func() { }) }) + Describe("InstallPip", func() { + Describe("BP_PIP_VERSION not set", func() { + BeforeEach(func() { + Expect(os.Unsetenv("BP_PIP_VERSION")).To(Succeed()) + }) + + It("skips install", func() { + mockInstaller.EXPECT().InstallOnlyVersion(gomock.Any(), gomock.Any()).Times(0) + mockCommand.EXPECT().Execute(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + mockStager.EXPECT().LinkDirectoryInDepDir(gomock.Any(), gomock.Any()).Times(0) + + Expect(supplier.InstallPip()).To(Succeed()) + }) + + It("uses python's pip module for installs", func() { + Expect(ioutil.WriteFile(filepath.Join(buildDir, "requirements.txt"), []byte{}, 0644)).To(Succeed()) + mockStager.EXPECT().LinkDirectoryInDepDir(gomock.Any(), gomock.Any()) + + mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "python", "-m", "pip", "install", "-r", filepath.Join(buildDir, "requirements.txt"), "--ignore-installed", "--exists-action=w", fmt.Sprintf("--src=%s/src", depDir), "--disable-pip-version-check") + Expect(supplier.RunPipUnvendored()).To(Succeed()) + }) + }) + + Describe("BP_PIP_VERSION set to 'latest'", func() { + BeforeEach(func() { + Expect(os.Setenv("BP_PIP_VERSION", "latest")).To(Succeed()) + }) + + AfterEach(func() { + Expect(os.Unsetenv("BP_PIP_VERSION")).To(Succeed()) + }) + + It("installs latest from manifest", func() { + mockInstaller.EXPECT().InstallOnlyVersion("pip", "/tmp/pip") + mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "python", "-m", "pip", "install", "pip", "--exists-action=w", "--no-index", "--ignore-installed", "--find-links=/tmp/pip") + mockStager.EXPECT().LinkDirectoryInDepDir(filepath.Join(filepath.Join(depDir, "python"), "bin"), "bin") + + Expect(supplier.InstallPip()).To(Succeed()) + }) + + It("uses installed pip for installs", func() { + Expect(ioutil.WriteFile(filepath.Join(buildDir, "requirements.txt"), []byte{}, 0644)).To(Succeed()) + mockStager.EXPECT().LinkDirectoryInDepDir(gomock.Any(), gomock.Any()) + + mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "pip", "install", "-r", filepath.Join(buildDir, "requirements.txt"), "--ignore-installed", "--exists-action=w", fmt.Sprintf("--src=%s/src", depDir), "--disable-pip-version-check") + Expect(supplier.RunPipUnvendored()).To(Succeed()) + }) + }) + + Describe("BP_PIP_VERSION is invalid", func() { + BeforeEach(func() { + Expect(os.Setenv("BP_PIP_VERSION", "something-else")).To(Succeed()) + }) + + AfterEach(func() { + Expect(os.Unsetenv("BP_PIP_VERSION")).To(Succeed()) + }) + + It("returns an error without installing", func() { + mockInstaller.EXPECT().InstallOnlyVersion(gomock.Any(), gomock.Any()).Times(0) + mockCommand.EXPECT().Execute(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + mockStager.EXPECT().LinkDirectoryInDepDir(gomock.Any(), gomock.Any()).Times(0) + + Expect(supplier.InstallPip()).To(MatchError("invalid pip version: something-else")) + }) + }) + }) + Describe("HandlePipfile", func() { BeforeEach(func() { Expect(os.MkdirAll(depDir, 0755)).To(Succeed())