[pip] Add pip support (via venv) DEV-1146

README.md

@@ -142,7 +142,7 @@ Devbox makes it easy to package your application into an OCI-compliant container
 Devbox currently detects the following languages/stacks:
 
 - Go
-- Python (Poetry)
+- Python ([See notes](planner/languages/python/))
 - NodeJS
 - PHP
 - static nginx (Experimental - [see notes](planner/languages/nginx))

devbox.go

@@ -140,7 +140,7 @@ func (d *Devbox) Shell() error {
 		return errors.WithStack(err)
 	}
 	nixDir := filepath.Join(d.srcDir, ".devbox/gen/shell.nix")
-	sh, err := nix.DetectShell(nix.WithWelcomeMessage(plan.ShellWelcomeMessage))
+	sh, err := nix.DetectShell(nix.WithPlanInitHook(plan.ShellInitHook))
 	if err != nil {
 		// Fall back to using a plain Nix shell.
 		sh = &nix.Shell{}

devbox_test.go

@@ -12,6 +12,7 @@ import (
 )
 
 func TestDevbox(t *testing.T) {
+	t.Setenv("TMPDIR", "/tmp")
 	testPaths, err := doublestar.FilepathGlob("./testdata/**/devbox.json")
 	assert.NoError(t, err, "Reading testdata/ should not fail")
 
@@ -89,7 +90,7 @@ func assertPlansMatch(t *testing.T, expected *plansdk.Plan, actual *plansdk.Plan
 	)
 
 	assert.ElementsMatch(expected.Definitions, actual.Definitions, "Definitions should match")
-	assert.Equal(expected.ShellWelcomeMessage, actual.ShellWelcomeMessage, "ShellWelcomeMessage should match")
+	assert.Equal(expected.ShellInitHook, actual.ShellInitHook, "ShellInitHook should match")
 	if expected.GeneratedFiles != nil {
 		assert.Equal(expected.GeneratedFiles, actual.GeneratedFiles, "GeneratedFiles should match")
 	}

nix/shell.go

@@ -37,10 +37,10 @@ type Shell struct {
 	name            name
 	binPath         string
 	userShellrcPath string
+	planInitHook    string
 
 	// UserInitHook contains commands that will run at shell startup.
-	UserInitHook   string
-	welcomeMessage string
+	UserInitHook string
 }
 
 type ShellOption func(*Shell)
@@ -91,9 +91,9 @@ func DetectShell(opts ...ShellOption) (*Shell, error) {
 	return sh, nil
 }
 
-func WithWelcomeMessage(message string) ShellOption {
+func WithPlanInitHook(hook string) ShellOption {
 	return func(s *Shell) {
-		s.welcomeMessage = message
+		s.planInitHook = hook
 	}
 }
 
@@ -249,12 +249,12 @@ func (s *Shell) writeDevboxShellrc() (path string, err error) {
 		OriginalInit     string
 		OriginalInitPath string
 		UserHook         string
-		WelcomeMessage   string
+		PlanInitHook     string
 	}{
 		OriginalInit:     string(bytes.TrimSpace(userShellrc)),
 		OriginalInitPath: filepath.Clean(s.userShellrcPath),
 		UserHook:         strings.TrimSpace(s.UserInitHook),
-		WelcomeMessage:   strings.TrimSpace(s.welcomeMessage),
+		PlanInitHook:     strings.TrimSpace(s.planInitHook),
 	})
 	if err != nil {
 		return "", fmt.Errorf("execute shellrc template: %v", err)

nix/shell_test.go

@@ -51,7 +51,7 @@ func TestWriteDevboxShellrc(t *testing.T) {
 			s := &Shell{
 				userShellrcPath: test.shellrcPath,
 				UserInitHook:    test.hook,
-				welcomeMessage:  "Welcome to the devbox!",
+				planInitHook:    `echo "Welcome to the devbox!"`,
 			}
 			gotPath, err := s.writeDevboxShellrc()
 			if err != nil {

nix/shellrc.tmpl

@@ -75,12 +75,12 @@ export PS1="(devbox) $PS1"
 
 {{- end }}
 
-# Begin Shell Welcome Message
+# Begin Plan Init Hook
 
-{{- if .WelcomeMessage }}
+{{- if .PlanInitHook }}
 
-echo "{{ .WelcomeMessage }}"
+{{ .PlanInitHook }}
 
 {{- end }}
 
-# End Shell Welcome Message
+# End Plan Init Hook

nix/testdata/shellrc/basic/shellrc.golden

@@ -85,8 +85,8 @@ echo "Hello from a devbox shell hook!"
 
 # End Devbox User Hook
 
-# Begin Shell Welcome Message
+# Begin Plan Init Hook
 
 echo "Welcome to the devbox!"
 
-# End Shell Welcome Message
+# End Plan Init Hook

nix/testdata/shellrc/nohook/shellrc.golden

@@ -79,8 +79,8 @@ export PS1="(devbox) $PS1"
 
 # End Devbox Post-init Hook
 
-# Begin Shell Welcome Message
+# Begin Plan Init Hook
 
 echo "Welcome to the devbox!"
 
-# End Shell Welcome Message
+# End Plan Init Hook

nix/testdata/shellrc/noshellrc/shellrc.golden

@@ -43,8 +43,8 @@ echo "Hello from a devbox shell hook!"
 
 # End Devbox User Hook
 
-# Begin Shell Welcome Message
+# Begin Plan Init Hook
 
 echo "Welcome to the devbox!"
 
-# End Shell Welcome Message
+# End Plan Init Hook

planner/languages/nginx/nginx_planner.go

@@ -29,7 +29,9 @@ func (p *Planner) IsRelevant(srcDir string) bool {
 
 func (p *Planner) GetPlan(srcDir string) *plansdk.Plan {
 	return &plansdk.Plan{
-		ShellWelcomeMessage: fmt.Sprintf(welcomeMessage, p.shellConfig(srcDir)),
+		ShellInitHook: plansdk.WelcomeMessage(
+			fmt.Sprintf(welcomeMessage, p.shellConfig(srcDir)),
+		),
 		DevPackages: []string{
 			"nginx",
 			"shell-nginx",

planner/languages/python/README.md

@@ -0,0 +1,41 @@
+# Python Planner
+
+* We currently support shell and build using poetry and pip. We generally recommend using poetry, but will do our best to support both. 
+
+# Python Poetry Planner
+
+## Detection
+
+* This planner looks for `poetry.lock` or `pyproject.toml` in your devbox.json directory.
+
+## Shell
+
+* poetry by default uses virtual environment so there should be no changes to dev workflow.
+
+## Build
+
+This planner uses pex to build an executable that has all dependencies. It looks for an entrypoint in the following order:
+
+* If there's a module with same name as the project, it uses that as the entrypoint.
+* If there's a script with same name as project, it uses that as the entrypoint.
+* Use first script in alphabetical order as the entrypoint.
+
+# Python Pip Planner
+
+## Detection
+
+* Looks for `requirements.txt` in your devbox.json directory. We default to python3 as provided by nix packages.
+
+## Shell
+
+* Uses venv (automatically created in .venv) to create a virtual environment.
+
+## Build
+
+* Uses pex to build an executable that has all dependencies. 
+* Requires barebones `setup.py`
+* Uses `__main__` module in package that matches the lowercase `setup.py` name. Any dashes in the name are replaced with underscores.
+
+# Limitations
+
+Currently `build` does not support projects that depend on libraries with native extensions (e.g. `pandas`).

planner/languages/python/python_pip_planner.go

@@ -0,0 +1,85 @@
+// Copyright 2022 Jetpack Technologies Inc and contributors. All rights reserved.
+// Use of this source code is governed by the license in the LICENSE file.
+
+package python
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"go.jetpack.io/devbox/boxcli/usererr"
+	"go.jetpack.io/devbox/planner/plansdk"
+)
+
+// TODO: Doesn't work with libraries like Pandas that have C extensions
+// We get error
+// ImportError: libstdc++.so.6: cannot open shared object file: No such file or directory
+// possible solution is to set $LD_LIBRARY_PATH
+// https://nixos.wiki/wiki/Packaging/Quirks_and_Caveats
+type PIPPlanner struct{}
+
+// PythonPoetryPlanner implements interface Planner (compile-time check)
+var _ plansdk.Planner = (*PIPPlanner)(nil)
+
+func (p *PIPPlanner) Name() string {
+	return "python.Planner"
+}
+
+func (p *PIPPlanner) IsRelevant(srcDir string) bool {
+	return plansdk.FileExists(filepath.Join(srcDir, "requirements.txt"))
+}
+func (p *PIPPlanner) GetPlan(srcDir string) *plansdk.Plan {
+	plan := &plansdk.Plan{
+		DevPackages: []string{
+			"python3",
+		},
+		RuntimePackages: []string{
+			`python3`,
+		},
+		ShellInitHook: p.shellInitHook(srcDir),
+	}
+	if err := p.isBuildable(srcDir); err != nil {
+		return plan.WithError(err)
+	}
+	plan.InstallStage = &plansdk.Stage{
+		Command:    "python -m venv .venv && source .venv/bin/activate && pip install -r requirements.txt",
+		InputFiles: plansdk.AllFiles(),
+	}
+	plan.BuildStage = &plansdk.Stage{Command: pipBuildCommand}
+	plan.StartStage = &plansdk.Stage{
+		Command:    "python ./app.pex",
+		InputFiles: []string{"app.pex"},
+	}
+	return plan
+}
+
+func (p *PIPPlanner) isBuildable(srcDir string) error {
+	if plansdk.FileExists(filepath.Join(srcDir, "setup.py")) {
+		return nil
+	}
+
+	return usererr.New(
+		"setup.py not found. Please create a setup.py file to build your project." +
+			" The distribution name must be a case-insensitive match of the package" +
+			" (dir) name. Dashes are converted to underscores.",
+	)
+}
+
+func (p *PIPPlanner) shellInitHook(srcDir string) string {
+	venvPath := filepath.Join(srcDir, ".venv")
+	venvActivatePath := filepath.Join(srcDir, ".venv", "bin", "activate")
+	script := strings.TrimSpace(`
+echo "Creating/Using virtual environment in %[1]s";
+python -m venv "%[1]s";
+source "%[2]s";`)
+	return fmt.Sprintf(script, venvPath, venvActivatePath)
+}
+
+var pipBuildCommand = strings.TrimSpace(`
+source .venv/bin/activate && \
+pip install pex && \
+PACKAGE_NAME=$(python setup.py --name |  tr '[:upper:]-' '[:lower:]_') && \
+pex . -o app.pex -m $PACKAGE_NAME -r requirements.txt
+`,
+)

planner/languages/python/python_poetry_planner.go

@@ -16,21 +16,21 @@ import (
 	"golang.org/x/exp/slices"
 )
 
-type Planner struct{}
+type PoetryPlanner struct{}
 
 // PythonPoetryPlanner implements interface Planner (compile-time check)
-var _ plansdk.Planner = (*Planner)(nil)
+var _ plansdk.Planner = (*PoetryPlanner)(nil)
 
-func (p *Planner) Name() string {
+func (p *PoetryPlanner) Name() string {
 	return "python.Planner"
 }
 
-func (p *Planner) IsRelevant(srcDir string) bool {
+func (p *PoetryPlanner) IsRelevant(srcDir string) bool {
 	return plansdk.FileExists(filepath.Join(srcDir, "poetry.lock")) ||
 		plansdk.FileExists(filepath.Join(srcDir, "pyproject.toml"))
 }
 
-func (p *Planner) GetPlan(srcDir string) *plansdk.Plan {
+func (p *PoetryPlanner) GetPlan(srcDir string) *plansdk.Plan {
 	version := p.PythonVersion(srcDir)
 	pythonPkg := fmt.Sprintf("python%s", version.MajorMinorConcatenated())
 	plan := &plansdk.Plan{
@@ -62,7 +62,7 @@ func (p *Planner) GetPlan(srcDir string) *plansdk.Plan {
 }
 
 // TODO: This can be generalized to all python planners
-func (p *Planner) PythonVersion(srcDir string) *plansdk.Version {
+func (p *PoetryPlanner) PythonVersion(srcDir string) *plansdk.Version {
 	defaultVersion, _ := plansdk.NewVersion("3.10.6")
 	project := p.PyProject(srcDir)
 
@@ -76,7 +76,7 @@ func (p *Planner) PythonVersion(srcDir string) *plansdk.Version {
 	return defaultVersion
 }
 
-func (p *Planner) buildCommand(srcDir string) string {
+func (p *PoetryPlanner) buildCommand(srcDir string) string {
 	project := p.PyProject(srcDir)
 	// Assume name follows https://peps.python.org/pep-0508/#names
 	// Do simple replacement "-" -> "_" and check if any script matches name.
@@ -113,7 +113,7 @@ type pyProject struct {
 	} `toml:"tool"`
 }
 
-func (p *Planner) PyProject(srcDir string) *pyProject {
+func (p *PoetryPlanner) PyProject(srcDir string) *pyProject {
 	pyProjectPath := filepath.Join(srcDir, "pyproject.toml")
 	content, err := os.ReadFile(pyProjectPath)
 	if err != nil {
@@ -124,7 +124,7 @@ func (p *Planner) PyProject(srcDir string) *pyProject {
 	return &proj
 }
 
-func (p *Planner) isBuildable(srcDir string) (bool, error) {
+func (p *PoetryPlanner) isBuildable(srcDir string) (bool, error) {
 	project := p.PyProject(srcDir)
 	if project == nil {
 		return false, usererr.New("Could not build container for python " +
@@ -164,7 +164,7 @@ func (p *Planner) isBuildable(srcDir string) (bool, error) {
 	return true, nil
 }
 
-func (p *Planner) formatBuildCommand(module, script string) string {
+func (p *PoetryPlanner) formatBuildCommand(module, script string) string {
 
 	// If no scripts, just run the module directly always.
 	if script == "" {

planner/planner.go

@@ -59,7 +59,8 @@ var PLANNERS = []plansdk.Planner{
 	&ocaml.Planner{},
 	&perl.Planner{},
 	&php.Planner{},
-	&python.Planner{},
+	&python.PoetryPlanner{},
+	&python.PIPPlanner{},
 	&ruby.Planner{},
 	&rust.Planner{},
 	&scala.Planner{},

planner/plansdk/plansdk.go

@@ -5,6 +5,7 @@ package plansdk
 
 import (
 	"encoding/json"
+	"fmt"
 	"os"
 
 	"github.com/imdario/mergo"
@@ -26,7 +27,7 @@ type PlanError struct {
 
 // Plan tells devbox how to start shells and build projects.
 type Plan struct {
-	ShellWelcomeMessage string `json:"shell_welcome_message,omitempty"`
+	ShellInitHook string `json:"shell_init_hook,omitempty"`
 
 	NixOverlays []string `cur:"[...string]" json:"nix_overlays,omitempty"`
 
@@ -183,3 +184,7 @@ func FileExists(path string) bool {
 	_, err := os.Stat(path)
 	return err == nil
 }
+
+func WelcomeMessage(s string) string {
+	return fmt.Sprintf(`echo "%s";`, s)
+}

testdata/nginx/plan.json

@@ -20,5 +20,8 @@
   "definitions": [
     "\nshell-nginx = pkgs.writeShellScriptBin \"shell-nginx\" ''\n\necho \"Starting nginx with command:\"\necho \"nginx -p testdata/nginx -c shell-nginx.conf -e /tmp/error.log -g \\\"pid /tmp/mynginx.pid;daemon off;\\\"\"\nnginx -p testdata/nginx -c shell-nginx.conf -e /tmp/error.log -g \"pid /tmp/shell-nginx.pid;daemon off;\"\n'';"
   ],
-  "shell_welcome_message": "\n##### WARNING: nginx planner is experimental #####\n\nYou may need to add\n\n\\\"include ./.devbox/gen/shell-helper-nginx.conf;\\\"\n\nto your shell-nginx.conf file to ensure the server can start in the nix shell.\n\nUse \\\"shell-nginx\\\" to start the server\n"
+  "shell_init_hook": "echo \"\n##### WARNING: nginx planner is experimental #####\n\nYou may need to add\n\n\\\"include ./.devbox/gen/shell-helper-nginx.conf;\\\"\n\nto your shell-nginx.conf file to ensure the server can start in the nix shell.\n\nUse \\\"shell-nginx\\\" to start the server\n\";",
+  "generated_files": {
+    "shell-helper-nginx.conf": "access_log /tmp/access.log;\nclient_body_temp_path /tmp/client_body;\nproxy_temp_path /tmp/proxy;\nfastcgi_temp_path /tmp/fastcgi;\nuwsgi_temp_path /tmp/uwsgi;\nscgi_temp_path /tmp/scgi;\n"
+  }
 }

testdata/python/pip-example/.gitignore

@@ -0,0 +1 @@
+.venv

testdata/python/pip-example/devbox.json

@@ -0,0 +1,6 @@
+{
+  "packages": [],
+  "shell": {
+    "init_hook": null
+  }
+}