Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

speed-up builds with buildGoCache #155

Merged
merged 5 commits into from
Mar 21, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions flake.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,14 @@
inputs.nix-editor.url = "github:replit/nix-editor";
inputs.nix-editor.inputs.nixpkgs.follows = "nixpkgs";

inputs.build-go-cache.url = "github:numtide/build-go-cache";
inputs.build-go-cache.inputs.nixpkgs.follows = "nixpkgs";

outputs = {
self,
nixpkgs,
nix-editor,
build-go-cache,
}: let
systems = [
"aarch64-darwin"
Expand All @@ -27,6 +31,7 @@
packages = eachSystem (system:
import ./nix {
inherit self nixpkgs rev system nix-editor;
inherit (build-go-cache.legacyPackages.${system}) buildGoCache;
});
devShells = eachSystem (system: {
default = self.packages.${system}.devShell;
Expand Down
3 changes: 2 additions & 1 deletion nix/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,13 @@
rev,
self,
system,
buildGoCache
}: let
pkgs = nixpkgs.legacyPackages.${system};
nix-editor-pkg = nix-editor.packages.${system}.nix-editor;
in rec {
default = upm;
devShell = pkgs.callPackage ./devshell {nix-editor = nix-editor-pkg;};
fmt = pkgs.callPackage ./fmt {};
upm = pkgs.callPackage ./upm {inherit rev;};
upm = pkgs.callPackage ./upm {inherit rev; inherit buildGoCache;};
}
29 changes: 23 additions & 6 deletions nix/upm/default.nix
Original file line number Diff line number Diff line change
@@ -1,15 +1,31 @@
{
lib,
buildGoModule,
rev,
makeWrapper,
buildGoCache,
lib,
runCommand,
}:
let
vendorHash = "sha256-7t87/28pBxU7A+SRvw6xxJQ2wErZ/76OJC8feNEQ80E=";

goCache = buildGoCache {
# keep this up-to-date in CI with:
# $ nix run 'github:numtide/build-go-cache#get-external-imports' -- ./. imported-packages
importPackagesFile = ./imported-packages;
# FIXME: Somehow we get cache invalidation everytime if we don't do this, also it uses a source filter
src = runCommand "go-mod" {} ''
install -D ${../../go.mod} $out/go.mod
install -D ${../../go.sum} $out/go.sum
'';
inherit vendorHash;
};
in
buildGoModule rec {
pname = "upm";
version = rev;

src = builtins.path {
name = "${pname}-${version}-src";
name = "${pname}-src";
path = ../../.;
filter = path: _:
builtins.all (block: (builtins.baseNameOf path) != block) [
Expand All @@ -24,8 +40,6 @@ buildGoModule rec {
];
};

vendorHash = "sha256-7t87/28pBxU7A+SRvw6xxJQ2wErZ/76OJC8feNEQ80E=";

ldflags = [
"-X github.com/replit/upm/internal/cli.version=${rev}"
];
Expand All @@ -34,7 +48,7 @@ buildGoModule rec {
go generate ./internal/backends/python
'';

buildInputs = [makeWrapper];
buildInputs = [makeWrapper goCache];

subPackages = ["cmd/upm"];

Expand All @@ -46,5 +60,8 @@ buildGoModule rec {
--set PYPI_MAP_DB "$out/pypi_map.sqlite"
'';

inherit vendorHash;
proxyVendor = true; # we only support proxyVendor with buildGoCache just now

doCheck = false;
}
53 changes: 53 additions & 0 deletions nix/upm/imported-packages
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
archive/tar
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My recommendation for this would be to have a github action that just updates this file after pushes to master automatically. After all it's a performance optimization and it causes too much friction/merge conflicts to have this being updated in pull requests.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Something like that but for updating this file: https://gist.github.com/Broxzier/1ed980b8b3822d213e98042fc6a92040

archive/zip
bufio
bytes
cloud.google.com/go/bigquery
compress/gzip
context
crypto/md5
database/sql
embed
encoding/binary
encoding/hex
encoding/json
encoding/xml
errors
flag
fmt
github.com/BurntSushi/toml
github.com/hashicorp/go-version
github.com/kballard/go-shellquote
github.com/mattn/go-sqlite3
github.com/natefinch/atomic
github.com/smacker/go-tree-sitter
github.com/smacker/go-tree-sitter/javascript
github.com/smacker/go-tree-sitter/python
github.com/smacker/go-tree-sitter/typescript/tsx
github.com/smacker/go-tree-sitter/typescript/typescript
github.com/spf13/cobra
golang.org/x/net/context
golang.org/x/net/html
golang.org/x/term
google.golang.org/api/iterator
gopkg.in/DataDog/dd-trace-go.v1/ddtrace
gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer
gopkg.in/yaml.v2
io
io/fs
log
net/http
net/url
os
os/exec
path
path/filepath
reflect
regexp
runtime
sort
strconv
strings
sync
testing
time
Loading