From df7a08893e4402182ec64178ffdb3130aa228943 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Tue, 18 Jun 2013 23:07:52 -0400 Subject: [PATCH] pkg2 wip: package caching works; install, update and remove in progress. --- base/pkg2.jl | 52 +++++++++++++++++++++++++++++++++++++++------- base/pkg2/cache.jl | 24 +++++++++++++++++++++ base/pkg2/dir.jl | 22 ++------------------ base/pkg2/query.jl | 19 ++++++++--------- base/pkg2/read.jl | 5 +++++ base/pkg2/write.jl | 34 +++++++++++++++++++++++++++++- 6 files changed, 117 insertions(+), 39 deletions(-) create mode 100644 base/pkg2/cache.jl diff --git a/base/pkg2.jl b/base/pkg2.jl index 97cf8e431ec8b..4a4f38521211d 100644 --- a/base/pkg2.jl +++ b/base/pkg2.jl @@ -6,11 +6,22 @@ include("pkg2/reqs.jl") include("pkg2/read.jl") include("pkg2/query.jl") include("pkg2/resolve.jl") +include("pkg2/cache.jl") include("pkg2/write.jl") using .Types +add(pkg::String, vers::VersionSet) = Dir.cd() do + Write.edit("REQUIRE", Reqs.add, pkg, vers) +end +add(pkg::String, vers::VersionNumber...) = add(pkg, VersionSet(vers...)) + +rm(pkg::String) = Dir.cd() do + Write.edit("REQUIRE", Reqs.rm, pkg) +end + resolve() = Dir.cd() do + # figure out what should be installed reqs = Reqs.parse("REQUIRE") avail = Read.available() fixed = Read.fixed(avail) @@ -18,16 +29,41 @@ resolve() = Dir.cd() do deps = Query.dependencies(avail,fixed) want = Resolve.resolve(reqs,deps) have = Read.free(avail) - diff = Query.diff(have,want) -end -add(pkg::ByteString, vers::VersionSet) = Dir.cd() do - Write.update_file(Reqs.add, "REQUIRE", pkg, vers) -end -add(pkg::ByteString, vers::VersionNumber...) = add(pkg, VersionSet(vers...)) + # compare what is installed with what should be + install, update, remove = Query.diff(have, want) + + # prefetch phase isolates network activity, nothing to roll back + for (pkg,ver) in install + Cache.fetch(pkg, Read.url(pkg), ver, Read.sha1(pkg,ver)) + end + for (pkg,(_,ver)) in update + Cache.fetch(pkg, Read.url(pkg), ver, Read.sha1(pkg,ver)) + end -rm(pkg::ByteString) = Dir.cd() do - Write.update_file(Reqs.rm, "REQUIRE", pkg) + # try applying changes, roll back everything if anything fails + try + for (pkg,ver) in install + Write.install(pkg,ver) + end + for (pkg,(v1,v2)) in update + Write.update(pkg,v1,v2) + end + for (pkg,ver) in remove + Write.remove(pkg) + end + catch + for (pkg,ver) in remove + Write.install!(pkg,ver) + end + for (pkg,(v1,v2)) in update + Write.update!(pkg,v2,v1) + end + for (pkg,ver) in install + Write.remove!(pkg) + end + rethrow() + end end end # module diff --git a/base/pkg2/cache.jl b/base/pkg2/cache.jl new file mode 100644 index 0000000000000..e20f934addacd --- /dev/null +++ b/base/pkg2/cache.jl @@ -0,0 +1,24 @@ +module Cache + +using Base.Git + +const cache = ".cache" + +function fetch(pkg::String, url::String, ver::VersionNumber, sha1::String) + isdir(cache) || mkpath(cache) + cd(cache) do + if !isdir(pkg) + run(`git clone -q --bare $url $pkg`) + cd(pkg) do + run(`git config --remove-section remote.origin`) + end + end + cd(pkg) do + run(`git fetch -q --tags $url`) + Git.iscommit(sha1) || error("$pkg version $ver [$(sha1[1:8])] not found") + end + end + nothing +end + +end # module diff --git a/base/pkg2/dir.jl b/base/pkg2/dir.jl index 55bc5b7d79325..9c412722430e6 100644 --- a/base/pkg2/dir.jl +++ b/base/pkg2/dir.jl @@ -2,7 +2,7 @@ module Dir using Base.Git -const DEFAULT_META = "git://github.com/JuliaLang/METADATA.jl.git" +const DEFAULT_META = "git@github.com:JuliaLang/METADATA.jl.git" @unix_only const DIR_NAME = ".julia" @windows_only const DIR_NAME = "packages" @@ -36,26 +36,8 @@ function init(meta::String=DEFAULT_META) try run(`mkdir -p $d`) cd() do - # create & configure - run(`git init`) - run(`git commit --allow-empty -m "Initial empty commit"`) - run(`git remote add origin .`) - if success(`git config --global github.user`) - base = basename(d) - user = readchomp(`git config --global github.user`) - run(`git config remote.origin.url git@github.com:$user/$base`) - else - run(`git config --unset remote.origin.url`) - end - run(`git config branch.master.remote origin`) - run(`git config branch.master.merge refs/heads/master`) - # initial content + run(`git clone $meta METADATA`) run(`touch REQUIRE`) - run(`git add REQUIRE`) - run(`git submodule add -b devel $meta METADATA`) - run(`git commit -m "Empty package repo"`) - cd(Git.autoconfig_pushurl,"METADATA") - Metadata.gen_hashes() end catch e run(`rm -rf $d`) diff --git a/base/pkg2/query.jl b/base/pkg2/query.jl index 8fb5735def4d9..8a7b990b33ada 100644 --- a/base/pkg2/query.jl +++ b/base/pkg2/query.jl @@ -12,6 +12,7 @@ function requirements(reqs::Dict, fix::Dict) merge_requires!(reqs, f2.requires) end end + reqs = copy(reqs) for (p,f) in fix delete!(reqs, p, nothing) end @@ -19,12 +20,10 @@ function requirements(reqs::Dict, fix::Dict) end function dependencies(avail::Dict, fix::Dict) + avail = deepcopy(avail) for (fp,fx) in fix - if haskey(avail, fp) - delete!(avail, fp) - continue - end - for (ap,av) in avail, (v,a) in av + delete!(avail, fp, nothing) + for (ap,av) in avail, (v,a) in copy(av) if satisfies(fp, fx.version, a.requires) delete!(a.requires, fp, nothing) else @@ -39,24 +38,24 @@ function dependencies(avail::Dict, fix::Dict) end function diff(have::Dict, want::Dict) - remove = Dict{ByteString,VersionNumber}() - update = Dict{ByteString,(VersionNumber,VersionNumber)}() install = Dict{ByteString,VersionNumber}() + update = Dict{ByteString,(VersionNumber,VersionNumber)}() + remove = Dict{ByteString,VersionNumber}() for pkg in sort!(union(keys(have),keys(want))) h, w = haskey(have,pkg), haskey(want,pkg) if h && w if have[pkg] != want[pkg] - update[pkg] = (free[pkg], want[pkg]) + update[pkg] = (have[pkg], want[pkg]) end elseif h - remove[pkg] = free[pkg] + remove[pkg] = have[pkg] elseif w install[pkg] = want[pkg] end end - remove, update, install + install, update, remove end end # module diff --git a/base/pkg2/read.jl b/base/pkg2/read.jl index a0b2cce32bcaa..91cc1830df80f 100644 --- a/base/pkg2/read.jl +++ b/base/pkg2/read.jl @@ -2,6 +2,11 @@ module Read using ..Types, ..Reqs, Base.Git +readstrip(path...) = strip(readall(joinpath(path...))) + +url(pkg::String) = readstrip("METADATA", pkg, "url") +sha1(pkg::String, ver::VersionNumber) = readstrip("METADATA", pkg, "versions", string(ver), "sha1") + function available(names=readdir("METADATA")) pkgs = Dict{ByteString,Dict{VersionNumber,Available}}() for pkg in names diff --git a/base/pkg2/write.jl b/base/pkg2/write.jl index 7f22a34b5a3f9..a3fcecfe52aa1 100644 --- a/base/pkg2/write.jl +++ b/base/pkg2/write.jl @@ -2,7 +2,7 @@ module Write using ..Types, ..Reqs -function update_file(f::Function, file::String, args...) +function edit(f::Function, file::String, args...) tmp = "$file.$(randstring()).tmp" ispath(tmp) && error("tempfile $tmp already exists!?") try @@ -17,5 +17,37 @@ function update_file(f::Function, file::String, args...) ispath(tmp) && rm(tmp) end end +edit(file::String, f::Function, args...) = edit(f, file, args...) + +function install(pkg::String, ver::VersionNumber) + info("Installing $pkg v$ver") + if ispath(pkg) + error("Path $pkg already exists! Please remove to allow installation.") + end + run(`git clone --reference . $url $pkg`) + cd(pkg) do + if !success(`git checkout -q $ver`) + run(`git fetch -q`) + try run(`git checkout -q $ver`) + catch + error("An invalid SHA1 hash seems to be registered for $pkg. Please contact the package maintainer.") + end + end + end +end + +function update(pkg::String, A::VersionNumber, B::VersionNumber) + info("$(A <= B ? "Up" : "Down")grading $pkg: v$A => v$B") + cd(pkg) do + Git.transact() do + run(`git checkout -q $B`) + end + end +end + +function remove(pkg::String) + info("Removing $pkg v$ver") + run(`rm -rf -- $pkg`) +end end # module