pkg2 wip: package caching works; install, update and remove in progress.

base/pkg2.jl

@@ -6,28 +6,64 @@ include("pkg2/reqs.jl")
 include("pkg2/read.jl")
 include("pkg2/query.jl")
 include("pkg2/resolve.jl")
+include("pkg2/cache.jl")
 include("pkg2/write.jl")
 
 using .Types
 
+add(pkg::String, vers::VersionSet) = Dir.cd() do
+    Write.edit("REQUIRE", Reqs.add, pkg, vers)
+end
+add(pkg::String, vers::VersionNumber...) = add(pkg, VersionSet(vers...))
+
+rm(pkg::String) = Dir.cd() do
+    Write.edit("REQUIRE", Reqs.rm, pkg)
+end
+
 resolve() = Dir.cd() do
+    # figure out what should be installed
     reqs  = Reqs.parse("REQUIRE")
     avail = Read.available()
     fixed = Read.fixed(avail)
     reqs  = Query.requirements(reqs,fixed)
     deps  = Query.dependencies(avail,fixed)
     want  = Resolve.resolve(reqs,deps)
     have  = Read.free(avail)
-    diff  = Query.diff(have,want)
-end
 
-add(pkg::ByteString, vers::VersionSet) = Dir.cd() do
-    Write.update_file(Reqs.add, "REQUIRE", pkg, vers)
-end
-add(pkg::ByteString, vers::VersionNumber...) = add(pkg, VersionSet(vers...))
+    # compare what is installed with what should be
+    install, update, remove = Query.diff(have, want)
+
+    # prefetch phase isolates network activity, nothing to roll back
+    for (pkg,ver) in install
+        Cache.fetch(pkg, Read.url(pkg), ver, Read.sha1(pkg,ver))
+    end
+    for (pkg,(_,ver)) in update
+        Cache.fetch(pkg, Read.url(pkg), ver, Read.sha1(pkg,ver))
+    end
 
-rm(pkg::ByteString) = Dir.cd() do
-    Write.update_file(Reqs.rm, "REQUIRE", pkg)
+    # try applying changes, roll back everything if anything fails
+    try
+        for (pkg,ver) in install
+            Write.install(pkg,ver)
+        end
+        for (pkg,(v1,v2)) in update
+            Write.update(pkg,v1,v2)
+        end
+        for (pkg,ver) in remove
+            Write.remove(pkg)
+        end
+    catch
+        for (pkg,ver) in remove
+            Write.install!(pkg,ver)
+        end
+        for (pkg,(v1,v2)) in update
+            Write.update!(pkg,v2,v1)
+        end
+        for (pkg,ver) in install
+            Write.remove!(pkg)
+        end
+        rethrow()
+    end
 end
 
 end # module

base/pkg2/cache.jl

@@ -0,0 +1,24 @@
+module Cache
+
+using Base.Git
+
+const cache = ".cache"
+
+function fetch(pkg::String, url::String, ver::VersionNumber, sha1::String)
+	isdir(cache) || mkpath(cache)
+	cd(cache) do
+		if !isdir(pkg)
+			run(`git clone -q --bare $url $pkg`)
+			cd(pkg) do
+				run(`git config --remove-section remote.origin`)
+			end
+		end
+		cd(pkg) do
+			run(`git fetch -q --tags $url`)
+			Git.iscommit(sha1) || error("$pkg version $ver [$(sha1[1:8])] not found")
+		end
+	end
+	nothing
+end
+
+end # module

base/pkg2/dir.jl

@@ -2,7 +2,7 @@ module Dir
 
 using Base.Git
 
-const DEFAULT_META = "git://github.com/JuliaLang/METADATA.jl.git"
+const DEFAULT_META = "git@github.com:JuliaLang/METADATA.jl.git"
 
 @unix_only const DIR_NAME = ".julia"
 @windows_only const DIR_NAME = "packages"
@@ -36,26 +36,8 @@ function init(meta::String=DEFAULT_META)
     try
         run(`mkdir -p $d`)
         cd() do
-            # create & configure
-            run(`git init`)
-            run(`git commit --allow-empty -m "Initial empty commit"`)
-            run(`git remote add origin .`)
-            if success(`git config --global github.user`)
-                base = basename(d)
-                user = readchomp(`git config --global github.user`)
-                run(`git config remote.origin.url git@github.com:$user/$base`)
-            else
-                run(`git config --unset remote.origin.url`)
-            end
-            run(`git config branch.master.remote origin`)
-            run(`git config branch.master.merge refs/heads/master`)
-            # initial content
+            run(`git clone $meta METADATA`)
             run(`touch REQUIRE`)
-            run(`git add REQUIRE`)
-            run(`git submodule add -b devel $meta METADATA`)
-            run(`git commit -m "Empty package repo"`)
-            cd(Git.autoconfig_pushurl,"METADATA")
-            Metadata.gen_hashes()
         end
     catch e
         run(`rm -rf $d`)

base/pkg2/query.jl

@@ -12,19 +12,18 @@ function requirements(reqs::Dict, fix::Dict)
             merge_requires!(reqs, f2.requires)
         end
     end
+    reqs = copy(reqs)
     for (p,f) in fix
         delete!(reqs, p, nothing)
     end
     reqs
 end
 
 function dependencies(avail::Dict, fix::Dict)
+    avail = deepcopy(avail)
     for (fp,fx) in fix
-        if haskey(avail, fp)
-            delete!(avail, fp)
-            continue
-        end
-        for (ap,av) in avail, (v,a) in av
+        delete!(avail, fp, nothing)
+        for (ap,av) in avail, (v,a) in copy(av)
             if satisfies(fp, fx.version, a.requires)
                 delete!(a.requires, fp, nothing)
             else
@@ -39,24 +38,24 @@ function dependencies(avail::Dict, fix::Dict)
 end
 
 function diff(have::Dict, want::Dict)
-    remove = Dict{ByteString,VersionNumber}()
-    update = Dict{ByteString,(VersionNumber,VersionNumber)}()
     install = Dict{ByteString,VersionNumber}()
+    update  = Dict{ByteString,(VersionNumber,VersionNumber)}()
+    remove  = Dict{ByteString,VersionNumber}()
 
     for pkg in sort!(union(keys(have),keys(want)))
         h, w = haskey(have,pkg), haskey(want,pkg)
         if h && w
             if have[pkg] != want[pkg]
-                update[pkg] = (free[pkg], want[pkg])
+                update[pkg] = (have[pkg], want[pkg])
             end
         elseif h
-            remove[pkg] = free[pkg]
+            remove[pkg] = have[pkg]
         elseif w
             install[pkg] = want[pkg]
         end
     end
 
-    remove, update, install
+    install, update, remove
 end
 
 end # module

base/pkg2/read.jl

@@ -2,6 +2,11 @@ module Read
 
 using ..Types, ..Reqs, Base.Git
 
+readstrip(path...) = strip(readall(joinpath(path...)))
+
+url(pkg::String) = readstrip("METADATA", pkg, "url")
+sha1(pkg::String, ver::VersionNumber) = readstrip("METADATA", pkg, "versions", string(ver), "sha1")
+
 function available(names=readdir("METADATA"))
     pkgs = Dict{ByteString,Dict{VersionNumber,Available}}()
     for pkg in names

base/pkg2/write.jl

@@ -2,7 +2,7 @@ module Write
 
 using ..Types, ..Reqs
 
-function update_file(f::Function, file::String, args...)
+function edit(f::Function, file::String, args...)
     tmp = "$file.$(randstring()).tmp"
     ispath(tmp) && error("tempfile $tmp already exists!?")
     try
@@ -17,5 +17,37 @@ function update_file(f::Function, file::String, args...)
         ispath(tmp) && rm(tmp)
     end
 end
+edit(file::String, f::Function, args...) = edit(f, file, args...)
+
+function install(pkg::String, ver::VersionNumber)
+    info("Installing $pkg v$ver")
+    if ispath(pkg)
+        error("Path $pkg already exists! Please remove to allow installation.")
+    end
+    run(`git clone --reference . $url $pkg`)
+    cd(pkg) do
+        if !success(`git checkout -q $ver`)
+            run(`git fetch -q`)
+            try run(`git checkout -q $ver`)
+            catch
+                error("An invalid SHA1 hash seems to be registered for $pkg. Please contact the package maintainer.")
+            end
+        end
+    end
+end
+
+function update(pkg::String, A::VersionNumber, B::VersionNumber)
+    info("$(A <= B ? "Up" : "Down")grading $pkg: v$A => v$B")
+    cd(pkg) do
+        Git.transact() do
+            run(`git checkout -q $B`)
+        end
+    end
+end
+
+function remove(pkg::String)
+    info("Removing $pkg v$ver")
+    run(`rm -rf -- $pkg`)
+end
 
 end # module